From 60e83cc208083b7759fded7302c193d34c02cb1a Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Thu, 6 Jul 2023 23:09:08 +0200 Subject: Patch from Aaron Boxer adding initial support for GPU-powered J2K encoding via his tool "grok". --- src/lib/config.cc | 21 + src/lib/config.h | 50 ++- src/lib/dcp_encoder.cc | 9 +- src/lib/dcp_encoder.h | 2 + src/lib/dcp_video.cc | 21 + src/lib/dcp_video.h | 7 + src/lib/encoder.h | 2 + src/lib/grok_context.h | 245 ++++++++++++ src/lib/grok_messenger.h | 930 +++++++++++++++++++++++++++++++++++++++++++ src/lib/j2k_encoder.cc | 133 +++++-- src/lib/j2k_encoder.h | 11 +- src/lib/job.cc | 4 +- src/lib/job.h | 3 +- src/lib/transcode_job.cc | 8 + src/lib/transcode_job.h | 2 + src/tools/dcpomatic.cc | 1 + src/tools/dcpomatic_batch.cc | 1 + src/tools/dcpomatic_disk.cc | 1 + src/wx/about_dialog.cc | 3 +- src/wx/full_config_dialog.cc | 2 + src/wx/gpu_config_panel.h | 186 +++++++++ 21 files changed, 1591 insertions(+), 51 deletions(-) create mode 100644 src/lib/grok_context.h create mode 100644 src/lib/grok_messenger.h create mode 100644 src/wx/gpu_config_panel.h diff --git a/src/lib/config.cc b/src/lib/config.cc index 190817cbc..bbb3eef27 100644 --- a/src/lib/config.cc +++ b/src/lib/config.cc @@ -212,6 +212,13 @@ Config::set_defaults () set_notification_email_to_default (); set_cover_sheet_to_default (); + _gpu_binary_location = ""; + _enable_gpu = false; + _selected_gpu = 0; + _gpu_license_server = ""; + _gpu_license_port = 5000; + _gpu_license = ""; + _main_divider_sash_position = {}; _main_content_divider_sash_position = {}; @@ -633,6 +640,13 @@ try _allow_smpte_bv20 = f.optional_bool_child("AllowSMPTEBv20").get_value_or(false); _isdcf_name_part_length = f.optional_number_child("ISDCFNamePartLength").get_value_or(14); + _enable_gpu = f.optional_bool_child("EnableGpu").get_value_or(false); + _gpu_binary_location = f.string_child("GpuBinaryLocation"); + _selected_gpu = f.number_child("SelectedGpu"); + _gpu_license_server = f.string_child ("GpuLicenseServer"); + _gpu_license_port = f.number_child ("GpuLicensePort"); + _gpu_license = f.string_child("GpuLicense"); + _export.read(f.optional_node_child("Export")); } catch (...) { @@ -1119,6 +1133,13 @@ Config::write_config () const /* [XML] ISDCFNamePartLength Maximum length of the "name" part of an ISDCF name, which should be 14 according to the standard */ root->add_child("ISDCFNamePartLength")->add_child_text(raw_convert(_isdcf_name_part_length)); + root->add_child("GpuBinaryLocation")->add_child_text (_gpu_binary_location); + root->add_child("EnableGpu")->add_child_text ((_enable_gpu ? "1" : "0")); + root->add_child("SelectedGpu")->add_child_text (raw_convert (_selected_gpu)); + root->add_child("GpuLicenseServer")->add_child_text (_gpu_license_server); + root->add_child("GpuLicensePort")->add_child_text (raw_convert (_gpu_license_port)); + root->add_child("GpuLicense")->add_child_text (_gpu_license); + _export.write(root->add_child("Export")); auto target = config_write_file(); diff --git a/src/lib/config.h b/src/lib/config.h index 0a332bcbb..0c9affbb6 100644 --- a/src/lib/config.h +++ b/src/lib/config.h @@ -618,6 +618,28 @@ public: return _allow_smpte_bv20; } + std::string gpu_binary_location () const { + return _gpu_binary_location; + } + + bool enable_gpu () const { + return _enable_gpu; + } + + int selected_gpu () const { + return _selected_gpu; + } + std::string gpu_license_server () const { + return _gpu_license_server; + } + + int gpu_license_port () const { + return _gpu_license_port; + } + std::string gpu_license () const { + return _gpu_license; + } + int isdcf_name_part_length() const { return _isdcf_name_part_length; } @@ -1198,11 +1220,29 @@ public: void set_allow_smpte_bv20(bool allow) { maybe_set(_allow_smpte_bv20, allow, ALLOW_SMPTE_BV20); } - + void set_gpu_binary_location (std::string location) { + maybe_set (_gpu_binary_location, location); + } + void set_enable_gpu (bool enable) { + maybe_set (_enable_gpu, enable); + } + void set_selected_gpu (int selected) { + maybe_set (_selected_gpu, selected); + } + void set_gpu_license_server (std::string s) { + maybe_set (_gpu_license_server, s); + } + void set_gpu_license_port (int p) { + maybe_set (_gpu_license_port, p); + } + void set_gpu_license (std::string p) { + maybe_set (_gpu_license, p); + } void set_isdcf_name_part_length(int length) { maybe_set(_isdcf_name_part_length, length, ISDCF_NAME_PART_LENGTH); } + void changed (Property p = OTHER); boost::signals2::signal Changed; /** Emitted if read() failed on an existing Config file. There is nothing @@ -1443,6 +1483,14 @@ private: bool _allow_smpte_bv20; int _isdcf_name_part_length; + /* GPU */ + bool _enable_gpu; + std::string _gpu_binary_location; + int _selected_gpu; + std::string _gpu_license_server; + int _gpu_license_port; + std::string _gpu_license; + ExportConfig _export; static int const _current_version; diff --git a/src/lib/dcp_encoder.cc b/src/lib/dcp_encoder.cc index 9a840c8ab..7088225cd 100644 --- a/src/lib/dcp_encoder.cc +++ b/src/lib/dcp_encoder.cc @@ -114,10 +114,17 @@ DCPEncoder::go () } _finishing = true; - _j2k_encoder.end(); + _j2k_encoder.end(true); _writer.finish(_film->dir(_film->dcp_name())); } +void DCPEncoder::pause(void) { + _j2k_encoder.pause(); +} + +void DCPEncoder::resume(void) { + _j2k_encoder.resume(); +} void DCPEncoder::video (shared_ptr data, DCPTime time) { diff --git a/src/lib/dcp_encoder.h b/src/lib/dcp_encoder.h index ad77f6951..a8043887e 100644 --- a/src/lib/dcp_encoder.h +++ b/src/lib/dcp_encoder.h @@ -52,6 +52,8 @@ public: bool finishing () const override { return _finishing; } + void pause(void) override; + void resume(void) override; private: diff --git a/src/lib/dcp_video.cc b/src/lib/dcp_video.cc index 8eb76fdd6..78e973ca3 100644 --- a/src/lib/dcp_video.cc +++ b/src/lib/dcp_video.cc @@ -118,6 +118,27 @@ DCPVideo::convert_to_xyz (shared_ptr frame, dcp::NoteHandler return xyz; } +dcp::Size +DCPVideo::get_size(void) { + auto image = _frame->image (bind(&PlayerVideo::keep_xyz_or_rgb, _1), VideoRange::FULL, false); + return image->size(); +} + +void +DCPVideo::convert_to_xyz (uint16_t *dst) { + + auto image = _frame->image (bind(&PlayerVideo::keep_xyz_or_rgb, _1), VideoRange::FULL, false); + if (_frame->colour_conversion()) { + dcp::rgb_to_xyz ( + image->data()[0], + dst, + image->size(), + image->stride()[0], + _frame->colour_conversion().get() + ); + } +} + /** J2K-encode this frame on the local host. * @return Encoded data. */ diff --git a/src/lib/dcp_video.h b/src/lib/dcp_video.h index 33df0942c..c09442d16 100644 --- a/src/lib/dcp_video.h +++ b/src/lib/dcp_video.h @@ -17,6 +17,8 @@ along with DCP-o-matic. If not, see . */ +#ifndef DCPOMATIC_DCP_VIDEO_H +#define DCPOMATIC_DCP_VIDEO_H #include "encode_server_description.h" @@ -47,6 +49,7 @@ class PlayerVideo; class DCPVideo { public: + DCPVideo (void) : DCPVideo(nullptr,0,0,0,Resolution::TWO_K){} DCPVideo (std::shared_ptr, int index, int dcp_fps, int bandwidth, Resolution r); DCPVideo (std::shared_ptr, cxml::ConstNodePtr); @@ -66,6 +69,8 @@ public: static std::shared_ptr convert_to_xyz (std::shared_ptr frame, dcp::NoteHandler note); + void convert_to_xyz (uint16_t *dst); + dcp::Size get_size(void); private: void add_metadata (xmlpp::Element *) const; @@ -76,3 +81,5 @@ private: int _j2k_bandwidth; ///< J2K bandwidth to use Resolution _resolution; ///< Resolution (2K or 4K) }; + +#endif diff --git a/src/lib/encoder.h b/src/lib/encoder.h index 9b67720d3..f921fcb51 100644 --- a/src/lib/encoder.h +++ b/src/lib/encoder.h @@ -58,6 +58,8 @@ public: /** @return the number of frames that are done */ virtual Frame frames_done () const = 0; virtual bool finishing () const = 0; + virtual void pause(void) {} + virtual void resume(void) {} protected: std::shared_ptr _film; diff --git a/src/lib/grok_context.h b/src/lib/grok_context.h new file mode 100644 index 000000000..1f9726aae --- /dev/null +++ b/src/lib/grok_context.h @@ -0,0 +1,245 @@ +/* + Copyright (C) 2023 Grok Image Compression Inc. + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ + +#pragma once + +#include "config.h" +#include "log.h" +#include "dcpomatic_log.h" +#include "writer.h" +#include "grok_messenger.h" + +class Film; +using dcp::Data; +using namespace dcpomatic; + +static std::mutex launchMutex; + +namespace grk_plugin +{ + +struct GrokLogger : public MessengerLogger { + explicit GrokLogger(const std::string &preamble) : MessengerLogger(preamble) + {} + virtual ~GrokLogger() = default; + void info(const char* fmt, ...) override{ + va_list arg; + va_start(arg, fmt); + dcpomatic_log->log(preamble_ + log_message(fmt, arg),LogEntry::TYPE_GENERAL); + va_end(arg); + } + void warn(const char* fmt, ...) override{ + va_list arg; + va_start(arg, fmt); + dcpomatic_log->log(preamble_ + log_message(fmt, arg),LogEntry::TYPE_WARNING); + va_end(arg); + } + void error(const char* fmt, ...) override{ + va_list arg; + va_start(arg, fmt); + dcpomatic_log->log(preamble_ + log_message(fmt, arg),LogEntry::TYPE_ERROR); + va_end(arg); + } +}; + +struct GrokInitializer { + GrokInitializer(void) { + setMessengerLogger(new GrokLogger("[GROK] ")); + } + ~GrokInitializer() = default; +}; + +struct FrameProxy { + FrameProxy(void) : FrameProxy(0,Eyes::LEFT,DCPVideo()) + {} + FrameProxy(int index, Eyes eyes, DCPVideo dcpv) : index_(index), eyes_(eyes), vf(dcpv) + {} + int index() const { + return index_; + } + Eyes eyes(void) const { + return eyes_; + } + int index_; + Eyes eyes_; + DCPVideo vf; +}; + +struct DcpomaticContext { + DcpomaticContext(std::shared_ptr film, Writer& writer, + EventHistory &history, const std::string &location) : + film_(film), writer_(writer), + history_(history), location_(location), + width_(0), height_(0) + {} + void setDimensions(uint32_t w, uint32_t h) { + width_ = w; + height_ = h; + } + std::shared_ptr film_; + Writer& writer_; + EventHistory &history_; + std::string location_; + uint32_t width_; + uint32_t height_; +}; + +class GrokContext { +public: + explicit GrokContext(const DcpomaticContext &dcpomaticContext) : + dcpomaticContext_(dcpomaticContext), + messenger_(nullptr), + launched_(false) + { + struct CompressedData : public dcp::Data { + explicit CompressedData(int dataLen) : data_(new uint8_t[dataLen]), dataLen_(dataLen) + {} + ~CompressedData(void){ + delete[] data_; + } + uint8_t const * data () const override { + return data_; + } + uint8_t * data () override { + return data_; + } + int size () const override { + return dataLen_; + } + uint8_t *data_; + int dataLen_; + }; + if (Config::instance()->enable_gpu ()) { + boost::filesystem::path folder(dcpomaticContext_.location_); + boost::filesystem::path binaryPath = folder / "grk_compress"; + if (!boost::filesystem::exists(binaryPath)) { + getMessengerLogger()->error("Invalid binary location %s", + dcpomaticContext_.location_.c_str()); + return; + } + auto proc = [this](const std::string& str) { + try { + Msg msg(str); + auto tag = msg.next(); + if(tag == GRK_MSGR_BATCH_SUBMIT_COMPRESSED) + { + auto clientFrameId = msg.nextUint(); + auto compressedFrameId = msg.nextUint(); + (void)compressedFrameId; + auto compressedFrameLength = msg.nextUint(); + auto processor = + [this](FrameProxy srcFrame, uint8_t* compressed, uint32_t compressedFrameLength) + { + auto compressedData = std::make_shared(compressedFrameLength); + memcpy(compressedData->data_,compressed,compressedFrameLength ); + dcpomaticContext_.writer_.write(compressedData, srcFrame.index(), srcFrame.eyes()); + frame_done (); + }; + int const minimum_size = 16384; + bool needsRecompression = compressedFrameLength < minimum_size; + messenger_->processCompressed(str, processor, needsRecompression); + if (needsRecompression) { + bool success = false; + auto fp = messenger_->retrieve(clientFrameId, success); + if (!success) + return; + + auto encoded = std::make_shared(fp.vf.encode_locally()); + dcpomaticContext_.writer_.write(encoded, fp.vf.index(), fp.vf.eyes()); + frame_done (); + } + } + } catch (std::exception &ex){ + getMessengerLogger()->error("%s",ex.what()); + } + }; + auto clientInit = + MessengerInit(clientToGrokMessageBuf, clientSentSynch, grokReceiveReadySynch, + grokToClientMessageBuf, grokSentSynch, clientReceiveReadySynch, proc, + std::thread::hardware_concurrency()); + messenger_ = new ScheduledMessenger(clientInit); + } + } + ~GrokContext(void) { + shutdown(); + } + bool launch(DCPVideo dcpv, int device){ + if (!messenger_ ) + return false; + if (launched_) + return true; + std::unique_lock lk_global(launchMutex); + if (!messenger_) + return false; + if (launched_) + return true; + if (MessengerInit::firstLaunch(true)) { + auto s = dcpv.get_size(); + dcpomaticContext_.setDimensions(s.width, s.height); + auto config = Config::instance(); + messenger_->launchGrok(dcpomaticContext_.location_, + dcpomaticContext_.width_,dcpomaticContext_.width_, + dcpomaticContext_.height_, + 3, 12, device, + dcpomaticContext_.film_->resolution() == Resolution::FOUR_K, + dcpomaticContext_.film_->video_frame_rate(), + dcpomaticContext_.film_->j2k_bandwidth(), + config->gpu_license_server(), + config->gpu_license_port(), + config->gpu_license()); + } + launched_ = messenger_->waitForClientInit(); + + return launched_; + } + bool scheduleCompress(const DCPVideo &vf){ + if (!messenger_) + return false; + + auto fp = FrameProxy(vf.index(),vf.eyes(),vf); + auto cvt = [this, &fp](BufferSrc src){ + // xyz conversion + fp.vf.convert_to_xyz((uint16_t*)src.framePtr_); + }; + return messenger_->scheduleCompress(fp, cvt); + } + void shutdown(void){ + if (!messenger_) + return; + + std::unique_lock lk_global(launchMutex); + if (!messenger_) + return; + if (launched_) + messenger_->shutdown(); + delete messenger_; + messenger_ = nullptr; + } + void frame_done () { + dcpomaticContext_.history_.event (); + } +private: + DcpomaticContext dcpomaticContext_; + ScheduledMessenger *messenger_; + bool launched_; +}; + +} + diff --git a/src/lib/grok_messenger.h b/src/lib/grok_messenger.h new file mode 100644 index 000000000..45ee752e5 --- /dev/null +++ b/src/lib/grok_messenger.h @@ -0,0 +1,930 @@ +/* + Copyright (C) 2023 Grok Image Compression Inc. + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#include +#pragma warning(disable : 4100) +#else +#include +#include +#include +#include +#include +#endif + +namespace grk_plugin +{ +static std::string grokToClientMessageBuf = "Global\\grok_to_client_message"; +static std::string grokSentSynch = "Global\\grok_sent"; +static std::string clientReceiveReadySynch = "Global\\client_receive_ready"; +static std::string clientToGrokMessageBuf = "Global\\client_to_grok_message"; +static std::string clientSentSynch = "Global\\client_sent"; +static std::string grokReceiveReadySynch = "Global\\grok_receive_ready"; +static std::string grokUncompressedBuf = "Global\\grok_uncompressed_buf"; +static std::string grokCompressedBuf = "Global\\grok_compressed_buf"; +static const std::string GRK_MSGR_BATCH_IMAGE = "GRK_MSGR_BATCH_IMAGE"; +static const std::string GRK_MSGR_BATCH_COMPRESS_INIT = "GRK_MSGR_BATCH_COMPRESS_INIT"; +static const std::string GRK_MSGR_BATCH_SUBMIT_UNCOMPRESSED = "GRK_MSGR_BATCH_SUBMIT_UNCOMPRESSED"; +static const std::string GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED = + "GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED"; +static const std::string GRK_MSGR_BATCH_SUBMIT_COMPRESSED = "GRK_MSGR_BATCH_SUBMIT_COMPRESSED"; +static const std::string GRK_MSGR_BATCH_PROCESSSED_COMPRESSED = + "GRK_MSGR_BATCH_PROCESSSED_COMPRESSED"; +static const std::string GRK_MSGR_BATCH_SHUTDOWN = "GRK_MSGR_BATCH_SHUTDOWN"; +static const std::string GRK_MSGR_BATCH_FLUSH = "GRK_MSGR_BATCH_FLUSH"; +static const size_t messageBufferLen = 256; +struct IMessengerLogger +{ + virtual ~IMessengerLogger(void) = default; + virtual void info(const char* fmt, ...) = 0; + virtual void warn(const char* fmt, ...) = 0; + virtual void error(const char* fmt, ...) = 0; + + protected: + template + std::string log_message(char const* const format, Args&... args) noexcept + { + constexpr size_t message_size = 512; + char message[message_size]; + + std::snprintf(message, message_size, format, args...); + return std::string(message); + } +}; +struct MessengerLogger : public IMessengerLogger +{ + explicit MessengerLogger(const std::string &preamble) : preamble_(preamble) {} + virtual ~MessengerLogger() = default; + virtual void info(const char* fmt, ...) override + { + va_list args; + std::string new_fmt = preamble_ + fmt + "\n"; + va_start(args, fmt); + vfprintf(stdout, new_fmt.c_str(), args); + va_end(args); + } + virtual void warn(const char* fmt, ...) override + { + va_list args; + std::string new_fmt = preamble_ + fmt + "\n"; + va_start(args, fmt); + vfprintf(stdout, new_fmt.c_str(), args); + va_end(args); + } + virtual void error(const char* fmt, ...) override + { + va_list args; + std::string new_fmt = preamble_ + fmt + "\n"; + va_start(args, fmt); + vfprintf(stderr, new_fmt.c_str(), args); + va_end(args); + } + + protected: + std::string preamble_; +}; + +static IMessengerLogger* sLogger = nullptr; +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" +#endif +static void setMessengerLogger(IMessengerLogger* logger) +{ + delete sLogger; + sLogger = logger; +} +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif +static IMessengerLogger* getMessengerLogger(void) +{ + return sLogger; +} +struct MessengerInit +{ + MessengerInit(const std::string &outBuf, const std::string &outSent, + const std::string &outReceiveReady, const std::string &inBuf, + const std::string &inSent, + const std::string &inReceiveReady, + std::function processor, + size_t numProcessingThreads) + : outboundMessageBuf(outBuf), outboundSentSynch(outSent), + outboundReceiveReadySynch(outReceiveReady), inboundMessageBuf(inBuf), + inboundSentSynch(inSent), inboundReceiveReadySynch(inReceiveReady), processor_(processor), + numProcessingThreads_(numProcessingThreads), + uncompressedFrameSize_(0), compressedFrameSize_(0), + numFrames_(0) + { + if(firstLaunch(true)) + unlink(); + } + void unlink(void) + { +#ifndef _WIN32 + shm_unlink(grokToClientMessageBuf.c_str()); + shm_unlink(clientToGrokMessageBuf.c_str()); +#endif + } + static bool firstLaunch(bool isClient) + { + bool debugGrok = false; + return debugGrok != isClient; + } + std::string outboundMessageBuf; + std::string outboundSentSynch; + std::string outboundReceiveReadySynch; + + std::string inboundMessageBuf; + std::string inboundSentSynch; + std::string inboundReceiveReadySynch; + + std::function processor_; + size_t numProcessingThreads_; + + size_t uncompressedFrameSize_; + size_t compressedFrameSize_; + size_t numFrames_; +}; + +/*************************** Synchronization *******************************/ +enum SynchDirection +{ + SYNCH_SENT, + SYNCH_RECEIVE_READY +}; + +typedef int grk_handle; +struct Synch +{ + Synch(const std::string &sentSemName, const std::string &receiveReadySemName) + : sentSemName_(sentSemName), receiveReadySemName_(receiveReadySemName) + { + // unlink semaphores in case of previous crash + if(MessengerInit::firstLaunch(true)) + unlink(); + open(); + } + ~Synch() + { + close(); + if(MessengerInit::firstLaunch(true)) + unlink(); + } + void post(SynchDirection dir) + { + auto sem = (dir == SYNCH_SENT ? sentSem_ : receiveReadySem_); + int rc = sem_post(sem); + if(rc) + getMessengerLogger()->error("Error posting to semaphore: %s", strerror(errno)); + } + void wait(SynchDirection dir) + { + auto sem = dir == SYNCH_SENT ? sentSem_ : receiveReadySem_; + int rc = sem_wait(sem); + if(rc) + getMessengerLogger()->error("Error waiting for semaphore: %s", strerror(errno)); + } + void open(void) + { + sentSem_ = sem_open(sentSemName_.c_str(), O_CREAT, 0666, 0); + if(!sentSem_) + getMessengerLogger()->error("Error opening shared memory: %s", strerror(errno)); + receiveReadySem_ = sem_open(receiveReadySemName_.c_str(), O_CREAT, 0666, 1); + if(!receiveReadySem_) + getMessengerLogger()->error("Error opening shared memory: %s", strerror(errno)); + } + void close(void) + { + int rc = sem_close(sentSem_); + if(rc) + getMessengerLogger()->error("Error closing semaphore %s: %s", sentSemName_.c_str(), + strerror(errno)); + rc = sem_close(receiveReadySem_); + if(rc) + getMessengerLogger()->error("Error closing semaphore %s: %s", + receiveReadySemName_.c_str(), strerror(errno)); + } + void unlink(void) + { + int rc = sem_unlink(sentSemName_.c_str()); + if(rc == -1 && errno != ENOENT) + getMessengerLogger()->error("Error unlinking semaphore %s: %s", sentSemName_.c_str(), + strerror(errno)); + rc = sem_unlink(receiveReadySemName_.c_str()); + if(rc == -1 && errno != ENOENT) + getMessengerLogger()->error("Error unlinking semaphore %s: %s", + receiveReadySemName_.c_str(), strerror(errno)); + } + sem_t* sentSem_; + sem_t* receiveReadySem_; + + private: + std::string sentSemName_; + std::string receiveReadySemName_; +}; +struct SharedMemoryManager +{ + static bool initShm(const std::string &name, size_t len, grk_handle* shm_fd, char** buffer) + { + *shm_fd = shm_open(name.c_str(), O_CREAT | O_RDWR, 0666); + if(*shm_fd < 0) + { + getMessengerLogger()->error("Error opening shared memory: %s", strerror(errno)); + return false; + } + int rc = ftruncate(*shm_fd, sizeof(char) * len); + if(rc) + { + getMessengerLogger()->error("Error truncating shared memory: %s", strerror(errno)); + rc = close(*shm_fd); + if(rc) + getMessengerLogger()->error("Error closing shared memory: %s", strerror(errno)); + rc = shm_unlink(name.c_str()); + if(rc) + getMessengerLogger()->error("Error unlinking shared memory: %s", strerror(errno)); + return false; + } + *buffer = static_cast(mmap(0, len, PROT_WRITE, MAP_SHARED, *shm_fd, 0)); + if(!*buffer) + { + getMessengerLogger()->error("Error mapping shared memory: %s", strerror(errno)); + rc = close(*shm_fd); + if(rc) + getMessengerLogger()->error("Error closing shared memory: %s", strerror(errno)); + rc = shm_unlink(name.c_str()); + if(rc) + getMessengerLogger()->error("Error unlinking shared memory: %s", strerror(errno)); + } + + return *buffer != nullptr; + } + static bool deinitShm(const std::string &name, size_t len, grk_handle &shm_fd, char** buffer) + { + if (!*buffer || !shm_fd) + return true; + + int rc = munmap(*buffer, len); + *buffer = nullptr; + if(rc) + getMessengerLogger()->error("Error unmapping shared memory %s: %s", name.c_str(), strerror(errno)); + rc = close(shm_fd); + shm_fd = 0; + if(rc) + getMessengerLogger()->error("Error closing shared memory %s: %s", name.c_str(), strerror(errno)); + rc = shm_unlink(name.c_str()); + if(rc) + fprintf(stderr,"Error unlinking shared memory %s : %s\n", name.c_str(), strerror(errno)); + + return true; + } +}; + +template +class MessengerBlockingQueue +{ + public: + explicit MessengerBlockingQueue(size_t max) : active_(true), max_size_(max) {} + MessengerBlockingQueue() : MessengerBlockingQueue(UINT_MAX) {} + size_t size() const + { + return queue_.size(); + } + // deactivate and clear queue + void deactivate() + { + { + std::lock_guard lk(mutex_); + active_ = false; + while(!queue_.empty()) + queue_.pop(); + } + + // release all waiting threads + can_pop_.notify_all(); + can_push_.notify_all(); + } + void activate() + { + std::lock_guard lk(mutex_); + active_ = true; + } + bool push(Data const& value) + { + bool rc; + { + std::unique_lock lk(mutex_); + rc = push_(value); + } + if(rc) + can_pop_.notify_one(); + + return rc; + } + bool waitAndPush(Data& value) + { + bool rc; + { + std::unique_lock lk(mutex_); + if(!active_) + return false; + // in case of spurious wakeup, loop until predicate in lambda + // is satisfied. + can_push_.wait(lk, [this] { return queue_.size() < max_size_ || !active_; }); + rc = push_(value); + } + if(rc) + can_pop_.notify_one(); + + return rc; + } + bool pop(Data& value) + { + bool rc; + { + std::unique_lock lk(mutex_); + rc = pop_(value); + } + if(rc) + can_push_.notify_one(); + + return rc; + } + bool waitAndPop(Data& value) + { + bool rc; + { + std::unique_lock lk(mutex_); + if(!active_) + return false; + // in case of spurious wakeup, loop until predicate in lambda + // is satisfied. + can_pop_.wait(lk, [this] { return !queue_.empty() || !active_; }); + rc = pop_(value); + } + if(rc) + can_push_.notify_one(); + + return rc; + } + + private: + bool push_(Data const& value) + { + if(queue_.size() == max_size_ || !active_) + return false; + queue_.push(value); + + return true; + } + bool pop_(Data& value) + { + if(queue_.empty() || !active_) + return false; + value = queue_.front(); + queue_.pop(); + + return true; + } + std::queue queue_; + mutable std::mutex mutex_; + std::condition_variable can_pop_; + std::condition_variable can_push_; + bool active_; + size_t max_size_; +}; +struct BufferSrc +{ + BufferSrc(void) : BufferSrc("") {} + explicit BufferSrc(const std::string &file) : file_(file), clientFrameId_(0), frameId_(0), framePtr_(nullptr) + {} + BufferSrc(size_t clientFrameId, size_t frameId, uint8_t* framePtr) + : file_(""), clientFrameId_(clientFrameId), frameId_(frameId), framePtr_(framePtr) + {} + bool fromDisk(void) + { + return !file_.empty() && framePtr_ == nullptr; + } + size_t index() const + { + return clientFrameId_; + } + std::string file_; + size_t clientFrameId_; + size_t frameId_; + uint8_t* framePtr_; +}; + +struct Messenger; +static void outboundThread(Messenger* messenger, const std::string &sendBuf, Synch* synch); +static void inboundThread(Messenger* messenger, const std::string &receiveBuf, Synch* synch); +static void processorThread(Messenger* messenger, std::function processor); + +struct Messenger +{ + explicit Messenger(MessengerInit init) + : running(true), initialized_(false), shutdown_(false), init_(init), + outboundSynch_(nullptr), + inboundSynch_(nullptr), uncompressed_buffer_(nullptr), compressed_buffer_(nullptr), + uncompressed_fd_(0), compressed_fd_(0) + {} + virtual ~Messenger(void) + { + running = false; + sendQueue.deactivate(); + receiveQueue.deactivate(); + + if (outboundSynch_) { + outboundSynch_->post(SYNCH_RECEIVE_READY); + outbound.join(); + } + + if (inboundSynch_) { + inboundSynch_->post(SYNCH_SENT); + inbound.join(); + } + + for(auto& p : processors_) + p.join(); + + delete outboundSynch_; + delete inboundSynch_; + + deinitShm(); + } + void startThreads(void) { + outboundSynch_ = + new Synch(init_.outboundSentSynch, init_.outboundReceiveReadySynch); + outbound = std::thread(outboundThread, this, init_.outboundMessageBuf, outboundSynch_); + + inboundSynch_ = + new Synch(init_.inboundSentSynch, init_.inboundReceiveReadySynch); + inbound = std::thread(inboundThread, this, init_.inboundMessageBuf, inboundSynch_); + + for(size_t i = 0; i < init_.numProcessingThreads_; ++i) + processors_.push_back(std::thread(processorThread, this, init_.processor_)); + } + size_t serialize(const std::string &dir, size_t clientFrameId, uint8_t* compressedPtr, + size_t compressedLength) + { + char fname[512]; + if(!compressedPtr || !compressedLength) + return 0; + sprintf(fname, "%s/test_%d.j2k", dir.c_str(), (int)clientFrameId); + auto fp = fopen(fname, "wb"); + if(!fp) + return 0; + size_t written = fwrite(compressedPtr, 1, compressedLength, fp); + if(written != compressedLength) + { + fclose(fp); + return 0; + } + fflush(fp); + fclose(fp); + + return written; + } + bool initBuffers(void) + { + bool rc = true; + if(init_.uncompressedFrameSize_) + { + rc = rc && SharedMemoryManager::initShm(grokUncompressedBuf, + init_.uncompressedFrameSize_ * init_.numFrames_, + &uncompressed_fd_, &uncompressed_buffer_); + } + if(init_.compressedFrameSize_) + { + rc = rc && SharedMemoryManager::initShm(grokCompressedBuf, + init_.compressedFrameSize_ * init_.numFrames_, + &compressed_fd_, &compressed_buffer_); + } + + return rc; + } + + bool deinitShm(void) + { + bool rc = SharedMemoryManager::deinitShm(grokUncompressedBuf, + init_.uncompressedFrameSize_ * init_.numFrames_, + uncompressed_fd_, &uncompressed_buffer_); + rc = rc && SharedMemoryManager::deinitShm(grokCompressedBuf, + init_.compressedFrameSize_ * init_.numFrames_, + compressed_fd_, &compressed_buffer_); + + return rc; + } + template + void send(const std::string& str, Args... args) + { + std::ostringstream oss; + oss << str; + int dummy[] = {0, ((void)(oss << ',' << args), 0)...}; + static_cast(dummy); + + sendQueue.push(oss.str()); + } + static pid_t get_pid_by_process_name(const char* name) + { + char command[256]; + snprintf(command, sizeof(command), "pgrep %s", name); + auto pgrep = popen(command, "r"); + if(!pgrep) + return -1; + pid_t pid; + if(fscanf(pgrep, "%d", &pid) != 1) + pid = -1; + pclose(pgrep); + + return pid; + } + static bool terminate_process(const char* name) + { + auto pid = get_pid_by_process_name(name); + + return (pid != -1 && kill(pid, SIGTERM) != -1); + } + static bool kill_process(const char* name) + { + auto pid = get_pid_by_process_name(name); + + return (pid != -1 && kill(pid, SIGKILL) != -1); + } + void launchGrok(const std::string &dir, uint32_t width, uint32_t stride, + uint32_t height, uint32_t samplesPerPixel, uint32_t depth, + int device, bool is4K, uint32_t fps, uint32_t bandwidth, + const std::string server, uint32_t port, + const std::string license) + { + + std::unique_lock lk(shutdownMutex_); + if (async_result_.valid()) + return; + if(MessengerInit::firstLaunch(true)) + init_.unlink(); + startThreads(); + char _cmd[4096]; + auto fullServer = server + ":" + std::to_string(port); + sprintf(_cmd, + "./grk_compress -batch_src %s,%d,%d,%d,%d,%d -out_fmt j2k -out_dir - -k 1 " + "-G %d -%s %d,%d -j %s -J %s", + GRK_MSGR_BATCH_IMAGE.c_str(), width, stride, height, samplesPerPixel, depth, + device, is4K ? "cinema4K" : "cinema2K", fps, bandwidth, + license.c_str(), fullServer.c_str()); + launch(_cmd, dir); + } + void initClient(size_t uncompressedFrameSize, size_t compressedFrameSize, size_t numFrames) + { + // client fills queue with pending uncompressed buffers + init_.uncompressedFrameSize_ = uncompressedFrameSize; + init_.compressedFrameSize_ = compressedFrameSize; + init_.numFrames_ = numFrames; + initBuffers(); + auto ptr = uncompressed_buffer_; + for(size_t i = 0; i < init_.numFrames_; ++i) + { + availableBuffers_.push(BufferSrc(0, i, (uint8_t*)ptr)); + ptr += init_.uncompressedFrameSize_; + } + + std::unique_lock lk(shutdownMutex_); + initialized_ = true; + clientInitializedCondition_.notify_all(); + } + bool waitForClientInit(void) + { + if(initialized_) + return true; + + std::unique_lock lk(shutdownMutex_); + if(initialized_) + return true; + else if (shutdown_) + return false; + clientInitializedCondition_.wait(lk, [this]{return initialized_ || shutdown_;}); + + return initialized_ && !shutdown_; + } + static size_t uncompressedFrameSize(uint32_t w, uint32_t h, uint32_t samplesPerPixel) + { + return sizeof(uint16_t) * w * h * samplesPerPixel; + } + void reclaimCompressed(size_t frameId) + { + availableBuffers_.push(BufferSrc(0, frameId, getCompressedFrame(frameId))); + } + void reclaimUncompressed(size_t frameId) + { + availableBuffers_.push(BufferSrc(0, frameId, getUncompressedFrame(frameId))); + } + uint8_t* getUncompressedFrame(size_t frameId) + { + assert(frameId < init_.numFrames_); + if(frameId >= init_.numFrames_) + return nullptr; + + return (uint8_t*)(uncompressed_buffer_ + frameId * init_.uncompressedFrameSize_); + } + uint8_t* getCompressedFrame(size_t frameId) + { + assert(frameId < init_.numFrames_); + if(frameId >= init_.numFrames_) + return nullptr; + + return (uint8_t*)(compressed_buffer_ + frameId * init_.compressedFrameSize_); + } + std::atomic_bool running; + bool initialized_; + bool shutdown_; + MessengerBlockingQueue sendQueue; + MessengerBlockingQueue receiveQueue; + MessengerBlockingQueue availableBuffers_; + MessengerInit init_; + std::string cmd_; + std::future async_result_; + std::mutex shutdownMutex_; + std::condition_variable shutdownCondition_; + + protected: + std::condition_variable clientInitializedCondition_; + private: + void launch(const std::string &cmd, const std::string &dir) + { + // Change the working directory + if(!dir.empty()) + { + if(chdir(dir.c_str()) != 0) + { + getMessengerLogger()->error("Error: failed to change the working directory"); + return; + } + } + // Execute the command using std::async and std::system + cmd_ = cmd; + async_result_ = std::async(std::launch::async, [this]() { return std::system(cmd_.c_str()); }); + } + std::thread outbound; + Synch* outboundSynch_; + + std::thread inbound; + Synch* inboundSynch_; + + std::vector processors_; + char* uncompressed_buffer_; + char* compressed_buffer_; + + grk_handle uncompressed_fd_; + grk_handle compressed_fd_; +}; + +static void outboundThread(Messenger* messenger, const std::string &sendBuf, Synch* synch) +{ + grk_handle shm_fd = 0; + char* send_buffer = nullptr; + + if(!SharedMemoryManager::initShm(sendBuf, messageBufferLen, &shm_fd, &send_buffer)) + return; + while(messenger->running) + { + synch->wait(SYNCH_RECEIVE_READY); + if(!messenger->running) + break; + std::string message; + if(!messenger->sendQueue.waitAndPop(message)) + break; + if(!messenger->running) + break; + memcpy(send_buffer, message.c_str(), message.size() + 1); + synch->post(SYNCH_SENT); + } + SharedMemoryManager::deinitShm(sendBuf, messageBufferLen, shm_fd, &send_buffer); +} + +static void inboundThread(Messenger* messenger, const std::string &receiveBuf, Synch* synch) +{ + grk_handle shm_fd = 0; + char* receive_buffer = nullptr; + + if(!SharedMemoryManager::initShm(receiveBuf, messageBufferLen, &shm_fd, &receive_buffer)) + return; + while(messenger->running) + { + synch->wait(SYNCH_SENT); + if(!messenger->running) + break; + auto message = std::string(receive_buffer); + synch->post(SYNCH_RECEIVE_READY); + messenger->receiveQueue.push(message); + } + SharedMemoryManager::deinitShm(receiveBuf, messageBufferLen, shm_fd, &receive_buffer); +} +struct Msg +{ + explicit Msg(const std::string &msg) : ct_(0) + { + std::stringstream ss(msg); + while(ss.good()) + { + std::string substr; + std::getline(ss, substr, ','); + cs_.push_back(substr); + } + } + std::string next() + { + if(ct_ == cs_.size()) + { + getMessengerLogger()->error("Msg: comma separated list exhausted. returning empty."); + return ""; + } + return cs_[ct_++]; + } + + uint32_t nextUint(void) + { + return (uint32_t)std::stoi(next()); + } + + std::vector cs_; + size_t ct_; +}; +static void processorThread(Messenger* messenger, std::function processor) +{ + while(messenger->running) + { + std::string message; + if(!messenger->receiveQueue.waitAndPop(message)) + break; + if(!messenger->running) + break; + Msg msg(message); + auto tag = msg.next(); + if(tag == GRK_MSGR_BATCH_COMPRESS_INIT) + { + auto width = msg.nextUint(); + auto stride = msg.nextUint(); + (void)stride; + auto height = msg.nextUint(); + auto samplesPerPixel = msg.nextUint(); + auto depth = msg.nextUint(); + (void)depth; + messenger->init_.uncompressedFrameSize_ = + Messenger::uncompressedFrameSize(width, height, samplesPerPixel); + auto compressedFrameSize = msg.nextUint(); + auto numFrames = msg.nextUint(); + messenger->initClient(compressedFrameSize, compressedFrameSize, numFrames); + } + else if(tag == GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED) + { + messenger->reclaimUncompressed(msg.nextUint()); + } + else if(tag == GRK_MSGR_BATCH_PROCESSSED_COMPRESSED) + { + messenger->reclaimCompressed(msg.nextUint()); + } + processor(message); + } +} + +template +struct ScheduledFrames +{ + void store(F& val) + { + std::unique_lock lk(mapMutex_); + auto it = map_.find(val.index()); + if (it == map_.end()) + map_[val.index()] = val; + } + F retrieve(size_t index, bool &success) + { + std::unique_lock lk(mapMutex_); + success = false; + auto it = map_.find(index); + if(it == map_.end()) + return F(); + + success = true; + F val = it->second; + map_.erase(index); + + return val; + } + + private: + std::mutex mapMutex_; + std::map map_; +}; + +template +struct ScheduledMessenger : public Messenger +{ + explicit ScheduledMessenger(MessengerInit init) : Messenger(init), + framesScheduled_(0), + framesCompressed_(0) + {} + ~ScheduledMessenger(void) { + shutdown(); + } + bool scheduleCompress(F proxy, std::function converter){ + size_t frameSize = init_.uncompressedFrameSize_; + assert(frameSize >= init_.uncompressedFrameSize_); + BufferSrc src; + if(!availableBuffers_.waitAndPop(src)) + return false; + converter(src); + scheduledFrames_.store(proxy); + framesScheduled_++; + send(GRK_MSGR_BATCH_SUBMIT_UNCOMPRESSED, proxy.index(), src.frameId_); + + return true; + } + void processCompressed(const std::string &message, std::function processor, bool needsRecompression) { + Msg msg(message); + msg.next(); + auto clientFrameId = msg.nextUint(); + auto compressedFrameId = msg.nextUint(); + auto compressedFrameLength = msg.nextUint(); + if (!needsRecompression) { + bool success = false; + auto srcFrame = scheduledFrames_.retrieve(clientFrameId,success); + if (!success) + return; + processor(srcFrame, getCompressedFrame(compressedFrameId),compressedFrameLength); + } + ++framesCompressed_; + send(GRK_MSGR_BATCH_PROCESSSED_COMPRESSED, compressedFrameId); + if (shutdown_ && framesCompressed_ == framesScheduled_) + shutdownCondition_.notify_all(); + } + void shutdown(void){ + try { + std::unique_lock lk(shutdownMutex_); + if (!async_result_.valid()) + return; + shutdown_ = true; + if (framesScheduled_) { + uint32_t scheduled = framesScheduled_; + send(GRK_MSGR_BATCH_FLUSH, scheduled); + shutdownCondition_.wait(lk, [this] { return framesScheduled_ == framesCompressed_; }); + } + availableBuffers_.deactivate(); + send(GRK_MSGR_BATCH_SHUTDOWN); + int result = async_result_.get(); + if(result != 0) + getMessengerLogger()->error("Accelerator failed with return code: %d\n",result); + } catch (std::exception &ex) { + getMessengerLogger()->error("%s",ex.what()); + } + + } + F retrieve(size_t index, bool &success) { + return scheduledFrames_.retrieve(index, success); + } + void store(F& val) { + scheduledFrames_.store(val); + } + +private: + ScheduledFrames scheduledFrames_; + std::atomic framesScheduled_; + std::atomic framesCompressed_; +}; + +} // namespace grk_plugin diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index d2e840f85..11f403d95 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -53,6 +53,7 @@ using boost::optional; using dcp::Data; using namespace dcpomatic; +static grk_plugin::GrokInitializer grokInitializer; /** @param film Film that we are encoding. * @param writer Writer that we are using. @@ -60,7 +61,9 @@ using namespace dcpomatic; J2KEncoder::J2KEncoder(shared_ptr film, Writer& writer) : _film (film) , _history (200) - , _writer (writer) + , _writer (writer) , + dcpomaticContext_(film,writer,_history, Config::instance()->gpu_binary_location ()), + context_(Config::instance()->enable_gpu () ? new grk_plugin::GrokContext(dcpomaticContext_) : nullptr) { servers_list_changed (); } @@ -70,10 +73,13 @@ J2KEncoder::~J2KEncoder () { _server_found_connection.disconnect(); + { boost::mutex::scoped_lock lm (_threads_mutex); terminate_threads (); -} + } + delete context_; +} void J2KEncoder::begin () @@ -83,23 +89,35 @@ J2KEncoder::begin () ); } +void J2KEncoder::pause(void){ + if (Config::instance()->enable_gpu ()) + end(false); +} + +void J2KEncoder::resume(void){ + if (Config::instance()->enable_gpu ()) { + context_ = new grk_plugin::GrokContext(dcpomaticContext_); + servers_list_changed (); + } +} void -J2KEncoder::end () +J2KEncoder::end (bool isFinal) { - boost::mutex::scoped_lock lock (_queue_mutex); + if (isFinal) { + boost::mutex::scoped_lock lock (_queue_mutex); - LOG_GENERAL (N_("Clearing queue of %1"), _queue.size ()); + LOG_GENERAL (N_("Clearing queue of %1"), _queue.size ()); - /* Keep waking workers until the queue is empty */ - while (!_queue.empty ()) { - rethrow (); - _empty_condition.notify_all (); - _full_condition.wait (lock); + /* Keep waking workers until the queue is empty */ + while (!_queue.empty ()) { + rethrow (); + _empty_condition.notify_all (); + _full_condition.wait (lock); + } + lock.unlock (); } - lock.unlock (); - LOG_GENERAL_NC (N_("Terminating encoder threads")); { @@ -113,27 +131,38 @@ J2KEncoder::end () LOG_GENERAL (N_("Mopping up %1"), _queue.size()); /* The following sequence of events can occur in the above code: - 1. a remote worker takes the last image off the queue - 2. the loop above terminates - 3. the remote worker fails to encode the image and puts it back on the queue - 4. the remote worker is then terminated by terminate_threads + 1. a remote worker takes the last image off the queue + 2. the loop above terminates + 3. the remote worker fails to encode the image and puts it back on the queue + 4. the remote worker is then terminated by terminate_threads - So just mop up anything left in the queue here. + So just mop up anything left in the queue here. */ - - for (auto const& i: _queue) { - LOG_GENERAL(N_("Encode left-over frame %1"), i.index()); - try { - _writer.write( - make_shared(i.encode_locally()), - i.index(), - i.eyes() - ); - frame_done (); - } catch (std::exception& e) { - LOG_ERROR (N_("Local encode failed (%1)"), e.what ()); + if (isFinal) { + for (auto & i: _queue) { + if (Config::instance()->enable_gpu ()) { + if (!context_->scheduleCompress(i)){ + LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), i.index()); + // handle error + } + } + else { + LOG_GENERAL(N_("Encode left-over frame %1"), i.index()); + try { + _writer.write( + make_shared(i.encode_locally()), + i.index(), + i.eyes() + ); + frame_done (); + } catch (std::exception& e) { + LOG_ERROR (N_("Local encode failed (%1)"), e.what ()); + } + } } } + delete context_; + context_ = nullptr; } @@ -183,7 +212,10 @@ J2KEncoder::encode (shared_ptr pv, DCPTime time) size_t threads = 0; { boost::mutex::scoped_lock lm (_threads_mutex); - threads = _threads->size(); + if (_threads) + threads = _threads->size(); + else + threads = std::thread::hardware_concurrency(); } boost::mutex::scoped_lock queue_lock (_queue_mutex); @@ -223,13 +255,14 @@ J2KEncoder::encode (shared_ptr pv, DCPTime time) LOG_DEBUG_ENCODE("Frame @ %1 ENCODE", to_string(time)); /* Queue this new frame for encoding */ LOG_TIMING ("add-frame-to-queue queue=%1", _queue.size ()); - _queue.push_back (DCPVideo( + auto dcpv = DCPVideo( pv, position, _film->video_frame_rate(), _film->j2k_bandwidth(), _film->resolution() - )); + ); + _queue.push_back (dcpv); /* The queue might not be empty any more, so notify anything which is waiting on that. @@ -269,6 +302,8 @@ void J2KEncoder::encoder_thread (optional server) try { + auto config = Config::instance (); + start_of_thread ("J2KEncoder"); if (server) { @@ -332,14 +367,22 @@ try } } else { - try { - LOG_TIMING ("start-local-encode thread=%1 frame=%2", thread_id(), vf.index()); - encoded = make_shared(vf.encode_locally()); - LOG_TIMING ("finish-local-encode thread=%1 frame=%2", thread_id(), vf.index()); - } catch (std::exception& e) { - /* This is very bad, so don't cope with it, just pass it on */ - LOG_ERROR (N_("Local encode failed (%1)"), e.what ()); - throw; + if (context_) { + if (!context_->launch(vf, config->selected_gpu()) || !context_->scheduleCompress(vf)) { + LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index()); + _queue.push_front (vf); + } + + } else { + try { + LOG_TIMING ("start-local-encode thread=%1 frame=%2", thread_id(), vf.index()); + encoded = make_shared(vf.encode_locally()); + LOG_TIMING ("finish-local-encode thread=%1 frame=%2", thread_id(), vf.index()); + } catch (std::exception& e) { + /* This is very bad, so don't cope with it, just pass it on */ + LOG_ERROR (N_("Local encode failed (%1)"), e.what ()); + throw; + } } } @@ -347,10 +390,12 @@ try _writer.write(encoded, vf.index(), vf.eyes()); frame_done (); } else { - lock.lock (); - LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index()); - _queue.push_front (vf); - lock.unlock (); + if (!Config::instance()->enable_gpu ()) { + lock.lock (); + LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index()); + _queue.push_front (vf); + lock.unlock (); + } } } diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index 63228a6b8..6df30a3f7 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -29,6 +29,7 @@ #include "cross.h" +#include "dcp_video.h" #include "enum_indexed_vector.h" #include "event_history.h" #include "exception_store.h" @@ -41,8 +42,8 @@ #include #include +#include "grok_context.h" -class DCPVideo; class EncodeServerDescription; class Film; class Job; @@ -70,8 +71,11 @@ public: /** Called to pass a bit of video to be encoded as the next DCP frame */ void encode (std::shared_ptr pv, dcpomatic::DCPTime time); + void pause(void); + void resume(void); + /** Called when a processing run has finished */ - void end (); + void end (bool isFinal); boost::optional current_encoding_rate () const; int video_frames_enqueued () const; @@ -107,6 +111,9 @@ private: boost::optional _last_player_video_time; boost::signals2::scoped_connection _server_found_connection; + + grk_plugin::DcpomaticContext dcpomaticContext_; + grk_plugin::GrokContext *context_; }; diff --git a/src/lib/job.cc b/src/lib/job.cc index 727456523..e900a427c 100644 --- a/src/lib/job.cc +++ b/src/lib/job.cc @@ -655,7 +655,7 @@ void Job::cancel () { if (_thread.joinable()) { - resume(); + Job::resume(); _thread.interrupt (); _thread.join (); @@ -682,6 +682,7 @@ Job::pause_by_user () } if (paused) { + pause(); _pause_changed.notify_all (); } @@ -694,6 +695,7 @@ Job::pause_by_priority () { if (running ()) { set_state (PAUSED_BY_PRIORITY); + pause(); _pause_changed.notify_all (); } } diff --git a/src/lib/job.h b/src/lib/job.h index dc5f7bc34..a4b7a319d 100644 --- a/src/lib/job.h +++ b/src/lib/job.h @@ -56,9 +56,10 @@ public: } void start (); + virtual void pause() {} bool pause_by_user (); void pause_by_priority (); - void resume (); + virtual void resume (); void cancel (); bool is_new () const; diff --git a/src/lib/transcode_job.cc b/src/lib/transcode_job.cc index 1b2d2ddd5..0e47aa24c 100644 --- a/src/lib/transcode_job.cc +++ b/src/lib/transcode_job.cc @@ -147,6 +147,14 @@ TranscodeJob::run () } } +void TranscodeJob::pause() { + _encoder->pause(); +} + +void TranscodeJob::resume() { + _encoder->resume(); + Job::resume(); +} string TranscodeJob::status () const diff --git a/src/lib/transcode_job.h b/src/lib/transcode_job.h index b05b20a16..794cf7016 100644 --- a/src/lib/transcode_job.h +++ b/src/lib/transcode_job.h @@ -56,6 +56,8 @@ public: std::string name () const override; std::string json_name () const override; void run () override; + void pause() override; + void resume() override; std::string status () const override; bool enable_notify () const override { return true; diff --git a/src/tools/dcpomatic.cc b/src/tools/dcpomatic.cc index 979672fb4..8f34b6836 100644 --- a/src/tools/dcpomatic.cc +++ b/src/tools/dcpomatic.cc @@ -1180,6 +1180,7 @@ private: FontConfig::drop(); ev.Skip (); + JobManager::drop (); } void active_jobs_changed() diff --git a/src/tools/dcpomatic_batch.cc b/src/tools/dcpomatic_batch.cc index dc092bf8c..16da43410 100644 --- a/src/tools/dcpomatic_batch.cc +++ b/src/tools/dcpomatic_batch.cc @@ -288,6 +288,7 @@ private: } ev.Skip (); + JobManager::drop (); } void file_add_film () diff --git a/src/tools/dcpomatic_disk.cc b/src/tools/dcpomatic_disk.cc index 5941d6d70..989fe3385 100644 --- a/src/tools/dcpomatic_disk.cc +++ b/src/tools/dcpomatic_disk.cc @@ -269,6 +269,7 @@ private: } ev.Skip (); + JobManager::drop (); } void copy () diff --git a/src/wx/about_dialog.cc b/src/wx/about_dialog.cc index fbb89bfd6..51d49ce6c 100644 --- a/src/wx/about_dialog.cc +++ b/src/wx/about_dialog.cc @@ -86,7 +86,7 @@ AboutDialog::AboutDialog (wxWindow* parent) t = new StaticText ( this, - _("(C) 2012-2023 Carl Hetherington, Terrence Meiczinger\n Ole Laursen"), + _("(C) 2012-2023 Carl Hetherington, Terrence Meiczinger\nOle Laursen, Aaron Boxer"), wxDefaultPosition, wxDefaultSize, wxALIGN_CENTER ); @@ -99,6 +99,7 @@ AboutDialog::AboutDialog (wxWindow* parent) written_by.Add (wxT ("Terrence Meiczinger")); written_by.Add (wxT ("Mart Jansink")); written_by.Add (wxT ("Ole Laursen")); + written_by.Add (wxT ("Aaron Boxer")); add_section (_("Written by"), written_by); wxArrayString with_help_from; diff --git a/src/wx/full_config_dialog.cc b/src/wx/full_config_dialog.cc index 175d78730..df974c1fa 100644 --- a/src/wx/full_config_dialog.cc +++ b/src/wx/full_config_dialog.cc @@ -45,6 +45,7 @@ #include "send_test_email_dialog.h" #include "server_dialog.h" #include "static_text.h" +#include "gpu_config_panel.h" #include "wx_util.h" #include "lib/config.h" #include "lib/cross.h" @@ -1944,6 +1945,7 @@ create_full_config_dialog () e->AddPage (new SoundPage (ps, border)); e->AddPage (new DefaultsPage (ps, border)); e->AddPage (new EncodingServersPage(ps, border)); + e->AddPage (new GPUPage (ps, border)); e->AddPage (new KeysPage (ps, border)); e->AddPage (new TMSPage (ps, border)); e->AddPage (new EmailPage (ps, border)); diff --git a/src/wx/gpu_config_panel.h b/src/wx/gpu_config_panel.h new file mode 100644 index 000000000..1478434be --- /dev/null +++ b/src/wx/gpu_config_panel.h @@ -0,0 +1,186 @@ +#pragma once + +static std::vector get_gpu_names(std::string binary, std::string filename) +{ + // Execute the GPU listing program and redirect its output to a file + std::system((binary + " > " + filename).c_str()); + + std::vector gpu_names; + std::ifstream file(filename); + if (file.is_open()) + { + std::string line; + while (std::getline(file, line)) + gpu_names.push_back(line); + file.close(); + } + + return gpu_names; +} + +class GpuList : public wxPanel +{ +public: + GpuList(wxPanel* parent) : wxPanel(parent, wxID_ANY), selection(0) { + comboBox = new wxComboBox(this, wxID_ANY, "", wxDefaultPosition, wxSize(400, -1)); + comboBox->Bind(wxEVT_COMBOBOX, &GpuList::OnComboBox, this); + update(); + + wxBoxSizer* sizer = new wxBoxSizer(wxHORIZONTAL); + + sizer->Add(comboBox, 0, wxALIGN_CENTER_VERTICAL); // Vertically center the comboBox + + this->SetSizerAndFit(sizer); + } + void update(void) { + auto cfg = Config::instance(); + auto lister_binary = cfg->gpu_binary_location() + "/" + "gpu_lister"; + auto lister_file = cfg->gpu_binary_location () + "/" + "gpus.txt"; + if (boost::filesystem::exists(lister_binary)) { + auto gpu_names = get_gpu_names(lister_binary, lister_file); + + comboBox->Clear(); + for (const auto& name : gpu_names) + comboBox->Append(name); + } + } + + int getSelection(void) { + return selection; + } + void setSelection(int sel) { + if ((int)comboBox->GetCount() > sel) + comboBox->SetSelection(sel); + } + +private: + void OnComboBox([[maybe_unused]] wxCommandEvent& event) { + selection = comboBox->GetSelection(); + if (selection != wxNOT_FOUND) + Config::instance ()->set_selected_gpu(selection); + } + + wxComboBox* comboBox; + int selection; +}; + +class GPUPage : public Page +{ +public: + GPUPage (wxSize panel_size, int border) + : Page (panel_size, border), + _enable_gpu(nullptr), _binary_location(nullptr), _gpu_list_control(nullptr) + {} + + wxString GetName () const override + { + return _("GPU"); + } + +#ifdef DCPOMATIC_OSX + wxBitmap GetLargeIcon () const override + { + return wxBitmap(icon_path("tms"), wxBITMAP_TYPE_PNG); + } +#endif + +private: + void setup () override + { + auto config = Config::instance (); + + _enable_gpu = new CheckBox (_panel, _("Enable GPU Acceleration")); + _panel->GetSizer()->Add (_enable_gpu, 0, wxALL | wxEXPAND, _border); + + wxFlexGridSizer* table = new wxFlexGridSizer (2, DCPOMATIC_SIZER_X_GAP, DCPOMATIC_SIZER_Y_GAP); + table->AddGrowableCol (1, 1); + _panel->GetSizer()->Add (table, 1, wxALL | wxEXPAND, _border); + + add_label_to_sizer (table, _panel, _("Acceleration Binary Folder"), true, 0, wxLEFT | wxLEFT | wxALIGN_CENTRE_VERTICAL); + _binary_location = new wxDirPickerCtrl (_panel, wxDD_DIR_MUST_EXIST); + table->Add (_binary_location, 1, wxEXPAND); + + add_label_to_sizer (table, _panel, _("GPU Selection"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + _gpu_list_control = new GpuList(_panel); + table->Add (_gpu_list_control, 1, wxEXPAND); + + add_label_to_sizer (table, _panel, _("License Server"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + _server = new wxTextCtrl (_panel, wxID_ANY); + table->Add (_server, 1, wxEXPAND | wxALL); + + add_label_to_sizer (table, _panel, _("Port"), false, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + _port = new wxSpinCtrl (_panel, wxID_ANY); + _port->SetRange (0, 65535); + table->Add (_port); + + add_label_to_sizer (table, _panel, _("License"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + _license = new PasswordEntry (_panel); + table->Add (_license->get_panel(), 1, wxEXPAND | wxALL); + + _enable_gpu->bind(&GPUPage::enable_gpu_changed, this); + _binary_location->Bind (wxEVT_DIRPICKER_CHANGED, boost::bind (&GPUPage::binary_location_changed, this)); + _server->Bind (wxEVT_TEXT, boost::bind(&GPUPage::server_changed, this)); + _port->Bind (wxEVT_SPINCTRL, boost::bind(&GPUPage::port_changed, this)); + _license->Changed.connect (boost::bind(&GPUPage::license_changed, this)); + + _binary_location->Enable(config->enable_gpu()); + _gpu_list_control->Enable(config->enable_gpu()); + _server->Enable(config->enable_gpu()); + _port->Enable(config->enable_gpu()); + _license->get_panel()->Enable(config->enable_gpu()); + } + + + void config_changed () override + { + auto config = Config::instance (); + + checked_set (_enable_gpu, config->enable_gpu()); + _binary_location->SetPath(config->gpu_binary_location ()); + _gpu_list_control->update(); + _gpu_list_control->setSelection(config->selected_gpu()); + checked_set (_server, config->gpu_license_server()); + checked_set (_port, config->gpu_license_port()); + checked_set (_license, config->gpu_license()); + } + + void enable_gpu_changed () + { + auto config = Config::instance (); + + config->set_enable_gpu (_enable_gpu->GetValue()); + _binary_location->Enable(config->enable_gpu()); + _gpu_list_control->Enable(config->enable_gpu()); + _server->Enable(config->enable_gpu()); + _port->Enable(config->enable_gpu()); + _license->get_panel()->Enable(config->enable_gpu()); + } + + void binary_location_changed () + { + Config::instance()->set_gpu_binary_location (wx_to_std (_binary_location->GetPath ())); + _gpu_list_control->update(); + } + + void server_changed () + { + Config::instance()->set_gpu_license_server(wx_to_std(_server->GetValue())); + } + + void port_changed () + { + Config::instance()->set_gpu_license_port(_port->GetValue()); + } + + void license_changed () + { + Config::instance()->set_gpu_license(_license->get()); + } + + CheckBox* _enable_gpu; + wxDirPickerCtrl* _binary_location; + GpuList *_gpu_list_control; + wxTextCtrl* _server; + wxSpinCtrl* _port; + PasswordEntry* _license; +}; -- cgit v1.2.3 From 444d0aa066758da92f000f434a53a1ba043dbfad Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Fri, 7 Jul 2023 23:24:45 +0200 Subject: Move grok headers into src/lib/grok --- src/lib/grok/context.h | 246 +++++++++++++ src/lib/grok/messenger.h | 930 +++++++++++++++++++++++++++++++++++++++++++++++ src/lib/grok_context.h | 245 ------------- src/lib/grok_messenger.h | 930 ----------------------------------------------- src/lib/j2k_encoder.h | 3 +- 5 files changed, 1177 insertions(+), 1177 deletions(-) create mode 100644 src/lib/grok/context.h create mode 100644 src/lib/grok/messenger.h delete mode 100644 src/lib/grok_context.h delete mode 100644 src/lib/grok_messenger.h diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h new file mode 100644 index 000000000..7a447d35a --- /dev/null +++ b/src/lib/grok/context.h @@ -0,0 +1,246 @@ +/* + Copyright (C) 2023 Grok Image Compression Inc. + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ + +#pragma once + +#include "../config.h" +#include "../dcp_video.h" +#include "../log.h" +#include "../dcpomatic_log.h" +#include "../writer.h" +#include "messenger.h" + +class Film; +using dcp::Data; +using namespace dcpomatic; + +static std::mutex launchMutex; + +namespace grk_plugin +{ + +struct GrokLogger : public MessengerLogger { + explicit GrokLogger(const std::string &preamble) : MessengerLogger(preamble) + {} + virtual ~GrokLogger() = default; + void info(const char* fmt, ...) override{ + va_list arg; + va_start(arg, fmt); + dcpomatic_log->log(preamble_ + log_message(fmt, arg),LogEntry::TYPE_GENERAL); + va_end(arg); + } + void warn(const char* fmt, ...) override{ + va_list arg; + va_start(arg, fmt); + dcpomatic_log->log(preamble_ + log_message(fmt, arg),LogEntry::TYPE_WARNING); + va_end(arg); + } + void error(const char* fmt, ...) override{ + va_list arg; + va_start(arg, fmt); + dcpomatic_log->log(preamble_ + log_message(fmt, arg),LogEntry::TYPE_ERROR); + va_end(arg); + } +}; + +struct GrokInitializer { + GrokInitializer(void) { + setMessengerLogger(new GrokLogger("[GROK] ")); + } + ~GrokInitializer() = default; +}; + +struct FrameProxy { + FrameProxy(void) : FrameProxy(0,Eyes::LEFT,DCPVideo()) + {} + FrameProxy(int index, Eyes eyes, DCPVideo dcpv) : index_(index), eyes_(eyes), vf(dcpv) + {} + int index() const { + return index_; + } + Eyes eyes(void) const { + return eyes_; + } + int index_; + Eyes eyes_; + DCPVideo vf; +}; + +struct DcpomaticContext { + DcpomaticContext(std::shared_ptr film, Writer& writer, + EventHistory &history, const std::string &location) : + film_(film), writer_(writer), + history_(history), location_(location), + width_(0), height_(0) + {} + void setDimensions(uint32_t w, uint32_t h) { + width_ = w; + height_ = h; + } + std::shared_ptr film_; + Writer& writer_; + EventHistory &history_; + std::string location_; + uint32_t width_; + uint32_t height_; +}; + +class GrokContext { +public: + explicit GrokContext(const DcpomaticContext &dcpomaticContext) : + dcpomaticContext_(dcpomaticContext), + messenger_(nullptr), + launched_(false) + { + struct CompressedData : public dcp::Data { + explicit CompressedData(int dataLen) : data_(new uint8_t[dataLen]), dataLen_(dataLen) + {} + ~CompressedData(void){ + delete[] data_; + } + uint8_t const * data () const override { + return data_; + } + uint8_t * data () override { + return data_; + } + int size () const override { + return dataLen_; + } + uint8_t *data_; + int dataLen_; + }; + if (Config::instance()->enable_gpu ()) { + boost::filesystem::path folder(dcpomaticContext_.location_); + boost::filesystem::path binaryPath = folder / "grk_compress"; + if (!boost::filesystem::exists(binaryPath)) { + getMessengerLogger()->error("Invalid binary location %s", + dcpomaticContext_.location_.c_str()); + return; + } + auto proc = [this](const std::string& str) { + try { + Msg msg(str); + auto tag = msg.next(); + if(tag == GRK_MSGR_BATCH_SUBMIT_COMPRESSED) + { + auto clientFrameId = msg.nextUint(); + auto compressedFrameId = msg.nextUint(); + (void)compressedFrameId; + auto compressedFrameLength = msg.nextUint(); + auto processor = + [this](FrameProxy srcFrame, uint8_t* compressed, uint32_t compressedFrameLength) + { + auto compressedData = std::make_shared(compressedFrameLength); + memcpy(compressedData->data_,compressed,compressedFrameLength ); + dcpomaticContext_.writer_.write(compressedData, srcFrame.index(), srcFrame.eyes()); + frame_done (); + }; + int const minimum_size = 16384; + bool needsRecompression = compressedFrameLength < minimum_size; + messenger_->processCompressed(str, processor, needsRecompression); + if (needsRecompression) { + bool success = false; + auto fp = messenger_->retrieve(clientFrameId, success); + if (!success) + return; + + auto encoded = std::make_shared(fp.vf.encode_locally()); + dcpomaticContext_.writer_.write(encoded, fp.vf.index(), fp.vf.eyes()); + frame_done (); + } + } + } catch (std::exception &ex){ + getMessengerLogger()->error("%s",ex.what()); + } + }; + auto clientInit = + MessengerInit(clientToGrokMessageBuf, clientSentSynch, grokReceiveReadySynch, + grokToClientMessageBuf, grokSentSynch, clientReceiveReadySynch, proc, + std::thread::hardware_concurrency()); + messenger_ = new ScheduledMessenger(clientInit); + } + } + ~GrokContext(void) { + shutdown(); + } + bool launch(DCPVideo dcpv, int device){ + if (!messenger_ ) + return false; + if (launched_) + return true; + std::unique_lock lk_global(launchMutex); + if (!messenger_) + return false; + if (launched_) + return true; + if (MessengerInit::firstLaunch(true)) { + auto s = dcpv.get_size(); + dcpomaticContext_.setDimensions(s.width, s.height); + auto config = Config::instance(); + messenger_->launchGrok(dcpomaticContext_.location_, + dcpomaticContext_.width_,dcpomaticContext_.width_, + dcpomaticContext_.height_, + 3, 12, device, + dcpomaticContext_.film_->resolution() == Resolution::FOUR_K, + dcpomaticContext_.film_->video_frame_rate(), + dcpomaticContext_.film_->j2k_bandwidth(), + config->gpu_license_server(), + config->gpu_license_port(), + config->gpu_license()); + } + launched_ = messenger_->waitForClientInit(); + + return launched_; + } + bool scheduleCompress(const DCPVideo &vf){ + if (!messenger_) + return false; + + auto fp = FrameProxy(vf.index(),vf.eyes(),vf); + auto cvt = [this, &fp](BufferSrc src){ + // xyz conversion + fp.vf.convert_to_xyz((uint16_t*)src.framePtr_); + }; + return messenger_->scheduleCompress(fp, cvt); + } + void shutdown(void){ + if (!messenger_) + return; + + std::unique_lock lk_global(launchMutex); + if (!messenger_) + return; + if (launched_) + messenger_->shutdown(); + delete messenger_; + messenger_ = nullptr; + } + void frame_done () { + dcpomaticContext_.history_.event (); + } +private: + DcpomaticContext dcpomaticContext_; + ScheduledMessenger *messenger_; + bool launched_; +}; + +} + diff --git a/src/lib/grok/messenger.h b/src/lib/grok/messenger.h new file mode 100644 index 000000000..45ee752e5 --- /dev/null +++ b/src/lib/grok/messenger.h @@ -0,0 +1,930 @@ +/* + Copyright (C) 2023 Grok Image Compression Inc. + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#include +#pragma warning(disable : 4100) +#else +#include +#include +#include +#include +#include +#endif + +namespace grk_plugin +{ +static std::string grokToClientMessageBuf = "Global\\grok_to_client_message"; +static std::string grokSentSynch = "Global\\grok_sent"; +static std::string clientReceiveReadySynch = "Global\\client_receive_ready"; +static std::string clientToGrokMessageBuf = "Global\\client_to_grok_message"; +static std::string clientSentSynch = "Global\\client_sent"; +static std::string grokReceiveReadySynch = "Global\\grok_receive_ready"; +static std::string grokUncompressedBuf = "Global\\grok_uncompressed_buf"; +static std::string grokCompressedBuf = "Global\\grok_compressed_buf"; +static const std::string GRK_MSGR_BATCH_IMAGE = "GRK_MSGR_BATCH_IMAGE"; +static const std::string GRK_MSGR_BATCH_COMPRESS_INIT = "GRK_MSGR_BATCH_COMPRESS_INIT"; +static const std::string GRK_MSGR_BATCH_SUBMIT_UNCOMPRESSED = "GRK_MSGR_BATCH_SUBMIT_UNCOMPRESSED"; +static const std::string GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED = + "GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED"; +static const std::string GRK_MSGR_BATCH_SUBMIT_COMPRESSED = "GRK_MSGR_BATCH_SUBMIT_COMPRESSED"; +static const std::string GRK_MSGR_BATCH_PROCESSSED_COMPRESSED = + "GRK_MSGR_BATCH_PROCESSSED_COMPRESSED"; +static const std::string GRK_MSGR_BATCH_SHUTDOWN = "GRK_MSGR_BATCH_SHUTDOWN"; +static const std::string GRK_MSGR_BATCH_FLUSH = "GRK_MSGR_BATCH_FLUSH"; +static const size_t messageBufferLen = 256; +struct IMessengerLogger +{ + virtual ~IMessengerLogger(void) = default; + virtual void info(const char* fmt, ...) = 0; + virtual void warn(const char* fmt, ...) = 0; + virtual void error(const char* fmt, ...) = 0; + + protected: + template + std::string log_message(char const* const format, Args&... args) noexcept + { + constexpr size_t message_size = 512; + char message[message_size]; + + std::snprintf(message, message_size, format, args...); + return std::string(message); + } +}; +struct MessengerLogger : public IMessengerLogger +{ + explicit MessengerLogger(const std::string &preamble) : preamble_(preamble) {} + virtual ~MessengerLogger() = default; + virtual void info(const char* fmt, ...) override + { + va_list args; + std::string new_fmt = preamble_ + fmt + "\n"; + va_start(args, fmt); + vfprintf(stdout, new_fmt.c_str(), args); + va_end(args); + } + virtual void warn(const char* fmt, ...) override + { + va_list args; + std::string new_fmt = preamble_ + fmt + "\n"; + va_start(args, fmt); + vfprintf(stdout, new_fmt.c_str(), args); + va_end(args); + } + virtual void error(const char* fmt, ...) override + { + va_list args; + std::string new_fmt = preamble_ + fmt + "\n"; + va_start(args, fmt); + vfprintf(stderr, new_fmt.c_str(), args); + va_end(args); + } + + protected: + std::string preamble_; +}; + +static IMessengerLogger* sLogger = nullptr; +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" +#endif +static void setMessengerLogger(IMessengerLogger* logger) +{ + delete sLogger; + sLogger = logger; +} +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif +static IMessengerLogger* getMessengerLogger(void) +{ + return sLogger; +} +struct MessengerInit +{ + MessengerInit(const std::string &outBuf, const std::string &outSent, + const std::string &outReceiveReady, const std::string &inBuf, + const std::string &inSent, + const std::string &inReceiveReady, + std::function processor, + size_t numProcessingThreads) + : outboundMessageBuf(outBuf), outboundSentSynch(outSent), + outboundReceiveReadySynch(outReceiveReady), inboundMessageBuf(inBuf), + inboundSentSynch(inSent), inboundReceiveReadySynch(inReceiveReady), processor_(processor), + numProcessingThreads_(numProcessingThreads), + uncompressedFrameSize_(0), compressedFrameSize_(0), + numFrames_(0) + { + if(firstLaunch(true)) + unlink(); + } + void unlink(void) + { +#ifndef _WIN32 + shm_unlink(grokToClientMessageBuf.c_str()); + shm_unlink(clientToGrokMessageBuf.c_str()); +#endif + } + static bool firstLaunch(bool isClient) + { + bool debugGrok = false; + return debugGrok != isClient; + } + std::string outboundMessageBuf; + std::string outboundSentSynch; + std::string outboundReceiveReadySynch; + + std::string inboundMessageBuf; + std::string inboundSentSynch; + std::string inboundReceiveReadySynch; + + std::function processor_; + size_t numProcessingThreads_; + + size_t uncompressedFrameSize_; + size_t compressedFrameSize_; + size_t numFrames_; +}; + +/*************************** Synchronization *******************************/ +enum SynchDirection +{ + SYNCH_SENT, + SYNCH_RECEIVE_READY +}; + +typedef int grk_handle; +struct Synch +{ + Synch(const std::string &sentSemName, const std::string &receiveReadySemName) + : sentSemName_(sentSemName), receiveReadySemName_(receiveReadySemName) + { + // unlink semaphores in case of previous crash + if(MessengerInit::firstLaunch(true)) + unlink(); + open(); + } + ~Synch() + { + close(); + if(MessengerInit::firstLaunch(true)) + unlink(); + } + void post(SynchDirection dir) + { + auto sem = (dir == SYNCH_SENT ? sentSem_ : receiveReadySem_); + int rc = sem_post(sem); + if(rc) + getMessengerLogger()->error("Error posting to semaphore: %s", strerror(errno)); + } + void wait(SynchDirection dir) + { + auto sem = dir == SYNCH_SENT ? sentSem_ : receiveReadySem_; + int rc = sem_wait(sem); + if(rc) + getMessengerLogger()->error("Error waiting for semaphore: %s", strerror(errno)); + } + void open(void) + { + sentSem_ = sem_open(sentSemName_.c_str(), O_CREAT, 0666, 0); + if(!sentSem_) + getMessengerLogger()->error("Error opening shared memory: %s", strerror(errno)); + receiveReadySem_ = sem_open(receiveReadySemName_.c_str(), O_CREAT, 0666, 1); + if(!receiveReadySem_) + getMessengerLogger()->error("Error opening shared memory: %s", strerror(errno)); + } + void close(void) + { + int rc = sem_close(sentSem_); + if(rc) + getMessengerLogger()->error("Error closing semaphore %s: %s", sentSemName_.c_str(), + strerror(errno)); + rc = sem_close(receiveReadySem_); + if(rc) + getMessengerLogger()->error("Error closing semaphore %s: %s", + receiveReadySemName_.c_str(), strerror(errno)); + } + void unlink(void) + { + int rc = sem_unlink(sentSemName_.c_str()); + if(rc == -1 && errno != ENOENT) + getMessengerLogger()->error("Error unlinking semaphore %s: %s", sentSemName_.c_str(), + strerror(errno)); + rc = sem_unlink(receiveReadySemName_.c_str()); + if(rc == -1 && errno != ENOENT) + getMessengerLogger()->error("Error unlinking semaphore %s: %s", + receiveReadySemName_.c_str(), strerror(errno)); + } + sem_t* sentSem_; + sem_t* receiveReadySem_; + + private: + std::string sentSemName_; + std::string receiveReadySemName_; +}; +struct SharedMemoryManager +{ + static bool initShm(const std::string &name, size_t len, grk_handle* shm_fd, char** buffer) + { + *shm_fd = shm_open(name.c_str(), O_CREAT | O_RDWR, 0666); + if(*shm_fd < 0) + { + getMessengerLogger()->error("Error opening shared memory: %s", strerror(errno)); + return false; + } + int rc = ftruncate(*shm_fd, sizeof(char) * len); + if(rc) + { + getMessengerLogger()->error("Error truncating shared memory: %s", strerror(errno)); + rc = close(*shm_fd); + if(rc) + getMessengerLogger()->error("Error closing shared memory: %s", strerror(errno)); + rc = shm_unlink(name.c_str()); + if(rc) + getMessengerLogger()->error("Error unlinking shared memory: %s", strerror(errno)); + return false; + } + *buffer = static_cast(mmap(0, len, PROT_WRITE, MAP_SHARED, *shm_fd, 0)); + if(!*buffer) + { + getMessengerLogger()->error("Error mapping shared memory: %s", strerror(errno)); + rc = close(*shm_fd); + if(rc) + getMessengerLogger()->error("Error closing shared memory: %s", strerror(errno)); + rc = shm_unlink(name.c_str()); + if(rc) + getMessengerLogger()->error("Error unlinking shared memory: %s", strerror(errno)); + } + + return *buffer != nullptr; + } + static bool deinitShm(const std::string &name, size_t len, grk_handle &shm_fd, char** buffer) + { + if (!*buffer || !shm_fd) + return true; + + int rc = munmap(*buffer, len); + *buffer = nullptr; + if(rc) + getMessengerLogger()->error("Error unmapping shared memory %s: %s", name.c_str(), strerror(errno)); + rc = close(shm_fd); + shm_fd = 0; + if(rc) + getMessengerLogger()->error("Error closing shared memory %s: %s", name.c_str(), strerror(errno)); + rc = shm_unlink(name.c_str()); + if(rc) + fprintf(stderr,"Error unlinking shared memory %s : %s\n", name.c_str(), strerror(errno)); + + return true; + } +}; + +template +class MessengerBlockingQueue +{ + public: + explicit MessengerBlockingQueue(size_t max) : active_(true), max_size_(max) {} + MessengerBlockingQueue() : MessengerBlockingQueue(UINT_MAX) {} + size_t size() const + { + return queue_.size(); + } + // deactivate and clear queue + void deactivate() + { + { + std::lock_guard lk(mutex_); + active_ = false; + while(!queue_.empty()) + queue_.pop(); + } + + // release all waiting threads + can_pop_.notify_all(); + can_push_.notify_all(); + } + void activate() + { + std::lock_guard lk(mutex_); + active_ = true; + } + bool push(Data const& value) + { + bool rc; + { + std::unique_lock lk(mutex_); + rc = push_(value); + } + if(rc) + can_pop_.notify_one(); + + return rc; + } + bool waitAndPush(Data& value) + { + bool rc; + { + std::unique_lock lk(mutex_); + if(!active_) + return false; + // in case of spurious wakeup, loop until predicate in lambda + // is satisfied. + can_push_.wait(lk, [this] { return queue_.size() < max_size_ || !active_; }); + rc = push_(value); + } + if(rc) + can_pop_.notify_one(); + + return rc; + } + bool pop(Data& value) + { + bool rc; + { + std::unique_lock lk(mutex_); + rc = pop_(value); + } + if(rc) + can_push_.notify_one(); + + return rc; + } + bool waitAndPop(Data& value) + { + bool rc; + { + std::unique_lock lk(mutex_); + if(!active_) + return false; + // in case of spurious wakeup, loop until predicate in lambda + // is satisfied. + can_pop_.wait(lk, [this] { return !queue_.empty() || !active_; }); + rc = pop_(value); + } + if(rc) + can_push_.notify_one(); + + return rc; + } + + private: + bool push_(Data const& value) + { + if(queue_.size() == max_size_ || !active_) + return false; + queue_.push(value); + + return true; + } + bool pop_(Data& value) + { + if(queue_.empty() || !active_) + return false; + value = queue_.front(); + queue_.pop(); + + return true; + } + std::queue queue_; + mutable std::mutex mutex_; + std::condition_variable can_pop_; + std::condition_variable can_push_; + bool active_; + size_t max_size_; +}; +struct BufferSrc +{ + BufferSrc(void) : BufferSrc("") {} + explicit BufferSrc(const std::string &file) : file_(file), clientFrameId_(0), frameId_(0), framePtr_(nullptr) + {} + BufferSrc(size_t clientFrameId, size_t frameId, uint8_t* framePtr) + : file_(""), clientFrameId_(clientFrameId), frameId_(frameId), framePtr_(framePtr) + {} + bool fromDisk(void) + { + return !file_.empty() && framePtr_ == nullptr; + } + size_t index() const + { + return clientFrameId_; + } + std::string file_; + size_t clientFrameId_; + size_t frameId_; + uint8_t* framePtr_; +}; + +struct Messenger; +static void outboundThread(Messenger* messenger, const std::string &sendBuf, Synch* synch); +static void inboundThread(Messenger* messenger, const std::string &receiveBuf, Synch* synch); +static void processorThread(Messenger* messenger, std::function processor); + +struct Messenger +{ + explicit Messenger(MessengerInit init) + : running(true), initialized_(false), shutdown_(false), init_(init), + outboundSynch_(nullptr), + inboundSynch_(nullptr), uncompressed_buffer_(nullptr), compressed_buffer_(nullptr), + uncompressed_fd_(0), compressed_fd_(0) + {} + virtual ~Messenger(void) + { + running = false; + sendQueue.deactivate(); + receiveQueue.deactivate(); + + if (outboundSynch_) { + outboundSynch_->post(SYNCH_RECEIVE_READY); + outbound.join(); + } + + if (inboundSynch_) { + inboundSynch_->post(SYNCH_SENT); + inbound.join(); + } + + for(auto& p : processors_) + p.join(); + + delete outboundSynch_; + delete inboundSynch_; + + deinitShm(); + } + void startThreads(void) { + outboundSynch_ = + new Synch(init_.outboundSentSynch, init_.outboundReceiveReadySynch); + outbound = std::thread(outboundThread, this, init_.outboundMessageBuf, outboundSynch_); + + inboundSynch_ = + new Synch(init_.inboundSentSynch, init_.inboundReceiveReadySynch); + inbound = std::thread(inboundThread, this, init_.inboundMessageBuf, inboundSynch_); + + for(size_t i = 0; i < init_.numProcessingThreads_; ++i) + processors_.push_back(std::thread(processorThread, this, init_.processor_)); + } + size_t serialize(const std::string &dir, size_t clientFrameId, uint8_t* compressedPtr, + size_t compressedLength) + { + char fname[512]; + if(!compressedPtr || !compressedLength) + return 0; + sprintf(fname, "%s/test_%d.j2k", dir.c_str(), (int)clientFrameId); + auto fp = fopen(fname, "wb"); + if(!fp) + return 0; + size_t written = fwrite(compressedPtr, 1, compressedLength, fp); + if(written != compressedLength) + { + fclose(fp); + return 0; + } + fflush(fp); + fclose(fp); + + return written; + } + bool initBuffers(void) + { + bool rc = true; + if(init_.uncompressedFrameSize_) + { + rc = rc && SharedMemoryManager::initShm(grokUncompressedBuf, + init_.uncompressedFrameSize_ * init_.numFrames_, + &uncompressed_fd_, &uncompressed_buffer_); + } + if(init_.compressedFrameSize_) + { + rc = rc && SharedMemoryManager::initShm(grokCompressedBuf, + init_.compressedFrameSize_ * init_.numFrames_, + &compressed_fd_, &compressed_buffer_); + } + + return rc; + } + + bool deinitShm(void) + { + bool rc = SharedMemoryManager::deinitShm(grokUncompressedBuf, + init_.uncompressedFrameSize_ * init_.numFrames_, + uncompressed_fd_, &uncompressed_buffer_); + rc = rc && SharedMemoryManager::deinitShm(grokCompressedBuf, + init_.compressedFrameSize_ * init_.numFrames_, + compressed_fd_, &compressed_buffer_); + + return rc; + } + template + void send(const std::string& str, Args... args) + { + std::ostringstream oss; + oss << str; + int dummy[] = {0, ((void)(oss << ',' << args), 0)...}; + static_cast(dummy); + + sendQueue.push(oss.str()); + } + static pid_t get_pid_by_process_name(const char* name) + { + char command[256]; + snprintf(command, sizeof(command), "pgrep %s", name); + auto pgrep = popen(command, "r"); + if(!pgrep) + return -1; + pid_t pid; + if(fscanf(pgrep, "%d", &pid) != 1) + pid = -1; + pclose(pgrep); + + return pid; + } + static bool terminate_process(const char* name) + { + auto pid = get_pid_by_process_name(name); + + return (pid != -1 && kill(pid, SIGTERM) != -1); + } + static bool kill_process(const char* name) + { + auto pid = get_pid_by_process_name(name); + + return (pid != -1 && kill(pid, SIGKILL) != -1); + } + void launchGrok(const std::string &dir, uint32_t width, uint32_t stride, + uint32_t height, uint32_t samplesPerPixel, uint32_t depth, + int device, bool is4K, uint32_t fps, uint32_t bandwidth, + const std::string server, uint32_t port, + const std::string license) + { + + std::unique_lock lk(shutdownMutex_); + if (async_result_.valid()) + return; + if(MessengerInit::firstLaunch(true)) + init_.unlink(); + startThreads(); + char _cmd[4096]; + auto fullServer = server + ":" + std::to_string(port); + sprintf(_cmd, + "./grk_compress -batch_src %s,%d,%d,%d,%d,%d -out_fmt j2k -out_dir - -k 1 " + "-G %d -%s %d,%d -j %s -J %s", + GRK_MSGR_BATCH_IMAGE.c_str(), width, stride, height, samplesPerPixel, depth, + device, is4K ? "cinema4K" : "cinema2K", fps, bandwidth, + license.c_str(), fullServer.c_str()); + launch(_cmd, dir); + } + void initClient(size_t uncompressedFrameSize, size_t compressedFrameSize, size_t numFrames) + { + // client fills queue with pending uncompressed buffers + init_.uncompressedFrameSize_ = uncompressedFrameSize; + init_.compressedFrameSize_ = compressedFrameSize; + init_.numFrames_ = numFrames; + initBuffers(); + auto ptr = uncompressed_buffer_; + for(size_t i = 0; i < init_.numFrames_; ++i) + { + availableBuffers_.push(BufferSrc(0, i, (uint8_t*)ptr)); + ptr += init_.uncompressedFrameSize_; + } + + std::unique_lock lk(shutdownMutex_); + initialized_ = true; + clientInitializedCondition_.notify_all(); + } + bool waitForClientInit(void) + { + if(initialized_) + return true; + + std::unique_lock lk(shutdownMutex_); + if(initialized_) + return true; + else if (shutdown_) + return false; + clientInitializedCondition_.wait(lk, [this]{return initialized_ || shutdown_;}); + + return initialized_ && !shutdown_; + } + static size_t uncompressedFrameSize(uint32_t w, uint32_t h, uint32_t samplesPerPixel) + { + return sizeof(uint16_t) * w * h * samplesPerPixel; + } + void reclaimCompressed(size_t frameId) + { + availableBuffers_.push(BufferSrc(0, frameId, getCompressedFrame(frameId))); + } + void reclaimUncompressed(size_t frameId) + { + availableBuffers_.push(BufferSrc(0, frameId, getUncompressedFrame(frameId))); + } + uint8_t* getUncompressedFrame(size_t frameId) + { + assert(frameId < init_.numFrames_); + if(frameId >= init_.numFrames_) + return nullptr; + + return (uint8_t*)(uncompressed_buffer_ + frameId * init_.uncompressedFrameSize_); + } + uint8_t* getCompressedFrame(size_t frameId) + { + assert(frameId < init_.numFrames_); + if(frameId >= init_.numFrames_) + return nullptr; + + return (uint8_t*)(compressed_buffer_ + frameId * init_.compressedFrameSize_); + } + std::atomic_bool running; + bool initialized_; + bool shutdown_; + MessengerBlockingQueue sendQueue; + MessengerBlockingQueue receiveQueue; + MessengerBlockingQueue availableBuffers_; + MessengerInit init_; + std::string cmd_; + std::future async_result_; + std::mutex shutdownMutex_; + std::condition_variable shutdownCondition_; + + protected: + std::condition_variable clientInitializedCondition_; + private: + void launch(const std::string &cmd, const std::string &dir) + { + // Change the working directory + if(!dir.empty()) + { + if(chdir(dir.c_str()) != 0) + { + getMessengerLogger()->error("Error: failed to change the working directory"); + return; + } + } + // Execute the command using std::async and std::system + cmd_ = cmd; + async_result_ = std::async(std::launch::async, [this]() { return std::system(cmd_.c_str()); }); + } + std::thread outbound; + Synch* outboundSynch_; + + std::thread inbound; + Synch* inboundSynch_; + + std::vector processors_; + char* uncompressed_buffer_; + char* compressed_buffer_; + + grk_handle uncompressed_fd_; + grk_handle compressed_fd_; +}; + +static void outboundThread(Messenger* messenger, const std::string &sendBuf, Synch* synch) +{ + grk_handle shm_fd = 0; + char* send_buffer = nullptr; + + if(!SharedMemoryManager::initShm(sendBuf, messageBufferLen, &shm_fd, &send_buffer)) + return; + while(messenger->running) + { + synch->wait(SYNCH_RECEIVE_READY); + if(!messenger->running) + break; + std::string message; + if(!messenger->sendQueue.waitAndPop(message)) + break; + if(!messenger->running) + break; + memcpy(send_buffer, message.c_str(), message.size() + 1); + synch->post(SYNCH_SENT); + } + SharedMemoryManager::deinitShm(sendBuf, messageBufferLen, shm_fd, &send_buffer); +} + +static void inboundThread(Messenger* messenger, const std::string &receiveBuf, Synch* synch) +{ + grk_handle shm_fd = 0; + char* receive_buffer = nullptr; + + if(!SharedMemoryManager::initShm(receiveBuf, messageBufferLen, &shm_fd, &receive_buffer)) + return; + while(messenger->running) + { + synch->wait(SYNCH_SENT); + if(!messenger->running) + break; + auto message = std::string(receive_buffer); + synch->post(SYNCH_RECEIVE_READY); + messenger->receiveQueue.push(message); + } + SharedMemoryManager::deinitShm(receiveBuf, messageBufferLen, shm_fd, &receive_buffer); +} +struct Msg +{ + explicit Msg(const std::string &msg) : ct_(0) + { + std::stringstream ss(msg); + while(ss.good()) + { + std::string substr; + std::getline(ss, substr, ','); + cs_.push_back(substr); + } + } + std::string next() + { + if(ct_ == cs_.size()) + { + getMessengerLogger()->error("Msg: comma separated list exhausted. returning empty."); + return ""; + } + return cs_[ct_++]; + } + + uint32_t nextUint(void) + { + return (uint32_t)std::stoi(next()); + } + + std::vector cs_; + size_t ct_; +}; +static void processorThread(Messenger* messenger, std::function processor) +{ + while(messenger->running) + { + std::string message; + if(!messenger->receiveQueue.waitAndPop(message)) + break; + if(!messenger->running) + break; + Msg msg(message); + auto tag = msg.next(); + if(tag == GRK_MSGR_BATCH_COMPRESS_INIT) + { + auto width = msg.nextUint(); + auto stride = msg.nextUint(); + (void)stride; + auto height = msg.nextUint(); + auto samplesPerPixel = msg.nextUint(); + auto depth = msg.nextUint(); + (void)depth; + messenger->init_.uncompressedFrameSize_ = + Messenger::uncompressedFrameSize(width, height, samplesPerPixel); + auto compressedFrameSize = msg.nextUint(); + auto numFrames = msg.nextUint(); + messenger->initClient(compressedFrameSize, compressedFrameSize, numFrames); + } + else if(tag == GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED) + { + messenger->reclaimUncompressed(msg.nextUint()); + } + else if(tag == GRK_MSGR_BATCH_PROCESSSED_COMPRESSED) + { + messenger->reclaimCompressed(msg.nextUint()); + } + processor(message); + } +} + +template +struct ScheduledFrames +{ + void store(F& val) + { + std::unique_lock lk(mapMutex_); + auto it = map_.find(val.index()); + if (it == map_.end()) + map_[val.index()] = val; + } + F retrieve(size_t index, bool &success) + { + std::unique_lock lk(mapMutex_); + success = false; + auto it = map_.find(index); + if(it == map_.end()) + return F(); + + success = true; + F val = it->second; + map_.erase(index); + + return val; + } + + private: + std::mutex mapMutex_; + std::map map_; +}; + +template +struct ScheduledMessenger : public Messenger +{ + explicit ScheduledMessenger(MessengerInit init) : Messenger(init), + framesScheduled_(0), + framesCompressed_(0) + {} + ~ScheduledMessenger(void) { + shutdown(); + } + bool scheduleCompress(F proxy, std::function converter){ + size_t frameSize = init_.uncompressedFrameSize_; + assert(frameSize >= init_.uncompressedFrameSize_); + BufferSrc src; + if(!availableBuffers_.waitAndPop(src)) + return false; + converter(src); + scheduledFrames_.store(proxy); + framesScheduled_++; + send(GRK_MSGR_BATCH_SUBMIT_UNCOMPRESSED, proxy.index(), src.frameId_); + + return true; + } + void processCompressed(const std::string &message, std::function processor, bool needsRecompression) { + Msg msg(message); + msg.next(); + auto clientFrameId = msg.nextUint(); + auto compressedFrameId = msg.nextUint(); + auto compressedFrameLength = msg.nextUint(); + if (!needsRecompression) { + bool success = false; + auto srcFrame = scheduledFrames_.retrieve(clientFrameId,success); + if (!success) + return; + processor(srcFrame, getCompressedFrame(compressedFrameId),compressedFrameLength); + } + ++framesCompressed_; + send(GRK_MSGR_BATCH_PROCESSSED_COMPRESSED, compressedFrameId); + if (shutdown_ && framesCompressed_ == framesScheduled_) + shutdownCondition_.notify_all(); + } + void shutdown(void){ + try { + std::unique_lock lk(shutdownMutex_); + if (!async_result_.valid()) + return; + shutdown_ = true; + if (framesScheduled_) { + uint32_t scheduled = framesScheduled_; + send(GRK_MSGR_BATCH_FLUSH, scheduled); + shutdownCondition_.wait(lk, [this] { return framesScheduled_ == framesCompressed_; }); + } + availableBuffers_.deactivate(); + send(GRK_MSGR_BATCH_SHUTDOWN); + int result = async_result_.get(); + if(result != 0) + getMessengerLogger()->error("Accelerator failed with return code: %d\n",result); + } catch (std::exception &ex) { + getMessengerLogger()->error("%s",ex.what()); + } + + } + F retrieve(size_t index, bool &success) { + return scheduledFrames_.retrieve(index, success); + } + void store(F& val) { + scheduledFrames_.store(val); + } + +private: + ScheduledFrames scheduledFrames_; + std::atomic framesScheduled_; + std::atomic framesCompressed_; +}; + +} // namespace grk_plugin diff --git a/src/lib/grok_context.h b/src/lib/grok_context.h deleted file mode 100644 index 1f9726aae..000000000 --- a/src/lib/grok_context.h +++ /dev/null @@ -1,245 +0,0 @@ -/* - Copyright (C) 2023 Grok Image Compression Inc. - - This file is part of DCP-o-matic. - - DCP-o-matic is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - DCP-o-matic is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with DCP-o-matic. If not, see . - -*/ - -#pragma once - -#include "config.h" -#include "log.h" -#include "dcpomatic_log.h" -#include "writer.h" -#include "grok_messenger.h" - -class Film; -using dcp::Data; -using namespace dcpomatic; - -static std::mutex launchMutex; - -namespace grk_plugin -{ - -struct GrokLogger : public MessengerLogger { - explicit GrokLogger(const std::string &preamble) : MessengerLogger(preamble) - {} - virtual ~GrokLogger() = default; - void info(const char* fmt, ...) override{ - va_list arg; - va_start(arg, fmt); - dcpomatic_log->log(preamble_ + log_message(fmt, arg),LogEntry::TYPE_GENERAL); - va_end(arg); - } - void warn(const char* fmt, ...) override{ - va_list arg; - va_start(arg, fmt); - dcpomatic_log->log(preamble_ + log_message(fmt, arg),LogEntry::TYPE_WARNING); - va_end(arg); - } - void error(const char* fmt, ...) override{ - va_list arg; - va_start(arg, fmt); - dcpomatic_log->log(preamble_ + log_message(fmt, arg),LogEntry::TYPE_ERROR); - va_end(arg); - } -}; - -struct GrokInitializer { - GrokInitializer(void) { - setMessengerLogger(new GrokLogger("[GROK] ")); - } - ~GrokInitializer() = default; -}; - -struct FrameProxy { - FrameProxy(void) : FrameProxy(0,Eyes::LEFT,DCPVideo()) - {} - FrameProxy(int index, Eyes eyes, DCPVideo dcpv) : index_(index), eyes_(eyes), vf(dcpv) - {} - int index() const { - return index_; - } - Eyes eyes(void) const { - return eyes_; - } - int index_; - Eyes eyes_; - DCPVideo vf; -}; - -struct DcpomaticContext { - DcpomaticContext(std::shared_ptr film, Writer& writer, - EventHistory &history, const std::string &location) : - film_(film), writer_(writer), - history_(history), location_(location), - width_(0), height_(0) - {} - void setDimensions(uint32_t w, uint32_t h) { - width_ = w; - height_ = h; - } - std::shared_ptr film_; - Writer& writer_; - EventHistory &history_; - std::string location_; - uint32_t width_; - uint32_t height_; -}; - -class GrokContext { -public: - explicit GrokContext(const DcpomaticContext &dcpomaticContext) : - dcpomaticContext_(dcpomaticContext), - messenger_(nullptr), - launched_(false) - { - struct CompressedData : public dcp::Data { - explicit CompressedData(int dataLen) : data_(new uint8_t[dataLen]), dataLen_(dataLen) - {} - ~CompressedData(void){ - delete[] data_; - } - uint8_t const * data () const override { - return data_; - } - uint8_t * data () override { - return data_; - } - int size () const override { - return dataLen_; - } - uint8_t *data_; - int dataLen_; - }; - if (Config::instance()->enable_gpu ()) { - boost::filesystem::path folder(dcpomaticContext_.location_); - boost::filesystem::path binaryPath = folder / "grk_compress"; - if (!boost::filesystem::exists(binaryPath)) { - getMessengerLogger()->error("Invalid binary location %s", - dcpomaticContext_.location_.c_str()); - return; - } - auto proc = [this](const std::string& str) { - try { - Msg msg(str); - auto tag = msg.next(); - if(tag == GRK_MSGR_BATCH_SUBMIT_COMPRESSED) - { - auto clientFrameId = msg.nextUint(); - auto compressedFrameId = msg.nextUint(); - (void)compressedFrameId; - auto compressedFrameLength = msg.nextUint(); - auto processor = - [this](FrameProxy srcFrame, uint8_t* compressed, uint32_t compressedFrameLength) - { - auto compressedData = std::make_shared(compressedFrameLength); - memcpy(compressedData->data_,compressed,compressedFrameLength ); - dcpomaticContext_.writer_.write(compressedData, srcFrame.index(), srcFrame.eyes()); - frame_done (); - }; - int const minimum_size = 16384; - bool needsRecompression = compressedFrameLength < minimum_size; - messenger_->processCompressed(str, processor, needsRecompression); - if (needsRecompression) { - bool success = false; - auto fp = messenger_->retrieve(clientFrameId, success); - if (!success) - return; - - auto encoded = std::make_shared(fp.vf.encode_locally()); - dcpomaticContext_.writer_.write(encoded, fp.vf.index(), fp.vf.eyes()); - frame_done (); - } - } - } catch (std::exception &ex){ - getMessengerLogger()->error("%s",ex.what()); - } - }; - auto clientInit = - MessengerInit(clientToGrokMessageBuf, clientSentSynch, grokReceiveReadySynch, - grokToClientMessageBuf, grokSentSynch, clientReceiveReadySynch, proc, - std::thread::hardware_concurrency()); - messenger_ = new ScheduledMessenger(clientInit); - } - } - ~GrokContext(void) { - shutdown(); - } - bool launch(DCPVideo dcpv, int device){ - if (!messenger_ ) - return false; - if (launched_) - return true; - std::unique_lock lk_global(launchMutex); - if (!messenger_) - return false; - if (launched_) - return true; - if (MessengerInit::firstLaunch(true)) { - auto s = dcpv.get_size(); - dcpomaticContext_.setDimensions(s.width, s.height); - auto config = Config::instance(); - messenger_->launchGrok(dcpomaticContext_.location_, - dcpomaticContext_.width_,dcpomaticContext_.width_, - dcpomaticContext_.height_, - 3, 12, device, - dcpomaticContext_.film_->resolution() == Resolution::FOUR_K, - dcpomaticContext_.film_->video_frame_rate(), - dcpomaticContext_.film_->j2k_bandwidth(), - config->gpu_license_server(), - config->gpu_license_port(), - config->gpu_license()); - } - launched_ = messenger_->waitForClientInit(); - - return launched_; - } - bool scheduleCompress(const DCPVideo &vf){ - if (!messenger_) - return false; - - auto fp = FrameProxy(vf.index(),vf.eyes(),vf); - auto cvt = [this, &fp](BufferSrc src){ - // xyz conversion - fp.vf.convert_to_xyz((uint16_t*)src.framePtr_); - }; - return messenger_->scheduleCompress(fp, cvt); - } - void shutdown(void){ - if (!messenger_) - return; - - std::unique_lock lk_global(launchMutex); - if (!messenger_) - return; - if (launched_) - messenger_->shutdown(); - delete messenger_; - messenger_ = nullptr; - } - void frame_done () { - dcpomaticContext_.history_.event (); - } -private: - DcpomaticContext dcpomaticContext_; - ScheduledMessenger *messenger_; - bool launched_; -}; - -} - diff --git a/src/lib/grok_messenger.h b/src/lib/grok_messenger.h deleted file mode 100644 index 45ee752e5..000000000 --- a/src/lib/grok_messenger.h +++ /dev/null @@ -1,930 +0,0 @@ -/* - Copyright (C) 2023 Grok Image Compression Inc. - - This file is part of DCP-o-matic. - - DCP-o-matic is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - DCP-o-matic is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with DCP-o-matic. If not, see . - -*/ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#include -#include -#pragma warning(disable : 4100) -#else -#include -#include -#include -#include -#include -#endif - -namespace grk_plugin -{ -static std::string grokToClientMessageBuf = "Global\\grok_to_client_message"; -static std::string grokSentSynch = "Global\\grok_sent"; -static std::string clientReceiveReadySynch = "Global\\client_receive_ready"; -static std::string clientToGrokMessageBuf = "Global\\client_to_grok_message"; -static std::string clientSentSynch = "Global\\client_sent"; -static std::string grokReceiveReadySynch = "Global\\grok_receive_ready"; -static std::string grokUncompressedBuf = "Global\\grok_uncompressed_buf"; -static std::string grokCompressedBuf = "Global\\grok_compressed_buf"; -static const std::string GRK_MSGR_BATCH_IMAGE = "GRK_MSGR_BATCH_IMAGE"; -static const std::string GRK_MSGR_BATCH_COMPRESS_INIT = "GRK_MSGR_BATCH_COMPRESS_INIT"; -static const std::string GRK_MSGR_BATCH_SUBMIT_UNCOMPRESSED = "GRK_MSGR_BATCH_SUBMIT_UNCOMPRESSED"; -static const std::string GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED = - "GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED"; -static const std::string GRK_MSGR_BATCH_SUBMIT_COMPRESSED = "GRK_MSGR_BATCH_SUBMIT_COMPRESSED"; -static const std::string GRK_MSGR_BATCH_PROCESSSED_COMPRESSED = - "GRK_MSGR_BATCH_PROCESSSED_COMPRESSED"; -static const std::string GRK_MSGR_BATCH_SHUTDOWN = "GRK_MSGR_BATCH_SHUTDOWN"; -static const std::string GRK_MSGR_BATCH_FLUSH = "GRK_MSGR_BATCH_FLUSH"; -static const size_t messageBufferLen = 256; -struct IMessengerLogger -{ - virtual ~IMessengerLogger(void) = default; - virtual void info(const char* fmt, ...) = 0; - virtual void warn(const char* fmt, ...) = 0; - virtual void error(const char* fmt, ...) = 0; - - protected: - template - std::string log_message(char const* const format, Args&... args) noexcept - { - constexpr size_t message_size = 512; - char message[message_size]; - - std::snprintf(message, message_size, format, args...); - return std::string(message); - } -}; -struct MessengerLogger : public IMessengerLogger -{ - explicit MessengerLogger(const std::string &preamble) : preamble_(preamble) {} - virtual ~MessengerLogger() = default; - virtual void info(const char* fmt, ...) override - { - va_list args; - std::string new_fmt = preamble_ + fmt + "\n"; - va_start(args, fmt); - vfprintf(stdout, new_fmt.c_str(), args); - va_end(args); - } - virtual void warn(const char* fmt, ...) override - { - va_list args; - std::string new_fmt = preamble_ + fmt + "\n"; - va_start(args, fmt); - vfprintf(stdout, new_fmt.c_str(), args); - va_end(args); - } - virtual void error(const char* fmt, ...) override - { - va_list args; - std::string new_fmt = preamble_ + fmt + "\n"; - va_start(args, fmt); - vfprintf(stderr, new_fmt.c_str(), args); - va_end(args); - } - - protected: - std::string preamble_; -}; - -static IMessengerLogger* sLogger = nullptr; -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#endif -static void setMessengerLogger(IMessengerLogger* logger) -{ - delete sLogger; - sLogger = logger; -} -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic pop -#endif -static IMessengerLogger* getMessengerLogger(void) -{ - return sLogger; -} -struct MessengerInit -{ - MessengerInit(const std::string &outBuf, const std::string &outSent, - const std::string &outReceiveReady, const std::string &inBuf, - const std::string &inSent, - const std::string &inReceiveReady, - std::function processor, - size_t numProcessingThreads) - : outboundMessageBuf(outBuf), outboundSentSynch(outSent), - outboundReceiveReadySynch(outReceiveReady), inboundMessageBuf(inBuf), - inboundSentSynch(inSent), inboundReceiveReadySynch(inReceiveReady), processor_(processor), - numProcessingThreads_(numProcessingThreads), - uncompressedFrameSize_(0), compressedFrameSize_(0), - numFrames_(0) - { - if(firstLaunch(true)) - unlink(); - } - void unlink(void) - { -#ifndef _WIN32 - shm_unlink(grokToClientMessageBuf.c_str()); - shm_unlink(clientToGrokMessageBuf.c_str()); -#endif - } - static bool firstLaunch(bool isClient) - { - bool debugGrok = false; - return debugGrok != isClient; - } - std::string outboundMessageBuf; - std::string outboundSentSynch; - std::string outboundReceiveReadySynch; - - std::string inboundMessageBuf; - std::string inboundSentSynch; - std::string inboundReceiveReadySynch; - - std::function processor_; - size_t numProcessingThreads_; - - size_t uncompressedFrameSize_; - size_t compressedFrameSize_; - size_t numFrames_; -}; - -/*************************** Synchronization *******************************/ -enum SynchDirection -{ - SYNCH_SENT, - SYNCH_RECEIVE_READY -}; - -typedef int grk_handle; -struct Synch -{ - Synch(const std::string &sentSemName, const std::string &receiveReadySemName) - : sentSemName_(sentSemName), receiveReadySemName_(receiveReadySemName) - { - // unlink semaphores in case of previous crash - if(MessengerInit::firstLaunch(true)) - unlink(); - open(); - } - ~Synch() - { - close(); - if(MessengerInit::firstLaunch(true)) - unlink(); - } - void post(SynchDirection dir) - { - auto sem = (dir == SYNCH_SENT ? sentSem_ : receiveReadySem_); - int rc = sem_post(sem); - if(rc) - getMessengerLogger()->error("Error posting to semaphore: %s", strerror(errno)); - } - void wait(SynchDirection dir) - { - auto sem = dir == SYNCH_SENT ? sentSem_ : receiveReadySem_; - int rc = sem_wait(sem); - if(rc) - getMessengerLogger()->error("Error waiting for semaphore: %s", strerror(errno)); - } - void open(void) - { - sentSem_ = sem_open(sentSemName_.c_str(), O_CREAT, 0666, 0); - if(!sentSem_) - getMessengerLogger()->error("Error opening shared memory: %s", strerror(errno)); - receiveReadySem_ = sem_open(receiveReadySemName_.c_str(), O_CREAT, 0666, 1); - if(!receiveReadySem_) - getMessengerLogger()->error("Error opening shared memory: %s", strerror(errno)); - } - void close(void) - { - int rc = sem_close(sentSem_); - if(rc) - getMessengerLogger()->error("Error closing semaphore %s: %s", sentSemName_.c_str(), - strerror(errno)); - rc = sem_close(receiveReadySem_); - if(rc) - getMessengerLogger()->error("Error closing semaphore %s: %s", - receiveReadySemName_.c_str(), strerror(errno)); - } - void unlink(void) - { - int rc = sem_unlink(sentSemName_.c_str()); - if(rc == -1 && errno != ENOENT) - getMessengerLogger()->error("Error unlinking semaphore %s: %s", sentSemName_.c_str(), - strerror(errno)); - rc = sem_unlink(receiveReadySemName_.c_str()); - if(rc == -1 && errno != ENOENT) - getMessengerLogger()->error("Error unlinking semaphore %s: %s", - receiveReadySemName_.c_str(), strerror(errno)); - } - sem_t* sentSem_; - sem_t* receiveReadySem_; - - private: - std::string sentSemName_; - std::string receiveReadySemName_; -}; -struct SharedMemoryManager -{ - static bool initShm(const std::string &name, size_t len, grk_handle* shm_fd, char** buffer) - { - *shm_fd = shm_open(name.c_str(), O_CREAT | O_RDWR, 0666); - if(*shm_fd < 0) - { - getMessengerLogger()->error("Error opening shared memory: %s", strerror(errno)); - return false; - } - int rc = ftruncate(*shm_fd, sizeof(char) * len); - if(rc) - { - getMessengerLogger()->error("Error truncating shared memory: %s", strerror(errno)); - rc = close(*shm_fd); - if(rc) - getMessengerLogger()->error("Error closing shared memory: %s", strerror(errno)); - rc = shm_unlink(name.c_str()); - if(rc) - getMessengerLogger()->error("Error unlinking shared memory: %s", strerror(errno)); - return false; - } - *buffer = static_cast(mmap(0, len, PROT_WRITE, MAP_SHARED, *shm_fd, 0)); - if(!*buffer) - { - getMessengerLogger()->error("Error mapping shared memory: %s", strerror(errno)); - rc = close(*shm_fd); - if(rc) - getMessengerLogger()->error("Error closing shared memory: %s", strerror(errno)); - rc = shm_unlink(name.c_str()); - if(rc) - getMessengerLogger()->error("Error unlinking shared memory: %s", strerror(errno)); - } - - return *buffer != nullptr; - } - static bool deinitShm(const std::string &name, size_t len, grk_handle &shm_fd, char** buffer) - { - if (!*buffer || !shm_fd) - return true; - - int rc = munmap(*buffer, len); - *buffer = nullptr; - if(rc) - getMessengerLogger()->error("Error unmapping shared memory %s: %s", name.c_str(), strerror(errno)); - rc = close(shm_fd); - shm_fd = 0; - if(rc) - getMessengerLogger()->error("Error closing shared memory %s: %s", name.c_str(), strerror(errno)); - rc = shm_unlink(name.c_str()); - if(rc) - fprintf(stderr,"Error unlinking shared memory %s : %s\n", name.c_str(), strerror(errno)); - - return true; - } -}; - -template -class MessengerBlockingQueue -{ - public: - explicit MessengerBlockingQueue(size_t max) : active_(true), max_size_(max) {} - MessengerBlockingQueue() : MessengerBlockingQueue(UINT_MAX) {} - size_t size() const - { - return queue_.size(); - } - // deactivate and clear queue - void deactivate() - { - { - std::lock_guard lk(mutex_); - active_ = false; - while(!queue_.empty()) - queue_.pop(); - } - - // release all waiting threads - can_pop_.notify_all(); - can_push_.notify_all(); - } - void activate() - { - std::lock_guard lk(mutex_); - active_ = true; - } - bool push(Data const& value) - { - bool rc; - { - std::unique_lock lk(mutex_); - rc = push_(value); - } - if(rc) - can_pop_.notify_one(); - - return rc; - } - bool waitAndPush(Data& value) - { - bool rc; - { - std::unique_lock lk(mutex_); - if(!active_) - return false; - // in case of spurious wakeup, loop until predicate in lambda - // is satisfied. - can_push_.wait(lk, [this] { return queue_.size() < max_size_ || !active_; }); - rc = push_(value); - } - if(rc) - can_pop_.notify_one(); - - return rc; - } - bool pop(Data& value) - { - bool rc; - { - std::unique_lock lk(mutex_); - rc = pop_(value); - } - if(rc) - can_push_.notify_one(); - - return rc; - } - bool waitAndPop(Data& value) - { - bool rc; - { - std::unique_lock lk(mutex_); - if(!active_) - return false; - // in case of spurious wakeup, loop until predicate in lambda - // is satisfied. - can_pop_.wait(lk, [this] { return !queue_.empty() || !active_; }); - rc = pop_(value); - } - if(rc) - can_push_.notify_one(); - - return rc; - } - - private: - bool push_(Data const& value) - { - if(queue_.size() == max_size_ || !active_) - return false; - queue_.push(value); - - return true; - } - bool pop_(Data& value) - { - if(queue_.empty() || !active_) - return false; - value = queue_.front(); - queue_.pop(); - - return true; - } - std::queue queue_; - mutable std::mutex mutex_; - std::condition_variable can_pop_; - std::condition_variable can_push_; - bool active_; - size_t max_size_; -}; -struct BufferSrc -{ - BufferSrc(void) : BufferSrc("") {} - explicit BufferSrc(const std::string &file) : file_(file), clientFrameId_(0), frameId_(0), framePtr_(nullptr) - {} - BufferSrc(size_t clientFrameId, size_t frameId, uint8_t* framePtr) - : file_(""), clientFrameId_(clientFrameId), frameId_(frameId), framePtr_(framePtr) - {} - bool fromDisk(void) - { - return !file_.empty() && framePtr_ == nullptr; - } - size_t index() const - { - return clientFrameId_; - } - std::string file_; - size_t clientFrameId_; - size_t frameId_; - uint8_t* framePtr_; -}; - -struct Messenger; -static void outboundThread(Messenger* messenger, const std::string &sendBuf, Synch* synch); -static void inboundThread(Messenger* messenger, const std::string &receiveBuf, Synch* synch); -static void processorThread(Messenger* messenger, std::function processor); - -struct Messenger -{ - explicit Messenger(MessengerInit init) - : running(true), initialized_(false), shutdown_(false), init_(init), - outboundSynch_(nullptr), - inboundSynch_(nullptr), uncompressed_buffer_(nullptr), compressed_buffer_(nullptr), - uncompressed_fd_(0), compressed_fd_(0) - {} - virtual ~Messenger(void) - { - running = false; - sendQueue.deactivate(); - receiveQueue.deactivate(); - - if (outboundSynch_) { - outboundSynch_->post(SYNCH_RECEIVE_READY); - outbound.join(); - } - - if (inboundSynch_) { - inboundSynch_->post(SYNCH_SENT); - inbound.join(); - } - - for(auto& p : processors_) - p.join(); - - delete outboundSynch_; - delete inboundSynch_; - - deinitShm(); - } - void startThreads(void) { - outboundSynch_ = - new Synch(init_.outboundSentSynch, init_.outboundReceiveReadySynch); - outbound = std::thread(outboundThread, this, init_.outboundMessageBuf, outboundSynch_); - - inboundSynch_ = - new Synch(init_.inboundSentSynch, init_.inboundReceiveReadySynch); - inbound = std::thread(inboundThread, this, init_.inboundMessageBuf, inboundSynch_); - - for(size_t i = 0; i < init_.numProcessingThreads_; ++i) - processors_.push_back(std::thread(processorThread, this, init_.processor_)); - } - size_t serialize(const std::string &dir, size_t clientFrameId, uint8_t* compressedPtr, - size_t compressedLength) - { - char fname[512]; - if(!compressedPtr || !compressedLength) - return 0; - sprintf(fname, "%s/test_%d.j2k", dir.c_str(), (int)clientFrameId); - auto fp = fopen(fname, "wb"); - if(!fp) - return 0; - size_t written = fwrite(compressedPtr, 1, compressedLength, fp); - if(written != compressedLength) - { - fclose(fp); - return 0; - } - fflush(fp); - fclose(fp); - - return written; - } - bool initBuffers(void) - { - bool rc = true; - if(init_.uncompressedFrameSize_) - { - rc = rc && SharedMemoryManager::initShm(grokUncompressedBuf, - init_.uncompressedFrameSize_ * init_.numFrames_, - &uncompressed_fd_, &uncompressed_buffer_); - } - if(init_.compressedFrameSize_) - { - rc = rc && SharedMemoryManager::initShm(grokCompressedBuf, - init_.compressedFrameSize_ * init_.numFrames_, - &compressed_fd_, &compressed_buffer_); - } - - return rc; - } - - bool deinitShm(void) - { - bool rc = SharedMemoryManager::deinitShm(grokUncompressedBuf, - init_.uncompressedFrameSize_ * init_.numFrames_, - uncompressed_fd_, &uncompressed_buffer_); - rc = rc && SharedMemoryManager::deinitShm(grokCompressedBuf, - init_.compressedFrameSize_ * init_.numFrames_, - compressed_fd_, &compressed_buffer_); - - return rc; - } - template - void send(const std::string& str, Args... args) - { - std::ostringstream oss; - oss << str; - int dummy[] = {0, ((void)(oss << ',' << args), 0)...}; - static_cast(dummy); - - sendQueue.push(oss.str()); - } - static pid_t get_pid_by_process_name(const char* name) - { - char command[256]; - snprintf(command, sizeof(command), "pgrep %s", name); - auto pgrep = popen(command, "r"); - if(!pgrep) - return -1; - pid_t pid; - if(fscanf(pgrep, "%d", &pid) != 1) - pid = -1; - pclose(pgrep); - - return pid; - } - static bool terminate_process(const char* name) - { - auto pid = get_pid_by_process_name(name); - - return (pid != -1 && kill(pid, SIGTERM) != -1); - } - static bool kill_process(const char* name) - { - auto pid = get_pid_by_process_name(name); - - return (pid != -1 && kill(pid, SIGKILL) != -1); - } - void launchGrok(const std::string &dir, uint32_t width, uint32_t stride, - uint32_t height, uint32_t samplesPerPixel, uint32_t depth, - int device, bool is4K, uint32_t fps, uint32_t bandwidth, - const std::string server, uint32_t port, - const std::string license) - { - - std::unique_lock lk(shutdownMutex_); - if (async_result_.valid()) - return; - if(MessengerInit::firstLaunch(true)) - init_.unlink(); - startThreads(); - char _cmd[4096]; - auto fullServer = server + ":" + std::to_string(port); - sprintf(_cmd, - "./grk_compress -batch_src %s,%d,%d,%d,%d,%d -out_fmt j2k -out_dir - -k 1 " - "-G %d -%s %d,%d -j %s -J %s", - GRK_MSGR_BATCH_IMAGE.c_str(), width, stride, height, samplesPerPixel, depth, - device, is4K ? "cinema4K" : "cinema2K", fps, bandwidth, - license.c_str(), fullServer.c_str()); - launch(_cmd, dir); - } - void initClient(size_t uncompressedFrameSize, size_t compressedFrameSize, size_t numFrames) - { - // client fills queue with pending uncompressed buffers - init_.uncompressedFrameSize_ = uncompressedFrameSize; - init_.compressedFrameSize_ = compressedFrameSize; - init_.numFrames_ = numFrames; - initBuffers(); - auto ptr = uncompressed_buffer_; - for(size_t i = 0; i < init_.numFrames_; ++i) - { - availableBuffers_.push(BufferSrc(0, i, (uint8_t*)ptr)); - ptr += init_.uncompressedFrameSize_; - } - - std::unique_lock lk(shutdownMutex_); - initialized_ = true; - clientInitializedCondition_.notify_all(); - } - bool waitForClientInit(void) - { - if(initialized_) - return true; - - std::unique_lock lk(shutdownMutex_); - if(initialized_) - return true; - else if (shutdown_) - return false; - clientInitializedCondition_.wait(lk, [this]{return initialized_ || shutdown_;}); - - return initialized_ && !shutdown_; - } - static size_t uncompressedFrameSize(uint32_t w, uint32_t h, uint32_t samplesPerPixel) - { - return sizeof(uint16_t) * w * h * samplesPerPixel; - } - void reclaimCompressed(size_t frameId) - { - availableBuffers_.push(BufferSrc(0, frameId, getCompressedFrame(frameId))); - } - void reclaimUncompressed(size_t frameId) - { - availableBuffers_.push(BufferSrc(0, frameId, getUncompressedFrame(frameId))); - } - uint8_t* getUncompressedFrame(size_t frameId) - { - assert(frameId < init_.numFrames_); - if(frameId >= init_.numFrames_) - return nullptr; - - return (uint8_t*)(uncompressed_buffer_ + frameId * init_.uncompressedFrameSize_); - } - uint8_t* getCompressedFrame(size_t frameId) - { - assert(frameId < init_.numFrames_); - if(frameId >= init_.numFrames_) - return nullptr; - - return (uint8_t*)(compressed_buffer_ + frameId * init_.compressedFrameSize_); - } - std::atomic_bool running; - bool initialized_; - bool shutdown_; - MessengerBlockingQueue sendQueue; - MessengerBlockingQueue receiveQueue; - MessengerBlockingQueue availableBuffers_; - MessengerInit init_; - std::string cmd_; - std::future async_result_; - std::mutex shutdownMutex_; - std::condition_variable shutdownCondition_; - - protected: - std::condition_variable clientInitializedCondition_; - private: - void launch(const std::string &cmd, const std::string &dir) - { - // Change the working directory - if(!dir.empty()) - { - if(chdir(dir.c_str()) != 0) - { - getMessengerLogger()->error("Error: failed to change the working directory"); - return; - } - } - // Execute the command using std::async and std::system - cmd_ = cmd; - async_result_ = std::async(std::launch::async, [this]() { return std::system(cmd_.c_str()); }); - } - std::thread outbound; - Synch* outboundSynch_; - - std::thread inbound; - Synch* inboundSynch_; - - std::vector processors_; - char* uncompressed_buffer_; - char* compressed_buffer_; - - grk_handle uncompressed_fd_; - grk_handle compressed_fd_; -}; - -static void outboundThread(Messenger* messenger, const std::string &sendBuf, Synch* synch) -{ - grk_handle shm_fd = 0; - char* send_buffer = nullptr; - - if(!SharedMemoryManager::initShm(sendBuf, messageBufferLen, &shm_fd, &send_buffer)) - return; - while(messenger->running) - { - synch->wait(SYNCH_RECEIVE_READY); - if(!messenger->running) - break; - std::string message; - if(!messenger->sendQueue.waitAndPop(message)) - break; - if(!messenger->running) - break; - memcpy(send_buffer, message.c_str(), message.size() + 1); - synch->post(SYNCH_SENT); - } - SharedMemoryManager::deinitShm(sendBuf, messageBufferLen, shm_fd, &send_buffer); -} - -static void inboundThread(Messenger* messenger, const std::string &receiveBuf, Synch* synch) -{ - grk_handle shm_fd = 0; - char* receive_buffer = nullptr; - - if(!SharedMemoryManager::initShm(receiveBuf, messageBufferLen, &shm_fd, &receive_buffer)) - return; - while(messenger->running) - { - synch->wait(SYNCH_SENT); - if(!messenger->running) - break; - auto message = std::string(receive_buffer); - synch->post(SYNCH_RECEIVE_READY); - messenger->receiveQueue.push(message); - } - SharedMemoryManager::deinitShm(receiveBuf, messageBufferLen, shm_fd, &receive_buffer); -} -struct Msg -{ - explicit Msg(const std::string &msg) : ct_(0) - { - std::stringstream ss(msg); - while(ss.good()) - { - std::string substr; - std::getline(ss, substr, ','); - cs_.push_back(substr); - } - } - std::string next() - { - if(ct_ == cs_.size()) - { - getMessengerLogger()->error("Msg: comma separated list exhausted. returning empty."); - return ""; - } - return cs_[ct_++]; - } - - uint32_t nextUint(void) - { - return (uint32_t)std::stoi(next()); - } - - std::vector cs_; - size_t ct_; -}; -static void processorThread(Messenger* messenger, std::function processor) -{ - while(messenger->running) - { - std::string message; - if(!messenger->receiveQueue.waitAndPop(message)) - break; - if(!messenger->running) - break; - Msg msg(message); - auto tag = msg.next(); - if(tag == GRK_MSGR_BATCH_COMPRESS_INIT) - { - auto width = msg.nextUint(); - auto stride = msg.nextUint(); - (void)stride; - auto height = msg.nextUint(); - auto samplesPerPixel = msg.nextUint(); - auto depth = msg.nextUint(); - (void)depth; - messenger->init_.uncompressedFrameSize_ = - Messenger::uncompressedFrameSize(width, height, samplesPerPixel); - auto compressedFrameSize = msg.nextUint(); - auto numFrames = msg.nextUint(); - messenger->initClient(compressedFrameSize, compressedFrameSize, numFrames); - } - else if(tag == GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED) - { - messenger->reclaimUncompressed(msg.nextUint()); - } - else if(tag == GRK_MSGR_BATCH_PROCESSSED_COMPRESSED) - { - messenger->reclaimCompressed(msg.nextUint()); - } - processor(message); - } -} - -template -struct ScheduledFrames -{ - void store(F& val) - { - std::unique_lock lk(mapMutex_); - auto it = map_.find(val.index()); - if (it == map_.end()) - map_[val.index()] = val; - } - F retrieve(size_t index, bool &success) - { - std::unique_lock lk(mapMutex_); - success = false; - auto it = map_.find(index); - if(it == map_.end()) - return F(); - - success = true; - F val = it->second; - map_.erase(index); - - return val; - } - - private: - std::mutex mapMutex_; - std::map map_; -}; - -template -struct ScheduledMessenger : public Messenger -{ - explicit ScheduledMessenger(MessengerInit init) : Messenger(init), - framesScheduled_(0), - framesCompressed_(0) - {} - ~ScheduledMessenger(void) { - shutdown(); - } - bool scheduleCompress(F proxy, std::function converter){ - size_t frameSize = init_.uncompressedFrameSize_; - assert(frameSize >= init_.uncompressedFrameSize_); - BufferSrc src; - if(!availableBuffers_.waitAndPop(src)) - return false; - converter(src); - scheduledFrames_.store(proxy); - framesScheduled_++; - send(GRK_MSGR_BATCH_SUBMIT_UNCOMPRESSED, proxy.index(), src.frameId_); - - return true; - } - void processCompressed(const std::string &message, std::function processor, bool needsRecompression) { - Msg msg(message); - msg.next(); - auto clientFrameId = msg.nextUint(); - auto compressedFrameId = msg.nextUint(); - auto compressedFrameLength = msg.nextUint(); - if (!needsRecompression) { - bool success = false; - auto srcFrame = scheduledFrames_.retrieve(clientFrameId,success); - if (!success) - return; - processor(srcFrame, getCompressedFrame(compressedFrameId),compressedFrameLength); - } - ++framesCompressed_; - send(GRK_MSGR_BATCH_PROCESSSED_COMPRESSED, compressedFrameId); - if (shutdown_ && framesCompressed_ == framesScheduled_) - shutdownCondition_.notify_all(); - } - void shutdown(void){ - try { - std::unique_lock lk(shutdownMutex_); - if (!async_result_.valid()) - return; - shutdown_ = true; - if (framesScheduled_) { - uint32_t scheduled = framesScheduled_; - send(GRK_MSGR_BATCH_FLUSH, scheduled); - shutdownCondition_.wait(lk, [this] { return framesScheduled_ == framesCompressed_; }); - } - availableBuffers_.deactivate(); - send(GRK_MSGR_BATCH_SHUTDOWN); - int result = async_result_.get(); - if(result != 0) - getMessengerLogger()->error("Accelerator failed with return code: %d\n",result); - } catch (std::exception &ex) { - getMessengerLogger()->error("%s",ex.what()); - } - - } - F retrieve(size_t index, bool &success) { - return scheduledFrames_.retrieve(index, success); - } - void store(F& val) { - scheduledFrames_.store(val); - } - -private: - ScheduledFrames scheduledFrames_; - std::atomic framesScheduled_; - std::atomic framesCompressed_; -}; - -} // namespace grk_plugin diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index 6df30a3f7..a3ea9f3e8 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -27,6 +27,7 @@ * @brief J2KEncoder class. */ +#include "grok/context.h" #include "cross.h" #include "dcp_video.h" @@ -42,8 +43,6 @@ #include #include -#include "grok_context.h" - class EncodeServerDescription; class Film; class Job; -- cgit v1.2.3 From bf770122a39965599cdd23e876dfd8e7db64a639 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Fri, 7 Jul 2023 23:42:08 +0200 Subject: Move grok headers into src/wx/grok --- src/wx/full_config_dialog.cc | 2 +- src/wx/gpu_config_panel.h | 186 ----------------------------------------- src/wx/grok/gpu_config_panel.h | 186 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 187 deletions(-) delete mode 100644 src/wx/gpu_config_panel.h create mode 100644 src/wx/grok/gpu_config_panel.h diff --git a/src/wx/full_config_dialog.cc b/src/wx/full_config_dialog.cc index df974c1fa..119091af3 100644 --- a/src/wx/full_config_dialog.cc +++ b/src/wx/full_config_dialog.cc @@ -45,7 +45,7 @@ #include "send_test_email_dialog.h" #include "server_dialog.h" #include "static_text.h" -#include "gpu_config_panel.h" +#include "grok/gpu_config_panel.h" #include "wx_util.h" #include "lib/config.h" #include "lib/cross.h" diff --git a/src/wx/gpu_config_panel.h b/src/wx/gpu_config_panel.h deleted file mode 100644 index 1478434be..000000000 --- a/src/wx/gpu_config_panel.h +++ /dev/null @@ -1,186 +0,0 @@ -#pragma once - -static std::vector get_gpu_names(std::string binary, std::string filename) -{ - // Execute the GPU listing program and redirect its output to a file - std::system((binary + " > " + filename).c_str()); - - std::vector gpu_names; - std::ifstream file(filename); - if (file.is_open()) - { - std::string line; - while (std::getline(file, line)) - gpu_names.push_back(line); - file.close(); - } - - return gpu_names; -} - -class GpuList : public wxPanel -{ -public: - GpuList(wxPanel* parent) : wxPanel(parent, wxID_ANY), selection(0) { - comboBox = new wxComboBox(this, wxID_ANY, "", wxDefaultPosition, wxSize(400, -1)); - comboBox->Bind(wxEVT_COMBOBOX, &GpuList::OnComboBox, this); - update(); - - wxBoxSizer* sizer = new wxBoxSizer(wxHORIZONTAL); - - sizer->Add(comboBox, 0, wxALIGN_CENTER_VERTICAL); // Vertically center the comboBox - - this->SetSizerAndFit(sizer); - } - void update(void) { - auto cfg = Config::instance(); - auto lister_binary = cfg->gpu_binary_location() + "/" + "gpu_lister"; - auto lister_file = cfg->gpu_binary_location () + "/" + "gpus.txt"; - if (boost::filesystem::exists(lister_binary)) { - auto gpu_names = get_gpu_names(lister_binary, lister_file); - - comboBox->Clear(); - for (const auto& name : gpu_names) - comboBox->Append(name); - } - } - - int getSelection(void) { - return selection; - } - void setSelection(int sel) { - if ((int)comboBox->GetCount() > sel) - comboBox->SetSelection(sel); - } - -private: - void OnComboBox([[maybe_unused]] wxCommandEvent& event) { - selection = comboBox->GetSelection(); - if (selection != wxNOT_FOUND) - Config::instance ()->set_selected_gpu(selection); - } - - wxComboBox* comboBox; - int selection; -}; - -class GPUPage : public Page -{ -public: - GPUPage (wxSize panel_size, int border) - : Page (panel_size, border), - _enable_gpu(nullptr), _binary_location(nullptr), _gpu_list_control(nullptr) - {} - - wxString GetName () const override - { - return _("GPU"); - } - -#ifdef DCPOMATIC_OSX - wxBitmap GetLargeIcon () const override - { - return wxBitmap(icon_path("tms"), wxBITMAP_TYPE_PNG); - } -#endif - -private: - void setup () override - { - auto config = Config::instance (); - - _enable_gpu = new CheckBox (_panel, _("Enable GPU Acceleration")); - _panel->GetSizer()->Add (_enable_gpu, 0, wxALL | wxEXPAND, _border); - - wxFlexGridSizer* table = new wxFlexGridSizer (2, DCPOMATIC_SIZER_X_GAP, DCPOMATIC_SIZER_Y_GAP); - table->AddGrowableCol (1, 1); - _panel->GetSizer()->Add (table, 1, wxALL | wxEXPAND, _border); - - add_label_to_sizer (table, _panel, _("Acceleration Binary Folder"), true, 0, wxLEFT | wxLEFT | wxALIGN_CENTRE_VERTICAL); - _binary_location = new wxDirPickerCtrl (_panel, wxDD_DIR_MUST_EXIST); - table->Add (_binary_location, 1, wxEXPAND); - - add_label_to_sizer (table, _panel, _("GPU Selection"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); - _gpu_list_control = new GpuList(_panel); - table->Add (_gpu_list_control, 1, wxEXPAND); - - add_label_to_sizer (table, _panel, _("License Server"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); - _server = new wxTextCtrl (_panel, wxID_ANY); - table->Add (_server, 1, wxEXPAND | wxALL); - - add_label_to_sizer (table, _panel, _("Port"), false, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); - _port = new wxSpinCtrl (_panel, wxID_ANY); - _port->SetRange (0, 65535); - table->Add (_port); - - add_label_to_sizer (table, _panel, _("License"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); - _license = new PasswordEntry (_panel); - table->Add (_license->get_panel(), 1, wxEXPAND | wxALL); - - _enable_gpu->bind(&GPUPage::enable_gpu_changed, this); - _binary_location->Bind (wxEVT_DIRPICKER_CHANGED, boost::bind (&GPUPage::binary_location_changed, this)); - _server->Bind (wxEVT_TEXT, boost::bind(&GPUPage::server_changed, this)); - _port->Bind (wxEVT_SPINCTRL, boost::bind(&GPUPage::port_changed, this)); - _license->Changed.connect (boost::bind(&GPUPage::license_changed, this)); - - _binary_location->Enable(config->enable_gpu()); - _gpu_list_control->Enable(config->enable_gpu()); - _server->Enable(config->enable_gpu()); - _port->Enable(config->enable_gpu()); - _license->get_panel()->Enable(config->enable_gpu()); - } - - - void config_changed () override - { - auto config = Config::instance (); - - checked_set (_enable_gpu, config->enable_gpu()); - _binary_location->SetPath(config->gpu_binary_location ()); - _gpu_list_control->update(); - _gpu_list_control->setSelection(config->selected_gpu()); - checked_set (_server, config->gpu_license_server()); - checked_set (_port, config->gpu_license_port()); - checked_set (_license, config->gpu_license()); - } - - void enable_gpu_changed () - { - auto config = Config::instance (); - - config->set_enable_gpu (_enable_gpu->GetValue()); - _binary_location->Enable(config->enable_gpu()); - _gpu_list_control->Enable(config->enable_gpu()); - _server->Enable(config->enable_gpu()); - _port->Enable(config->enable_gpu()); - _license->get_panel()->Enable(config->enable_gpu()); - } - - void binary_location_changed () - { - Config::instance()->set_gpu_binary_location (wx_to_std (_binary_location->GetPath ())); - _gpu_list_control->update(); - } - - void server_changed () - { - Config::instance()->set_gpu_license_server(wx_to_std(_server->GetValue())); - } - - void port_changed () - { - Config::instance()->set_gpu_license_port(_port->GetValue()); - } - - void license_changed () - { - Config::instance()->set_gpu_license(_license->get()); - } - - CheckBox* _enable_gpu; - wxDirPickerCtrl* _binary_location; - GpuList *_gpu_list_control; - wxTextCtrl* _server; - wxSpinCtrl* _port; - PasswordEntry* _license; -}; diff --git a/src/wx/grok/gpu_config_panel.h b/src/wx/grok/gpu_config_panel.h new file mode 100644 index 000000000..1478434be --- /dev/null +++ b/src/wx/grok/gpu_config_panel.h @@ -0,0 +1,186 @@ +#pragma once + +static std::vector get_gpu_names(std::string binary, std::string filename) +{ + // Execute the GPU listing program and redirect its output to a file + std::system((binary + " > " + filename).c_str()); + + std::vector gpu_names; + std::ifstream file(filename); + if (file.is_open()) + { + std::string line; + while (std::getline(file, line)) + gpu_names.push_back(line); + file.close(); + } + + return gpu_names; +} + +class GpuList : public wxPanel +{ +public: + GpuList(wxPanel* parent) : wxPanel(parent, wxID_ANY), selection(0) { + comboBox = new wxComboBox(this, wxID_ANY, "", wxDefaultPosition, wxSize(400, -1)); + comboBox->Bind(wxEVT_COMBOBOX, &GpuList::OnComboBox, this); + update(); + + wxBoxSizer* sizer = new wxBoxSizer(wxHORIZONTAL); + + sizer->Add(comboBox, 0, wxALIGN_CENTER_VERTICAL); // Vertically center the comboBox + + this->SetSizerAndFit(sizer); + } + void update(void) { + auto cfg = Config::instance(); + auto lister_binary = cfg->gpu_binary_location() + "/" + "gpu_lister"; + auto lister_file = cfg->gpu_binary_location () + "/" + "gpus.txt"; + if (boost::filesystem::exists(lister_binary)) { + auto gpu_names = get_gpu_names(lister_binary, lister_file); + + comboBox->Clear(); + for (const auto& name : gpu_names) + comboBox->Append(name); + } + } + + int getSelection(void) { + return selection; + } + void setSelection(int sel) { + if ((int)comboBox->GetCount() > sel) + comboBox->SetSelection(sel); + } + +private: + void OnComboBox([[maybe_unused]] wxCommandEvent& event) { + selection = comboBox->GetSelection(); + if (selection != wxNOT_FOUND) + Config::instance ()->set_selected_gpu(selection); + } + + wxComboBox* comboBox; + int selection; +}; + +class GPUPage : public Page +{ +public: + GPUPage (wxSize panel_size, int border) + : Page (panel_size, border), + _enable_gpu(nullptr), _binary_location(nullptr), _gpu_list_control(nullptr) + {} + + wxString GetName () const override + { + return _("GPU"); + } + +#ifdef DCPOMATIC_OSX + wxBitmap GetLargeIcon () const override + { + return wxBitmap(icon_path("tms"), wxBITMAP_TYPE_PNG); + } +#endif + +private: + void setup () override + { + auto config = Config::instance (); + + _enable_gpu = new CheckBox (_panel, _("Enable GPU Acceleration")); + _panel->GetSizer()->Add (_enable_gpu, 0, wxALL | wxEXPAND, _border); + + wxFlexGridSizer* table = new wxFlexGridSizer (2, DCPOMATIC_SIZER_X_GAP, DCPOMATIC_SIZER_Y_GAP); + table->AddGrowableCol (1, 1); + _panel->GetSizer()->Add (table, 1, wxALL | wxEXPAND, _border); + + add_label_to_sizer (table, _panel, _("Acceleration Binary Folder"), true, 0, wxLEFT | wxLEFT | wxALIGN_CENTRE_VERTICAL); + _binary_location = new wxDirPickerCtrl (_panel, wxDD_DIR_MUST_EXIST); + table->Add (_binary_location, 1, wxEXPAND); + + add_label_to_sizer (table, _panel, _("GPU Selection"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + _gpu_list_control = new GpuList(_panel); + table->Add (_gpu_list_control, 1, wxEXPAND); + + add_label_to_sizer (table, _panel, _("License Server"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + _server = new wxTextCtrl (_panel, wxID_ANY); + table->Add (_server, 1, wxEXPAND | wxALL); + + add_label_to_sizer (table, _panel, _("Port"), false, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + _port = new wxSpinCtrl (_panel, wxID_ANY); + _port->SetRange (0, 65535); + table->Add (_port); + + add_label_to_sizer (table, _panel, _("License"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + _license = new PasswordEntry (_panel); + table->Add (_license->get_panel(), 1, wxEXPAND | wxALL); + + _enable_gpu->bind(&GPUPage::enable_gpu_changed, this); + _binary_location->Bind (wxEVT_DIRPICKER_CHANGED, boost::bind (&GPUPage::binary_location_changed, this)); + _server->Bind (wxEVT_TEXT, boost::bind(&GPUPage::server_changed, this)); + _port->Bind (wxEVT_SPINCTRL, boost::bind(&GPUPage::port_changed, this)); + _license->Changed.connect (boost::bind(&GPUPage::license_changed, this)); + + _binary_location->Enable(config->enable_gpu()); + _gpu_list_control->Enable(config->enable_gpu()); + _server->Enable(config->enable_gpu()); + _port->Enable(config->enable_gpu()); + _license->get_panel()->Enable(config->enable_gpu()); + } + + + void config_changed () override + { + auto config = Config::instance (); + + checked_set (_enable_gpu, config->enable_gpu()); + _binary_location->SetPath(config->gpu_binary_location ()); + _gpu_list_control->update(); + _gpu_list_control->setSelection(config->selected_gpu()); + checked_set (_server, config->gpu_license_server()); + checked_set (_port, config->gpu_license_port()); + checked_set (_license, config->gpu_license()); + } + + void enable_gpu_changed () + { + auto config = Config::instance (); + + config->set_enable_gpu (_enable_gpu->GetValue()); + _binary_location->Enable(config->enable_gpu()); + _gpu_list_control->Enable(config->enable_gpu()); + _server->Enable(config->enable_gpu()); + _port->Enable(config->enable_gpu()); + _license->get_panel()->Enable(config->enable_gpu()); + } + + void binary_location_changed () + { + Config::instance()->set_gpu_binary_location (wx_to_std (_binary_location->GetPath ())); + _gpu_list_control->update(); + } + + void server_changed () + { + Config::instance()->set_gpu_license_server(wx_to_std(_server->GetValue())); + } + + void port_changed () + { + Config::instance()->set_gpu_license_port(_port->GetValue()); + } + + void license_changed () + { + Config::instance()->set_gpu_license(_license->get()); + } + + CheckBox* _enable_gpu; + wxDirPickerCtrl* _binary_location; + GpuList *_gpu_list_control; + wxTextCtrl* _server; + wxSpinCtrl* _port; + PasswordEntry* _license; +}; -- cgit v1.2.3 From 2d724c547843aff1345fd762b1075249676dcb3e Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sat, 8 Jul 2023 00:20:15 +0200 Subject: Formatting, variable name tidying and some const correctness. --- src/lib/dcp_encoder.cc | 11 +++++++++-- src/lib/dcp_encoder.h | 5 +++-- src/lib/dcp_video.cc | 13 ++++++++----- src/lib/dcp_video.h | 7 ++++--- src/lib/encoder.h | 4 ++-- src/lib/j2k_encoder.cc | 37 ++++++++++++++++++++++--------------- src/lib/j2k_encoder.h | 8 ++++---- src/lib/transcode_job.cc | 10 ++++++++-- 8 files changed, 60 insertions(+), 35 deletions(-) diff --git a/src/lib/dcp_encoder.cc b/src/lib/dcp_encoder.cc index 7088225cd..a03f689b0 100644 --- a/src/lib/dcp_encoder.cc +++ b/src/lib/dcp_encoder.cc @@ -118,13 +118,20 @@ DCPEncoder::go () _writer.finish(_film->dir(_film->dcp_name())); } -void DCPEncoder::pause(void) { + +void +DCPEncoder::pause() +{ _j2k_encoder.pause(); } -void DCPEncoder::resume(void) { + +void +DCPEncoder::resume() +{ _j2k_encoder.resume(); } + void DCPEncoder::video (shared_ptr data, DCPTime time) { diff --git a/src/lib/dcp_encoder.h b/src/lib/dcp_encoder.h index a8043887e..771679a98 100644 --- a/src/lib/dcp_encoder.h +++ b/src/lib/dcp_encoder.h @@ -52,8 +52,9 @@ public: bool finishing () const override { return _finishing; } - void pause(void) override; - void resume(void) override; + + void pause() override; + void resume() override; private: diff --git a/src/lib/dcp_video.cc b/src/lib/dcp_video.cc index 78e973ca3..940ca31d2 100644 --- a/src/lib/dcp_video.cc +++ b/src/lib/dcp_video.cc @@ -119,15 +119,17 @@ DCPVideo::convert_to_xyz (shared_ptr frame, dcp::NoteHandler } dcp::Size -DCPVideo::get_size(void) { - auto image = _frame->image (bind(&PlayerVideo::keep_xyz_or_rgb, _1), VideoRange::FULL, false); +DCPVideo::get_size() const +{ + auto image = _frame->image(bind(&PlayerVideo::keep_xyz_or_rgb, _1), VideoRange::FULL, false); return image->size(); } -void -DCPVideo::convert_to_xyz (uint16_t *dst) { - auto image = _frame->image (bind(&PlayerVideo::keep_xyz_or_rgb, _1), VideoRange::FULL, false); +void +DCPVideo::convert_to_xyz(uint16_t* dst) const +{ + auto image = _frame->image(bind(&PlayerVideo::keep_xyz_or_rgb, _1), VideoRange::FULL, false); if (_frame->colour_conversion()) { dcp::rgb_to_xyz ( image->data()[0], @@ -139,6 +141,7 @@ DCPVideo::convert_to_xyz (uint16_t *dst) { } } + /** J2K-encode this frame on the local host. * @return Encoded data. */ diff --git a/src/lib/dcp_video.h b/src/lib/dcp_video.h index c09442d16..b351fda1e 100644 --- a/src/lib/dcp_video.h +++ b/src/lib/dcp_video.h @@ -49,7 +49,7 @@ class PlayerVideo; class DCPVideo { public: - DCPVideo (void) : DCPVideo(nullptr,0,0,0,Resolution::TWO_K){} + DCPVideo() : DCPVideo(nullptr, 0, 0, 0, Resolution::TWO_K) {} DCPVideo (std::shared_ptr, int index, int dcp_fps, int bandwidth, Resolution r); DCPVideo (std::shared_ptr, cxml::ConstNodePtr); @@ -69,8 +69,9 @@ public: static std::shared_ptr convert_to_xyz (std::shared_ptr frame, dcp::NoteHandler note); - void convert_to_xyz (uint16_t *dst); - dcp::Size get_size(void); + void convert_to_xyz(uint16_t* dst) const; + dcp::Size get_size() const; + private: void add_metadata (xmlpp::Element *) const; diff --git a/src/lib/encoder.h b/src/lib/encoder.h index f921fcb51..aeaf7f620 100644 --- a/src/lib/encoder.h +++ b/src/lib/encoder.h @@ -58,8 +58,8 @@ public: /** @return the number of frames that are done */ virtual Frame frames_done () const = 0; virtual bool finishing () const = 0; - virtual void pause(void) {} - virtual void resume(void) {} + virtual void pause() {} + virtual void resume() {} protected: std::shared_ptr _film; diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index 11f403d95..7480e4541 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -61,9 +61,9 @@ static grk_plugin::GrokInitializer grokInitializer; J2KEncoder::J2KEncoder(shared_ptr film, Writer& writer) : _film (film) , _history (200) - , _writer (writer) , - dcpomaticContext_(film,writer,_history, Config::instance()->gpu_binary_location ()), - context_(Config::instance()->enable_gpu () ? new grk_plugin::GrokContext(dcpomaticContext_) : nullptr) + , _writer (writer) + , _dcpomatic_context(film, writer, _history, Config::instance()->gpu_binary_location()) + , _context(Config::instance()->enable_gpu() ? new grk_plugin::GrokContext(_dcpomatic_context) : nullptr) { servers_list_changed (); } @@ -78,7 +78,7 @@ J2KEncoder::~J2KEncoder () terminate_threads (); } - delete context_; + delete _context; } void @@ -89,18 +89,25 @@ J2KEncoder::begin () ); } -void J2KEncoder::pause(void){ - if (Config::instance()->enable_gpu ()) + +void +J2KEncoder::pause() +{ + if (Config::instance()->enable_gpu()) { end(false); + } } -void J2KEncoder::resume(void){ - if (Config::instance()->enable_gpu ()) { - context_ = new grk_plugin::GrokContext(dcpomaticContext_); - servers_list_changed (); + +void J2KEncoder::resume() +{ + if (Config::instance()->enable_gpu()) { + _context = new grk_plugin::GrokContext(_dcpomatic_context); + servers_list_changed(); } } + void J2KEncoder::end (bool isFinal) { @@ -141,7 +148,7 @@ J2KEncoder::end (bool isFinal) if (isFinal) { for (auto & i: _queue) { if (Config::instance()->enable_gpu ()) { - if (!context_->scheduleCompress(i)){ + if (!_context->scheduleCompress(i)){ LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), i.index()); // handle error } @@ -161,8 +168,8 @@ J2KEncoder::end (bool isFinal) } } } - delete context_; - context_ = nullptr; + delete _context; + _context = nullptr; } @@ -367,8 +374,8 @@ try } } else { - if (context_) { - if (!context_->launch(vf, config->selected_gpu()) || !context_->scheduleCompress(vf)) { + if (_context) { + if (!_context->launch(vf, config->selected_gpu()) || !_context->scheduleCompress(vf)) { LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index()); _queue.push_front (vf); } diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index a3ea9f3e8..1d31274b3 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -70,8 +70,8 @@ public: /** Called to pass a bit of video to be encoded as the next DCP frame */ void encode (std::shared_ptr pv, dcpomatic::DCPTime time); - void pause(void); - void resume(void); + void pause(); + void resume(); /** Called when a processing run has finished */ void end (bool isFinal); @@ -111,8 +111,8 @@ private: boost::signals2::scoped_connection _server_found_connection; - grk_plugin::DcpomaticContext dcpomaticContext_; - grk_plugin::GrokContext *context_; + grk_plugin::DcpomaticContext _dcpomatic_context; + grk_plugin::GrokContext *_context; }; diff --git a/src/lib/transcode_job.cc b/src/lib/transcode_job.cc index 0e47aa24c..7c842a99f 100644 --- a/src/lib/transcode_job.cc +++ b/src/lib/transcode_job.cc @@ -147,15 +147,21 @@ TranscodeJob::run () } } -void TranscodeJob::pause() { + +void +TranscodeJob::pause() +{ _encoder->pause(); } -void TranscodeJob::resume() { + +void TranscodeJob::resume() +{ _encoder->resume(); Job::resume(); } + string TranscodeJob::status () const { -- cgit v1.2.3 From a651a17dd5e242d09111bee87e5e4864420dcdcd Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sat, 8 Jul 2023 01:13:04 +0200 Subject: Remove default constructor from DCPVideo. --- src/lib/dcp_video.h | 1 - src/lib/grok/context.h | 16 +++++++--------- src/lib/grok/messenger.h | 26 +++++++++++++------------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/lib/dcp_video.h b/src/lib/dcp_video.h index b351fda1e..5f43dabb8 100644 --- a/src/lib/dcp_video.h +++ b/src/lib/dcp_video.h @@ -49,7 +49,6 @@ class PlayerVideo; class DCPVideo { public: - DCPVideo() : DCPVideo(nullptr, 0, 0, 0, Resolution::TWO_K) {} DCPVideo (std::shared_ptr, int index, int dcp_fps, int bandwidth, Resolution r); DCPVideo (std::shared_ptr, cxml::ConstNodePtr); diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index 7a447d35a..cb913f58d 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -68,8 +68,6 @@ struct GrokInitializer { }; struct FrameProxy { - FrameProxy(void) : FrameProxy(0,Eyes::LEFT,DCPVideo()) - {} FrameProxy(int index, Eyes eyes, DCPVideo dcpv) : index_(index), eyes_(eyes), vf(dcpv) {} int index() const { @@ -157,13 +155,13 @@ public: bool needsRecompression = compressedFrameLength < minimum_size; messenger_->processCompressed(str, processor, needsRecompression); if (needsRecompression) { - bool success = false; - auto fp = messenger_->retrieve(clientFrameId, success); - if (!success) + auto fp = messenger_->retrieve(clientFrameId); + if (!fp) { return; + } - auto encoded = std::make_shared(fp.vf.encode_locally()); - dcpomaticContext_.writer_.write(encoded, fp.vf.index(), fp.vf.eyes()); + auto encoded = std::make_shared(fp->vf.encode_locally()); + dcpomaticContext_.writer_.write(encoded, fp->vf.index(), fp->vf.eyes()); frame_done (); } } @@ -210,11 +208,11 @@ public: return launched_; } - bool scheduleCompress(const DCPVideo &vf){ + bool scheduleCompress(DCPVideo const& vf){ if (!messenger_) return false; - auto fp = FrameProxy(vf.index(),vf.eyes(),vf); + auto fp = FrameProxy(vf.index(), vf.eyes(), vf); auto cvt = [this, &fp](BufferSrc src){ // xyz conversion fp.vf.convert_to_xyz((uint16_t*)src.framePtr_); diff --git a/src/lib/grok/messenger.h b/src/lib/grok/messenger.h index 45ee752e5..86a50d530 100644 --- a/src/lib/grok/messenger.h +++ b/src/lib/grok/messenger.h @@ -825,22 +825,20 @@ static void processorThread(Messenger* messenger, std::function struct ScheduledFrames { - void store(F& val) + void store(F const& val) { std::unique_lock lk(mapMutex_); auto it = map_.find(val.index()); if (it == map_.end()) - map_[val.index()] = val; + map_.emplace(std::make_pair(val.index(), val)); } - F retrieve(size_t index, bool &success) + boost::optional retrieve(size_t index) { std::unique_lock lk(mapMutex_); - success = false; auto it = map_.find(index); if(it == map_.end()) - return F(); + return {}; - success = true; F val = it->second; map_.erase(index); @@ -862,7 +860,7 @@ struct ScheduledMessenger : public Messenger ~ScheduledMessenger(void) { shutdown(); } - bool scheduleCompress(F proxy, std::function converter){ + bool scheduleCompress(F const& proxy, std::function converter){ size_t frameSize = init_.uncompressedFrameSize_; assert(frameSize >= init_.uncompressedFrameSize_); BufferSrc src; @@ -882,11 +880,11 @@ struct ScheduledMessenger : public Messenger auto compressedFrameId = msg.nextUint(); auto compressedFrameLength = msg.nextUint(); if (!needsRecompression) { - bool success = false; - auto srcFrame = scheduledFrames_.retrieve(clientFrameId,success); - if (!success) + auto src_frame = scheduledFrames_.retrieve(clientFrameId); + if (!src_frame) { return; - processor(srcFrame, getCompressedFrame(compressedFrameId),compressedFrameLength); + } + processor(*src_frame, getCompressedFrame(compressedFrameId),compressedFrameLength); } ++framesCompressed_; send(GRK_MSGR_BATCH_PROCESSSED_COMPRESSED, compressedFrameId); @@ -914,9 +912,11 @@ struct ScheduledMessenger : public Messenger } } - F retrieve(size_t index, bool &success) { - return scheduledFrames_.retrieve(index, success); + + boost::optional retrieve(size_t index) { + return scheduledFrames_.retrieve(index); } + void store(F& val) { scheduledFrames_.store(val); } -- cgit v1.2.3 From 2d12d76bced748c5f85833ee3e88d96376ee21ec Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 9 Jul 2023 23:18:40 +0200 Subject: Change end() to only do one thing, and copy the required stuff into pause() --- src/lib/dcp_encoder.cc | 2 +- src/lib/j2k_encoder.cc | 82 ++++++++++++++++++++++++++++---------------------- src/lib/j2k_encoder.h | 2 +- 3 files changed, 48 insertions(+), 38 deletions(-) diff --git a/src/lib/dcp_encoder.cc b/src/lib/dcp_encoder.cc index a03f689b0..1b1b117be 100644 --- a/src/lib/dcp_encoder.cc +++ b/src/lib/dcp_encoder.cc @@ -114,7 +114,7 @@ DCPEncoder::go () } _finishing = true; - _j2k_encoder.end(true); + _j2k_encoder.end(); _writer.finish(_film->dir(_film->dcp_name())); } diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index 7480e4541..48883bec4 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -93,37 +93,48 @@ J2KEncoder::begin () void J2KEncoder::pause() { - if (Config::instance()->enable_gpu()) { - end(false); + if (!Config::instance()->enable_gpu()) { + return; } + + { + boost::mutex::scoped_lock lm (_threads_mutex); + terminate_threads (); + } + + /* Something might have been thrown during terminate_threads */ + rethrow (); + + delete _context; + _context = nullptr; } void J2KEncoder::resume() { - if (Config::instance()->enable_gpu()) { - _context = new grk_plugin::GrokContext(_dcpomatic_context); - servers_list_changed(); + if (!Config::instance()->enable_gpu()) { + return; } + + _context = new grk_plugin::GrokContext(_dcpomatic_context); + servers_list_changed(); } void -J2KEncoder::end (bool isFinal) +J2KEncoder::end() { - if (isFinal) { - boost::mutex::scoped_lock lock (_queue_mutex); + boost::mutex::scoped_lock lock (_queue_mutex); - LOG_GENERAL (N_("Clearing queue of %1"), _queue.size ()); + LOG_GENERAL (N_("Clearing queue of %1"), _queue.size ()); - /* Keep waking workers until the queue is empty */ - while (!_queue.empty ()) { - rethrow (); - _empty_condition.notify_all (); - _full_condition.wait (lock); - } - lock.unlock (); - } + /* Keep waking workers until the queue is empty */ + while (!_queue.empty ()) { + rethrow (); + _empty_condition.notify_all (); + _full_condition.wait (lock); + } + lock.unlock (); LOG_GENERAL_NC (N_("Terminating encoder threads")); @@ -145,29 +156,28 @@ J2KEncoder::end (bool isFinal) So just mop up anything left in the queue here. */ - if (isFinal) { - for (auto & i: _queue) { - if (Config::instance()->enable_gpu ()) { - if (!_context->scheduleCompress(i)){ - LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), i.index()); - // handle error - } + for (auto & i: _queue) { + if (Config::instance()->enable_gpu ()) { + if (!_context->scheduleCompress(i)){ + LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), i.index()); + // handle error } - else { - LOG_GENERAL(N_("Encode left-over frame %1"), i.index()); - try { - _writer.write( - make_shared(i.encode_locally()), - i.index(), - i.eyes() - ); - frame_done (); - } catch (std::exception& e) { - LOG_ERROR (N_("Local encode failed (%1)"), e.what ()); - } + } + else { + LOG_GENERAL(N_("Encode left-over frame %1"), i.index()); + try { + _writer.write( + make_shared(i.encode_locally()), + i.index(), + i.eyes() + ); + frame_done (); + } catch (std::exception& e) { + LOG_ERROR (N_("Local encode failed (%1)"), e.what ()); } } } + delete _context; _context = nullptr; } diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index 1d31274b3..433a0498f 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -74,7 +74,7 @@ public: void resume(); /** Called when a processing run has finished */ - void end (bool isFinal); + void end(); boost::optional current_encoding_rate () const; int video_frames_enqueued () const; -- cgit v1.2.3 From 66e124d1aa3b8df16a92146c4b0803f072f6f106 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 9 Jul 2023 23:22:20 +0200 Subject: Cleanup: white space tidying. --- src/lib/j2k_encoder.cc | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index 48883bec4..b94626fcd 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -129,11 +129,11 @@ J2KEncoder::end() LOG_GENERAL (N_("Clearing queue of %1"), _queue.size ()); /* Keep waking workers until the queue is empty */ - while (!_queue.empty ()) { - rethrow (); - _empty_condition.notify_all (); - _full_condition.wait (lock); - } + while (!_queue.empty ()) { + rethrow (); + _empty_condition.notify_all (); + _full_condition.wait (lock); + } lock.unlock (); LOG_GENERAL_NC (N_("Terminating encoder threads")); @@ -149,12 +149,12 @@ J2KEncoder::end() LOG_GENERAL (N_("Mopping up %1"), _queue.size()); /* The following sequence of events can occur in the above code: - 1. a remote worker takes the last image off the queue - 2. the loop above terminates - 3. the remote worker fails to encode the image and puts it back on the queue - 4. the remote worker is then terminated by terminate_threads + 1. a remote worker takes the last image off the queue + 2. the loop above terminates + 3. the remote worker fails to encode the image and puts it back on the queue + 4. the remote worker is then terminated by terminate_threads - So just mop up anything left in the queue here. + So just mop up anything left in the queue here. */ for (auto & i: _queue) { if (Config::instance()->enable_gpu ()) { @@ -389,7 +389,6 @@ try LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index()); _queue.push_front (vf); } - } else { try { LOG_TIMING ("start-local-encode thread=%1 frame=%2", thread_id(), vf.index()); -- cgit v1.2.3 From 1ee5e03a53149d9e3db032de39943b83c4bfa45c Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 9 Jul 2023 23:25:36 +0200 Subject: Tidy up some includes / forward declarations. --- src/lib/grok/context.h | 1 + src/lib/j2k_encoder.h | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index cb913f58d..da2fd78de 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -22,6 +22,7 @@ #include "../config.h" #include "../dcp_video.h" +#include "../film.h" #include "../log.h" #include "../dcpomatic_log.h" #include "../writer.h" diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index 433a0498f..840602dbb 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -27,10 +27,9 @@ * @brief J2KEncoder class. */ -#include "grok/context.h" +#include "grok/context.h" #include "cross.h" -#include "dcp_video.h" #include "enum_indexed_vector.h" #include "event_history.h" #include "exception_store.h" @@ -43,6 +42,8 @@ #include #include + +class DCPVideo; class EncodeServerDescription; class Film; class Job; -- cgit v1.2.3 From 30cefe57adea0653cc85f6e04346355ec8ca739a Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Mon, 4 Sep 2023 00:58:19 +0200 Subject: Further patch from Aaron. --- src/lib/grok/messenger.h | 58 ++++++------------------------------------------ 1 file changed, 7 insertions(+), 51 deletions(-) diff --git a/src/lib/grok/messenger.h b/src/lib/grok/messenger.h index 86a50d530..008e58dd2 100644 --- a/src/lib/grok/messenger.h +++ b/src/lib/grok/messenger.h @@ -276,7 +276,8 @@ struct SharedMemoryManager if(rc) getMessengerLogger()->error("Error closing shared memory: %s", strerror(errno)); rc = shm_unlink(name.c_str()); - if(rc) + // 2 == No such file or directory + if(rc && errno != 2) getMessengerLogger()->error("Error unlinking shared memory: %s", strerror(errno)); return false; } @@ -288,7 +289,8 @@ struct SharedMemoryManager if(rc) getMessengerLogger()->error("Error closing shared memory: %s", strerror(errno)); rc = shm_unlink(name.c_str()); - if(rc) + // 2 == No such file or directory + if(rc && errno != 2) getMessengerLogger()->error("Error unlinking shared memory: %s", strerror(errno)); } @@ -308,7 +310,8 @@ struct SharedMemoryManager if(rc) getMessengerLogger()->error("Error closing shared memory %s: %s", name.c_str(), strerror(errno)); rc = shm_unlink(name.c_str()); - if(rc) + // 2 == No such file or directory + if(rc && errno != 2) fprintf(stderr,"Error unlinking shared memory %s : %s\n", name.c_str(), strerror(errno)); return true; @@ -499,27 +502,6 @@ struct Messenger for(size_t i = 0; i < init_.numProcessingThreads_; ++i) processors_.push_back(std::thread(processorThread, this, init_.processor_)); } - size_t serialize(const std::string &dir, size_t clientFrameId, uint8_t* compressedPtr, - size_t compressedLength) - { - char fname[512]; - if(!compressedPtr || !compressedLength) - return 0; - sprintf(fname, "%s/test_%d.j2k", dir.c_str(), (int)clientFrameId); - auto fp = fopen(fname, "wb"); - if(!fp) - return 0; - size_t written = fwrite(compressedPtr, 1, compressedLength, fp); - if(written != compressedLength) - { - fclose(fp); - return 0; - } - fflush(fp); - fclose(fp); - - return written; - } bool initBuffers(void) { bool rc = true; @@ -560,32 +542,6 @@ struct Messenger sendQueue.push(oss.str()); } - static pid_t get_pid_by_process_name(const char* name) - { - char command[256]; - snprintf(command, sizeof(command), "pgrep %s", name); - auto pgrep = popen(command, "r"); - if(!pgrep) - return -1; - pid_t pid; - if(fscanf(pgrep, "%d", &pid) != 1) - pid = -1; - pclose(pgrep); - - return pid; - } - static bool terminate_process(const char* name) - { - auto pid = get_pid_by_process_name(name); - - return (pid != -1 && kill(pid, SIGTERM) != -1); - } - static bool kill_process(const char* name) - { - auto pid = get_pid_by_process_name(name); - - return (pid != -1 && kill(pid, SIGKILL) != -1); - } void launchGrok(const std::string &dir, uint32_t width, uint32_t stride, uint32_t height, uint32_t samplesPerPixel, uint32_t depth, int device, bool is4K, uint32_t fps, uint32_t bandwidth, @@ -602,7 +558,7 @@ struct Messenger char _cmd[4096]; auto fullServer = server + ":" + std::to_string(port); sprintf(_cmd, - "./grk_compress -batch_src %s,%d,%d,%d,%d,%d -out_fmt j2k -out_dir - -k 1 " + "./grk_compress -batch_src %s,%d,%d,%d,%d,%d -out_fmt j2k -k 1 " "-G %d -%s %d,%d -j %s -J %s", GRK_MSGR_BATCH_IMAGE.c_str(), width, stride, height, samplesPerPixel, depth, device, is4K ? "cinema4K" : "cinema2K", fps, bandwidth, -- cgit v1.2.3 From 0e7f9fbebfce24d943b8fec4a564602a756d44e0 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 10 Sep 2023 23:45:56 +0200 Subject: Cleanup: white space. --- src/lib/j2k_encoder.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index b94626fcd..4b53ee113 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -74,8 +74,8 @@ J2KEncoder::~J2KEncoder () _server_found_connection.disconnect(); { - boost::mutex::scoped_lock lm (_threads_mutex); - terminate_threads (); + boost::mutex::scoped_lock lm(_threads_mutex); + terminate_threads(); } delete _context; -- cgit v1.2.3 From 574b2871b30cc87dbf1bef1e784ec185ff8e19e6 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 10 Sep 2023 23:46:19 +0200 Subject: Remove check for _threads being null, as I think it should not happen. --- src/lib/j2k_encoder.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index 4b53ee113..4447b7be2 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -229,10 +229,7 @@ J2KEncoder::encode (shared_ptr pv, DCPTime time) size_t threads = 0; { boost::mutex::scoped_lock lm (_threads_mutex); - if (_threads) - threads = _threads->size(); - else - threads = std::thread::hardware_concurrency(); + threads = _threads->size(); } boost::mutex::scoped_lock queue_lock (_queue_mutex); -- cgit v1.2.3 From 2ed3dadd6287859551fcbeaf85e09b0b3f1e8ff5 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 00:34:15 +0200 Subject: Rearrange encoder threading. Soon we'll add a new encoder type, and the existing structure was already creaking a bit at the seams while handling local and remote encodes. Here we split out an encoder thread and introduce the concept of a "sync" thread (which blocks while the encoding is happening). Later we'll have another type which submits the encode request to a GPU and receives the reply back later. --- src/lib/cpu_j2k_encoder_thread.cc | 42 ++++++ src/lib/cpu_j2k_encoder_thread.h | 16 ++ src/lib/dcp_encoder.h | 4 + src/lib/encode_server.cc | 3 + src/lib/encode_server.h | 6 + src/lib/grok_j2k_encoder_thread.cc | 46 ++++++ src/lib/grok_j2k_encoder_thread.h | 23 +++ src/lib/j2k_encoder.cc | 278 +++++++++++++++-------------------- src/lib/j2k_encoder.h | 20 ++- src/lib/j2k_encoder_thread.cc | 38 +++++ src/lib/j2k_encoder_thread.h | 32 ++++ src/lib/j2k_sync_encoder_thread.cc | 44 ++++++ src/lib/j2k_sync_encoder_thread.h | 32 ++++ src/lib/make_dcp.cc | 6 +- src/lib/make_dcp.h | 2 +- src/lib/remote_j2k_encoder_thread.cc | 64 ++++++++ src/lib/remote_j2k_encoder_thread.h | 21 +++ src/lib/scope_guard.h | 10 +- src/lib/transcode_job.h | 4 + src/lib/wscript | 5 + test/client_server_test.cc | 27 +++- test/j2k_encode_threading_test.cc | 96 ++++++++++++ test/wscript | 1 + 23 files changed, 651 insertions(+), 169 deletions(-) create mode 100644 src/lib/cpu_j2k_encoder_thread.cc create mode 100644 src/lib/cpu_j2k_encoder_thread.h create mode 100644 src/lib/grok_j2k_encoder_thread.cc create mode 100644 src/lib/grok_j2k_encoder_thread.h create mode 100644 src/lib/j2k_encoder_thread.cc create mode 100644 src/lib/j2k_encoder_thread.h create mode 100644 src/lib/j2k_sync_encoder_thread.cc create mode 100644 src/lib/j2k_sync_encoder_thread.h create mode 100644 src/lib/remote_j2k_encoder_thread.cc create mode 100644 src/lib/remote_j2k_encoder_thread.h create mode 100644 test/j2k_encode_threading_test.cc diff --git a/src/lib/cpu_j2k_encoder_thread.cc b/src/lib/cpu_j2k_encoder_thread.cc new file mode 100644 index 000000000..70afac236 --- /dev/null +++ b/src/lib/cpu_j2k_encoder_thread.cc @@ -0,0 +1,42 @@ +#include "cpu_j2k_encoder_thread.h" +#include "cross.h" +#include "dcpomatic_log.h" +#include "dcp_video.h" +#include "j2k_encoder.h" +#include "scope_guard.h" +#include "util.h" + +#include "i18n.h" + + +using std::make_shared; +using std::shared_ptr; + + +CPUJ2KEncoderThread::CPUJ2KEncoderThread(J2KEncoder& encoder) + : J2KSyncEncoderThread(encoder) +{ + +} + + +void +CPUJ2KEncoderThread::log_thread_start() const +{ + start_of_thread("CPUJ2KEncoder"); + LOG_TIMING("start-encoder-thread thread=%1 server=localhost", thread_id()); +} + + +shared_ptr +CPUJ2KEncoderThread::encode(DCPVideo const& frame) +{ + try { + return make_shared(frame.encode_locally()); + } catch (std::exception& e) { + LOG_ERROR(N_("Local encode failed (%1)"), e.what()); + } + + return {}; +} + diff --git a/src/lib/cpu_j2k_encoder_thread.h b/src/lib/cpu_j2k_encoder_thread.h new file mode 100644 index 000000000..fb138f484 --- /dev/null +++ b/src/lib/cpu_j2k_encoder_thread.h @@ -0,0 +1,16 @@ +#include "j2k_sync_encoder_thread.h" +#include + + +class DCPVideo; + + +class CPUJ2KEncoderThread : public J2KSyncEncoderThread +{ +public: + CPUJ2KEncoderThread(J2KEncoder& encoder); + + void log_thread_start() const override; + std::shared_ptr encode(DCPVideo const& frame) override; +}; + diff --git a/src/lib/dcp_encoder.h b/src/lib/dcp_encoder.h index 771679a98..ce0b72204 100644 --- a/src/lib/dcp_encoder.h +++ b/src/lib/dcp_encoder.h @@ -35,6 +35,8 @@ class Job; class Player; class PlayerVideo; +struct frames_not_lost_when_threads_disappear; + /** @class DCPEncoder */ class DCPEncoder : public Encoder @@ -58,6 +60,8 @@ public: private: + friend struct ::frames_not_lost_when_threads_disappear; + void video (std::shared_ptr, dcpomatic::DCPTime); void audio (std::shared_ptr, dcpomatic::DCPTime); void text (PlayerText, TextType, boost::optional, dcpomatic::DCPTimePeriod); diff --git a/src/lib/encode_server.cc b/src/lib/encode_server.cc index 6501dcde1..11dbab628 100644 --- a/src/lib/encode_server.cc +++ b/src/lib/encode_server.cc @@ -81,6 +81,7 @@ EncodeServer::EncodeServer (bool verbose, int num_threads) #endif , _verbose (verbose) , _num_threads (num_threads) + , _frames_encoded(0) { } @@ -165,6 +166,8 @@ EncodeServer::process (shared_ptr socket, struct timeval& after_read, st throw; } + ++_frames_encoded; + return dcp_video_frame.index (); } diff --git a/src/lib/encode_server.h b/src/lib/encode_server.h index f93d66746..8059abd0f 100644 --- a/src/lib/encode_server.h +++ b/src/lib/encode_server.h @@ -32,6 +32,7 @@ #include "exception_store.h" #include "server.h" #include +#include #include #include #include @@ -53,6 +54,10 @@ public: void run () override; + int frames_encoded() const { + return _frames_encoded; + } + private: void handle (std::shared_ptr) override; void worker_thread (); @@ -67,6 +72,7 @@ private: bool _verbose; int _num_threads; Waker _waker; + boost::atomic _frames_encoded; struct Broadcast { diff --git a/src/lib/grok_j2k_encoder_thread.cc b/src/lib/grok_j2k_encoder_thread.cc new file mode 100644 index 000000000..8d0dee5a9 --- /dev/null +++ b/src/lib/grok_j2k_encoder_thread.cc @@ -0,0 +1,46 @@ +#include "cross.h" +#include "dcpomatic_log.h" +#include "dcp_video.h" +#include "grok_j2k_encoder_thread.h" +#include "j2k_encoder.h" +#include "scope_guard.h" +#include "util.h" + +#include "i18n.h" + + +using std::make_shared; +using std::shared_ptr; + + +GrokJ2KEncoderThread::GrokJ2KEncoderThread(J2KEncoder& encoder, grk_plugin::GrokContext* context) + : J2KEncoderThread(encoder) + , _context(context) +{ + +} + + +void +GrokJ2KEncoderThread::run() +try +{ + while (true) + { + LOG_TIMING("encoder-sleep thread=%1", thread_id()); + auto frame = _encoder.pop(); + + ScopeGuard frame_guard([this, &frame]() { + _encoder.retry(frame); + }); + + LOG_TIMING("encoder-pop thread=%1 frame=%2 eyes=%3", thread_id(), frame.index(), static_cast(frame.eyes())); + + if (_context->launch(frame, Config::instance()->selected_gpu()) && _context->scheduleCompress(frame)) { + frame_guard.cancel(); + } + } +} catch (boost::thread_interrupted& e) { +} catch (...) { + store_current(); +} diff --git a/src/lib/grok_j2k_encoder_thread.h b/src/lib/grok_j2k_encoder_thread.h new file mode 100644 index 000000000..8171830a9 --- /dev/null +++ b/src/lib/grok_j2k_encoder_thread.h @@ -0,0 +1,23 @@ +#include "exception_store.h" +#include "j2k_encoder_thread.h" +#include + + +class DCPVideo; + +namespace grk_plugin { + class GrokContext; +} + + +class GrokJ2KEncoderThread : public J2KEncoderThread, public ExceptionStore +{ +public: + GrokJ2KEncoderThread(J2KEncoder& encoder, grk_plugin::GrokContext* context); + + void run() override; + +private: + grk_plugin::GrokContext* _context; +}; + diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index 4447b7be2..7a01bfc4c 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -32,6 +32,9 @@ #include "encode_server_description.h" #include "encode_server_finder.h" #include "film.h" +#include "cpu_j2k_encoder_thread.h" +#include "grok_j2k_encoder_thread.h" +#include "remote_j2k_encoder_thread.h" #include "j2k_encoder.h" #include "log.h" #include "player_video.h" @@ -44,6 +47,7 @@ using std::cout; +using std::dynamic_pointer_cast; using std::exception; using std::list; using std::make_shared; @@ -53,6 +57,7 @@ using boost::optional; using dcp::Data; using namespace dcpomatic; + static grk_plugin::GrokInitializer grokInitializer; /** @param film Film that we are encoding. @@ -73,14 +78,24 @@ J2KEncoder::~J2KEncoder () { _server_found_connection.disconnect(); - { - boost::mutex::scoped_lock lm(_threads_mutex); - terminate_threads(); - } + terminate_threads(); delete _context; } + +void +J2KEncoder::servers_list_changed() +{ + auto config = Config::instance(); + + auto const cpu = (config->enable_gpu() || config->only_servers_encode()) ? 0 : config->master_encoding_threads(); + auto const gpu = config->enable_gpu() ? config->master_encoding_threads() : 0; + + remake_threads(cpu, gpu, EncodeServerFinder::instance()->servers()); +} + + void J2KEncoder::begin () { @@ -97,10 +112,7 @@ J2KEncoder::pause() return; } - { - boost::mutex::scoped_lock lm (_threads_mutex); - terminate_threads (); - } + terminate_threads (); /* Something might have been thrown during terminate_threads */ rethrow (); @@ -131,17 +143,13 @@ J2KEncoder::end() /* Keep waking workers until the queue is empty */ while (!_queue.empty ()) { rethrow (); - _empty_condition.notify_all (); _full_condition.wait (lock); } lock.unlock (); LOG_GENERAL_NC (N_("Terminating encoder threads")); - { - boost::mutex::scoped_lock lm (_threads_mutex); - terminate_threads (); - } + terminate_threads (); /* Something might have been thrown during terminate_threads */ rethrow (); @@ -167,7 +175,7 @@ J2KEncoder::end() LOG_GENERAL(N_("Encode left-over frame %1"), i.index()); try { _writer.write( - make_shared(i.encode_locally()), + make_shared(i.encode_locally()), i.index(), i.eyes() ); @@ -229,7 +237,7 @@ J2KEncoder::encode (shared_ptr pv, DCPTime time) size_t threads = 0; { boost::mutex::scoped_lock lm (_threads_mutex); - threads = _threads->size(); + threads = _threads.size(); } boost::mutex::scoped_lock queue_lock (_queue_mutex); @@ -289,181 +297,139 @@ J2KEncoder::encode (shared_ptr pv, DCPTime time) } -/** Caller must hold a lock on _threads_mutex */ void J2KEncoder::terminate_threads () { + boost::mutex::scoped_lock lm(_threads_mutex); boost::this_thread::disable_interruption dis; - if (!_threads) { - return; - } - - _threads->interrupt_all (); - try { - _threads->join_all (); - } catch (exception& e) { - LOG_ERROR ("join() threw an exception: %1", e.what()); - } catch (...) { - LOG_ERROR_NC ("join() threw an exception"); + for (auto& thread: _threads) { + thread->stop(); } - _threads.reset (); + _threads.clear(); + _ending = true; } void -J2KEncoder::encoder_thread (optional server) -try +J2KEncoder::remake_threads(int cpu, int gpu, list servers) { - auto config = Config::instance (); + boost::mutex::scoped_lock lm (_threads_mutex); + if (_ending) { + return; + } - start_of_thread ("J2KEncoder"); + auto remove_threads = [this](int wanted, int current, std::function)> predicate) { + for (auto i = wanted; i < current; ++i) { + auto iter = std::find_if(_threads.begin(), _threads.end(), predicate); + if (iter != _threads.end()) { + (*iter)->stop(); + _threads.erase(iter); + } + } + }; - if (server) { - LOG_TIMING ("start-encoder-thread thread=%1 server=%2", thread_id (), server->host_name ()); - } else { - LOG_TIMING ("start-encoder-thread thread=%1 server=localhost", thread_id ()); + + /* CPU */ + + auto const is_cpu_thread = [](shared_ptr thread) { + return static_cast(dynamic_pointer_cast(thread)); + }; + + auto const current_cpu_threads = std::count_if(_threads.begin(), _threads.end(), is_cpu_thread); + + for (auto i = current_cpu_threads; i < cpu; ++i) { + auto thread = make_shared(*this); + thread->start(); + _threads.push_back(thread); } - /* Number of seconds that we currently wait between attempts - to connect to the server; not relevant for localhost - encodings. - */ - int remote_backoff = 0; + remove_threads(cpu, current_cpu_threads, is_cpu_thread); + - while (true) { + /* GPU */ - LOG_TIMING ("encoder-sleep thread=%1", thread_id ()); - boost::mutex::scoped_lock lock (_queue_mutex); - while (_queue.empty ()) { - _empty_condition.wait (lock); + auto const is_grok_thread = [](shared_ptr thread) { + return static_cast(dynamic_pointer_cast(thread)); + }; + + auto const current_gpu_threads = std::count_if(_threads.begin(), _threads.end(), is_grok_thread); + + for (auto i = current_gpu_threads; i < gpu; ++i) { + auto thread = make_shared(*this, _context); + thread->start(); + _threads.push_back(thread); + } + + remove_threads(gpu, current_gpu_threads, is_grok_thread); + + + /* Remote */ + + for (auto const& server: servers) { + if (!server.current_link_version()) { + continue; } - LOG_TIMING ("encoder-wake thread=%1 queue=%2", thread_id(), _queue.size()); - auto vf = _queue.front (); + auto is_remote_thread = [server](shared_ptr thread) { + auto remote = dynamic_pointer_cast(thread); + return remote && remote->server().host_name() == server.host_name(); + }; - /* We're about to commit to either encoding this frame or putting it back onto the queue, - so we must not be interrupted until one or other of these things have happened. This - block has thread interruption disabled. - */ - { - boost::this_thread::disable_interruption dis; - - LOG_TIMING ("encoder-pop thread=%1 frame=%2 eyes=%3", thread_id(), vf.index(), static_cast(vf.eyes())); - _queue.pop_front (); - - lock.unlock (); - - shared_ptr encoded; - - /* We need to encode this input */ - if (server) { - try { - encoded = make_shared(vf.encode_remotely(server.get())); - - if (remote_backoff > 0) { - LOG_GENERAL ("%1 was lost, but now she is found; removing backoff", server->host_name ()); - } - - /* This job succeeded, so remove any backoff */ - remote_backoff = 0; - - } catch (std::exception& e) { - if (remote_backoff < 60) { - /* back off more */ - remote_backoff += 10; - } - LOG_ERROR ( - N_("Remote encode of %1 on %2 failed (%3); thread sleeping for %4s"), - vf.index(), server->host_name(), e.what(), remote_backoff - ); - } - - } else { - if (_context) { - if (!_context->launch(vf, config->selected_gpu()) || !_context->scheduleCompress(vf)) { - LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index()); - _queue.push_front (vf); - } - } else { - try { - LOG_TIMING ("start-local-encode thread=%1 frame=%2", thread_id(), vf.index()); - encoded = make_shared(vf.encode_locally()); - LOG_TIMING ("finish-local-encode thread=%1 frame=%2", thread_id(), vf.index()); - } catch (std::exception& e) { - /* This is very bad, so don't cope with it, just pass it on */ - LOG_ERROR (N_("Local encode failed (%1)"), e.what ()); - throw; - } - } - } + auto const current_threads = std::count_if(_threads.begin(), _threads.end(), is_remote_thread); - if (encoded) { - _writer.write(encoded, vf.index(), vf.eyes()); - frame_done (); - } else { - if (!Config::instance()->enable_gpu ()) { - lock.lock (); - LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index()); - _queue.push_front (vf); - lock.unlock (); - } - } + auto const wanted_threads = server.threads(); + + if (wanted_threads > current_threads) { + LOG_GENERAL(N_("Adding %1 worker threads for remote %2"), wanted_threads - current_threads, server.host_name()); + } else if (wanted_threads < current_threads) { + LOG_GENERAL(N_("Removing %1 worker threads for remote %2"), current_threads - wanted_threads, server.host_name()); } - if (remote_backoff > 0) { - boost::this_thread::sleep (boost::posix_time::seconds (remote_backoff)); + for (auto i = current_threads; i < wanted_threads; ++i) { + auto thread = make_shared(*this, server); + thread->start(); + _threads.push_back(thread); } - /* The queue might not be full any more, so notify anything that is waiting on that */ - lock.lock (); - _full_condition.notify_all (); + remove_threads(wanted_threads, current_threads, is_remote_thread); } + + _writer.set_encoder_threads(_threads.size()); } -catch (boost::thread_interrupted& e) { - /* Ignore these and just stop the thread */ - _full_condition.notify_all (); -} -catch (...) + + +DCPVideo +J2KEncoder::pop() { - store_current (); - /* Wake anything waiting on _full_condition so it can see the exception */ - _full_condition.notify_all (); + boost::mutex::scoped_lock lock(_queue_mutex); + while (_queue.empty()) { + _empty_condition.wait (lock); + } + + LOG_TIMING("encoder-wake thread=%1 queue=%2", thread_id(), _queue.size()); + + auto vf = _queue.front(); + _queue.pop_front(); + + _full_condition.notify_all(); + return vf; } void -J2KEncoder::servers_list_changed () +J2KEncoder::retry(DCPVideo video) { - boost::mutex::scoped_lock lm (_threads_mutex); - - terminate_threads (); - _threads = make_shared(); - - /* XXX: could re-use threads */ - - if (!Config::instance()->only_servers_encode ()) { - for (int i = 0; i < Config::instance()->master_encoding_threads (); ++i) { -#ifdef DCPOMATIC_LINUX - auto t = _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, optional())); - pthread_setname_np (t->native_handle(), "encode-worker"); -#else - _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, optional())); -#endif - } - } - - for (auto i: EncodeServerFinder::instance()->servers()) { - if (!i.current_link_version()) { - continue; - } + boost::mutex::scoped_lock lock(_queue_mutex); + _queue.push_front(video); + _empty_condition.notify_all(); +} - LOG_GENERAL (N_("Adding %1 worker threads for remote %2"), i.threads(), i.host_name ()); - for (int j = 0; j < i.threads(); ++j) { - _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, i)); - } - } - _writer.set_encoder_threads(_threads->size()); +void +J2KEncoder::write(shared_ptr data, int index, Eyes eyes) +{ + _writer.write(data, index, eyes); + frame_done(); } diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index 840602dbb..a6e190dcf 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -33,6 +33,7 @@ #include "enum_indexed_vector.h" #include "event_history.h" #include "exception_store.h" +#include "j2k_encoder_thread.h" #include "writer.h" #include #include @@ -49,6 +50,10 @@ class Film; class Job; class PlayerVideo; +struct local_threads_created_and_destroyed; +struct remote_threads_created_and_destroyed; +struct frames_not_lost_when_threads_disappear; + /** @class J2KEncoder * @brief Class to manage encoding to J2K. @@ -80,13 +85,18 @@ public: boost::optional current_encoding_rate () const; int video_frames_enqueued () const; - void servers_list_changed (); + DCPVideo pop(); + void retry(DCPVideo frame); + void write(std::shared_ptr data, int index, Eyes eyes); private: + friend struct ::local_threads_created_and_destroyed; + friend struct ::remote_threads_created_and_destroyed; + friend struct ::frames_not_lost_when_threads_disappear; void frame_done (); - - void encoder_thread (boost::optional); + void servers_list_changed (); + void remake_threads(int cpu, int gpu, std::list servers); void terminate_threads (); /** Film that we are encoding */ @@ -95,7 +105,7 @@ private: EventHistory _history; boost::mutex _threads_mutex; - std::shared_ptr _threads; + std::vector> _threads; mutable boost::mutex _queue_mutex; std::list _queue; @@ -114,6 +124,8 @@ private: grk_plugin::DcpomaticContext _dcpomatic_context; grk_plugin::GrokContext *_context; + + bool _ending = false; }; diff --git a/src/lib/j2k_encoder_thread.cc b/src/lib/j2k_encoder_thread.cc new file mode 100644 index 000000000..0199209be --- /dev/null +++ b/src/lib/j2k_encoder_thread.cc @@ -0,0 +1,38 @@ +#include "dcp_video.h" +#include "dcpomatic_log.h" +#include "j2k_encoder.h" +#include "j2k_encoder_thread.h" +#include "scope_guard.h" + + +J2KEncoderThread::J2KEncoderThread(J2KEncoder& encoder) + : _encoder(encoder) +{ + +} + + +void +J2KEncoderThread::start() +{ + _thread = boost::thread(boost::bind(&J2KEncoderThread::run, this)); +#ifdef DCPOMATIC_LINUX + pthread_setname_np(_thread.native_handle(), "encode-worker"); +#endif +} + + +void +J2KEncoderThread::stop() +{ + _thread.interrupt(); + try { + _thread.join(); + } catch (std::exception& e) { + LOG_ERROR("join() threw an exception: %1", e.what()); + } catch (...) { + LOG_ERROR_NC("join() threw an exception"); + } +} + + diff --git a/src/lib/j2k_encoder_thread.h b/src/lib/j2k_encoder_thread.h new file mode 100644 index 000000000..4a4b25f37 --- /dev/null +++ b/src/lib/j2k_encoder_thread.h @@ -0,0 +1,32 @@ +#ifndef DCPOMATIC_J2K_ENCODER_THREAD +#define DCPOMATIC_J2K_ENCODER_THREAD + + +#include + + +class J2KEncoder; + + +class J2KEncoderThread +{ +public: + J2KEncoderThread(J2KEncoder& encoder); + + J2KEncoderThread(J2KEncoderThread const&) = delete; + J2KEncoderThread& operator=(J2KEncoderThread const&) = delete; + + void start(); + void stop(); + + virtual void run() = 0; + +protected: + J2KEncoder& _encoder; + +private: + boost::thread _thread; +}; + + +#endif diff --git a/src/lib/j2k_sync_encoder_thread.cc b/src/lib/j2k_sync_encoder_thread.cc new file mode 100644 index 000000000..9398bcb85 --- /dev/null +++ b/src/lib/j2k_sync_encoder_thread.cc @@ -0,0 +1,44 @@ +#include "dcp_video.h" +#include "dcpomatic_log.h" +#include "j2k_encoder.h" +#include "j2k_sync_encoder_thread.h" +#include "scope_guard.h" + + +J2KSyncEncoderThread::J2KSyncEncoderThread(J2KEncoder& encoder) + : J2KEncoderThread(encoder) +{ + +} + + +void +J2KSyncEncoderThread::run() +try +{ + log_thread_start(); + + while (true) { + LOG_TIMING("encoder-sleep thread=%1", thread_id()); + auto frame = _encoder.pop(); + + ScopeGuard frame_guard([this, &frame]() { + boost::this_thread::disable_interruption dis; + _encoder.retry(frame); + }); + + LOG_TIMING("encoder-pop thread=%1 frame=%2 eyes=%3", thread_id(), frame.index(), static_cast(frame.eyes())); + + auto encoded = encode(frame); + + if (encoded) { + boost::this_thread::disable_interruption dis; + frame_guard.cancel(); + _encoder.write(encoded, frame.index(), frame.eyes()); + } + } +} catch (boost::thread_interrupted& e) { +} catch (...) { + store_current(); +} + diff --git a/src/lib/j2k_sync_encoder_thread.h b/src/lib/j2k_sync_encoder_thread.h new file mode 100644 index 000000000..45222279e --- /dev/null +++ b/src/lib/j2k_sync_encoder_thread.h @@ -0,0 +1,32 @@ +#ifndef DCPOMATIC_J2K_SYNC_ENCODER_THREAD_H +#define DCPOMATIC_J2K_SYNC_ENCODER_THREAD_H + + +#include "exception_store.h" +#include "j2k_encoder_thread.h" +#include +#include + + +class DCPVideo; +class J2KEncoder; + + +class J2KSyncEncoderThread : public J2KEncoderThread, public ExceptionStore +{ +public: + J2KSyncEncoderThread(J2KEncoder& encoder); + + J2KSyncEncoderThread(J2KSyncEncoderThread const&) = delete; + J2KSyncEncoderThread& operator=(J2KSyncEncoderThread const&) = delete; + + virtual ~J2KSyncEncoderThread() {} + + void run() override; + + virtual void log_thread_start() const = 0; + virtual std::shared_ptr encode(DCPVideo const& frame) = 0; +}; + + +#endif diff --git a/src/lib/make_dcp.cc b/src/lib/make_dcp.cc index 17d45be46..d8d42f49a 100644 --- a/src/lib/make_dcp.cc +++ b/src/lib/make_dcp.cc @@ -40,8 +40,8 @@ using std::shared_ptr; using std::string; -/** Add suitable Jobs to the JobManager to create a DCP for a Film */ -void +/** Add suitable Job to the JobManager to create a DCP for a Film */ +shared_ptr make_dcp (shared_ptr film, TranscodeJob::ChangedBehaviour behaviour) { if (film->dcp_name().find("/") != string::npos) { @@ -101,5 +101,7 @@ make_dcp (shared_ptr film, TranscodeJob::ChangedBehaviour behaviour) auto tj = make_shared(film, behaviour); tj->set_encoder (make_shared(film, tj)); JobManager::instance()->add (tj); + + return tj; } diff --git a/src/lib/make_dcp.h b/src/lib/make_dcp.h index 9f5072782..fe0bcd2f6 100644 --- a/src/lib/make_dcp.h +++ b/src/lib/make_dcp.h @@ -25,5 +25,5 @@ class Film; -void make_dcp (std::shared_ptr film, TranscodeJob::ChangedBehaviour behaviour); +std::shared_ptr make_dcp(std::shared_ptr film, TranscodeJob::ChangedBehaviour behaviour); diff --git a/src/lib/remote_j2k_encoder_thread.cc b/src/lib/remote_j2k_encoder_thread.cc new file mode 100644 index 000000000..eac7bb538 --- /dev/null +++ b/src/lib/remote_j2k_encoder_thread.cc @@ -0,0 +1,64 @@ +#include "dcp_video.h" +#include "dcpomatic_log.h" +#include "j2k_encoder.h" +#include "remote_j2k_encoder_thread.h" +#include "scope_guard.h" +#include "util.h" + +#include "i18n.h" + + +using std::make_shared; +using std::shared_ptr; + + +RemoteJ2KEncoderThread::RemoteJ2KEncoderThread(J2KEncoder& encoder, EncodeServerDescription server) + : J2KSyncEncoderThread(encoder) + , _server(server) +{ + +} + + +void +RemoteJ2KEncoderThread::log_thread_start() const +{ + start_of_thread("RemoteJ2KEncoder"); + LOG_TIMING("start-encoder-thread thread=%1 server=%2", thread_id(), _server.host_name()); +} + + +shared_ptr +RemoteJ2KEncoderThread::encode(DCPVideo const& frame) +{ + shared_ptr encoded; + + try { + encoded = make_shared(frame.encode_remotely(_server)); + if (_remote_backoff > 0) { + LOG_GENERAL("%1 was lost, but now she is found; removing backoff", _server.host_name()); + _remote_backoff = 0; + } + } catch (std::exception& e) { + LOG_ERROR( + N_("Remote encode of %1 on %2 failed (%3); thread sleeping for %4s"), + frame.index(), _server.host_name(), e.what(), _remote_backoff + ); + } catch (...) { + LOG_ERROR( + N_("Remote encode of %1 on %2 failed; thread sleeping for %4s"), + frame.index(), _server.host_name(), _remote_backoff + ); + } + + if (!encoded) { + if (_remote_backoff < 60) { + /* back off more */ + _remote_backoff += 10; + } + boost::this_thread::sleep(boost::posix_time::seconds(_remote_backoff)); + } + + return encoded; +} + diff --git a/src/lib/remote_j2k_encoder_thread.h b/src/lib/remote_j2k_encoder_thread.h new file mode 100644 index 000000000..f3fe7f94a --- /dev/null +++ b/src/lib/remote_j2k_encoder_thread.h @@ -0,0 +1,21 @@ +#include "encode_server_description.h" +#include "j2k_sync_encoder_thread.h" + + +class RemoteJ2KEncoderThread : public J2KSyncEncoderThread +{ +public: + RemoteJ2KEncoderThread(J2KEncoder& encoder, EncodeServerDescription server); + + void log_thread_start() const override; + std::shared_ptr encode(DCPVideo const& frame) override; + + EncodeServerDescription server() const { + return _server; + } + +private: + EncodeServerDescription _server; + /** Number of seconds that we currently wait between attempts to connect to the server */ + int _remote_backoff = 0; +}; diff --git a/src/lib/scope_guard.h b/src/lib/scope_guard.h index ac60f9fea..e0d1e81fc 100644 --- a/src/lib/scope_guard.h +++ b/src/lib/scope_guard.h @@ -45,11 +45,19 @@ public: ~ScopeGuard () { - _function(); + if (!_cancelled) { + _function(); + } + } + + void cancel() + { + _cancelled = true; } private: std::function _function; + bool _cancelled = false; }; diff --git a/src/lib/transcode_job.h b/src/lib/transcode_job.h index 794cf7016..35870231d 100644 --- a/src/lib/transcode_job.h +++ b/src/lib/transcode_job.h @@ -37,6 +37,8 @@ class Encoder; +struct frames_not_lost_when_threads_disappear; + /** @class TranscodeJob * @brief A job which transcodes a Film to another format. @@ -66,6 +68,8 @@ public: void set_encoder (std::shared_ptr t); private: + friend struct ::frames_not_lost_when_threads_disappear; + virtual void post_transcode () {} float frames_per_second() const; diff --git a/src/lib/wscript b/src/lib/wscript index dad8947b1..e0cfaa79c 100644 --- a/src/lib/wscript +++ b/src/lib/wscript @@ -59,6 +59,7 @@ sources = """ content_factory.cc combine_dcp_job.cc copy_dcp_details_to_film.cc + cpu_j2k_encoder_thread.cc create_cli.cc crop.cc cross_common.cc @@ -122,6 +123,7 @@ sources = """ font_id_map.cc frame_interval_checker.cc frame_rate_change.cc + grok_j2k_encoder_thread.cc guess_crop.cc hints.cc internet.cc @@ -138,6 +140,8 @@ sources = """ job.cc job_manager.cc j2k_encoder.cc + j2k_encoder_thread.cc + j2k_sync_encoder_thread.cc json_server.cc kdm_cli.cc kdm_recipient.cc @@ -163,6 +167,7 @@ sources = """ referenced_reel_asset.cc release_notes.cc render_text.cc + remote_j2k_encoder_thread.cc resampler.cc resolution.cc rgba.cc diff --git a/test/client_server_test.cc b/test/client_server_test.cc index 4f5015fc8..1bfa4c5a6 100644 --- a/test/client_server_test.cc +++ b/test/client_server_test.cc @@ -20,20 +20,18 @@ /** @file test/client_server_test.cc - * @brief Test the server class. + * @brief Test the remote encoding code. * @ingroup feature - * - * Create a test image and then encode it using the standard mechanism - * and also using a EncodeServer object running on localhost. Compare the resulting - * encoded data to check that they are the same. */ +#include "lib/content_factory.h" #include "lib/cross.h" #include "lib/dcp_video.h" #include "lib/dcpomatic_log.h" #include "lib/encode_server.h" #include "lib/encode_server_description.h" +#include "lib/encode_server_finder.h" #include "lib/file_log.h" #include "lib/image.h" #include "lib/j2k_image_proxy.h" @@ -316,3 +314,22 @@ BOOST_AUTO_TEST_CASE (client_server_test_j2k) } +BOOST_AUTO_TEST_CASE(real_encode_with_server) +{ + auto content = content_factory(TestPaths::private_data() / "dolby_aurora.vob"); + auto film = new_test_film2("real_encode_with_server", content); + + EncodeServerFinder::instance(); + + EncodeServer server(true, 4); + thread server_thread(boost::bind(&EncodeServer::run, &server)); + + make_and_verify_dcp(film); + + server.stop(); + server_thread.join(); + + BOOST_CHECK(server.frames_encoded() > 0); + EncodeServerFinder::drop(); +} + diff --git a/test/j2k_encode_threading_test.cc b/test/j2k_encode_threading_test.cc new file mode 100644 index 000000000..ca9bc5cd9 --- /dev/null +++ b/test/j2k_encode_threading_test.cc @@ -0,0 +1,96 @@ +#include "lib/config.h" +#include "lib/content_factory.h" +#include "lib/dcp_encoder.h" +#include "lib/dcp_transcode_job.h" +#include "lib/encode_server_description.h" +#include "lib/film.h" +#include "lib/j2k_encoder.h" +#include "lib/job_manager.h" +#include "lib/make_dcp.h" +#include "lib/transcode_job.h" +#include "test.h" +#include +#include +#include +#include +#include + + +using std::dynamic_pointer_cast; +using std::list; + + +BOOST_AUTO_TEST_CASE(local_threads_created_and_destroyed) +{ + auto film = new_test_film2("local_threads_created_and_destroyed", {}); + Writer writer(film, {}); + J2KEncoder encoder(film, writer); + + encoder.remake_threads(32, 0, {}); + BOOST_CHECK_EQUAL(encoder._threads.size(), 32U); + + encoder.remake_threads(9, 0, {}); + BOOST_CHECK_EQUAL(encoder._threads.size(), 9U); + + encoder.end(); + BOOST_CHECK_EQUAL(encoder._threads.size(), 0U); +} + + +BOOST_AUTO_TEST_CASE(remote_threads_created_and_destroyed) +{ + auto film = new_test_film2("remote_threads_created_and_destroyed", {}); + Writer writer(film, {}); + J2KEncoder encoder(film, writer); + + list servers = { + { "fred", 7, SERVER_LINK_VERSION }, + { "jim", 2, SERVER_LINK_VERSION }, + { "sheila", 14, SERVER_LINK_VERSION }, + }; + + encoder.remake_threads(0, 0, servers); + BOOST_CHECK_EQUAL(encoder._threads.size(), 7U + 2U + 14U); + + servers = { + { "fred", 7, SERVER_LINK_VERSION }, + { "jim", 5, SERVER_LINK_VERSION }, + { "sheila", 14, SERVER_LINK_VERSION }, + }; + + encoder.remake_threads(0, 0, servers); + BOOST_CHECK_EQUAL(encoder._threads.size(), 7U + 5U + 14U); + + servers = { + { "fred", 0, SERVER_LINK_VERSION }, + { "jim", 0, SERVER_LINK_VERSION }, + { "sheila", 11, SERVER_LINK_VERSION }, + }; + + encoder.remake_threads(0, 0, servers); + BOOST_CHECK_EQUAL(encoder._threads.size(), 11U); +} + + +BOOST_AUTO_TEST_CASE(frames_not_lost_when_threads_disappear) +{ + auto content = content_factory(TestPaths::private_data() / "clapperboard.mp4"); + auto film = new_test_film2("frames_not_lost", content); + film->write_metadata(); + auto job = make_dcp(film, TranscodeJob::ChangedBehaviour::IGNORE); + auto& encoder = dynamic_pointer_cast(job->_encoder)->_j2k_encoder; + + while (JobManager::instance()->work_to_do()) { + encoder.remake_threads(rand() % 8, 0, {}); + dcpomatic_sleep_seconds(1); + } + + BOOST_CHECK(!JobManager::instance()->errors()); + + dcp::DCP dcp(film->dir(film->dcp_name())); + dcp.read(); + BOOST_REQUIRE_EQUAL(dcp.cpls().size(), 1U); + BOOST_REQUIRE_EQUAL(dcp.cpls()[0]->reels().size(), 1U); + BOOST_REQUIRE_EQUAL(dcp.cpls()[0]->reels()[0]->main_picture()->intrinsic_duration(), 423U); +} + diff --git a/test/wscript b/test/wscript index e8762009c..51df1b8ed 100644 --- a/test/wscript +++ b/test/wscript @@ -111,6 +111,7 @@ def build(bld): interrupt_encoder_test.cc isdcf_name_test.cc j2k_bandwidth_test.cc + j2k_encode_threading_test.cc job_manager_test.cc kdm_cli_test.cc kdm_naming_test.cc -- cgit v1.2.3 From 0053d931a1293d4f663489f874a54fbcfbd5783b Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 12:01:54 +0200 Subject: Remove unnecessary using statements. --- src/lib/grok/context.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index da2fd78de..8e03c1d54 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -29,8 +29,6 @@ #include "messenger.h" class Film; -using dcp::Data; -using namespace dcpomatic; static std::mutex launchMutex; -- cgit v1.2.3 From 5b4ff8ad597ab9b4ca5d5ee3190dce76c9451566 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 12:02:19 +0200 Subject: Explicitly set up Grok logger rather than relying on a static variable. --- src/lib/grok/context.h | 7 ------- src/lib/j2k_encoder.cc | 2 -- src/tools/dcpomatic.cc | 3 +++ src/tools/dcpomatic_batch.cc | 3 +++ src/tools/dcpomatic_server.cc | 3 +++ src/tools/dcpomatic_server_cli.cc | 3 +++ 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index 8e03c1d54..2513f8863 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -59,13 +59,6 @@ struct GrokLogger : public MessengerLogger { } }; -struct GrokInitializer { - GrokInitializer(void) { - setMessengerLogger(new GrokLogger("[GROK] ")); - } - ~GrokInitializer() = default; -}; - struct FrameProxy { FrameProxy(int index, Eyes eyes, DCPVideo dcpv) : index_(index), eyes_(eyes), vf(dcpv) {} diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index 7a01bfc4c..a3cab7c1c 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -58,8 +58,6 @@ using dcp::Data; using namespace dcpomatic; -static grk_plugin::GrokInitializer grokInitializer; - /** @param film Film that we are encoding. * @param writer Writer that we are using. */ diff --git a/src/tools/dcpomatic.cc b/src/tools/dcpomatic.cc index 8f34b6836..af864ad09 100644 --- a/src/tools/dcpomatic.cc +++ b/src/tools/dcpomatic.cc @@ -75,6 +75,7 @@ #include "lib/ffmpeg_encoder.h" #include "lib/film.h" #include "lib/font_config.h" +#include "lib/grok/context.h" #include "lib/hints.h" #include "lib/job_manager.h" #include "lib/kdm_with_metadata.h" @@ -1712,6 +1713,8 @@ private: notes.Centre(); notes.ShowModal(); } + + grk_plugin::setMessengerLogger(new grk_plugin::GrokLogger("[GROK] ")); } catch (exception& e) { diff --git a/src/tools/dcpomatic_batch.cc b/src/tools/dcpomatic_batch.cc index 16da43410..bf0ea5237 100644 --- a/src/tools/dcpomatic_batch.cc +++ b/src/tools/dcpomatic_batch.cc @@ -31,6 +31,7 @@ #include "lib/config.h" #include "lib/dcpomatic_socket.h" #include "lib/film.h" +#include "lib/grok/context.h" #include "lib/job.h" #include "lib/job_manager.h" #include "lib/make_dcp.h" @@ -496,6 +497,8 @@ class App : public wxApp } } + grk_plugin::setMessengerLogger(new grk_plugin::GrokLogger("[GROK] ")); + return true; } diff --git a/src/tools/dcpomatic_server.cc b/src/tools/dcpomatic_server.cc index e5e3a7e5a..528af8858 100644 --- a/src/tools/dcpomatic_server.cc +++ b/src/tools/dcpomatic_server.cc @@ -25,6 +25,7 @@ #include "lib/encoded_log_entry.h" #include "lib/encode_server.h" #include "lib/config.h" +#include "lib/grok/context.h" #include "lib/log.h" #include "lib/signaller.h" #include "lib/cross.h" @@ -326,6 +327,8 @@ private: SetExitOnFrameDelete (false); + grk_plugin::setMessengerLogger(new grk_plugin::GrokLogger("[GROK] ")); + return true; } diff --git a/src/tools/dcpomatic_server_cli.cc b/src/tools/dcpomatic_server_cli.cc index 6d7f6aba7..30f119a5e 100644 --- a/src/tools/dcpomatic_server_cli.cc +++ b/src/tools/dcpomatic_server_cli.cc @@ -25,6 +25,7 @@ #include "lib/config.h" #include "lib/image.h" #include "lib/file_log.h" +#include "lib/grok/context.h" #include "lib/null_log.h" #include "lib/version.h" #include "lib/encode_server.h" @@ -109,6 +110,8 @@ main (int argc, char* argv[]) dcpomatic_log.reset (new FileLog("dcpomatic_server_cli.log")); } + setMessengerLogger(new grk_plugin::GrokLogger("[GROK] ")); + EncodeServer server (verbose, num_threads); try { -- cgit v1.2.3 From 014cffeabbf9710c927bd9518b558211298f52e2 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 12:09:16 +0200 Subject: Only build grok for Ubuntu 22.04. --- cscript | 3 +++ src/lib/j2k_encoder.cc | 27 +++++++++++++++++++++++---- src/lib/j2k_encoder.h | 6 +++++- src/lib/writer.h | 1 + src/lib/wscript | 4 +++- src/tools/dcpomatic.cc | 4 ++++ src/tools/dcpomatic_batch.cc | 4 ++++ src/tools/dcpomatic_server.cc | 4 ++++ src/tools/dcpomatic_server_cli.cc | 4 ++++ src/wx/full_config_dialog.cc | 4 ++++ wscript | 5 +++++ 11 files changed, 60 insertions(+), 6 deletions(-) diff --git a/cscript b/cscript index e96ff0ae9..ad68da6e7 100644 --- a/cscript +++ b/cscript @@ -564,6 +564,9 @@ def configure_options(target, options, for_package=False): if target.platform == 'osx' and target.arch == 'arm64': opt += ' --target-macos-arm64 --wx-config=%s/wx-config' % target.bin + if target.platform == 'linux' and target.distro == 'ubuntu' and target.version in ['22.04']: + opt += ' --enable-grok' + return opt def build(target, options, for_package): diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index a3cab7c1c..22f2ea6d7 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -33,7 +33,9 @@ #include "encode_server_finder.h" #include "film.h" #include "cpu_j2k_encoder_thread.h" +#ifdef DCPOMATIC_GROK #include "grok_j2k_encoder_thread.h" +#endif #include "remote_j2k_encoder_thread.h" #include "j2k_encoder.h" #include "log.h" @@ -65,8 +67,10 @@ J2KEncoder::J2KEncoder(shared_ptr film, Writer& writer) : _film (film) , _history (200) , _writer (writer) +#ifdef DCPOMATIC_GROK , _dcpomatic_context(film, writer, _history, Config::instance()->gpu_binary_location()) , _context(Config::instance()->enable_gpu() ? new grk_plugin::GrokContext(_dcpomatic_context) : nullptr) +#endif { servers_list_changed (); } @@ -78,7 +82,9 @@ J2KEncoder::~J2KEncoder () terminate_threads(); +#ifdef DCPOMATIC_GROK delete _context; +#endif } @@ -115,8 +121,10 @@ J2KEncoder::pause() /* Something might have been thrown during terminate_threads */ rethrow (); +#ifdef DCPOMATIC_GROK delete _context; _context = nullptr; +#endif } @@ -126,7 +134,9 @@ void J2KEncoder::resume() return; } +#ifdef DCPOMATIC_GROK _context = new grk_plugin::GrokContext(_dcpomatic_context); +#endif servers_list_changed(); } @@ -163,13 +173,16 @@ J2KEncoder::end() So just mop up anything left in the queue here. */ for (auto & i: _queue) { +#ifdef DCPOMATIC_GROK if (Config::instance()->enable_gpu ()) { if (!_context->scheduleCompress(i)){ LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), i.index()); // handle error } - } - else { + } else { +#else + { +#endif LOG_GENERAL(N_("Encode left-over frame %1"), i.index()); try { _writer.write( @@ -184,8 +197,10 @@ J2KEncoder::end() } } +#ifdef DCPOMATIC_GROK delete _context; _context = nullptr; +#endif } @@ -311,7 +326,11 @@ J2KEncoder::terminate_threads () void +#ifdef DCPOMATIC_GROK J2KEncoder::remake_threads(int cpu, int gpu, list servers) +#else +J2KEncoder::remake_threads(int cpu, int, list servers) +#endif { boost::mutex::scoped_lock lm (_threads_mutex); if (_ending) { @@ -345,7 +364,7 @@ J2KEncoder::remake_threads(int cpu, int gpu, list serve remove_threads(cpu, current_cpu_threads, is_cpu_thread); - +#ifdef DCPOMATIC_GROK /* GPU */ auto const is_grok_thread = [](shared_ptr thread) { @@ -361,7 +380,7 @@ J2KEncoder::remake_threads(int cpu, int gpu, list serve } remove_threads(gpu, current_gpu_threads, is_grok_thread); - +#endif /* Remote */ diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index a6e190dcf..9d9d85894 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -28,11 +28,13 @@ */ -#include "grok/context.h" #include "cross.h" #include "enum_indexed_vector.h" #include "event_history.h" #include "exception_store.h" +#ifdef DCPOMATIC_GROK +#include "grok/context.h" +#endif #include "j2k_encoder_thread.h" #include "writer.h" #include @@ -122,8 +124,10 @@ private: boost::signals2::scoped_connection _server_found_connection; +#ifdef DCPOMATIC_GROK grk_plugin::DcpomaticContext _dcpomatic_context; grk_plugin::GrokContext *_context; +#endif bool _ending = false; }; diff --git a/src/lib/writer.h b/src/lib/writer.h index 1fbf7bbd5..0b38e9030 100644 --- a/src/lib/writer.h +++ b/src/lib/writer.h @@ -34,6 +34,7 @@ #include "exception_store.h" #include "font_id_map.h" #include "player_text.h" +#include "text_type.h" #include "weak_film.h" #include #include diff --git a/src/lib/wscript b/src/lib/wscript index e0cfaa79c..67c6b5869 100644 --- a/src/lib/wscript +++ b/src/lib/wscript @@ -123,7 +123,6 @@ sources = """ font_id_map.cc frame_interval_checker.cc frame_rate_change.cc - grok_j2k_encoder_thread.cc guess_crop.cc hints.cc internet.cc @@ -246,6 +245,9 @@ def build(bld): if bld.env.TARGET_LINUX: obj.uselib += ' POLKIT' + if bld.env.ENABLE_GROK: + obj.source += ' grok_j2k_encoder_thread.cc' + if bld.env.TARGET_WINDOWS_64 or bld.env.TARGET_WINDOWS_32: obj.uselib += ' WINSOCK2 DBGHELP SHLWAPI MSWSOCK BOOST_LOCALE SETUPAPI OLE32 UUID' obj.source += ' cross_windows.cc' diff --git a/src/tools/dcpomatic.cc b/src/tools/dcpomatic.cc index af864ad09..db18d1a2f 100644 --- a/src/tools/dcpomatic.cc +++ b/src/tools/dcpomatic.cc @@ -75,7 +75,9 @@ #include "lib/ffmpeg_encoder.h" #include "lib/film.h" #include "lib/font_config.h" +#ifdef DCPOMATIC_GROK #include "lib/grok/context.h" +#endif #include "lib/hints.h" #include "lib/job_manager.h" #include "lib/kdm_with_metadata.h" @@ -1714,7 +1716,9 @@ private: notes.ShowModal(); } +#ifdef DCPOMATIC_GROK grk_plugin::setMessengerLogger(new grk_plugin::GrokLogger("[GROK] ")); +#endif } catch (exception& e) { diff --git a/src/tools/dcpomatic_batch.cc b/src/tools/dcpomatic_batch.cc index bf0ea5237..d112f2060 100644 --- a/src/tools/dcpomatic_batch.cc +++ b/src/tools/dcpomatic_batch.cc @@ -31,7 +31,9 @@ #include "lib/config.h" #include "lib/dcpomatic_socket.h" #include "lib/film.h" +#ifdef DCPOMATIC_GROK #include "lib/grok/context.h" +#endif #include "lib/job.h" #include "lib/job_manager.h" #include "lib/make_dcp.h" @@ -497,7 +499,9 @@ class App : public wxApp } } +#ifdef DCPOMATIC_GROK grk_plugin::setMessengerLogger(new grk_plugin::GrokLogger("[GROK] ")); +#endif return true; } diff --git a/src/tools/dcpomatic_server.cc b/src/tools/dcpomatic_server.cc index 528af8858..b7100d62a 100644 --- a/src/tools/dcpomatic_server.cc +++ b/src/tools/dcpomatic_server.cc @@ -25,7 +25,9 @@ #include "lib/encoded_log_entry.h" #include "lib/encode_server.h" #include "lib/config.h" +#ifdef DCPOMATIC_GROK #include "lib/grok/context.h" +#endif #include "lib/log.h" #include "lib/signaller.h" #include "lib/cross.h" @@ -327,7 +329,9 @@ private: SetExitOnFrameDelete (false); +#ifdef DCPOMATIC_GROK grk_plugin::setMessengerLogger(new grk_plugin::GrokLogger("[GROK] ")); +#endif return true; } diff --git a/src/tools/dcpomatic_server_cli.cc b/src/tools/dcpomatic_server_cli.cc index 30f119a5e..9e4a8814f 100644 --- a/src/tools/dcpomatic_server_cli.cc +++ b/src/tools/dcpomatic_server_cli.cc @@ -25,7 +25,9 @@ #include "lib/config.h" #include "lib/image.h" #include "lib/file_log.h" +#ifdef DCPOMATIC_GROK #include "lib/grok/context.h" +#endif #include "lib/null_log.h" #include "lib/version.h" #include "lib/encode_server.h" @@ -110,7 +112,9 @@ main (int argc, char* argv[]) dcpomatic_log.reset (new FileLog("dcpomatic_server_cli.log")); } +#ifdef DCPOMATIC_GROK setMessengerLogger(new grk_plugin::GrokLogger("[GROK] ")); +#endif EncodeServer server (verbose, num_threads); diff --git a/src/wx/full_config_dialog.cc b/src/wx/full_config_dialog.cc index 119091af3..bc5b5de4e 100644 --- a/src/wx/full_config_dialog.cc +++ b/src/wx/full_config_dialog.cc @@ -45,7 +45,9 @@ #include "send_test_email_dialog.h" #include "server_dialog.h" #include "static_text.h" +#ifdef DCPOMATIC_GROK #include "grok/gpu_config_panel.h" +#endif #include "wx_util.h" #include "lib/config.h" #include "lib/cross.h" @@ -1945,7 +1947,9 @@ create_full_config_dialog () e->AddPage (new SoundPage (ps, border)); e->AddPage (new DefaultsPage (ps, border)); e->AddPage (new EncodingServersPage(ps, border)); +#ifdef DCPOMATIC_GROK e->AddPage (new GPUPage (ps, border)); +#endif e->AddPage (new KeysPage (ps, border)); e->AddPage (new TMSPage (ps, border)); e->AddPage (new EmailPage (ps, border)); diff --git a/wscript b/wscript index a6f3617b6..4629fb243 100644 --- a/wscript +++ b/wscript @@ -76,6 +76,7 @@ def options(opt): opt.add_option('--workaround-gssapi', action='store_true', default=False, help='link to gssapi_krb5') opt.add_option('--use-lld', action='store_true', default=False, help='use lld linker') opt.add_option('--enable-disk', action='store_true', default=False, help='build dcpomatic2_disk tool; requires Boost process, lwext4 and nanomsg libraries') + opt.add_option('--enable-grok', action='store_true', default=False, help='build with support for grok J2K encoder') opt.add_option('--warnings-are-errors', action='store_true', default=False, help='build with -Werror') opt.add_option('--wx-config', help='path to wx-config') @@ -96,6 +97,7 @@ def configure(conf): conf.env.DEBUG = conf.options.enable_debug conf.env.STATIC_DCPOMATIC = conf.options.static_dcpomatic conf.env.ENABLE_DISK = conf.options.enable_disk + conf.env.ENABLE_GROK = conf.options.enable_grok if conf.options.destdir == '': conf.env.INSTALL_PREFIX = conf.options.prefix else: @@ -140,6 +142,9 @@ def configure(conf): if conf.options.enable_disk: conf.env.append_value('CXXFLAGS', '-DDCPOMATIC_DISK') + if conf.options.enable_grok: + conf.env.append_value('CXXFLAGS', '-DDCPOMATIC_GROK') + if conf.options.use_lld: try: conf.find_program('ld.lld') -- cgit v1.2.3 From e14d0004b3dbe17ee7b0e9422ebf54470a4dcfc8 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 13:13:40 +0200 Subject: Hide compiler warning. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In file included from ../src/lib/grok/context.h:29, from ../src/lib/j2k_encoder.h:36, from ../src/lib/grok_j2k_encoder_thread.cc:5: In member function ‘void grk_plugin::Synch::unlink()’, inlined from ‘grk_plugin::Synch::Synch(const string&, const string&)’ at ../src/lib/grok/messenger.h:200:10: ../src/lib/grok/messenger.h:247:52: error: ‘this’ pointer is null [-Werror=nonnull] 247 | getMessengerLogger()->error("Error unlinking semaphore %s: %s", sentSemName_.c_str(), | ~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 248 | strerror(errno)); on Ubuntu 22.04. --- wscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wscript b/wscript index 4629fb243..bbd73f64c 100644 --- a/wscript +++ b/wscript @@ -112,6 +112,8 @@ def configure(conf): '-Wextra', '-Wwrite-strings', '-Wno-error=deprecated', + # getMessengerLogger() in the grok code triggers these warnings + '-Wno-nonnull', # I tried and failed to ignore these with _Pragma '-Wno-ignored-qualifiers', '-D_FILE_OFFSET_BITS=64', -- cgit v1.2.3 From ff240aa07b9ba3c07abb45ec349df9acd0bffc57 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 13:42:05 +0200 Subject: Bump libdcp for things that grok needs. --- cscript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cscript b/cscript index ad68da6e7..fbe36a452 100644 --- a/cscript +++ b/cscript @@ -508,7 +508,7 @@ def dependencies(target, options): # Use distro-provided FFmpeg on Arch deps = [] - deps.append(('libdcp', 'v1.8.85')) + deps.append(('libdcp', 'v1.8.87')) deps.append(('libsub', 'v1.6.44')) deps.append(('leqm-nrt', '30dcaea1373ac62fba050e02ce5b0c1085797a23')) deps.append(('rtaudio', 'f619b76')) -- cgit v1.2.3 From 114a4a1a3056aa077b7734887fbcbff107d80a51 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 16:39:27 +0200 Subject: Check for errors when calling the GPU lister. --- src/wx/grok/gpu_config_panel.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wx/grok/gpu_config_panel.h b/src/wx/grok/gpu_config_panel.h index 1478434be..2647c1ece 100644 --- a/src/wx/grok/gpu_config_panel.h +++ b/src/wx/grok/gpu_config_panel.h @@ -3,7 +3,9 @@ static std::vector get_gpu_names(std::string binary, std::string filename) { // Execute the GPU listing program and redirect its output to a file - std::system((binary + " > " + filename).c_str()); + if (std::system((binary + " > " + filename).c_str()) < 0) { + return {}; + } std::vector gpu_names; std::ifstream file(filename); -- cgit v1.2.3 From bd83d491b6881377ec58572767169f519f59735f Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 16:41:57 +0200 Subject: Copy-edit GUI strings. --- src/wx/grok/gpu_config_panel.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/wx/grok/gpu_config_panel.h b/src/wx/grok/gpu_config_panel.h index 2647c1ece..87af69ef4 100644 --- a/src/wx/grok/gpu_config_panel.h +++ b/src/wx/grok/gpu_config_panel.h @@ -91,22 +91,22 @@ private: { auto config = Config::instance (); - _enable_gpu = new CheckBox (_panel, _("Enable GPU Acceleration")); + _enable_gpu = new CheckBox(_panel, _("Enable GPU acceleration")); _panel->GetSizer()->Add (_enable_gpu, 0, wxALL | wxEXPAND, _border); wxFlexGridSizer* table = new wxFlexGridSizer (2, DCPOMATIC_SIZER_X_GAP, DCPOMATIC_SIZER_Y_GAP); table->AddGrowableCol (1, 1); _panel->GetSizer()->Add (table, 1, wxALL | wxEXPAND, _border); - add_label_to_sizer (table, _panel, _("Acceleration Binary Folder"), true, 0, wxLEFT | wxLEFT | wxALIGN_CENTRE_VERTICAL); + add_label_to_sizer(table, _panel, _("Acceleration binary folder"), true, 0, wxLEFT | wxLEFT | wxALIGN_CENTRE_VERTICAL); _binary_location = new wxDirPickerCtrl (_panel, wxDD_DIR_MUST_EXIST); table->Add (_binary_location, 1, wxEXPAND); - add_label_to_sizer (table, _panel, _("GPU Selection"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + add_label_to_sizer(table, _panel, _("GPU selection"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); _gpu_list_control = new GpuList(_panel); table->Add (_gpu_list_control, 1, wxEXPAND); - add_label_to_sizer (table, _panel, _("License Server"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + add_label_to_sizer(table, _panel, _("License server"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); _server = new wxTextCtrl (_panel, wxID_ANY); table->Add (_server, 1, wxEXPAND | wxALL); -- cgit v1.2.3 From 9a44789407978765651793053e65cc2ce8045d61 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 16:44:27 +0200 Subject: Remove C++17-ism. --- src/wx/grok/gpu_config_panel.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wx/grok/gpu_config_panel.h b/src/wx/grok/gpu_config_panel.h index 87af69ef4..d3088c924 100644 --- a/src/wx/grok/gpu_config_panel.h +++ b/src/wx/grok/gpu_config_panel.h @@ -56,7 +56,8 @@ public: } private: - void OnComboBox([[maybe_unused]] wxCommandEvent& event) { + void OnComboBox(wxCommandEvent&) + { selection = comboBox->GetSelection(); if (selection != wxNOT_FOUND) Config::instance ()->set_selected_gpu(selection); -- cgit v1.2.3 From a17efe392b844c32f790f91788f6d986e6fb1901 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 16:54:11 +0200 Subject: Add missing copyright banner. --- src/wx/grok/gpu_config_panel.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/wx/grok/gpu_config_panel.h b/src/wx/grok/gpu_config_panel.h index d3088c924..88163ae0c 100644 --- a/src/wx/grok/gpu_config_panel.h +++ b/src/wx/grok/gpu_config_panel.h @@ -1,3 +1,24 @@ +/* + Copyright (C) 2023 Grok Image Compression Inc. + + This file is part of DCP-o-matic. + + DCP-o-matic is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + DCP-o-matic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with DCP-o-matic. If not, see . + +*/ + + #pragma once static std::vector get_gpu_names(std::string binary, std::string filename) -- cgit v1.2.3 From d8f3f1426cdd5903fae3ef9fc118068f40310316 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 17:04:21 +0200 Subject: Cleanup: remove unnecessary forward declaration. --- src/lib/grok/context.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index 2513f8863..640ec2196 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -28,7 +28,6 @@ #include "../writer.h" #include "messenger.h" -class Film; static std::mutex launchMutex; -- cgit v1.2.3 From 9b3b2b5014c08ba5667ef4072b90e93a542b35d0 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 17:04:34 +0200 Subject: Cleanup: use dcp::ArrayData instead of home-grown version. --- src/lib/grok/context.h | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index 640ec2196..72e62c4fa 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -27,6 +27,7 @@ #include "../dcpomatic_log.h" #include "../writer.h" #include "messenger.h" +#include static std::mutex launchMutex; @@ -98,24 +99,6 @@ public: messenger_(nullptr), launched_(false) { - struct CompressedData : public dcp::Data { - explicit CompressedData(int dataLen) : data_(new uint8_t[dataLen]), dataLen_(dataLen) - {} - ~CompressedData(void){ - delete[] data_; - } - uint8_t const * data () const override { - return data_; - } - uint8_t * data () override { - return data_; - } - int size () const override { - return dataLen_; - } - uint8_t *data_; - int dataLen_; - }; if (Config::instance()->enable_gpu ()) { boost::filesystem::path folder(dcpomaticContext_.location_); boost::filesystem::path binaryPath = folder / "grk_compress"; @@ -137,9 +120,8 @@ public: auto processor = [this](FrameProxy srcFrame, uint8_t* compressed, uint32_t compressedFrameLength) { - auto compressedData = std::make_shared(compressedFrameLength); - memcpy(compressedData->data_,compressed,compressedFrameLength ); - dcpomaticContext_.writer_.write(compressedData, srcFrame.index(), srcFrame.eyes()); + auto compressed_data = std::make_shared(compressed, compressedFrameLength); + dcpomaticContext_.writer_.write(compressed_data, srcFrame.index(), srcFrame.eyes()); frame_done (); }; int const minimum_size = 16384; -- cgit v1.2.3 From 9ba80f8c08dbfbf1b4eafa8fcb40001a55bbbf0e Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 17:23:57 +0200 Subject: Use boost::filesystem::path for gpu_binary_location(). --- src/lib/config.cc | 2 +- src/lib/config.h | 13 +++++++--- src/lib/grok/context.h | 55 ++++++++++++++++++++++++++---------------- src/lib/grok/messenger.h | 28 +++++++++++++++------ src/wx/grok/gpu_config_panel.h | 12 ++++----- 5 files changed, 71 insertions(+), 39 deletions(-) diff --git a/src/lib/config.cc b/src/lib/config.cc index bbb3eef27..a4ee4ad99 100644 --- a/src/lib/config.cc +++ b/src/lib/config.cc @@ -1133,7 +1133,7 @@ Config::write_config () const /* [XML] ISDCFNamePartLength Maximum length of the "name" part of an ISDCF name, which should be 14 according to the standard */ root->add_child("ISDCFNamePartLength")->add_child_text(raw_convert(_isdcf_name_part_length)); - root->add_child("GpuBinaryLocation")->add_child_text (_gpu_binary_location); + root->add_child("GpuBinaryLocation")->add_child_text (_gpu_binary_location.string()); root->add_child("EnableGpu")->add_child_text ((_enable_gpu ? "1" : "0")); root->add_child("SelectedGpu")->add_child_text (raw_convert (_selected_gpu)); root->add_child("GpuLicenseServer")->add_child_text (_gpu_license_server); diff --git a/src/lib/config.h b/src/lib/config.h index 0c9affbb6..9aba9e9d2 100644 --- a/src/lib/config.h +++ b/src/lib/config.h @@ -618,7 +618,7 @@ public: return _allow_smpte_bv20; } - std::string gpu_binary_location () const { + boost::filesystem::path gpu_binary_location() const { return _gpu_binary_location; } @@ -1220,24 +1220,31 @@ public: void set_allow_smpte_bv20(bool allow) { maybe_set(_allow_smpte_bv20, allow, ALLOW_SMPTE_BV20); } - void set_gpu_binary_location (std::string location) { + + void set_gpu_binary_location(boost::filesystem::path location) { maybe_set (_gpu_binary_location, location); } + void set_enable_gpu (bool enable) { maybe_set (_enable_gpu, enable); } + void set_selected_gpu (int selected) { maybe_set (_selected_gpu, selected); } + void set_gpu_license_server (std::string s) { maybe_set (_gpu_license_server, s); } + void set_gpu_license_port (int p) { maybe_set (_gpu_license_port, p); } + void set_gpu_license (std::string p) { maybe_set (_gpu_license, p); } + void set_isdcf_name_part_length(int length) { maybe_set(_isdcf_name_part_length, length, ISDCF_NAME_PART_LENGTH); } @@ -1485,7 +1492,7 @@ private: /* GPU */ bool _enable_gpu; - std::string _gpu_binary_location; + boost::filesystem::path _gpu_binary_location; int _selected_gpu; std::string _gpu_license_server; int _gpu_license_port; diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index 72e62c4fa..3e4144d07 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -74,12 +74,22 @@ struct FrameProxy { }; struct DcpomaticContext { - DcpomaticContext(std::shared_ptr film, Writer& writer, - EventHistory &history, const std::string &location) : - film_(film), writer_(writer), - history_(history), location_(location), - width_(0), height_(0) - {} + DcpomaticContext( + std::shared_ptr film, + Writer& writer, + EventHistory& history, + boost::filesystem::path const& location + ) + : film_(film) + , writer_(writer) + , history_(history) + , _location(location) + , width_(0) + , height_(0) + { + + } + void setDimensions(uint32_t w, uint32_t h) { width_ = w; height_ = h; @@ -87,7 +97,7 @@ struct DcpomaticContext { std::shared_ptr film_; Writer& writer_; EventHistory &history_; - std::string location_; + boost::filesystem::path _location; uint32_t width_; uint32_t height_; }; @@ -100,12 +110,13 @@ public: launched_(false) { if (Config::instance()->enable_gpu ()) { - boost::filesystem::path folder(dcpomaticContext_.location_); + boost::filesystem::path folder(dcpomaticContext_._location); boost::filesystem::path binaryPath = folder / "grk_compress"; if (!boost::filesystem::exists(binaryPath)) { - getMessengerLogger()->error("Invalid binary location %s", - dcpomaticContext_.location_.c_str()); - return; + getMessengerLogger()->error( + "Invalid binary location %s", dcpomaticContext_._location.c_str() + ); + return; } auto proc = [this](const std::string& str) { try { @@ -166,16 +177,18 @@ public: auto s = dcpv.get_size(); dcpomaticContext_.setDimensions(s.width, s.height); auto config = Config::instance(); - messenger_->launchGrok(dcpomaticContext_.location_, - dcpomaticContext_.width_,dcpomaticContext_.width_, - dcpomaticContext_.height_, - 3, 12, device, - dcpomaticContext_.film_->resolution() == Resolution::FOUR_K, - dcpomaticContext_.film_->video_frame_rate(), - dcpomaticContext_.film_->j2k_bandwidth(), - config->gpu_license_server(), - config->gpu_license_port(), - config->gpu_license()); + messenger_->launchGrok( + dcpomaticContext_._location, + dcpomaticContext_.width_,dcpomaticContext_.width_, + dcpomaticContext_.height_, + 3, 12, device, + dcpomaticContext_.film_->resolution() == Resolution::FOUR_K, + dcpomaticContext_.film_->video_frame_rate(), + dcpomaticContext_.film_->j2k_bandwidth(), + config->gpu_license_server(), + config->gpu_license_port(), + config->gpu_license() + ); } launched_ = messenger_->waitForClientInit(); diff --git a/src/lib/grok/messenger.h b/src/lib/grok/messenger.h index 008e58dd2..5cb21a1cd 100644 --- a/src/lib/grok/messenger.h +++ b/src/lib/grok/messenger.h @@ -542,11 +542,22 @@ struct Messenger sendQueue.push(oss.str()); } - void launchGrok(const std::string &dir, uint32_t width, uint32_t stride, - uint32_t height, uint32_t samplesPerPixel, uint32_t depth, - int device, bool is4K, uint32_t fps, uint32_t bandwidth, - const std::string server, uint32_t port, - const std::string license) + + void launchGrok( + boost::filesystem::path const& dir, + uint32_t width, + uint32_t stride, + uint32_t height, + uint32_t samplesPerPixel, + uint32_t depth, + int device, + bool is4K, + uint32_t fps, + uint32_t bandwidth, + const std::string server, + uint32_t port, + const std::string license + ) { std::unique_lock lk(shutdownMutex_); @@ -640,13 +651,14 @@ struct Messenger protected: std::condition_variable clientInitializedCondition_; private: - void launch(const std::string &cmd, const std::string &dir) + void launch(std::string const& cmd, boost::filesystem::path const& dir) { // Change the working directory if(!dir.empty()) { - if(chdir(dir.c_str()) != 0) - { + boost::system::error_code ec; + boost::filesystem::current_path(dir, ec); + if (ec) { getMessengerLogger()->error("Error: failed to change the working directory"); return; } diff --git a/src/wx/grok/gpu_config_panel.h b/src/wx/grok/gpu_config_panel.h index 88163ae0c..a0f2a1f7f 100644 --- a/src/wx/grok/gpu_config_panel.h +++ b/src/wx/grok/gpu_config_panel.h @@ -21,15 +21,15 @@ #pragma once -static std::vector get_gpu_names(std::string binary, std::string filename) +static std::vector get_gpu_names(boost::filesystem::path binary, boost::filesystem::path filename) { // Execute the GPU listing program and redirect its output to a file - if (std::system((binary + " > " + filename).c_str()) < 0) { + if (std::system((binary.string() + " > " + filename.string()).c_str()) < 0) { return {}; } std::vector gpu_names; - std::ifstream file(filename); + std::ifstream file(filename.c_str()); if (file.is_open()) { std::string line; @@ -57,8 +57,8 @@ public: } void update(void) { auto cfg = Config::instance(); - auto lister_binary = cfg->gpu_binary_location() + "/" + "gpu_lister"; - auto lister_file = cfg->gpu_binary_location () + "/" + "gpus.txt"; + auto lister_binary = cfg->gpu_binary_location() / "gpu_lister"; + auto lister_file = cfg->gpu_binary_location () / "gpus.txt"; if (boost::filesystem::exists(lister_binary)) { auto gpu_names = get_gpu_names(lister_binary, lister_file); @@ -160,7 +160,7 @@ private: auto config = Config::instance (); checked_set (_enable_gpu, config->enable_gpu()); - _binary_location->SetPath(config->gpu_binary_location ()); + _binary_location->SetPath(std_to_wx(config->gpu_binary_location().string())); _gpu_list_control->update(); _gpu_list_control->setSelection(config->selected_gpu()); checked_set (_server, config->gpu_license_server()); -- cgit v1.2.3 From 0dafc0dbef39a3a37335fa0853d41d19da7b15fe Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 24 Sep 2023 17:26:31 +0200 Subject: Use GPU not Gpu for config element names. --- src/lib/config.cc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/lib/config.cc b/src/lib/config.cc index a4ee4ad99..3a7442c39 100644 --- a/src/lib/config.cc +++ b/src/lib/config.cc @@ -640,12 +640,12 @@ try _allow_smpte_bv20 = f.optional_bool_child("AllowSMPTEBv20").get_value_or(false); _isdcf_name_part_length = f.optional_number_child("ISDCFNamePartLength").get_value_or(14); - _enable_gpu = f.optional_bool_child("EnableGpu").get_value_or(false); - _gpu_binary_location = f.string_child("GpuBinaryLocation"); - _selected_gpu = f.number_child("SelectedGpu"); - _gpu_license_server = f.string_child ("GpuLicenseServer"); - _gpu_license_port = f.number_child ("GpuLicensePort"); - _gpu_license = f.string_child("GpuLicense"); + _enable_gpu = f.optional_bool_child("EnableGPU").get_value_or(false); + _gpu_binary_location = f.string_child("GPUBinaryLocation"); + _selected_gpu = f.number_child("SelectedGPU"); + _gpu_license_server = f.string_child ("GPULicenseServer"); + _gpu_license_port = f.number_child ("GPULicensePort"); + _gpu_license = f.string_child("GPULicense"); _export.read(f.optional_node_child("Export")); } @@ -1133,12 +1133,12 @@ Config::write_config () const /* [XML] ISDCFNamePartLength Maximum length of the "name" part of an ISDCF name, which should be 14 according to the standard */ root->add_child("ISDCFNamePartLength")->add_child_text(raw_convert(_isdcf_name_part_length)); - root->add_child("GpuBinaryLocation")->add_child_text (_gpu_binary_location.string()); - root->add_child("EnableGpu")->add_child_text ((_enable_gpu ? "1" : "0")); - root->add_child("SelectedGpu")->add_child_text (raw_convert (_selected_gpu)); - root->add_child("GpuLicenseServer")->add_child_text (_gpu_license_server); - root->add_child("GpuLicensePort")->add_child_text (raw_convert (_gpu_license_port)); - root->add_child("GpuLicense")->add_child_text (_gpu_license); + root->add_child("GPUBinaryLocation")->add_child_text (_gpu_binary_location.string()); + root->add_child("EnableGPU")->add_child_text ((_enable_gpu ? "1" : "0")); + root->add_child("SelectedGPU")->add_child_text (raw_convert (_selected_gpu)); + root->add_child("GPULicenseServer")->add_child_text (_gpu_license_server); + root->add_child("GPULicensePort")->add_child_text (raw_convert (_gpu_license_port)); + root->add_child("GPULicense")->add_child_text (_gpu_license); _export.write(root->add_child("Export")); -- cgit v1.2.3 From aea08ecdc6ba8ba519f8d5183f1b53e9eb6cdfd2 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Tue, 26 Sep 2023 13:00:16 +0200 Subject: Cleanup: sorting. --- src/lib/dcp_encoder.cc | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/lib/dcp_encoder.cc b/src/lib/dcp_encoder.cc index 1b1b117be..bd78312fa 100644 --- a/src/lib/dcp_encoder.cc +++ b/src/lib/dcp_encoder.cc @@ -18,6 +18,7 @@ */ + /** @file src/dcp_encoder.cc * @brief A class which takes a Film and some Options, then uses those to encode the film into a DCP. * @@ -25,31 +26,33 @@ * as a parameter to the constructor. */ + +#include "audio_decoder.h" +#include "compose.hpp" #include "dcp_encoder.h" -#include "j2k_encoder.h" #include "film.h" -#include "video_decoder.h" -#include "audio_decoder.h" -#include "player.h" +#include "j2k_encoder.h" #include "job.h" -#include "writer.h" -#include "compose.hpp" +#include "player.h" +#include "player_video.h" #include "referenced_reel_asset.h" #include "text_content.h" -#include "player_video.h" +#include "video_decoder.h" +#include "writer.h" #include #include #include "i18n.h" -using std::string; + using std::cout; +using std::dynamic_pointer_cast; using std::list; -using std::vector; +using std::make_shared; using std::shared_ptr; +using std::string; +using std::vector; using std::weak_ptr; -using std::dynamic_pointer_cast; -using std::make_shared; using boost::optional; #if BOOST_VERSION >= 106100 using namespace boost::placeholders; -- cgit v1.2.3 From 6937ba070d10e07c5a0c31bf75edd1d2b9d15307 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Tue, 26 Sep 2023 13:11:36 +0200 Subject: Cleanup: reformat and rename dcpomaticContext_ -> _dcpomatic_context --- src/lib/grok/context.h | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index 3e4144d07..1aac6f177 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -102,19 +102,21 @@ struct DcpomaticContext { uint32_t height_; }; -class GrokContext { + +class GrokContext +{ public: - explicit GrokContext(const DcpomaticContext &dcpomaticContext) : - dcpomaticContext_(dcpomaticContext), - messenger_(nullptr), - launched_(false) + explicit GrokContext(DcpomaticContext const& dcpomatic_context) + : _dcpomatic_context(dcpomatic_context) + , messenger_(nullptr) + , launched_(false) { if (Config::instance()->enable_gpu ()) { - boost::filesystem::path folder(dcpomaticContext_._location); + boost::filesystem::path folder(_dcpomatic_context._location); boost::filesystem::path binaryPath = folder / "grk_compress"; if (!boost::filesystem::exists(binaryPath)) { getMessengerLogger()->error( - "Invalid binary location %s", dcpomaticContext_._location.c_str() + "Invalid binary location %s", _dcpomatic_context._location.c_str() ); return; } @@ -132,7 +134,7 @@ public: [this](FrameProxy srcFrame, uint8_t* compressed, uint32_t compressedFrameLength) { auto compressed_data = std::make_shared(compressed, compressedFrameLength); - dcpomaticContext_.writer_.write(compressed_data, srcFrame.index(), srcFrame.eyes()); + _dcpomatic_context.writer_.write(compressed_data, srcFrame.index(), srcFrame.eyes()); frame_done (); }; int const minimum_size = 16384; @@ -145,7 +147,7 @@ public: } auto encoded = std::make_shared(fp->vf.encode_locally()); - dcpomaticContext_.writer_.write(encoded, fp->vf.index(), fp->vf.eyes()); + _dcpomatic_context.writer_.write(encoded, fp->vf.index(), fp->vf.eyes()); frame_done (); } } @@ -175,16 +177,16 @@ public: return true; if (MessengerInit::firstLaunch(true)) { auto s = dcpv.get_size(); - dcpomaticContext_.setDimensions(s.width, s.height); + _dcpomatic_context.setDimensions(s.width, s.height); auto config = Config::instance(); messenger_->launchGrok( - dcpomaticContext_._location, - dcpomaticContext_.width_,dcpomaticContext_.width_, - dcpomaticContext_.height_, + _dcpomatic_context._location, + _dcpomatic_context.width_,_dcpomatic_context.width_, + _dcpomatic_context.height_, 3, 12, device, - dcpomaticContext_.film_->resolution() == Resolution::FOUR_K, - dcpomaticContext_.film_->video_frame_rate(), - dcpomaticContext_.film_->j2k_bandwidth(), + _dcpomatic_context.film_->resolution() == Resolution::FOUR_K, + _dcpomatic_context.film_->video_frame_rate(), + _dcpomatic_context.film_->j2k_bandwidth(), config->gpu_license_server(), config->gpu_license_port(), config->gpu_license() @@ -218,10 +220,10 @@ public: messenger_ = nullptr; } void frame_done () { - dcpomaticContext_.history_.event (); + _dcpomatic_context.history_.event(); } private: - DcpomaticContext dcpomaticContext_; + DcpomaticContext _dcpomatic_context; ScheduledMessenger *messenger_; bool launched_; }; -- cgit v1.2.3 From 0cab88e322be99b58317b74d2b10def745e8d183 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Tue, 26 Sep 2023 13:29:59 +0200 Subject: Create a single DcpomaticContext for all GrokContexts, rather than copying them. I think this makes sense, and also allows us to forward-declare the contexts in a forthcoming commit. --- src/lib/grok/context.h | 28 ++++++++++++++-------------- src/lib/j2k_encoder.cc | 3 ++- src/lib/j2k_encoder.h | 2 +- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index 1aac6f177..96477d597 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -106,17 +106,17 @@ struct DcpomaticContext { class GrokContext { public: - explicit GrokContext(DcpomaticContext const& dcpomatic_context) + explicit GrokContext(DcpomaticContext* dcpomatic_context) : _dcpomatic_context(dcpomatic_context) , messenger_(nullptr) , launched_(false) { if (Config::instance()->enable_gpu ()) { - boost::filesystem::path folder(_dcpomatic_context._location); + boost::filesystem::path folder(_dcpomatic_context->_location); boost::filesystem::path binaryPath = folder / "grk_compress"; if (!boost::filesystem::exists(binaryPath)) { getMessengerLogger()->error( - "Invalid binary location %s", _dcpomatic_context._location.c_str() + "Invalid binary location %s", _dcpomatic_context->_location.c_str() ); return; } @@ -134,7 +134,7 @@ public: [this](FrameProxy srcFrame, uint8_t* compressed, uint32_t compressedFrameLength) { auto compressed_data = std::make_shared(compressed, compressedFrameLength); - _dcpomatic_context.writer_.write(compressed_data, srcFrame.index(), srcFrame.eyes()); + _dcpomatic_context->writer_.write(compressed_data, srcFrame.index(), srcFrame.eyes()); frame_done (); }; int const minimum_size = 16384; @@ -147,7 +147,7 @@ public: } auto encoded = std::make_shared(fp->vf.encode_locally()); - _dcpomatic_context.writer_.write(encoded, fp->vf.index(), fp->vf.eyes()); + _dcpomatic_context->writer_.write(encoded, fp->vf.index(), fp->vf.eyes()); frame_done (); } } @@ -177,16 +177,16 @@ public: return true; if (MessengerInit::firstLaunch(true)) { auto s = dcpv.get_size(); - _dcpomatic_context.setDimensions(s.width, s.height); + _dcpomatic_context->setDimensions(s.width, s.height); auto config = Config::instance(); messenger_->launchGrok( - _dcpomatic_context._location, - _dcpomatic_context.width_,_dcpomatic_context.width_, - _dcpomatic_context.height_, + _dcpomatic_context->_location, + _dcpomatic_context->width_,_dcpomatic_context->width_, + _dcpomatic_context->height_, 3, 12, device, - _dcpomatic_context.film_->resolution() == Resolution::FOUR_K, - _dcpomatic_context.film_->video_frame_rate(), - _dcpomatic_context.film_->j2k_bandwidth(), + _dcpomatic_context->film_->resolution() == Resolution::FOUR_K, + _dcpomatic_context->film_->video_frame_rate(), + _dcpomatic_context->film_->j2k_bandwidth(), config->gpu_license_server(), config->gpu_license_port(), config->gpu_license() @@ -220,10 +220,10 @@ public: messenger_ = nullptr; } void frame_done () { - _dcpomatic_context.history_.event(); + _dcpomatic_context->history_.event(); } private: - DcpomaticContext _dcpomatic_context; + DcpomaticContext* _dcpomatic_context; ScheduledMessenger *messenger_; bool launched_; }; diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index 22f2ea6d7..fe63deacd 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -68,7 +68,7 @@ J2KEncoder::J2KEncoder(shared_ptr film, Writer& writer) , _history (200) , _writer (writer) #ifdef DCPOMATIC_GROK - , _dcpomatic_context(film, writer, _history, Config::instance()->gpu_binary_location()) + , _dcpomatic_context(new grk_plugin::DcpomaticContext(film, writer, _history, Config::instance()->gpu_binary_location())) , _context(Config::instance()->enable_gpu() ? new grk_plugin::GrokContext(_dcpomatic_context) : nullptr) #endif { @@ -84,6 +84,7 @@ J2KEncoder::~J2KEncoder () #ifdef DCPOMATIC_GROK delete _context; + delete _dcpomatic_context; #endif } diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index 9d9d85894..913beb5a9 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -125,7 +125,7 @@ private: boost::signals2::scoped_connection _server_found_connection; #ifdef DCPOMATIC_GROK - grk_plugin::DcpomaticContext _dcpomatic_context; + grk_plugin::DcpomaticContext* _dcpomatic_context; grk_plugin::GrokContext *_context; #endif -- cgit v1.2.3 From f0a03cacb2f2214b7452e42f2dec465888422e6f Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Tue, 26 Sep 2023 13:32:00 +0200 Subject: Forward-declare grk_plugin stuff. --- src/lib/grok_j2k_encoder_thread.cc | 2 ++ src/lib/j2k_encoder.cc | 1 + src/lib/j2k_encoder.h | 8 +++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/lib/grok_j2k_encoder_thread.cc b/src/lib/grok_j2k_encoder_thread.cc index 8d0dee5a9..d5256619d 100644 --- a/src/lib/grok_j2k_encoder_thread.cc +++ b/src/lib/grok_j2k_encoder_thread.cc @@ -1,6 +1,8 @@ +#include "config.h" #include "cross.h" #include "dcpomatic_log.h" #include "dcp_video.h" +#include "grok/context.h" #include "grok_j2k_encoder_thread.h" #include "j2k_encoder.h" #include "scope_guard.h" diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index fe63deacd..ad04ac0fa 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -34,6 +34,7 @@ #include "film.h" #include "cpu_j2k_encoder_thread.h" #ifdef DCPOMATIC_GROK +#include "grok/context.h" #include "grok_j2k_encoder_thread.h" #endif #include "remote_j2k_encoder_thread.h" diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index 913beb5a9..0dbe654a4 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -32,9 +32,6 @@ #include "enum_indexed_vector.h" #include "event_history.h" #include "exception_store.h" -#ifdef DCPOMATIC_GROK -#include "grok/context.h" -#endif #include "j2k_encoder_thread.h" #include "writer.h" #include @@ -52,6 +49,11 @@ class Film; class Job; class PlayerVideo; +namespace grk_plugin { + struct DcpomaticContext; + struct GrokContext; +} + struct local_threads_created_and_destroyed; struct remote_threads_created_and_destroyed; struct frames_not_lost_when_threads_disappear; -- cgit v1.2.3 From 02517b4be62a1056f18d1c337c8b404432560e19 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Tue, 26 Sep 2023 20:46:08 +0200 Subject: Hack to make sure the right things are rebuilt when grok headers change. --- wscript | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/wscript b/wscript index bbd73f64c..6f5380d29 100644 --- a/wscript +++ b/wscript @@ -613,6 +613,18 @@ def configure(conf): def build(bld): create_version_cc(VERSION, bld.env.CXXFLAGS) + # waf can't find these dependencies by itself because they are only included if DCPOMATIC_GROK is defined, + # and I can't find a way to pass that to waf's dependency scanner + if bld.env.ENABLE_GROK: + for dep in ( + 'src/lib/j2k_encoder.cc', + 'src/tools/dcpomatic.cc', + 'src/tools/dcpomatic_server.cc', + 'src/tools/dcpomatic_server_cli.cc', + 'src/tools/dcpomatic_batch.cc' + ): + bld.add_manual_dependency(bld.path.find_node(dep), bld.path.find_node('src/lib/grok/context.h')) + bld.recurse('src') bld.recurse('graphics') -- cgit v1.2.3 From e7d983c4a39b53cc6feae13701400904916eea94 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Tue, 26 Sep 2023 20:53:43 +0200 Subject: Cleanup: coding style. --- src/lib/grok/messenger.h | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/lib/grok/messenger.h b/src/lib/grok/messenger.h index 5cb21a1cd..50d44e251 100644 --- a/src/lib/grok/messenger.h +++ b/src/lib/grok/messenger.h @@ -461,7 +461,7 @@ static void processorThread(Messenger* messenger, std::function lk(shutdownMutex_); - initialized_ = true; + _initialized = true; clientInitializedCondition_.notify_all(); } - bool waitForClientInit(void) + + bool waitForClientInit() { - if(initialized_) + if (_initialized) { return true; + } std::unique_lock lk(shutdownMutex_); - if(initialized_) + + if (_initialized) { return true; - else if (shutdown_) + } else if (_shutdown) { return false; - clientInitializedCondition_.wait(lk, [this]{return initialized_ || shutdown_;}); + } + + clientInitializedCondition_.wait(lk, [this] { return _initialized || _shutdown; }); - return initialized_ && !shutdown_; + return _initialized && !_shutdown; } + static size_t uncompressedFrameSize(uint32_t w, uint32_t h, uint32_t samplesPerPixel) { return sizeof(uint16_t) * w * h * samplesPerPixel; @@ -637,8 +643,8 @@ struct Messenger return (uint8_t*)(compressed_buffer_ + frameId * init_.compressedFrameSize_); } std::atomic_bool running; - bool initialized_; - bool shutdown_; + bool _initialized; + bool _shutdown; MessengerBlockingQueue sendQueue; MessengerBlockingQueue receiveQueue; MessengerBlockingQueue availableBuffers_; @@ -754,11 +760,11 @@ struct Msg }; static void processorThread(Messenger* messenger, std::function processor) { - while(messenger->running) - { + while (messenger->running) { std::string message; - if(!messenger->receiveQueue.waitAndPop(message)) + if (!messenger->receiveQueue.waitAndPop(message)) { break; + } if(!messenger->running) break; Msg msg(message); @@ -856,7 +862,7 @@ struct ScheduledMessenger : public Messenger } ++framesCompressed_; send(GRK_MSGR_BATCH_PROCESSSED_COMPRESSED, compressedFrameId); - if (shutdown_ && framesCompressed_ == framesScheduled_) + if (_shutdown && framesCompressed_ == framesScheduled_) shutdownCondition_.notify_all(); } void shutdown(void){ @@ -864,7 +870,7 @@ struct ScheduledMessenger : public Messenger std::unique_lock lk(shutdownMutex_); if (!async_result_.valid()) return; - shutdown_ = true; + _shutdown = true; if (framesScheduled_) { uint32_t scheduled = framesScheduled_; send(GRK_MSGR_BATCH_FLUSH, scheduled); -- cgit v1.2.3 From 2f709134a4ce1a3668a0219f0a99369df3e89289 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Tue, 26 Sep 2023 20:56:19 +0200 Subject: Looks like we also need an explicit dep on messenger.h --- wscript | 1 + 1 file changed, 1 insertion(+) diff --git a/wscript b/wscript index 6f5380d29..164b53fca 100644 --- a/wscript +++ b/wscript @@ -624,6 +624,7 @@ def build(bld): 'src/tools/dcpomatic_batch.cc' ): bld.add_manual_dependency(bld.path.find_node(dep), bld.path.find_node('src/lib/grok/context.h')) + bld.add_manual_dependency(bld.path.find_node(dep), bld.path.find_node('src/lib/grok/messenger.h')) bld.recurse('src') bld.recurse('graphics') -- cgit v1.2.3 From a89d5d001f2ef44b86780a1dd074ac7458a4d284 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Tue, 26 Sep 2023 20:59:39 +0200 Subject: Cleanup: coding style. --- src/lib/grok/messenger.h | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/lib/grok/messenger.h b/src/lib/grok/messenger.h index 50d44e251..bb7ada213 100644 --- a/src/lib/grok/messenger.h +++ b/src/lib/grok/messenger.h @@ -758,6 +758,7 @@ struct Msg std::vector cs_; size_t ct_; }; + static void processorThread(Messenger* messenger, std::function processor) { while (messenger->running) { @@ -765,31 +766,26 @@ static void processorThread(Messenger* messenger, std::functionreceiveQueue.waitAndPop(message)) { break; } - if(!messenger->running) + + if (!messenger->running) { break; + } + Msg msg(message); auto tag = msg.next(); - if(tag == GRK_MSGR_BATCH_COMPRESS_INIT) - { + if (tag == GRK_MSGR_BATCH_COMPRESS_INIT) { auto width = msg.nextUint(); - auto stride = msg.nextUint(); - (void)stride; + msg.nextUint(); // stride auto height = msg.nextUint(); - auto samplesPerPixel = msg.nextUint(); - auto depth = msg.nextUint(); - (void)depth; - messenger->init_.uncompressedFrameSize_ = - Messenger::uncompressedFrameSize(width, height, samplesPerPixel); - auto compressedFrameSize = msg.nextUint(); - auto numFrames = msg.nextUint(); - messenger->initClient(compressedFrameSize, compressedFrameSize, numFrames); - } - else if(tag == GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED) - { + auto samples_per_pixel = msg.nextUint(); + msg.nextUint(); // depth + messenger->init_.uncompressedFrameSize_ = Messenger::uncompressedFrameSize(width, height, samples_per_pixel); + auto compressed_frame_size = msg.nextUint(); + auto num_frames = msg.nextUint(); + messenger->initClient(compressed_frame_size, compressed_frame_size, num_frames); + } else if (tag == GRK_MSGR_BATCH_PROCESSED_UNCOMPRESSED) { messenger->reclaimUncompressed(msg.nextUint()); - } - else if(tag == GRK_MSGR_BATCH_PROCESSSED_COMPRESSED) - { + } else if (tag == GRK_MSGR_BATCH_PROCESSSED_COMPRESSED) { messenger->reclaimCompressed(msg.nextUint()); } processor(message); -- cgit v1.2.3 From ed89aa948cb7426d066213f53ab69ec9d9f5b0ce Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Wed, 4 Oct 2023 15:36:55 +0200 Subject: Log failure to schedule a frame with grok. --- src/lib/grok_j2k_encoder_thread.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/grok_j2k_encoder_thread.cc b/src/lib/grok_j2k_encoder_thread.cc index d5256619d..79b80f694 100644 --- a/src/lib/grok_j2k_encoder_thread.cc +++ b/src/lib/grok_j2k_encoder_thread.cc @@ -33,6 +33,7 @@ try auto frame = _encoder.pop(); ScopeGuard frame_guard([this, &frame]() { + LOG_ERROR("Failed to schedule encode of %1 using grok", frame.index()); _encoder.retry(frame); }); -- cgit v1.2.3 From ec1758d615a0f6ca837e960e1d2c0492a546523d Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Wed, 4 Oct 2023 19:11:45 +0200 Subject: Another patch from Aaron. --- src/lib/grok/context.h | 48 ++++++++++++++++++++++++------------ src/lib/grok/messenger.h | 63 +++++++++++++++++++++++++----------------------- src/lib/j2k_encoder.cc | 27 +++++++++++++++++++++ 3 files changed, 92 insertions(+), 46 deletions(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index 96477d597..e7b5dbcbc 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -20,6 +20,9 @@ #pragma once +#include +namespace fs = boost::filesystem; + #include "../config.h" #include "../dcp_video.h" #include "../film.h" @@ -106,10 +109,11 @@ struct DcpomaticContext { class GrokContext { public: - explicit GrokContext(DcpomaticContext* dcpomatic_context) - : _dcpomatic_context(dcpomatic_context) - , messenger_(nullptr) - , launched_(false) + explicit GrokContext(DcpomaticContext* dcpomatic_context) : + _dcpomatic_context(dcpomatic_context), + messenger_(nullptr), + launched_(false), + launchFailed_(false) { if (Config::instance()->enable_gpu ()) { boost::filesystem::path folder(_dcpomatic_context->_location); @@ -170,29 +174,40 @@ public: return false; if (launched_) return true; + if (launchFailed_) + return false; std::unique_lock lk_global(launchMutex); if (!messenger_) return false; if (launched_) return true; + if (launchFailed_) + return false; if (MessengerInit::firstLaunch(true)) { + + if (!fs::exists(_dcpomatic_context->_location) || !fs::is_directory(_dcpomatic_context->_location)) { + getMessengerLogger()->error("Invalid directory %s", _dcpomatic_context->_location.c_str()); + return false; + } auto s = dcpv.get_size(); _dcpomatic_context->setDimensions(s.width, s.height); auto config = Config::instance(); - messenger_->launchGrok( - _dcpomatic_context->_location, - _dcpomatic_context->width_,_dcpomatic_context->width_, - _dcpomatic_context->height_, - 3, 12, device, - _dcpomatic_context->film_->resolution() == Resolution::FOUR_K, - _dcpomatic_context->film_->video_frame_rate(), - _dcpomatic_context->film_->j2k_bandwidth(), - config->gpu_license_server(), - config->gpu_license_port(), - config->gpu_license() - ); + if (!messenger_->launchGrok(_dcpomatic_context->_location, + _dcpomatic_context->width_,_dcpomatic_context->width_, + _dcpomatic_context->height_, + 3, 12, device, + _dcpomatic_context->film_->resolution() == Resolution::FOUR_K, + _dcpomatic_context->film_->video_frame_rate(), + _dcpomatic_context->film_->j2k_bandwidth(), + config->gpu_license_server(), + config->gpu_license_port(), + config->gpu_license())) { + launchFailed_ = true; + return false; + } } launched_ = messenger_->waitForClientInit(); + launchFailed_ = launched_; return launched_; } @@ -226,6 +241,7 @@ private: DcpomaticContext* _dcpomatic_context; ScheduledMessenger *messenger_; bool launched_; + bool launchFailed_; }; } diff --git a/src/lib/grok/messenger.h b/src/lib/grok/messenger.h index bb7ada213..51526aee3 100644 --- a/src/lib/grok/messenger.h +++ b/src/lib/grok/messenger.h @@ -119,23 +119,10 @@ struct MessengerLogger : public IMessengerLogger std::string preamble_; }; -static IMessengerLogger* sLogger = nullptr; -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#endif -static void setMessengerLogger(IMessengerLogger* logger) -{ - delete sLogger; - sLogger = logger; -} -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic pop -#endif -static IMessengerLogger* getMessengerLogger(void) -{ - return sLogger; -} +extern IMessengerLogger* sLogger; +void setMessengerLogger(IMessengerLogger* logger); +IMessengerLogger* getMessengerLogger(void); + struct MessengerInit { MessengerInit(const std::string &outBuf, const std::string &outSent, @@ -543,7 +530,7 @@ struct Messenger sendQueue.push(oss.str()); } - void launchGrok( + bool launchGrok( boost::filesystem::path const& dir, uint32_t width, uint32_t stride, @@ -562,7 +549,7 @@ struct Messenger std::unique_lock lk(shutdownMutex_); if (async_result_.valid()) - return; + return true; if(MessengerInit::firstLaunch(true)) init_.unlink(); startThreads(); @@ -570,11 +557,12 @@ struct Messenger auto fullServer = server + ":" + std::to_string(port); sprintf(_cmd, "./grk_compress -batch_src %s,%d,%d,%d,%d,%d -out_fmt j2k -k 1 " - "-G %d -%s %d,%d -j %s -J %s", + "-G %d -%s %d,%d -j %s -J %s -v", GRK_MSGR_BATCH_IMAGE.c_str(), width, stride, height, samplesPerPixel, depth, device, is4K ? "cinema4K" : "cinema2K", fps, bandwidth, license.c_str(), fullServer.c_str()); - launch(_cmd, dir); + + return launch(_cmd, dir); } void initClient(size_t uncompressedFrameSize, size_t compressedFrameSize, size_t numFrames) { @@ -597,19 +585,27 @@ struct Messenger bool waitForClientInit() { - if (_initialized) { + if(_initialized) return true; - } + else if (_shutdown) + return false; std::unique_lock lk(shutdownMutex_); - - if (_initialized) { + if(_initialized) return true; - } else if (_shutdown) { + else if (_shutdown) return false; - } - clientInitializedCondition_.wait(lk, [this] { return _initialized || _shutdown; }); + while (true) { + if (clientInitializedCondition_.wait_for(lk, std::chrono::seconds(1), [this]{ return _initialized || _shutdown; })) { + break; + } + auto status = async_result_.wait_for(std::chrono::milliseconds(100)); + if (status == std::future_status::ready) { + getMessengerLogger()->error("Grok exited unexpectedly during initialization"); + return false; + } + } return _initialized && !_shutdown; } @@ -657,7 +653,7 @@ struct Messenger protected: std::condition_variable clientInitializedCondition_; private: - void launch(std::string const& cmd, boost::filesystem::path const& dir) + bool launch(std::string const& cmd, boost::filesystem::path const& dir) { // Change the working directory if(!dir.empty()) @@ -666,12 +662,19 @@ struct Messenger boost::filesystem::current_path(dir, ec); if (ec) { getMessengerLogger()->error("Error: failed to change the working directory"); - return; + return false; } } // Execute the command using std::async and std::system cmd_ = cmd; + getMessengerLogger()->info(cmd.c_str()); async_result_ = std::async(std::launch::async, [this]() { return std::system(cmd_.c_str()); }); + bool success = async_result_.valid(); + if (!success) + getMessengerLogger()->error("Grok launch failed"); + + return success; + } std::thread outbound; Synch* outboundSynch_; diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index ad04ac0fa..3bb7ccaed 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -60,6 +60,33 @@ using boost::optional; using dcp::Data; using namespace dcpomatic; +#ifdef DCPOMATIC_GROK + +namespace grk_plugin { + +IMessengerLogger* sLogger = nullptr; + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" +#endif +void setMessengerLogger(grk_plugin::IMessengerLogger* logger) +{ + delete sLogger; + sLogger = logger; +} +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif +grk_plugin::IMessengerLogger* getMessengerLogger(void) +{ + return sLogger; +} + +} + +#endif + /** @param film Film that we are encoding. * @param writer Writer that we are using. -- cgit v1.2.3 From 7dd4fb0e9c8024c4ccd67e2aa5cda8eaccc52477 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Wed, 4 Oct 2023 19:47:32 +0200 Subject: Don't pollute the global namespace with a namespace called fs. --- src/lib/grok/context.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index e7b5dbcbc..85ed385bd 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -21,7 +21,6 @@ #pragma once #include -namespace fs = boost::filesystem; #include "../config.h" #include "../dcp_video.h" @@ -170,6 +169,8 @@ public: shutdown(); } bool launch(DCPVideo dcpv, int device){ + namespace fs = boost::filesystem; + if (!messenger_ ) return false; if (launched_) -- cgit v1.2.3 From 252f378dda893da1e1a7d94beceeed141c0c7d0b Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Wed, 4 Oct 2023 19:51:45 +0200 Subject: Cleanup: include ordering. --- src/lib/grok/context.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index 85ed385bd..6b27eb7a8 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -20,7 +20,6 @@ #pragma once -#include #include "../config.h" #include "../dcp_video.h" @@ -30,6 +29,7 @@ #include "../writer.h" #include "messenger.h" #include +#include static std::mutex launchMutex; -- cgit v1.2.3 From 3d897c8d329a96c3c7af2ad7e8d8973d732fc0c4 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Wed, 4 Oct 2023 19:51:54 +0200 Subject: Cleanup: whitespace to reduce the diff. --- src/lib/grok/context.h | 13 +++++++------ src/lib/grok/messenger.h | 11 +++++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index 6b27eb7a8..ecb9d1f98 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -108,11 +108,11 @@ struct DcpomaticContext { class GrokContext { public: - explicit GrokContext(DcpomaticContext* dcpomatic_context) : - _dcpomatic_context(dcpomatic_context), - messenger_(nullptr), - launched_(false), - launchFailed_(false) + explicit GrokContext(DcpomaticContext* dcpomatic_context) + : _dcpomatic_context(dcpomatic_context) + , messenger_(nullptr) + , launched_(false) + , launchFailed_(false) { if (Config::instance()->enable_gpu ()) { boost::filesystem::path folder(_dcpomatic_context->_location); @@ -193,7 +193,8 @@ public: auto s = dcpv.get_size(); _dcpomatic_context->setDimensions(s.width, s.height); auto config = Config::instance(); - if (!messenger_->launchGrok(_dcpomatic_context->_location, + if (!messenger_->launchGrok( + _dcpomatic_context->_location, _dcpomatic_context->width_,_dcpomatic_context->width_, _dcpomatic_context->height_, 3, 12, device, diff --git a/src/lib/grok/messenger.h b/src/lib/grok/messenger.h index 51526aee3..eb2fe9560 100644 --- a/src/lib/grok/messenger.h +++ b/src/lib/grok/messenger.h @@ -585,16 +585,19 @@ struct Messenger bool waitForClientInit() { - if(_initialized) + if (_initialized) { return true; - else if (_shutdown) + } else if (_shutdown) { return false; + } std::unique_lock lk(shutdownMutex_); - if(_initialized) + + if (_initialized) { return true; - else if (_shutdown) + } else if (_shutdown) { return false; + } while (true) { if (clientInitializedCondition_.wait_for(lk, std::chrono::seconds(1), [this]{ return _initialized || _shutdown; })) { -- cgit v1.2.3 From 8c68fd3d1b861ac6d5fd5451cd693f2230969cad Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Fri, 6 Oct 2023 21:27:39 +0200 Subject: Cleanup: whitespace / coding style. --- src/lib/grok/context.h | 262 ++++++++++++++++++++++++----------------- src/wx/grok/gpu_config_panel.h | 186 +++++++++++++++-------------- 2 files changed, 246 insertions(+), 202 deletions(-) diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index ecb9d1f98..d27caa5f6 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -75,33 +75,34 @@ struct FrameProxy { DCPVideo vf; }; -struct DcpomaticContext { +struct DcpomaticContext +{ DcpomaticContext( - std::shared_ptr film, - Writer& writer, - EventHistory& history, - boost::filesystem::path const& location + std::shared_ptr film_, + Writer& writer_, + EventHistory& history_, + boost::filesystem::path const& location_ ) - : film_(film) - , writer_(writer) - , history_(history) - , _location(location) - , width_(0) - , height_(0) + : film(film_) + , writer(writer_) + , history(history_) + , location(location_) { } - void setDimensions(uint32_t w, uint32_t h) { - width_ = w; - height_ = h; + void set_dimensions(uint32_t w, uint32_t h) + { + width = w; + height = h; } - std::shared_ptr film_; - Writer& writer_; - EventHistory &history_; - boost::filesystem::path _location; - uint32_t width_; - uint32_t height_; + + std::shared_ptr film; + Writer& writer; + EventHistory& history; + boost::filesystem::path location; + uint32_t width = 0; + uint32_t height = 0; }; @@ -110,140 +111,179 @@ class GrokContext public: explicit GrokContext(DcpomaticContext* dcpomatic_context) : _dcpomatic_context(dcpomatic_context) - , messenger_(nullptr) - , launched_(false) - , launchFailed_(false) { - if (Config::instance()->enable_gpu ()) { - boost::filesystem::path folder(_dcpomatic_context->_location); - boost::filesystem::path binaryPath = folder / "grk_compress"; - if (!boost::filesystem::exists(binaryPath)) { - getMessengerLogger()->error( - "Invalid binary location %s", _dcpomatic_context->_location.c_str() - ); - return; - } - auto proc = [this](const std::string& str) { - try { - Msg msg(str); - auto tag = msg.next(); - if(tag == GRK_MSGR_BATCH_SUBMIT_COMPRESSED) - { - auto clientFrameId = msg.nextUint(); - auto compressedFrameId = msg.nextUint(); - (void)compressedFrameId; - auto compressedFrameLength = msg.nextUint(); - auto processor = - [this](FrameProxy srcFrame, uint8_t* compressed, uint32_t compressedFrameLength) - { - auto compressed_data = std::make_shared(compressed, compressedFrameLength); - _dcpomatic_context->writer_.write(compressed_data, srcFrame.index(), srcFrame.eyes()); - frame_done (); - }; - int const minimum_size = 16384; - bool needsRecompression = compressedFrameLength < minimum_size; - messenger_->processCompressed(str, processor, needsRecompression); - if (needsRecompression) { - auto fp = messenger_->retrieve(clientFrameId); - if (!fp) { - return; - } - - auto encoded = std::make_shared(fp->vf.encode_locally()); - _dcpomatic_context->writer_.write(encoded, fp->vf.index(), fp->vf.eyes()); - frame_done (); + if (!Config::instance()->enable_gpu()) { + return; + } + + boost::filesystem::path folder(_dcpomatic_context->location); + boost::filesystem::path binary_path = folder / "grk_compress"; + if (!boost::filesystem::exists(binary_path)) { + getMessengerLogger()->error( + "Invalid binary location %s", _dcpomatic_context->location.c_str() + ); + return; + } + + auto proc = [this](const std::string& str) { + try { + Msg msg(str); + auto tag = msg.next(); + if (tag == GRK_MSGR_BATCH_SUBMIT_COMPRESSED) { + auto clientFrameId = msg.nextUint(); + msg.nextUint(); // compressed frame ID + auto compressedFrameLength = msg.nextUint(); + auto processor = [this](FrameProxy srcFrame, uint8_t* compressed, uint32_t compressedFrameLength) { + auto compressed_data = std::make_shared(compressed, compressedFrameLength); + _dcpomatic_context->writer.write(compressed_data, srcFrame.index(), srcFrame.eyes()); + frame_done (); + }; + + int const minimum_size = 16384; + + bool needsRecompression = compressedFrameLength < minimum_size; + _messenger->processCompressed(str, processor, needsRecompression); + + if (needsRecompression) { + auto fp = _messenger->retrieve(clientFrameId); + if (!fp) { + return; } + + auto encoded = std::make_shared(fp->vf.encode_locally()); + _dcpomatic_context->writer.write(encoded, fp->vf.index(), fp->vf.eyes()); + frame_done (); } - } catch (std::exception &ex){ - getMessengerLogger()->error("%s",ex.what()); } - }; - auto clientInit = - MessengerInit(clientToGrokMessageBuf, clientSentSynch, grokReceiveReadySynch, - grokToClientMessageBuf, grokSentSynch, clientReceiveReadySynch, proc, - std::thread::hardware_concurrency()); - messenger_ = new ScheduledMessenger(clientInit); - } + } catch (std::exception& ex) { + getMessengerLogger()->error("%s",ex.what()); + } + }; + + auto clientInit = MessengerInit( + clientToGrokMessageBuf, + clientSentSynch, + grokReceiveReadySynch, + grokToClientMessageBuf, + grokSentSynch, + clientReceiveReadySynch, + proc, + std::thread::hardware_concurrency() + ); + + _messenger = new ScheduledMessenger(clientInit); } - ~GrokContext(void) { + + ~GrokContext() + { shutdown(); } - bool launch(DCPVideo dcpv, int device){ + + bool launch(DCPVideo dcpv, int device) + { namespace fs = boost::filesystem; - if (!messenger_ ) + if (!_messenger) { return false; - if (launched_) + } + if (_launched) { return true; - if (launchFailed_) + } + if (_launch_failed) { return false; + } + std::unique_lock lk_global(launchMutex); - if (!messenger_) + + if (!_messenger) { return false; - if (launched_) + } + if (_launched) { return true; - if (launchFailed_) + } + if (_launch_failed) { return false; + } + if (MessengerInit::firstLaunch(true)) { - if (!fs::exists(_dcpomatic_context->_location) || !fs::is_directory(_dcpomatic_context->_location)) { - getMessengerLogger()->error("Invalid directory %s", _dcpomatic_context->_location.c_str()); + if (!fs::exists(_dcpomatic_context->location) || !fs::is_directory(_dcpomatic_context->location)) { + getMessengerLogger()->error("Invalid directory %s", _dcpomatic_context->location.c_str()); return false; - } + } + auto s = dcpv.get_size(); - _dcpomatic_context->setDimensions(s.width, s.height); + _dcpomatic_context->set_dimensions(s.width, s.height); auto config = Config::instance(); - if (!messenger_->launchGrok( - _dcpomatic_context->_location, - _dcpomatic_context->width_,_dcpomatic_context->width_, - _dcpomatic_context->height_, - 3, 12, device, - _dcpomatic_context->film_->resolution() == Resolution::FOUR_K, - _dcpomatic_context->film_->video_frame_rate(), - _dcpomatic_context->film_->j2k_bandwidth(), + if (!_messenger->launchGrok( + _dcpomatic_context->location, + _dcpomatic_context->width, + _dcpomatic_context->width, + _dcpomatic_context->height, + 3, + 12, + device, + _dcpomatic_context->film->resolution() == Resolution::FOUR_K, + _dcpomatic_context->film->video_frame_rate(), + _dcpomatic_context->film->j2k_bandwidth(), config->gpu_license_server(), config->gpu_license_port(), config->gpu_license())) { - launchFailed_ = true; + _launch_failed = true; return false; } } - launched_ = messenger_->waitForClientInit(); - launchFailed_ = launched_; - return launched_; + _launched = _messenger->waitForClientInit(); + _launch_failed = _launched; + + return _launched; } - bool scheduleCompress(DCPVideo const& vf){ - if (!messenger_) + + bool scheduleCompress(DCPVideo const& vf) + { + if (!_messenger) { return false; + } auto fp = FrameProxy(vf.index(), vf.eyes(), vf); - auto cvt = [this, &fp](BufferSrc src){ - // xyz conversion + auto cvt = [this, &fp](BufferSrc src) { fp.vf.convert_to_xyz((uint16_t*)src.framePtr_); }; - return messenger_->scheduleCompress(fp, cvt); + + return _messenger->scheduleCompress(fp, cvt); } - void shutdown(void){ - if (!messenger_) + + void shutdown() + { + if (!_messenger) { return; + } std::unique_lock lk_global(launchMutex); - if (!messenger_) + + if (!_messenger) { return; - if (launched_) - messenger_->shutdown(); - delete messenger_; - messenger_ = nullptr; + } + + if (_launched) { + _messenger->shutdown(); + } + + delete _messenger; + _messenger = nullptr; } - void frame_done () { - _dcpomatic_context->history_.event(); + + void frame_done() + { + _dcpomatic_context->history.event(); } + private: DcpomaticContext* _dcpomatic_context; - ScheduledMessenger *messenger_; - bool launched_; - bool launchFailed_; + ScheduledMessenger* _messenger = nullptr; + bool _launched = false; + bool _launch_failed = false; }; } diff --git a/src/wx/grok/gpu_config_panel.h b/src/wx/grok/gpu_config_panel.h index a0f2a1f7f..40e999181 100644 --- a/src/wx/grok/gpu_config_panel.h +++ b/src/wx/grok/gpu_config_panel.h @@ -41,111 +41,121 @@ static std::vector get_gpu_names(boost::filesystem::path binary, bo return gpu_names; } + class GpuList : public wxPanel { public: - GpuList(wxPanel* parent) : wxPanel(parent, wxID_ANY), selection(0) { - comboBox = new wxComboBox(this, wxID_ANY, "", wxDefaultPosition, wxSize(400, -1)); - comboBox->Bind(wxEVT_COMBOBOX, &GpuList::OnComboBox, this); - update(); - - wxBoxSizer* sizer = new wxBoxSizer(wxHORIZONTAL); + GpuList(wxPanel* parent) + : wxPanel(parent, wxID_ANY) + { + _combo_box = new wxComboBox(this, wxID_ANY, "", wxDefaultPosition, wxSize(400, -1)); + _combo_box->Bind(wxEVT_COMBOBOX, &GpuList::OnComboBox, this); + update(); - sizer->Add(comboBox, 0, wxALIGN_CENTER_VERTICAL); // Vertically center the comboBox + auto sizer = new wxBoxSizer(wxHORIZONTAL); + sizer->Add(_combo_box, 0, wxALIGN_CENTER_VERTICAL); + SetSizerAndFit(sizer); + } - this->SetSizerAndFit(sizer); - } - void update(void) { - auto cfg = Config::instance(); - auto lister_binary = cfg->gpu_binary_location() / "gpu_lister"; - auto lister_file = cfg->gpu_binary_location () / "gpus.txt"; - if (boost::filesystem::exists(lister_binary)) { + void update() + { + auto cfg = Config::instance(); + auto lister_binary = cfg->gpu_binary_location() / "gpu_lister"; + auto lister_file = cfg->gpu_binary_location () / "gpus.txt"; + if (boost::filesystem::exists(lister_binary)) { auto gpu_names = get_gpu_names(lister_binary, lister_file); - comboBox->Clear(); - for (const auto& name : gpu_names) - comboBox->Append(name); - } - } + _combo_box->Clear(); + for (auto const& name: gpu_names) { + _combo_box->Append(name); + } + } + } - int getSelection(void) { - return selection; - } - void setSelection(int sel) { - if ((int)comboBox->GetCount() > sel) - comboBox->SetSelection(sel); - } + void set_selection(int sel) + { + if (sel < static_cast(_combo_box->GetCount())) { + _combo_box->SetSelection(sel); + } + } private: - void OnComboBox(wxCommandEvent&) - { - selection = comboBox->GetSelection(); - if (selection != wxNOT_FOUND) - Config::instance ()->set_selected_gpu(selection); - } + void OnComboBox(wxCommandEvent&) + { + auto selection = _combo_box->GetSelection(); + if (selection != wxNOT_FOUND) { + Config::instance()->set_selected_gpu(selection); + } + } - wxComboBox* comboBox; - int selection; + wxComboBox* _combo_box; + int _selection = 0; }; + class GPUPage : public Page { public: - GPUPage (wxSize panel_size, int border) - : Page (panel_size, border), - _enable_gpu(nullptr), _binary_location(nullptr), _gpu_list_control(nullptr) + GPUPage(wxSize panel_size, int border) + : Page(panel_size, border) {} - wxString GetName () const override + wxString GetName() const override { return _("GPU"); } #ifdef DCPOMATIC_OSX - wxBitmap GetLargeIcon () const override + /* XXX: this icon does not exist */ + wxBitmap GetLargeIcon() const override { - return wxBitmap(icon_path("tms"), wxBITMAP_TYPE_PNG); + return wxBitmap(icon_path("gpu"), wxBITMAP_TYPE_PNG); } #endif private: - void setup () override + void setup() override { - auto config = Config::instance (); - _enable_gpu = new CheckBox(_panel, _("Enable GPU acceleration")); - _panel->GetSizer()->Add (_enable_gpu, 0, wxALL | wxEXPAND, _border); + _panel->GetSizer()->Add(_enable_gpu, 0, wxALL | wxEXPAND, _border); - wxFlexGridSizer* table = new wxFlexGridSizer (2, DCPOMATIC_SIZER_X_GAP, DCPOMATIC_SIZER_Y_GAP); - table->AddGrowableCol (1, 1); - _panel->GetSizer()->Add (table, 1, wxALL | wxEXPAND, _border); + wxFlexGridSizer* table = new wxFlexGridSizer(2, DCPOMATIC_SIZER_X_GAP, DCPOMATIC_SIZER_Y_GAP); + table->AddGrowableCol(1, 1); + _panel->GetSizer()->Add(table, 1, wxALL | wxEXPAND, _border); add_label_to_sizer(table, _panel, _("Acceleration binary folder"), true, 0, wxLEFT | wxLEFT | wxALIGN_CENTRE_VERTICAL); - _binary_location = new wxDirPickerCtrl (_panel, wxDD_DIR_MUST_EXIST); - table->Add (_binary_location, 1, wxEXPAND); + _binary_location = new wxDirPickerCtrl(_panel, wxDD_DIR_MUST_EXIST); + table->Add(_binary_location, 1, wxEXPAND); add_label_to_sizer(table, _panel, _("GPU selection"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); _gpu_list_control = new GpuList(_panel); - table->Add (_gpu_list_control, 1, wxEXPAND); + table->Add(_gpu_list_control, 1, wxEXPAND); add_label_to_sizer(table, _panel, _("License server"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); - _server = new wxTextCtrl (_panel, wxID_ANY); - table->Add (_server, 1, wxEXPAND | wxALL); + _server = new wxTextCtrl(_panel, wxID_ANY); + table->Add(_server, 1, wxEXPAND | wxALL); - add_label_to_sizer (table, _panel, _("Port"), false, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); - _port = new wxSpinCtrl (_panel, wxID_ANY); - _port->SetRange (0, 65535); - table->Add (_port); + add_label_to_sizer(table, _panel, _("Port"), false, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + _port = new wxSpinCtrl(_panel, wxID_ANY); + _port->SetRange(0, 65535); + table->Add(_port); - add_label_to_sizer (table, _panel, _("License"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); - _license = new PasswordEntry (_panel); - table->Add (_license->get_panel(), 1, wxEXPAND | wxALL); + add_label_to_sizer(table, _panel, _("License"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); + _license = new PasswordEntry(_panel); + table->Add(_license->get_panel(), 1, wxEXPAND | wxALL); _enable_gpu->bind(&GPUPage::enable_gpu_changed, this); - _binary_location->Bind (wxEVT_DIRPICKER_CHANGED, boost::bind (&GPUPage::binary_location_changed, this)); - _server->Bind (wxEVT_TEXT, boost::bind(&GPUPage::server_changed, this)); - _port->Bind (wxEVT_SPINCTRL, boost::bind(&GPUPage::port_changed, this)); - _license->Changed.connect (boost::bind(&GPUPage::license_changed, this)); + _binary_location->Bind(wxEVT_DIRPICKER_CHANGED, boost::bind (&GPUPage::binary_location_changed, this)); + _server->Bind(wxEVT_TEXT, boost::bind(&GPUPage::server_changed, this)); + _port->Bind(wxEVT_SPINCTRL, boost::bind(&GPUPage::port_changed, this)); + _license->Changed.connect(boost::bind(&GPUPage::license_changed, this)); + + setup_sensitivity(); + } + + void setup_sensitivity() + { + auto config = Config::instance(); _binary_location->Enable(config->enable_gpu()); _gpu_list_control->Enable(config->enable_gpu()); @@ -154,57 +164,51 @@ private: _license->get_panel()->Enable(config->enable_gpu()); } - - void config_changed () override + void config_changed() override { - auto config = Config::instance (); + auto config = Config::instance(); - checked_set (_enable_gpu, config->enable_gpu()); + checked_set(_enable_gpu, config->enable_gpu()); _binary_location->SetPath(std_to_wx(config->gpu_binary_location().string())); _gpu_list_control->update(); - _gpu_list_control->setSelection(config->selected_gpu()); - checked_set (_server, config->gpu_license_server()); - checked_set (_port, config->gpu_license_port()); - checked_set (_license, config->gpu_license()); + _gpu_list_control->set_selection(config->selected_gpu()); + checked_set(_server, config->gpu_license_server()); + checked_set(_port, config->gpu_license_port()); + checked_set(_license, config->gpu_license()); } - void enable_gpu_changed () + void enable_gpu_changed() { - auto config = Config::instance (); - - config->set_enable_gpu (_enable_gpu->GetValue()); - _binary_location->Enable(config->enable_gpu()); - _gpu_list_control->Enable(config->enable_gpu()); - _server->Enable(config->enable_gpu()); - _port->Enable(config->enable_gpu()); - _license->get_panel()->Enable(config->enable_gpu()); + auto config = Config::instance(); + config->set_enable_gpu(_enable_gpu->GetValue()); + setup_sensitivity(); } - void binary_location_changed () + void binary_location_changed() { - Config::instance()->set_gpu_binary_location (wx_to_std (_binary_location->GetPath ())); + Config::instance()->set_gpu_binary_location(wx_to_std(_binary_location->GetPath())); _gpu_list_control->update(); } - void server_changed () + void server_changed() { Config::instance()->set_gpu_license_server(wx_to_std(_server->GetValue())); } - void port_changed () + void port_changed() { Config::instance()->set_gpu_license_port(_port->GetValue()); } - void license_changed () + void license_changed() { Config::instance()->set_gpu_license(_license->get()); } - CheckBox* _enable_gpu; - wxDirPickerCtrl* _binary_location; - GpuList *_gpu_list_control; - wxTextCtrl* _server; - wxSpinCtrl* _port; - PasswordEntry* _license; + CheckBox* _enable_gpu = nullptr; + wxDirPickerCtrl* _binary_location = nullptr; + GpuList* _gpu_list_control = nullptr; + wxTextCtrl* _server = nullptr; + wxSpinCtrl* _port = nullptr; + PasswordEntry* _license = nullptr; }; -- cgit v1.2.3 From bdc861093c0af0a0a9a9da677b9ea366892ca408 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Fri, 6 Oct 2023 21:38:50 +0200 Subject: Add another manual dependency. --- wscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wscript b/wscript index 164b53fca..bb525c8d2 100644 --- a/wscript +++ b/wscript @@ -626,6 +626,8 @@ def build(bld): bld.add_manual_dependency(bld.path.find_node(dep), bld.path.find_node('src/lib/grok/context.h')) bld.add_manual_dependency(bld.path.find_node(dep), bld.path.find_node('src/lib/grok/messenger.h')) + bld.add_manual_dependency(bld.path.find_node('src/wx/full_config_dialog.cc'), bld.path.find_node('src/wx/grok/gpu_config_panel.h')) + bld.recurse('src') bld.recurse('graphics') -- cgit v1.2.3 From 0d3a24730ffc1c694b0ab34c2726e8cfcb89c765 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Fri, 6 Oct 2023 22:42:15 +0200 Subject: Remove a directory created by the config machinery to avoid confusion. --- test/map_cli_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/map_cli_test.cc b/test/map_cli_test.cc index a4056dc46..94eb50b5a 100644 --- a/test/map_cli_test.cc +++ b/test/map_cli_test.cc @@ -433,6 +433,7 @@ BOOST_AUTO_TEST_CASE(map_with_given_config) }; boost::filesystem::remove_all(out); + boost::filesystem::remove_all("test/data/map_with_given_config/2.18"); Config::instance()->drop(); vector output_messages; -- cgit v1.2.3 From 617b8cd303b4a96621a207724c55bed1d749b10c Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Fri, 6 Oct 2023 22:42:44 +0200 Subject: Clean up grok's presence in the config file and make sure it's optional. It should be allowed to not have any grok stuff in the config file, and we should generally call it grok rather than GPU in case other non-grok GPU stuff arrives in the future. --- src/lib/config.cc | 66 ++++++++++++++++++++++++---------- src/lib/config.h | 74 +++++++++++++------------------------- src/lib/grok/context.h | 11 +++--- src/lib/grok_j2k_encoder_thread.cc | 4 ++- src/lib/j2k_encoder.cc | 31 ++++++++++------ src/lib/j2k_encoder.h | 4 +-- src/wx/grok/gpu_config_panel.h | 69 ++++++++++++++++++++--------------- 7 files changed, 145 insertions(+), 114 deletions(-) diff --git a/src/lib/config.cc b/src/lib/config.cc index 3a7442c39..938be090c 100644 --- a/src/lib/config.cc +++ b/src/lib/config.cc @@ -212,12 +212,9 @@ Config::set_defaults () set_notification_email_to_default (); set_cover_sheet_to_default (); - _gpu_binary_location = ""; - _enable_gpu = false; - _selected_gpu = 0; - _gpu_license_server = ""; - _gpu_license_port = 5000; - _gpu_license = ""; +#ifdef DCPOMATIC_GROK + _grok = boost::none; +#endif _main_divider_sash_position = {}; _main_content_divider_sash_position = {}; @@ -640,12 +637,11 @@ try _allow_smpte_bv20 = f.optional_bool_child("AllowSMPTEBv20").get_value_or(false); _isdcf_name_part_length = f.optional_number_child("ISDCFNamePartLength").get_value_or(14); - _enable_gpu = f.optional_bool_child("EnableGPU").get_value_or(false); - _gpu_binary_location = f.string_child("GPUBinaryLocation"); - _selected_gpu = f.number_child("SelectedGPU"); - _gpu_license_server = f.string_child ("GPULicenseServer"); - _gpu_license_port = f.number_child ("GPULicensePort"); - _gpu_license = f.string_child("GPULicense"); +#ifdef DCPOMATIC_GROK + if (auto grok = f.optional_node_child("Grok")) { + _grok = Grok(grok); + } +#endif _export.read(f.optional_node_child("Export")); } @@ -1133,12 +1129,11 @@ Config::write_config () const /* [XML] ISDCFNamePartLength Maximum length of the "name" part of an ISDCF name, which should be 14 according to the standard */ root->add_child("ISDCFNamePartLength")->add_child_text(raw_convert(_isdcf_name_part_length)); - root->add_child("GPUBinaryLocation")->add_child_text (_gpu_binary_location.string()); - root->add_child("EnableGPU")->add_child_text ((_enable_gpu ? "1" : "0")); - root->add_child("SelectedGPU")->add_child_text (raw_convert (_selected_gpu)); - root->add_child("GPULicenseServer")->add_child_text (_gpu_license_server); - root->add_child("GPULicensePort")->add_child_text (raw_convert (_gpu_license_port)); - root->add_child("GPULicense")->add_child_text (_gpu_license); +#ifdef DCPOMATIC_GROK + if (_grok) { + _grok->as_xml(root->add_child("Grok")); + } +#endif _export.write(root->add_child("Export")); @@ -1660,3 +1655,38 @@ Config::initial_path(string id) const return iter->second; } + +#ifdef DCPOMATIC_GROK + +Config::Grok::Grok(cxml::ConstNodePtr node) + : enable(node->bool_child("Enable")) + , binary_location(node->string_child("BinaryLocation")) + , selected(node->number_child("Selected")) + , licence_server(node->string_child("LicenceServer")) + , licence_port(node->number_child("LicencePort")) + , licence(node->string_child("Licence")) +{ + +} + + +void +Config::Grok::as_xml(xmlpp::Element* node) const +{ + node->add_child("BinaryLocation")->add_child_text(binary_location.string()); + node->add_child("Enable")->add_child_text((enable ? "1" : "0")); + node->add_child("Selected")->add_child_text(raw_convert(selected)); + node->add_child("LicenceServer")->add_child_text(licence_server); + node->add_child("LicencePort")->add_child_text(raw_convert(licence_port)); + node->add_child("Licence")->add_child_text(licence); +} + + +void +Config::set_grok(Grok const& grok) +{ + _grok = grok; + changed(OTHER); +} + +#endif diff --git a/src/lib/config.h b/src/lib/config.h index 9aba9e9d2..eaf85451d 100644 --- a/src/lib/config.h +++ b/src/lib/config.h @@ -618,27 +618,27 @@ public: return _allow_smpte_bv20; } - boost::filesystem::path gpu_binary_location() const { - return _gpu_binary_location; - } - - bool enable_gpu () const { - return _enable_gpu; - } - - int selected_gpu () const { - return _selected_gpu; - } - std::string gpu_license_server () const { - return _gpu_license_server; - } +#ifdef DCPOMATIC_GROK + class Grok + { + public: + Grok() = default; + Grok(cxml::ConstNodePtr node); + + void as_xml(xmlpp::Element* node) const; + + bool enable = false; + boost::filesystem::path binary_location; + int selected = 0; + std::string licence_server; + int licence_port = 5000; + std::string licence; + }; - int gpu_license_port () const { - return _gpu_license_port; - } - std::string gpu_license () const { - return _gpu_license; + boost::optional grok() const { + return _grok; } +#endif int isdcf_name_part_length() const { return _isdcf_name_part_length; @@ -1221,29 +1221,9 @@ public: maybe_set(_allow_smpte_bv20, allow, ALLOW_SMPTE_BV20); } - void set_gpu_binary_location(boost::filesystem::path location) { - maybe_set (_gpu_binary_location, location); - } - - void set_enable_gpu (bool enable) { - maybe_set (_enable_gpu, enable); - } - - void set_selected_gpu (int selected) { - maybe_set (_selected_gpu, selected); - } - - void set_gpu_license_server (std::string s) { - maybe_set (_gpu_license_server, s); - } - - void set_gpu_license_port (int p) { - maybe_set (_gpu_license_port, p); - } - - void set_gpu_license (std::string p) { - maybe_set (_gpu_license, p); - } +#ifdef DCPOMATIC_GROK + void set_grok(Grok const& grok); +#endif void set_isdcf_name_part_length(int length) { maybe_set(_isdcf_name_part_length, length, ISDCF_NAME_PART_LENGTH); @@ -1490,13 +1470,9 @@ private: bool _allow_smpte_bv20; int _isdcf_name_part_length; - /* GPU */ - bool _enable_gpu; - boost::filesystem::path _gpu_binary_location; - int _selected_gpu; - std::string _gpu_license_server; - int _gpu_license_port; - std::string _gpu_license; +#ifdef DCPOMATIC_GROK + boost::optional _grok; +#endif ExportConfig _export; diff --git a/src/lib/grok/context.h b/src/lib/grok/context.h index d27caa5f6..521faae8d 100644 --- a/src/lib/grok/context.h +++ b/src/lib/grok/context.h @@ -112,7 +112,8 @@ public: explicit GrokContext(DcpomaticContext* dcpomatic_context) : _dcpomatic_context(dcpomatic_context) { - if (!Config::instance()->enable_gpu()) { + auto grok = Config::instance()->grok().get_value_or({}); + if (!grok.enable) { return; } @@ -214,7 +215,7 @@ public: auto s = dcpv.get_size(); _dcpomatic_context->set_dimensions(s.width, s.height); - auto config = Config::instance(); + auto grok = Config::instance()->grok().get_value_or({}); if (!_messenger->launchGrok( _dcpomatic_context->location, _dcpomatic_context->width, @@ -226,9 +227,9 @@ public: _dcpomatic_context->film->resolution() == Resolution::FOUR_K, _dcpomatic_context->film->video_frame_rate(), _dcpomatic_context->film->j2k_bandwidth(), - config->gpu_license_server(), - config->gpu_license_port(), - config->gpu_license())) { + grok.licence_server, + grok.licence_port, + grok.licence)) { _launch_failed = true; return false; } diff --git a/src/lib/grok_j2k_encoder_thread.cc b/src/lib/grok_j2k_encoder_thread.cc index 79b80f694..79fb1bbae 100644 --- a/src/lib/grok_j2k_encoder_thread.cc +++ b/src/lib/grok_j2k_encoder_thread.cc @@ -39,7 +39,9 @@ try LOG_TIMING("encoder-pop thread=%1 frame=%2 eyes=%3", thread_id(), frame.index(), static_cast(frame.eyes())); - if (_context->launch(frame, Config::instance()->selected_gpu()) && _context->scheduleCompress(frame)) { + auto grok = Config::instance()->grok().get_value_or({}); + + if (_context->launch(frame, grok.selected) && _context->scheduleCompress(frame)) { frame_guard.cancel(); } } diff --git a/src/lib/j2k_encoder.cc b/src/lib/j2k_encoder.cc index 3bb7ccaed..e68402483 100644 --- a/src/lib/j2k_encoder.cc +++ b/src/lib/j2k_encoder.cc @@ -95,11 +95,14 @@ J2KEncoder::J2KEncoder(shared_ptr film, Writer& writer) : _film (film) , _history (200) , _writer (writer) +{ #ifdef DCPOMATIC_GROK - , _dcpomatic_context(new grk_plugin::DcpomaticContext(film, writer, _history, Config::instance()->gpu_binary_location())) - , _context(Config::instance()->enable_gpu() ? new grk_plugin::GrokContext(_dcpomatic_context) : nullptr) + auto grok = Config::instance()->grok().get_value_or({}); + _dcpomatic_context = new grk_plugin::DcpomaticContext(film, writer, _history, grok.binary_location); + if (grok.enable) { + _context = new grk_plugin::GrokContext(_dcpomatic_context); + } #endif -{ servers_list_changed (); } @@ -121,9 +124,14 @@ void J2KEncoder::servers_list_changed() { auto config = Config::instance(); +#ifdef DCPOMATIC_GROK + auto const grok_enable = config->grok().get_value_or({}).enable; +#else + auto const grok_enable = false; +#endif - auto const cpu = (config->enable_gpu() || config->only_servers_encode()) ? 0 : config->master_encoding_threads(); - auto const gpu = config->enable_gpu() ? config->master_encoding_threads() : 0; + auto const cpu = (grok_enable || config->only_servers_encode()) ? 0 : config->master_encoding_threads(); + auto const gpu = grok_enable ? config->master_encoding_threads() : 0; remake_threads(cpu, gpu, EncodeServerFinder::instance()->servers()); } @@ -141,16 +149,17 @@ J2KEncoder::begin () void J2KEncoder::pause() { - if (!Config::instance()->enable_gpu()) { +#ifdef DCPOMATIC_GROK + if (!Config::instance()->grok().get_value_or({}).enable) { return; } + return; terminate_threads (); /* Something might have been thrown during terminate_threads */ rethrow (); -#ifdef DCPOMATIC_GROK delete _context; _context = nullptr; #endif @@ -159,14 +168,14 @@ J2KEncoder::pause() void J2KEncoder::resume() { - if (!Config::instance()->enable_gpu()) { +#ifdef DCPOMATIC_GROK + if (!Config::instance()->grok().get_value_or({}).enable) { return; } -#ifdef DCPOMATIC_GROK _context = new grk_plugin::GrokContext(_dcpomatic_context); -#endif servers_list_changed(); +#endif } @@ -203,7 +212,7 @@ J2KEncoder::end() */ for (auto & i: _queue) { #ifdef DCPOMATIC_GROK - if (Config::instance()->enable_gpu ()) { + if (Config::instance()->grok().get_value_or({}).enable) { if (!_context->scheduleCompress(i)){ LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), i.index()); // handle error diff --git a/src/lib/j2k_encoder.h b/src/lib/j2k_encoder.h index 0dbe654a4..6bfbaea49 100644 --- a/src/lib/j2k_encoder.h +++ b/src/lib/j2k_encoder.h @@ -127,8 +127,8 @@ private: boost::signals2::scoped_connection _server_found_connection; #ifdef DCPOMATIC_GROK - grk_plugin::DcpomaticContext* _dcpomatic_context; - grk_plugin::GrokContext *_context; + grk_plugin::DcpomaticContext* _dcpomatic_context = nullptr; + grk_plugin::GrokContext *_context = nullptr; #endif bool _ending = false; diff --git a/src/wx/grok/gpu_config_panel.h b/src/wx/grok/gpu_config_panel.h index 40e999181..cbf037592 100644 --- a/src/wx/grok/gpu_config_panel.h +++ b/src/wx/grok/gpu_config_panel.h @@ -59,9 +59,9 @@ public: void update() { - auto cfg = Config::instance(); - auto lister_binary = cfg->gpu_binary_location() / "gpu_lister"; - auto lister_file = cfg->gpu_binary_location () / "gpus.txt"; + auto grok = Config::instance()->grok().get_value_or({}); + auto lister_binary = grok.binary_location / "gpu_lister"; + auto lister_file = grok.binary_location / "gpus.txt"; if (boost::filesystem::exists(lister_binary)) { auto gpu_names = get_gpu_names(lister_binary, lister_file); @@ -84,7 +84,9 @@ private: { auto selection = _combo_box->GetSelection(); if (selection != wxNOT_FOUND) { - Config::instance()->set_selected_gpu(selection); + auto grok = Config::instance()->grok().get_value_or({}); + grok.selected = selection; + Config::instance()->set_grok(grok); } } @@ -141,68 +143,79 @@ private: table->Add(_port); add_label_to_sizer(table, _panel, _("License"), true, 0, wxLEFT | wxRIGHT | wxALIGN_CENTRE_VERTICAL); - _license = new PasswordEntry(_panel); - table->Add(_license->get_panel(), 1, wxEXPAND | wxALL); + _licence = new PasswordEntry(_panel); + table->Add(_licence->get_panel(), 1, wxEXPAND | wxALL); _enable_gpu->bind(&GPUPage::enable_gpu_changed, this); _binary_location->Bind(wxEVT_DIRPICKER_CHANGED, boost::bind (&GPUPage::binary_location_changed, this)); _server->Bind(wxEVT_TEXT, boost::bind(&GPUPage::server_changed, this)); _port->Bind(wxEVT_SPINCTRL, boost::bind(&GPUPage::port_changed, this)); - _license->Changed.connect(boost::bind(&GPUPage::license_changed, this)); + _licence->Changed.connect(boost::bind(&GPUPage::licence_changed, this)); setup_sensitivity(); } void setup_sensitivity() { - auto config = Config::instance(); + auto grok = Config::instance()->grok().get_value_or({}); - _binary_location->Enable(config->enable_gpu()); - _gpu_list_control->Enable(config->enable_gpu()); - _server->Enable(config->enable_gpu()); - _port->Enable(config->enable_gpu()); - _license->get_panel()->Enable(config->enable_gpu()); + _binary_location->Enable(grok.enable); + _gpu_list_control->Enable(grok.enable); + _server->Enable(grok.enable); + _port->Enable(grok.enable); + _licence->get_panel()->Enable(grok.enable); } void config_changed() override { - auto config = Config::instance(); + auto grok = Config::instance()->grok().get_value_or({}); - checked_set(_enable_gpu, config->enable_gpu()); - _binary_location->SetPath(std_to_wx(config->gpu_binary_location().string())); + checked_set(_enable_gpu, grok.enable); + _binary_location->SetPath(std_to_wx(grok.binary_location.string())); _gpu_list_control->update(); - _gpu_list_control->set_selection(config->selected_gpu()); - checked_set(_server, config->gpu_license_server()); - checked_set(_port, config->gpu_license_port()); - checked_set(_license, config->gpu_license()); + _gpu_list_control->set_selection(grok.selected); + checked_set(_server, grok.licence_server); + checked_set(_port, grok.licence_port); + checked_set(_licence, grok.licence); } void enable_gpu_changed() { - auto config = Config::instance(); - config->set_enable_gpu(_enable_gpu->GetValue()); + auto grok = Config::instance()->grok().get_value_or({}); + grok.enable = _enable_gpu->GetValue(); + Config::instance()->set_grok(grok); + setup_sensitivity(); } void binary_location_changed() { - Config::instance()->set_gpu_binary_location(wx_to_std(_binary_location->GetPath())); + auto grok = Config::instance()->grok().get_value_or({}); + grok.binary_location = wx_to_std(_binary_location->GetPath()); + Config::instance()->set_grok(grok); + _gpu_list_control->update(); } void server_changed() { - Config::instance()->set_gpu_license_server(wx_to_std(_server->GetValue())); + auto grok = Config::instance()->grok().get_value_or({}); + grok.licence_server = wx_to_std(_server->GetValue()); + Config::instance()->set_grok(grok); } void port_changed() { - Config::instance()->set_gpu_license_port(_port->GetValue()); + auto grok = Config::instance()->grok().get_value_or({}); + grok.licence_port = _port->GetValue(); + Config::instance()->set_grok(grok); } - void license_changed() + void licence_changed() { - Config::instance()->set_gpu_license(_license->get()); + auto grok = Config::instance()->grok().get_value_or({}); + grok.licence = wx_to_std(_licence->get()); + Config::instance()->set_grok(grok); } CheckBox* _enable_gpu = nullptr; @@ -210,5 +223,5 @@ private: GpuList* _gpu_list_control = nullptr; wxTextCtrl* _server = nullptr; wxSpinCtrl* _port = nullptr; - PasswordEntry* _license = nullptr; + PasswordEntry* _licence = nullptr; }; -- cgit v1.2.3 From b0c1482f98c7e00634c1bc3dd801e76ce69907e2 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sun, 8 Oct 2023 15:48:58 +0200 Subject: Make stop() private and use drop() instead. This avoids confusing situations where we call stop(), then try to restart the finder (unsuccessfully, because once it's stop()ped it will stay around, never to be restarted). --- src/lib/encode_server_finder.h | 3 +-- src/tools/dcpomatic_cli.cc | 2 +- test/test.cc | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lib/encode_server_finder.h b/src/lib/encode_server_finder.h index f8a30af54..c478387f9 100644 --- a/src/lib/encode_server_finder.h +++ b/src/lib/encode_server_finder.h @@ -50,8 +50,6 @@ public: static EncodeServerFinder* instance (); static void drop (); - void stop (); - std::list servers () const; /** Emitted whenever the list of servers changes */ @@ -62,6 +60,7 @@ private: ~EncodeServerFinder (); void start (); + void stop (); void search_thread (); void listen_thread (); diff --git a/src/tools/dcpomatic_cli.cc b/src/tools/dcpomatic_cli.cc index 96bf83086..2abc3a149 100644 --- a/src/tools/dcpomatic_cli.cc +++ b/src/tools/dcpomatic_cli.cc @@ -413,7 +413,7 @@ main (int argc, char* argv[]) signal_manager = new SignalManager (); if (no_remote || export_format) { - EncodeServerFinder::instance()->stop (); + EncodeServerFinder::drop(); } if (json_port) { diff --git a/test/test.cc b/test/test.cc index 615e9d3bc..3d8c91b26 100644 --- a/test/test.cc +++ b/test/test.cc @@ -161,7 +161,7 @@ struct TestConfig setup_test_config (); capture_ffmpeg_logs(); - EncodeServerFinder::instance()->stop (); + EncodeServerFinder::drop(); signal_manager = new TestSignalManager (); -- cgit v1.2.3