A whole load of UTF-8/-16 confusion fixes for Windows.

17 files changed, 120 insertions, 60 deletions

diff --git a/src/lib/audio_analysis.cc b/src/lib/audio_analysis.cc
index bc59bccca..1488f89fc 100644
--- a/src/lib/audio_analysis.cc
+++ b/src/lib/audio_analysis.cc
@@ -20,15 +20,14 @@
 #include <stdint.h>
 #include <cmath>
 #include <cassert>
-#include <fstream>
+#include <cstdio>
 #include <boost/filesystem.hpp>
 #include "audio_analysis.h"
+#include "cross.h"
 
 using std::ostream;
 using std::istream;
 using std::string;
-using std::ofstream;
-using std::ifstream;
 using std::vector;
 using std::cout;
 using std::max;
@@ -41,10 +40,10 @@ AudioPoint::AudioPoint ()
 	}
 }
 
-AudioPoint::AudioPoint (istream& s)
+AudioPoint::AudioPoint (FILE* f)
 {
 	for (int i = 0; i < COUNT; ++i) {
-		s >> _data[i];
+		fscanf (f, "%f", &_data[i]);
 	}
 }
 
@@ -70,10 +69,10 @@ AudioPoint::operator= (AudioPoint const & other)
 }
 
 void
-AudioPoint::write (ostream& s) const
+AudioPoint::write (FILE* f) const
 {
 	for (int i = 0; i < COUNT; ++i) {
-		s << _data[i] << "\n";
+		fprintf (f, "%f\n", _data[i]);
 	}
 }
 	
@@ -85,15 +84,15 @@ AudioAnalysis::AudioAnalysis (int channels)
 
 AudioAnalysis::AudioAnalysis (boost::filesystem::path filename)
 {
-	ifstream f (filename.string().c_str ());
+	FILE* f = fopen_boost (filename, "r");
 
 	int channels;
-	f >> channels;
+	fscanf (f, "%d", &channels);
 	_data.resize (channels);
 
 	for (int i = 0; i < channels; ++i) {
 		int points;
-		f >> points;
+		fscanf (f, "%d", &points);
 		for (int j = 0; j < points; ++j) {
 			_data[i].push_back (AudioPoint (f));
 		}
@@ -130,17 +129,19 @@ AudioAnalysis::points (int c) const
 void
 AudioAnalysis::write (boost::filesystem::path filename)
 {
-	string tmp = filename.string() + ".tmp";
+	boost::filesystem::path tmp = filename;
+	tmp.replace_extension (".tmp");
 
-	ofstream f (tmp.c_str ());
-	f << _data.size() << "\n";
+	FILE* f = fopen_boost (tmp, "w");
+
+	fprintf (f, "%ld\n", _data.size ());
 	for (vector<vector<AudioPoint> >::iterator i = _data.begin(); i != _data.end(); ++i) {
-		f << i->size () << "\n";
+		fprintf (f, "%ld\n", i->size ());
 		for (vector<AudioPoint>::iterator j = i->begin(); j != i->end(); ++j) {
 			j->write (f);
 		}
 	}
 
-	f.close ();
+	fclose (f);
 	boost::filesystem::rename (tmp, filename);
 }
diff --git a/src/lib/audio_analysis.h b/src/lib/audio_analysis.h
index cfc170c84..824472dda 100644
--- a/src/lib/audio_analysis.h
+++ b/src/lib/audio_analysis.h
@@ -20,7 +20,6 @@
 #ifndef DCPOMATIC_AUDIO_ANALYSIS_H
 #define DCPOMATIC_AUDIO_ANALYSIS_H
 
-#include <iostream>
 #include <vector>
 #include <list>
 #include <boost/filesystem.hpp>
@@ -35,11 +34,11 @@ public:
 	};
 
 	AudioPoint ();
-	AudioPoint (std::istream &);
+	AudioPoint (FILE *);
 	AudioPoint (AudioPoint const &);
 	AudioPoint& operator= (AudioPoint const &);
 
-	void write (std::ostream &) const;
+	void write (FILE *) const;
 	
 	float& operator[] (int t) {
 		return _data[t];
diff --git a/src/lib/config.cc b/src/lib/config.cc
index 4b6455f51..54b9168f2 100644
--- a/src/lib/config.cc
+++ b/src/lib/config.cc
@@ -41,7 +41,6 @@
 using std::vector;
 using std::ifstream;
 using std::string;
-using std::ofstream;
 using std::list;
 using std::max;
 using std::exception;
@@ -182,6 +181,7 @@ Config::read ()
 void
 Config::read_old_metadata ()
 {
+	/* XXX: this won't work with non-Latin filenames */
 	ifstream f (file(true).string().c_str ());
 	string line;
 
diff --git a/src/lib/cross.cc b/src/lib/cross.cc
index 41051ee2e..7436dbf26 100644
--- a/src/lib/cross.cc
+++ b/src/lib/cross.cc
@@ -70,6 +70,9 @@ cpu_info ()
 	string info;
 	
 #ifdef DCPOMATIC_LINUX
+	/* This use of ifstream is ok; the filename can never
+	   be non-Latin
+	*/
 	ifstream f ("/proc/cpuinfo");
 	while (f.good ()) {
 		string l;
@@ -269,3 +272,19 @@ openssl_path ()
 #endif
 
 }
+
+/* Apparently there is no way to create an ofstream using a UTF-8
+   filename under Windows.  We are hence reduced to using fopen
+   with this wrapper.
+*/
+FILE *
+fopen_boost (boost::filesystem::path p, string t)
+{
+#ifdef DCPOMATIC_WINDOWS
+        wstring w (t.begin(), t.end());
+	/* c_str() here should give a UTF-16 string */
+        return _wfopen (p.c_str(), w.c_str ());
+#else
+        return fopen (p.c_str(), t.c_str ());
+#endif
+}
diff --git a/src/lib/cross.h b/src/lib/cross.h
index 1fe34edbe..931e7d890 100644
--- a/src/lib/cross.h
+++ b/src/lib/cross.h
@@ -33,3 +33,4 @@ extern boost::filesystem::path openssl_path ();
 #ifdef DCPOMATIC_OSX
 extern boost::filesystem::path app_contents ();
 #endif
+extern FILE * fopen_boost (boost::filesystem::path, std::string);
diff --git a/src/lib/dcp_video_frame.cc b/src/lib/dcp_video_frame.cc
index 25abd6f0d..679a0490e 100644
--- a/src/lib/dcp_video_frame.cc
+++ b/src/lib/dcp_video_frame.cc
@@ -59,12 +59,12 @@
 #include "scaler.h"
 #include "image.h"
 #include "log.h"
+#include "cross.h"
 
 #include "i18n.h"
 
 using std::string;
 using std::stringstream;
-using std::ofstream;
 using std::cout;
 using boost::shared_ptr;
 using boost::lexical_cast;
@@ -379,8 +379,9 @@ void
 EncodedData::write_info (shared_ptr<const Film> film, int frame, Eyes eyes, libdcp::FrameInfo fin) const
 {
 	boost::filesystem::path const info = film->info_path (frame, eyes);
-	ofstream h (info.string().c_str());
+	FILE* h = fopen_boost (info, "w");
 	fin.write (h);
+	fclose (h);
 }
 
 /** Send this data to a socket.
diff --git a/src/lib/film.cc b/src/lib/film.cc
index eab91c7d2..b1b868984 100644
--- a/src/lib/film.cc
+++ b/src/lib/film.cc
@@ -64,8 +64,6 @@ using std::multimap;
 using std::pair;
 using std::map;
 using std::vector;
-using std::ifstream;
-using std::ofstream;
 using std::setfill;
 using std::min;
 using std::make_pair;
diff --git a/src/lib/log.cc b/src/lib/log.cc
index 86de21bdd..9ddf460d4 100644
--- a/src/lib/log.cc
+++ b/src/lib/log.cc
@@ -21,9 +21,10 @@
  *  @brief A very simple logging class.
  */
 
-#include <fstream>
 #include <time.h>
+#include <cstdio>
 #include "log.h"
+#include "cross.h"
 
 #include "i18n.h"
 
@@ -102,7 +103,8 @@ FileLog::FileLog (boost::filesystem::path file)
 void
 FileLog::do_log (string m)
 {
-	ofstream f (_file.string().c_str(), fstream::app);
-	f << m << N_("\n");
+	FILE* f = fopen_boost (_file, "a");
+	fprintf (f, "%s\n", m.c_str ());
+	fclose (f);
 }
 
diff --git a/src/lib/still_image_examiner.cc b/src/lib/still_image_examiner.cc
index 5f45d6365..d5e863634 100644
--- a/src/lib/still_image_examiner.cc
+++ b/src/lib/still_image_examiner.cc
@@ -26,6 +26,7 @@
 #include "i18n.h"
 
 using std::cout;
+using std::string;
 using boost::shared_ptr;
 
 StillImageExaminer::StillImageExaminer (shared_ptr<const Film> f, shared_ptr<const StillImageContent> c)
@@ -33,7 +34,7 @@ StillImageExaminer::StillImageExaminer (shared_ptr<const Film> f, shared_ptr<con
 	, _film (f)
 {
 	using namespace MagickCore;
-	Magick::Image* image = new Magick::Image (_still_image_content->path().string());
+	Magick::Image* image = new Magick::Image (_still_image_content->path().string ());
 	_video_size = libdcp::Size (image->columns(), image->rows());
 	delete image;
 }
diff --git a/src/lib/util.cc b/src/lib/util.cc
index 484c4fb9b..e70440271 100644
--- a/src/lib/util.cc
+++ b/src/lib/util.cc
@@ -39,6 +39,7 @@
 #include <boost/lexical_cast.hpp>
 #include <boost/thread.hpp>
 #include <boost/filesystem.hpp>
+#include <boost/locale.hpp>
 #include <glib.h>
 #include <openjpeg.h>
 #include <openssl/md5.h>
@@ -80,7 +81,6 @@ using std::endl;
 using std::vector;
 using std::hex;
 using std::setw;
-using std::ifstream;
 using std::ios;
 using std::min;
 using std::max;
@@ -89,7 +89,6 @@ using std::multimap;
 using std::istream;
 using std::numeric_limits;
 using std::pair;
-using std::ofstream;
 using std::cout;
 using boost::shared_ptr;
 using boost::thread;
@@ -260,8 +259,11 @@ seconds (struct timeval t)
 LONG WINAPI exception_handler(struct _EXCEPTION_POINTERS *)
 {
 	dbg::stack s;
-	ofstream f (backtrace_file.string().c_str());
-	std::copy(s.begin(), s.end(), std::ostream_iterator<dbg::stack_frame>(f, "\n"));
+	FILE* f = fopen_boost (backtrace_file, "w");
+	for (dbg::stack::const_iterator i = s.begin(); i != s.end(); ++i) {
+		fprintf (f, "%p %s %d %s", i->instruction, i->function.c_str(), i->line, i->module.c_str());
+	}
+	fclose (f);
 	return EXCEPTION_CONTINUE_SEARCH;
 }
 #endif
@@ -276,6 +278,21 @@ dcpomatic_setup ()
 	backtrace_file /= g_get_user_config_dir ();
 	backtrace_file /= "backtrace.txt";
 	SetUnhandledExceptionFilter(exception_handler);
+
+	/* Dark voodoo which, I think, gets boost::filesystem::path to
+	   correctly convert UTF-8 strings to paths, and also paths
+	   back to UTF-8 strings (on path::string()).
+
+	   After this, constructing boost::filesystem::paths from strings
+	   converts from UTF-8 to UTF-16 inside the path.  Then
+	   path::string().c_str() gives UTF-8 and
+	   path::c_str()          gives UTF-16.
+
+	   This is all Windows-only.  AFAICT Linux/OS X use UTF-8 everywhere,
+	   so things are much simpler.
+	*/
+	std::locale::global (boost::locale::generator().generate (""));
+	boost::filesystem::path::imbue (std::locale ());
 #endif	
 	
 	avfilter_register_all ();
@@ -392,29 +409,28 @@ md5_digest (void const * data, int size)
 string
 md5_digest (boost::filesystem::path file)
 {
-	ifstream f (file.string().c_str(), std::ios::binary);
-	if (!f.good ()) {
+	FILE* f = fopen_boost (file, "rb");
+	if (!f) {
 		throw OpenFileError (file.string());
 	}
-	
-	f.seekg (0, std::ios::end);
-	int bytes = f.tellg ();
-	f.seekg (0, std::ios::beg);
 
-	int const buffer_size = 64 * 1024;
+	boost::uintmax_t bytes = boost::filesystem::file_size (file);
+
+	boost::uintmax_t const buffer_size = 64 * 1024;
 	char buffer[buffer_size];
 
 	MD5_CTX md5_context;
 	MD5_Init (&md5_context);
 	while (bytes > 0) {
 		int const t = min (bytes, buffer_size);
-		f.read (buffer, t);
+		fread (buffer, 1, t, f);
 		MD5_Update (&md5_context, buffer, t);
 		bytes -= t;
 	}
 
 	unsigned char digest[MD5_DIGEST_LENGTH];
 	MD5_Final (digest, &md5_context);
+	fclose (f);
 
 	stringstream s;
 	for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
@@ -428,7 +444,7 @@ md5_digest (boost::filesystem::path file)
 string
 md5_digest_directory (boost::filesystem::path directory, shared_ptr<Job> job)
 {
-	int const buffer_size = 64 * 1024;
+	boost::uintmax_t const buffer_size = 64 * 1024;
 	char buffer[buffer_size];
 
 	MD5_CTX md5_context;
@@ -443,18 +459,16 @@ md5_digest_directory (boost::filesystem::path directory, shared_ptr<Job> job)
 
 	int j = 0;
 	for (boost::filesystem::directory_iterator i(directory); i != boost::filesystem::directory_iterator(); ++i) {
-		ifstream f (i->path().string().c_str(), std::ios::binary);
-		if (!f.good ()) {
+		FILE* f = fopen_boost (i->path(), "rb");
+		if (!f) {
 			throw OpenFileError (i->path().string());
 		}
-	
-		f.seekg (0, std::ios::end);
-		int bytes = f.tellg ();
-		f.seekg (0, std::ios::beg);
+
+		boost::uintmax_t bytes = boost::filesystem::file_size (i->path ());
 
 		while (bytes > 0) {
 			int const t = min (bytes, buffer_size);
-			f.read (buffer, t);
+			fread (buffer, 1, t, f);
 			MD5_Update (&md5_context, buffer, t);
 			bytes -= t;
 		}
@@ -463,6 +477,8 @@ md5_digest_directory (boost::filesystem::path directory, shared_ptr<Job> job)
 			job->set_progress (float (j) / files);
 			++j;
 		}
+
+		fclose (f);
 	}
 
 	unsigned char digest[MD5_DIGEST_LENGTH];
diff --git a/src/lib/writer.cc b/src/lib/writer.cc
index 103ac2ba1..f4128e6c5 100644
--- a/src/lib/writer.cc
+++ b/src/lib/writer.cc
@@ -36,13 +36,13 @@
 #include "audio_mapping.h"
 #include "config.h"
 #include "job.h"
+#include "cross.h"
 
 #include "i18n.h"
 
 using std::make_pair;
 using std::pair;
 using std::string;
-using std::ifstream;
 using std::list;
 using std::cout;
 using boost::shared_ptr;
@@ -141,8 +141,9 @@ Writer::fake_write (int frame, Eyes eyes)
 {
 	boost::mutex::scoped_lock lock (_mutex);
 
-	ifstream ifi (_film->info_path (frame, eyes).string().c_str());
+	FILE* ifi = fopen_boost (_film->info_path (frame, eyes), "r");
 	libdcp::FrameInfo info (ifi);
+	fclose (ifi);
 	
 	QueueItem qi;
 	qi.type = QueueItem::FAKE;
@@ -430,8 +431,9 @@ bool
 Writer::check_existing_picture_mxf_frame (FILE* mxf, int f, Eyes eyes)
 {
 	/* Read the frame info as written */
-	ifstream ifi (_film->info_path (f, eyes).string().c_str());
+	FILE* ifi = fopen_boost (_film->info_path (f, eyes), "r");
 	libdcp::FrameInfo info (ifi);
+	fclose (ifi);
 	if (info.size == 0) {
 		_film->log()->log (String::compose ("Existing frame %1 has no info file", f));
 		return false;
diff --git a/src/lib/wscript b/src/lib/wscript
index 852bb1aed..e317026f3 100644
--- a/src/lib/wscript
+++ b/src/lib/wscript
@@ -74,7 +74,7 @@ def build(bld):
     obj.export_includes = ['..']
     obj.uselib = """
                  AVCODEC AVUTIL AVFORMAT AVFILTER SWSCALE SWRESAMPLE 
-                 BOOST_FILESYSTEM BOOST_THREAD BOOST_DATETIME BOOST_SIGNALS2 
+                 BOOST_FILESYSTEM BOOST_THREAD BOOST_DATETIME BOOST_SIGNALS2 BOOST_LOCALE
                  SNDFILE OPENJPEG POSTPROC TIFF MAGICK SSH DCP CXML GLIB LZMA XML++
                  CURL ZIP QUICKMAIL
                  """
diff --git a/src/tools/dcpomatic.cc b/src/tools/dcpomatic.cc
index 1d01e4da8..2a0cb9cd3 100644
--- a/src/tools/dcpomatic.cc
+++ b/src/tools/dcpomatic.cc
@@ -62,7 +62,6 @@ using std::map;
 using std::make_pair;
 using std::list;
 using std::exception;
-using std::ofstream;
 using boost::shared_ptr;
 using boost::dynamic_pointer_cast;
 
@@ -247,6 +246,22 @@ public:
 		: wxFrame (NULL, -1, title)
 		, _servers_list_dialog (0)
 	{
+#ifdef DCPOMATIC_WINDOWS_CONSOLE		
+                AllocConsole();
+		
+		HANDLE handle_out = GetStdHandle(STD_OUTPUT_HANDLE);
+		int hCrt = _open_osfhandle((intptr_t) handle_out, _O_TEXT);
+		FILE* hf_out = _fdopen(hCrt, "w");
+		setvbuf(hf_out, NULL, _IONBF, 1);
+		*stdout = *hf_out;
+		
+		HANDLE handle_in = GetStdHandle(STD_INPUT_HANDLE);
+		hCrt = _open_osfhandle((intptr_t) handle_in, _O_TEXT);
+		FILE* hf_in = _fdopen(hCrt, "r");
+		setvbuf(hf_in, NULL, _IONBF, 128);
+		*stdin = *hf_in;
+#endif
+
 		wxMenuBar* bar = new wxMenuBar;
 		setup_menu (bar);
 		SetMenuBar (bar);
@@ -332,7 +347,7 @@ private:
 					    std_to_wx (
 						    String::compose (wx_to_std (_("The directory %1 already exists and is not empty.  "
 										  "Are you sure you want to use it?")),
-								     d->get_path().c_str())
+								     d->get_path().string().c_str())
 						    )
 					    )) {
 					return;
diff --git a/src/wx/film_editor.cc b/src/wx/film_editor.cc
index 9cf840614..ef3534939 100644
--- a/src/wx/film_editor.cc
+++ b/src/wx/film_editor.cc
@@ -731,11 +731,14 @@ FilmEditor::setup_content ()
 void
 FilmEditor::content_add_file_clicked ()
 {
-	wxFileDialog* d = new wxFileDialog (this, _("Choose a file or files"), wxT (""), wxT (""), wxT ("*.*"), wxFD_MULTIPLE);
+	/* The wxFD_CHANGE_DIR here prevents a `could not set working directory' error 123 on Windows when using
+	   non-Latin filenames or paths.
+	*/
+	wxFileDialog* d = new wxFileDialog (this, _("Choose a file or files"), wxT (""), wxT (""), wxT ("*.*"), wxFD_MULTIPLE | wxFD_CHANGE_DIR);
 	int const r = d->ShowModal ();
-	d->Destroy ();
 
 	if (r != wxID_OK) {
+		d->Destroy ();
 		return;
 	}
 
@@ -745,8 +748,10 @@ FilmEditor::content_add_file_clicked ()
 	/* XXX: check for lots of files here and do something */
 
 	for (unsigned int i = 0; i < paths.GetCount(); ++i) {
-		_film->examine_and_add_content (content_factory (_film, wx_to_std (paths[i])));
+		_film->examine_and_add_content (content_factory (_film, wx_to_std (d->GetPath ())));
 	}
+
+	d->Destroy ();
 }
 
 void
diff --git a/src/wx/new_film_dialog.cc b/src/wx/new_film_dialog.cc
index 2612a6afe..be5af999e 100644
--- a/src/wx/new_film_dialog.cc
+++ b/src/wx/new_film_dialog.cc
@@ -74,11 +74,11 @@ NewFilmDialog::~NewFilmDialog ()
 	_directory = wx_to_std (_folder->GetPath ());
 }
 
-string
+boost::filesystem::path
 NewFilmDialog::get_path () const
 {
 	filesystem::path p;
 	p /= wx_to_std (_folder->GetPath ());
 	p /= wx_to_std (_name->GetValue ());
-	return p.string ();
+	return p;
 }
diff --git a/src/wx/new_film_dialog.h b/src/wx/new_film_dialog.h
index 4176b060d..a835c7ceb 100644
--- a/src/wx/new_film_dialog.h
+++ b/src/wx/new_film_dialog.h
@@ -29,7 +29,7 @@ public:
 	NewFilmDialog (wxWindow *);
 	~NewFilmDialog ();
 
-	std::string get_path () const;
+	boost::filesystem::path get_path () const;
 
 private:
 	wxTextCtrl* _name;
diff --git a/src/wx/wx_util.cc b/src/wx/wx_util.cc
index 27a4554ca..103d36d00 100644
--- a/src/wx/wx_util.cc
+++ b/src/wx/wx_util.cc
@@ -105,7 +105,7 @@ confirm_dialog (wxWindow* parent, wxString m)
 string
 wx_to_std (wxString s)
 {
-	return string (s.mb_str ());
+	return string (s.ToUTF8 ());
 }
 
 /** @param s STL string.