+/*
+Copyright (c) 2005-2006, John Hurst
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+/*! \file kmfilegen.cpp
+ \version $Id$
+ \brief large file test program
+*/
+
+
+#include "AS_DCP.h"
+#include <iostream>
+#include <KM_fileio.h>
+#include <KM_prng.h>
+#include <openssl/aes.h>
+#include <assert.h>
+
+using namespace Kumu;
+
+// constants
+static const char* PACKAGE = "kmfilegen"; // program name for messages
+const ui32_t RNG_KEY_SIZE = 16;
+const ui32_t RNG_KEY_SIZE_BITS = 128;
+const ui32_t RNG_BLOCK_SIZE = 16;
+
+// globals
+ui32_t s_Nonce = 0;
+FortunaRNG s_RNG;
+
+
+//------------------------------------------------------------------------------------------
+//
+// command line option parser class
+
+// Increment the iterator, test for an additional non-option command line argument.
+// Causes the caller to return if there are no remaining arguments or if the next
+// argument begins with '-'.
+#define TEST_EXTRA_ARG(i,c) if ( ++i >= argc || argv[(i)][0] == '-' ) \
+ { \
+ fprintf(stderr, "Argument not found for option -%c.\n", (c)); \
+ return; \
+ }
+
+//
+void
+banner(FILE* stream = stdout)
+{
+ fprintf(stream, "\n\
+%s (asdcplib %s)\n\n\
+Copyright (c) 2005-2006 John Hurst\n\
+%s is part of the asdcplib DCP tools package.\n\
+asdcplib may be copied only under the terms of the license found at\n\
+the top of every file in the asdcplib distribution kit.\n\n\
+Specify the -h (help) option for further information about %s\n\n",
+ PACKAGE, ASDCP::Version(), PACKAGE, PACKAGE);
+}
+
+
+//
+void
+usage(FILE* stream = stdout)
+{
+ fprintf(stream, "\
+USAGE: %s [-c <file-size>][-o <fwd|rev|rand>][-w <filename>][-v]\n\
+ <filename>\n\
+\n\
+ %s [-h|-help] [-V]\n\
+\n\
+ -c <file-size> - Create a test file containing <file-size> megabytes of data\n\
+ -h | -help - Show help\n\
+ -o <order> - Specify order used when validating a file.\n\
+ One of fwd|rev|rand, default is rand\n\
+ -v - Verbose. Prints informative messages to stderr\n\
+ -V - Show version information\n\
+ -w - Read-Validate-Write - file is written to <filename>\n\
+ (sequential read only)\n\
+\n\
+ NOTES: o There is no option grouping, all options must be distinct arguments.\n\
+ o All option arguments must be separated from the option by whitespace.\n\
+\n", PACKAGE, PACKAGE);
+}
+
+enum MajorMode_t {
+ MMT_NONE,
+ MMT_CREATE,
+ MMT_VALIDATE,
+ MMT_VAL_WRITE
+};
+
+//
+class CommandOptions
+{
+ CommandOptions();
+
+public:
+ bool error_flag; // true if the given options are in error or not complete
+ const char* order; // one of fwd|rev|rand
+ bool verbose_flag; // true if the verbose option was selected
+ bool version_flag; // true if the version display option was selected
+ bool help_flag; // true if the help display option was selected
+ const char* filename; // filename to be processed
+ const char* write_filename; // filename to write with val_write_flag
+ ui32_t chunk_count;
+ MajorMode_t mode; // MajorMode selector
+
+ //
+ CommandOptions(int argc, const char** argv) :
+ error_flag(true), order(0), verbose_flag(false), version_flag(false), help_flag(false),
+ filename(""), write_filename(""), chunk_count(0), mode(MMT_VALIDATE)
+ {
+ order = "rand";
+
+ for ( int i = 1; i < argc; i++ )
+ {
+
+ if ( (strcmp( argv[i], "-help") == 0) )
+ {
+ help_flag = true;
+ continue;
+ }
+
+ if ( argv[i][0] == '-' && isalpha(argv[i][1]) && argv[i][2] == 0 )
+ {
+ switch ( argv[i][1] )
+ {
+ case 'c':
+ mode = MMT_CREATE;
+ TEST_EXTRA_ARG(i, 'c');
+ chunk_count = atoi(argv[i]);
+ break;
+
+ case 'V': version_flag = true; break;
+ case 'h': help_flag = true; break;
+ case 'v': verbose_flag = true; break;
+
+ case 'o':
+ TEST_EXTRA_ARG(i, 'o');
+ order = argv[i];
+
+ if ( strcmp(order, "fwd" ) != 0
+ && strcmp(order, "rev" ) != 0
+ && strcmp(order, "rand" ) != 0 )
+ {
+ fprintf(stderr, "Unexpected order token: %s, expecting fwd|rev|rand\n", order);
+ return;
+ }
+
+ break;
+
+ case 'w':
+ mode = MMT_VAL_WRITE;
+ TEST_EXTRA_ARG(i, 'w');
+ write_filename = argv[i];
+ break;
+
+ default:
+ fprintf(stderr, "Unrecognized option: %s\n", argv[i]);
+ return;
+ }
+ }
+ else
+ {
+ if (argv[i][0] != '-' )
+ {
+ if ( filename != "" )
+ {
+ fprintf(stderr, "Extra filename found: %s\n", argv[i]);
+ return;
+ }
+ else
+ filename = argv[i];
+ }
+ else
+ {
+ fprintf(stderr, "Unrecognized option: %s\n", argv[i]);
+ return;
+ }
+ }
+ }
+
+ if ( help_flag || version_flag )
+ return;
+
+ if ( strlen ( filename ) == 0 )
+ {
+ fprintf(stderr, "Filename required.\n");
+ return;
+ }
+
+ if ( strcmp ( filename, write_filename ) == 0 )
+ {
+ fprintf(stderr, "Output and input files must be different.\n");
+ return;
+ }
+
+ error_flag = false;
+ }
+};
+
+//------------------------------------------------------------------------------------------
+
+
+//
+#pragma pack(4)
+class CTR_Setup
+{
+ AES_KEY m_Context;
+ byte_t m_key[RNG_KEY_SIZE];
+ byte_t m_preamble[8];
+ ui32_t m_nonce;
+ ui32_t m_ctr;
+
+ KM_NO_COPY_CONSTRUCT(CTR_Setup);
+
+public:
+ CTR_Setup() {}
+ ~CTR_Setup() {}
+
+ inline ui32_t Nonce() { return KM_i32_LE(m_nonce); }
+ inline ui32_t WriteSize() { return ( sizeof(m_key) + sizeof(m_preamble)
+ + sizeof(m_nonce) + sizeof(m_ctr) ); }
+
+ //
+ void SetupWrite(byte_t* buf)
+ {
+ assert(buf);
+ s_RNG.FillRandom(m_key, WriteSize());
+ assert(s_Nonce > 0);
+ m_nonce = KM_i32_LE(s_Nonce--);
+ m_ctr &= KM_i32_LE(0x7fffffff); // make sure we have 2GB headroom
+ memcpy(buf, m_key, WriteSize());
+ AES_set_encrypt_key(m_key, RNG_KEY_SIZE_BITS, &m_Context);
+ }
+
+ //
+ void SetupRead(const byte_t* buf)
+ {
+ assert(buf);
+ memcpy(m_key, buf, WriteSize());
+ AES_set_encrypt_key(m_key, RNG_KEY_SIZE_BITS, &m_Context);
+ }
+
+ //
+ void FillRandom(byte_t* buf, ui32_t buf_len)
+ {
+ ui32_t gen_count = 0;
+ while ( gen_count + RNG_BLOCK_SIZE <= buf_len )
+ {
+ AES_encrypt(m_preamble, buf + gen_count, &m_Context);
+ m_ctr = KM_i32_LE(KM_i32_LE(m_ctr) + 1);
+ gen_count += RNG_BLOCK_SIZE;
+ }
+ }
+};
+
+//
+Result_t
+CreateLargeFile(CommandOptions& Options)
+{
+ ui32_t write_total = 0;
+ ui32_t write_count = 0;
+ FileWriter Writer;
+ ByteString FB;
+
+ FB.Capacity(Megabyte);
+ assert(FB.Capacity() == Megabyte);
+
+ fprintf(stderr, "Writing %lu chunks:\n", Options.chunk_count);
+ s_Nonce = Options.chunk_count;
+ Result_t result = Writer.OpenWrite(Options.filename);
+
+ while ( KM_SUCCESS(result) && write_total < Options.chunk_count )
+ {
+ if ( KM_SUCCESS(result))
+ {
+ CTR_Setup CTR;
+ CTR.SetupWrite(FB.Data());
+ CTR.FillRandom(FB.Data() + CTR.WriteSize(), Megabyte - CTR.WriteSize());
+ result = Writer.Write(FB.RoData(), Megabyte, &write_count);
+ assert(write_count == Megabyte);
+ fprintf(stderr, "\r%8lu ", ++write_total);
+ }
+ }
+
+ fputs("\n", stderr);
+
+ return result;
+}
+
+//
+Result_t
+validate_chunk(ByteString& FB, ByteString& CB, ui32_t* nonce_value)
+{
+ assert(nonce_value);
+ CTR_Setup CTR;
+ CTR.SetupRead(FB.RoData());
+
+ CTR.FillRandom(CB.Data() + CTR.WriteSize(),
+ Megabyte - CTR.WriteSize());
+
+ if ( memcmp(FB.RoData() + CTR.WriteSize(),
+ CB.RoData() + CTR.WriteSize(),
+ Megabyte - CTR.WriteSize()) != 0 )
+ {
+ fprintf(stderr, "Check data mismatched in chunk\n");
+ return RESULT_FAIL;
+ }
+
+ *nonce_value = CTR.Nonce();
+
+ return RESULT_OK;
+}
+
+//
+struct read_list_t
+{
+ ui32_t nonce;
+ Kumu::fpos_t position;
+};
+
+//
+void
+randomize_list(read_list_t* read_list, ui32_t check_total)
+{
+ static ui32_t tmp_ints[4];
+ static ui32_t seq = 0;
+
+ for ( ui32_t j = 0; j < check_total; j++ )
+ {
+ if ( seq > 3 )
+ seq = 0;
+
+ if ( seq == 0 )
+ s_RNG.FillRandom((byte_t*)tmp_ints, 16);
+
+ ui32_t i = tmp_ints[seq++] % (check_total - 1);
+
+ if ( i == j )
+ continue;
+
+ read_list_t t = read_list[i];
+ read_list[i] = read_list[j];
+ read_list[j] = t;
+ }
+}
+
+//
+Result_t
+ReadValidateWriteLargeFile(CommandOptions& Options)
+{
+ assert(Options.write_filename);
+ ui32_t check_total = 0;
+ ui32_t write_total = 0;
+ ui32_t read_count = 0;
+ ui32_t write_count = 0;
+ FileReader Reader;
+ FileWriter Writer;
+ ByteString FB, CB; // Frame Buffer and Check Buffer
+
+
+ FB.Capacity(Megabyte);
+ assert(FB.Capacity() == Megabyte);
+ CB.Capacity(Megabyte);
+ assert(CB.Capacity() == Megabyte);
+
+ Result_t result = Reader.OpenRead(Options.filename);
+
+ if ( KM_SUCCESS(result) )
+ result = Writer.OpenWrite(Options.write_filename);
+
+ // read the first chunk and get set up
+ while ( KM_SUCCESS(result) )
+ {
+ result = Reader.Read(FB.Data(), Megabyte, &read_count);
+
+ if ( KM_SUCCESS(result) )
+ {
+ if ( read_count < Megabyte )
+ {
+ fprintf(stderr, "Read() returned short buffer: %lu\n", read_count);
+ result = RESULT_FAIL;
+ }
+
+ result = validate_chunk(FB, CB, &check_total);
+
+ if ( KM_SUCCESS(result) )
+ {
+ result = Writer.Write(FB.RoData(), Megabyte, &write_count);
+ assert(write_count == Megabyte);
+ fprintf(stderr, "\r%8lu ", ++write_total);
+ }
+ }
+ else if ( result == RESULT_ENDOFFILE )
+ {
+ result = RESULT_OK;
+ break;
+ }
+ }
+
+ fputs("\n", stderr);
+ return result;
+}
+
+
+//
+Result_t
+ValidateLargeFile(CommandOptions& Options)
+{
+ ui32_t check_total = 0;
+ ui32_t read_count = 0;
+ ui32_t read_list_i = 0;
+ read_list_t* read_list = 0;
+ FileReader Reader;
+ ByteString FB, CB; // Frame Buffer and Check Buffer
+
+ FB.Capacity(Megabyte);
+ assert(FB.Capacity() == Megabyte);
+ CB.Capacity(Megabyte);
+ assert(CB.Capacity() == Megabyte);
+
+ Result_t result = Reader.OpenRead(Options.filename);
+
+ // read the first chunk and get set up
+ if ( KM_SUCCESS(result) )
+ {
+ result = Reader.Read(FB.Data(), Megabyte, &read_count);
+
+ if ( read_count < Megabyte )
+ {
+ fprintf(stderr, "Read() returned short buffer: %lu\n", read_count);
+ result = RESULT_FAIL;
+ }
+ else if ( KM_SUCCESS(result) )
+ result = validate_chunk(FB, CB, &check_total);
+
+ if ( KM_SUCCESS(result) )
+ {
+ fprintf(stderr, "Validating %lu chunk%s in %s order:\n",
+ check_total, (check_total == 1 ? "" : "s"), Options.order);
+ assert(read_list == 0);
+ read_list = (read_list_t*)malloc(check_total * sizeof(read_list_t));
+ assert(read_list);
+
+ // Set up an index to the chunks. The chunks are written
+ // to the file in order of descending nonce value.
+ if ( strcmp(Options.order, "fwd") == 0 )
+ {
+ for ( ui32_t i = 0; i < check_total; i++ )
+ {
+ read_list[i].nonce = check_total - i;
+ Kumu::fpos_t ofst = check_total - read_list[i].nonce;
+ read_list[i].position = ofst * (Kumu::fpos_t)Megabyte;
+ }
+ }
+ else
+ {
+ for ( ui32_t i = 0; i < check_total; i++ )
+ {
+ read_list[i].nonce = i + 1;
+ Kumu::fpos_t ofst = check_total - read_list[i].nonce;
+ read_list[i].position = ofst * (Kumu::fpos_t)Megabyte;
+ }
+
+ if ( strcmp(Options.order, "rand") == 0 )
+ randomize_list(read_list, check_total); // this makes it random
+ }
+ }
+ }
+
+ if ( KM_SUCCESS(result) )
+ {
+ assert(read_list);
+ ui32_t nonce = 0;
+
+ for ( read_list_i = 0;
+ read_list_i < check_total && KM_SUCCESS(result);
+ read_list_i++ )
+ {
+ fprintf(stderr, "\r%8lu [%8lu] ", read_list_i+1, read_list[read_list_i]);
+ result = Reader.Seek(read_list[read_list_i].position);
+
+ if ( KM_SUCCESS(result) )
+ result = Reader.Read(FB.Data(), Megabyte, &read_count);
+
+ if ( result == RESULT_ENDOFFILE )
+ break;
+
+ else if ( read_count < Megabyte )
+ {
+ fprintf(stderr, "Read() returned short buffer: %lu\n", read_count);
+ result = RESULT_FAIL;
+ }
+ else if ( KM_SUCCESS(result) )
+ {
+ result = validate_chunk(FB, CB, &nonce);
+
+ if ( nonce != read_list[read_list_i].nonce )
+ {
+ fprintf(stderr, "Nonce mismatch: expecting %lu, got %lu\n",
+ nonce, read_list[read_list_i].nonce);
+
+ return RESULT_FAIL;
+ }
+ }
+ }
+ }
+
+ fputs("\n", stderr);
+
+ if ( result == RESULT_ENDOFFILE )
+ {
+ if ( check_total == read_list_i )
+ result = RESULT_OK;
+ else
+ {
+ fprintf(stderr, "Unexpected chunk count, got %lu, wanted %lu\n",
+ read_list_i, check_total);
+ result = RESULT_FAIL;
+ }
+ }
+
+ return result;
+}
+
+//
+int
+main(int argc, const char **argv)
+{
+ Result_t result = RESULT_FAIL;
+ CommandOptions Options(argc, argv);
+
+ if ( Options.version_flag )
+ banner();
+
+ if ( Options.help_flag )
+ usage();
+
+ if ( Options.version_flag || Options.help_flag )
+ return 0;
+
+ if ( Options.error_flag )
+ {
+ fprintf(stderr, "There was a problem. Type %s -h for help.\n", PACKAGE);
+ return 3;
+ }
+
+ switch ( Options.mode )
+ {
+
+ case MMT_CREATE:
+ result = CreateLargeFile(Options);
+ break;
+
+ case MMT_VALIDATE:
+ result = ValidateLargeFile(Options);
+ break;
+
+ case MMT_VAL_WRITE:
+ result = ReadValidateWriteLargeFile(Options);
+ break;
+ }
+
+ if ( result != RESULT_OK )
+ {
+ fputs("Program stopped on error.\n", stderr);
+
+ if ( result != RESULT_FAIL )
+ {
+ fputs(result.Label(), stderr);
+ fputc('\n', stderr);
+ }
+
+ return 1;
+ }
+
+ return 0;
+}
+
+
+//
+// end kmfilegen.cpp
+//