2 Copyright (c) 2005-2009, John Hurst
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions
8 1. Redistributions of source code must retain the above copyright
9 notice, this list of conditions and the following disclaimer.
10 2. Redistributions in binary form must reproduce the above copyright
11 notice, this list of conditions and the following disclaimer in the
12 documentation and/or other materials provided with the distribution.
13 3. The name of the author may not be used to endorse or promote products
14 derived from this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 /*! \file kmfilegen.cpp
28 \version $Id: kmfilegen.cpp,v 1.8 2009/03/04 17:52:45 jhurst Exp $
29 \brief large file test program
35 #include <KM_fileio.h>
37 #include <openssl/aes.h>
43 static const char* PROGRAM_NAME = "kmfilegen"; // program name for messages
44 const ui32_t RNG_KEY_SIZE = 16;
45 const ui32_t RNG_KEY_SIZE_BITS = 128;
46 const ui32_t RNG_BLOCK_SIZE = 16;
53 //------------------------------------------------------------------------------------------
55 // command line option parser class
57 // Increment the iterator, test for an additional non-option command line argument.
58 // Causes the caller to return if there are no remaining arguments or if the next
59 // argument begins with '-'.
60 #define TEST_EXTRA_ARG(i,c) if ( ++i >= argc || argv[(i)][0] == '-' ) \
62 fprintf(stderr, "Argument not found for option -%c.\n", (c)); \
68 banner(FILE* stream = stdout)
72 Copyright (c) 2005-2009 John Hurst\n\
73 %s is part of the asdcplib DCP tools package.\n\
74 asdcplib may be copied only under the terms of the license found at\n\
75 the top of every file in the asdcplib distribution kit.\n\n\
76 Specify the -h (help) option for further information about %s\n\n",
77 PROGRAM_NAME, Kumu::Version(), PROGRAM_NAME, PROGRAM_NAME);
83 usage(FILE* stream = stdout)
86 USAGE: %s [-c <file-size>] [-v] <output-file>\n\
88 %s [-o <fwd|rev|rand>] [-v] <input-file>\n\
90 %s [-w <output-file>] [-v] <input-file>\n\
94 -c <file-size> - Create test file containing <file-size> megabytes of data\n\
95 -h | -help - Show help\n\
96 -o <fwd|rev|rand> - Specify order used when validating a file.\n\
97 One of fwd|rev|rand, default is rand\n\
98 -v - Verbose. Prints informative messages to stderr\n\
99 -V - Show version information\n\
100 -w <output-file> - Read-Validate-Write - file is written to <output-file>\n\
101 (sequential read only)\n\
103 NOTES: o There is no option grouping, all options must be distinct arguments.\n\
104 o All option arguments must be separated from the option by whitespace.\n\
105 \n", PROGRAM_NAME, PROGRAM_NAME, PROGRAM_NAME, PROGRAM_NAME);
121 bool error_flag; // true if the given options are in error or not complete
122 const char* order; // one of fwd|rev|rand
123 bool verbose_flag; // true if the verbose option was selected
124 bool version_flag; // true if the version display option was selected
125 bool help_flag; // true if the help display option was selected
126 const char* filename; // filename to be processed
127 const char* write_filename; // filename to write with val_write_flag
129 MajorMode_t mode; // MajorMode selector
132 CommandOptions(int argc, const char** argv) :
133 error_flag(true), order(""), verbose_flag(false), version_flag(false), help_flag(false),
134 filename(""), write_filename(""), chunk_count(0), mode(MMT_VALIDATE)
138 for ( int i = 1; i < argc; i++ )
141 if ( (strcmp( argv[i], "-help") == 0) )
147 if ( argv[i][0] == '-' && isalpha(argv[i][1]) && argv[i][2] == 0 )
149 switch ( argv[i][1] )
153 TEST_EXTRA_ARG(i, 'c');
154 chunk_count = atoi(argv[i]);
157 case 'V': version_flag = true; break;
158 case 'h': help_flag = true; break;
159 case 'v': verbose_flag = true; break;
162 TEST_EXTRA_ARG(i, 'o');
165 if ( strcmp(order, "fwd" ) != 0
166 && strcmp(order, "rev" ) != 0
167 && strcmp(order, "rand" ) != 0 )
169 fprintf(stderr, "Unexpected order token: %s, expecting fwd|rev|rand\n", order);
176 mode = MMT_VAL_WRITE;
177 TEST_EXTRA_ARG(i, 'w');
178 write_filename = argv[i];
182 fprintf(stderr, "Unrecognized option: %s\n", argv[i]);
188 if (argv[i][0] != '-' )
190 if ( filename != "" )
192 fprintf(stderr, "Extra filename found: %s\n", argv[i]);
200 fprintf(stderr, "Unrecognized option: %s\n", argv[i]);
206 if ( help_flag || version_flag )
209 if ( strlen ( filename ) == 0 )
211 fprintf(stderr, "Filename required.\n");
215 if ( mode != MMT_VALIDATE && strcmp(order, "") != 0 )
217 fprintf(stderr, "-o option not valid with -c or -w options.\n");
221 if ( strcmp(order, "") == 0 )
224 if ( strcmp ( filename, write_filename ) == 0 )
226 fprintf(stderr, "Output and input files must be different.\n");
234 //------------------------------------------------------------------------------------------
242 byte_t m_key[RNG_KEY_SIZE];
243 byte_t m_preamble[8];
247 KM_NO_COPY_CONSTRUCT(CTR_Setup);
253 inline ui32_t Nonce() { return KM_i32_LE(m_nonce); }
254 inline ui32_t WriteSize() { return ( sizeof(m_key) + sizeof(m_preamble)
255 + sizeof(m_nonce) + sizeof(m_ctr) ); }
258 void SetupWrite(byte_t* buf)
261 s_RNG.FillRandom(m_key, WriteSize());
263 m_nonce = KM_i32_LE(s_Nonce--);
264 m_ctr &= KM_i32_LE(0x7fffffff); // make sure we have 2GB headroom
265 memcpy(buf, m_key, WriteSize());
266 AES_set_encrypt_key(m_key, RNG_KEY_SIZE_BITS, &m_Context);
270 void SetupRead(const byte_t* buf)
273 memcpy(m_key, buf, WriteSize());
274 AES_set_encrypt_key(m_key, RNG_KEY_SIZE_BITS, &m_Context);
278 void FillRandom(byte_t* buf, ui32_t buf_len)
280 ui32_t gen_count = 0;
281 while ( gen_count + RNG_BLOCK_SIZE <= buf_len )
283 AES_encrypt(m_preamble, buf + gen_count, &m_Context);
284 m_ctr = KM_i32_LE(KM_i32_LE(m_ctr) + 1);
285 gen_count += RNG_BLOCK_SIZE;
292 CreateLargeFile(CommandOptions& Options)
294 ui32_t write_total = 0;
295 ui32_t write_count = 0;
299 FB.Capacity(Megabyte);
300 assert(FB.Capacity() == Megabyte);
302 fprintf(stderr, "Writing %u chunks:\n", Options.chunk_count);
303 s_Nonce = Options.chunk_count;
304 Result_t result = Writer.OpenWrite(Options.filename);
306 while ( KM_SUCCESS(result) && write_total < Options.chunk_count )
308 if ( KM_SUCCESS(result))
311 CTR.SetupWrite(FB.Data());
312 CTR.FillRandom(FB.Data() + CTR.WriteSize(), Megabyte - CTR.WriteSize());
313 result = Writer.Write(FB.RoData(), Megabyte, &write_count);
314 assert(write_count == Megabyte);
315 fprintf(stderr, "\r%8u ", ++write_total);
326 validate_chunk(ByteString& FB, ByteString& CB, ui32_t* nonce_value)
330 CTR.SetupRead(FB.RoData());
332 CTR.FillRandom(CB.Data() + CTR.WriteSize(),
333 Megabyte - CTR.WriteSize());
335 if ( memcmp(FB.RoData() + CTR.WriteSize(),
336 CB.RoData() + CTR.WriteSize(),
337 Megabyte - CTR.WriteSize()) != 0 )
339 fprintf(stderr, "Check data mismatched in chunk\n");
343 *nonce_value = CTR.Nonce();
352 Kumu::fpos_t position;
357 randomize_list(read_list_t* read_list, ui32_t check_total)
359 static ui32_t tmp_ints[4];
360 static ui32_t seq = 0;
362 for ( ui32_t j = 0; j < check_total; j++ )
368 s_RNG.FillRandom((byte_t*)tmp_ints, 16);
370 ui32_t i = tmp_ints[seq++] % (check_total - 1);
375 read_list_t t = read_list[i];
376 read_list[i] = read_list[j];
383 ReadValidateWriteLargeFile(CommandOptions& Options)
385 assert(Options.write_filename);
386 ui32_t check_total = 0;
387 ui32_t write_total = 0;
388 ui32_t read_count = 0;
389 ui32_t write_count = 0;
392 ByteString FB, CB; // Frame Buffer and Check Buffer
395 FB.Capacity(Megabyte);
396 assert(FB.Capacity() == Megabyte);
397 CB.Capacity(Megabyte);
398 assert(CB.Capacity() == Megabyte);
400 Result_t result = Reader.OpenRead(Options.filename);
402 if ( KM_SUCCESS(result) )
403 result = Writer.OpenWrite(Options.write_filename);
405 // read the first chunk and get set up
406 while ( KM_SUCCESS(result) )
408 result = Reader.Read(FB.Data(), Megabyte, &read_count);
410 if ( KM_SUCCESS(result) )
412 if ( read_count < Megabyte )
414 fprintf(stderr, "Read() returned short buffer: %u\n", read_count);
415 result = RESULT_FAIL;
418 result = validate_chunk(FB, CB, &check_total);
420 if ( KM_SUCCESS(result) )
422 result = Writer.Write(FB.RoData(), Megabyte, &write_count);
423 assert(write_count == Megabyte);
424 fprintf(stderr, "\r%8u ", ++write_total);
427 else if ( result == RESULT_ENDOFFILE )
441 ValidateLargeFile(CommandOptions& Options)
443 ui32_t check_total = 0;
444 ui32_t read_count = 0;
445 ui32_t read_list_i = 0;
446 read_list_t* read_list = 0;
448 ByteString FB, CB; // Frame Buffer and Check Buffer
450 FB.Capacity(Megabyte);
451 assert(FB.Capacity() == Megabyte);
452 CB.Capacity(Megabyte);
453 assert(CB.Capacity() == Megabyte);
455 Result_t result = Reader.OpenRead(Options.filename);
457 // read the first chunk and get set up
458 if ( KM_SUCCESS(result) )
460 result = Reader.Read(FB.Data(), Megabyte, &read_count);
462 if ( read_count < Megabyte )
464 fprintf(stderr, "Read() returned short buffer: %u\n", read_count);
465 result = RESULT_FAIL;
467 else if ( KM_SUCCESS(result) )
468 result = validate_chunk(FB, CB, &check_total);
470 if ( KM_SUCCESS(result) )
472 fprintf(stderr, "Validating %u chunk%s in %s order:\n",
473 check_total, (check_total == 1 ? "" : "s"), Options.order);
474 assert(read_list == 0);
475 read_list = (read_list_t*)malloc(check_total * sizeof(read_list_t));
478 // Set up an index to the chunks. The chunks are written
479 // to the file in order of descending nonce value.
480 if ( strcmp(Options.order, "fwd") == 0 )
482 for ( ui32_t i = 0; i < check_total; i++ )
484 read_list[i].nonce = check_total - i;
485 Kumu::fpos_t ofst = check_total - read_list[i].nonce;
486 read_list[i].position = ofst * (Kumu::fpos_t)Megabyte;
491 for ( ui32_t i = 0; i < check_total; i++ )
493 read_list[i].nonce = i + 1;
494 Kumu::fpos_t ofst = check_total - read_list[i].nonce;
495 read_list[i].position = ofst * (Kumu::fpos_t)Megabyte;
498 if ( strcmp(Options.order, "rand") == 0 )
499 randomize_list(read_list, check_total); // this makes it random
504 if ( KM_SUCCESS(result) )
509 for ( read_list_i = 0;
510 read_list_i < check_total && KM_SUCCESS(result);
513 fprintf(stderr, "\r%8u [%8u] ", read_list_i+1, read_list[read_list_i].nonce);
514 result = Reader.Seek(read_list[read_list_i].position);
516 if ( KM_SUCCESS(result) )
517 result = Reader.Read(FB.Data(), Megabyte, &read_count);
519 if ( result == RESULT_ENDOFFILE )
522 else if ( read_count < Megabyte )
524 fprintf(stderr, "Read() returned short buffer: %u\n", read_count);
525 result = RESULT_FAIL;
527 else if ( KM_SUCCESS(result) )
529 result = validate_chunk(FB, CB, &nonce);
531 if ( nonce != read_list[read_list_i].nonce )
533 fprintf(stderr, "Nonce mismatch: expecting %u, got %u\n",
534 nonce, read_list[read_list_i].nonce);
544 if ( result == RESULT_ENDOFFILE )
546 if ( check_total == read_list_i )
550 fprintf(stderr, "Unexpected chunk count, got %u, wanted %u\n",
551 read_list_i, check_total);
552 result = RESULT_FAIL;
561 main(int argc, const char **argv)
563 Result_t result = RESULT_FAIL;
564 CommandOptions Options(argc, argv);
566 if ( Options.version_flag )
569 if ( Options.help_flag )
572 if ( Options.version_flag || Options.help_flag )
575 if ( Options.error_flag )
577 fprintf(stderr, "There was a problem. Type %s -h for help.\n", PROGRAM_NAME);
581 switch ( Options.mode )
585 result = CreateLargeFile(Options);
589 result = ValidateLargeFile(Options);
593 result = ReadValidateWriteLargeFile(Options);
597 if ( result != RESULT_OK )
599 fputs("Program stopped on error.\n", stderr);
601 if ( result != RESULT_FAIL )
603 fputs(result.Label(), stderr);