2 Copyright (c) 2005-2009, John Hurst
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions
8 1. Redistributions of source code must retain the above copyright
9 notice, this list of conditions and the following disclaimer.
10 2. Redistributions in binary form must reproduce the above copyright
11 notice, this list of conditions and the following disclaimer in the
12 documentation and/or other materials provided with the distribution.
13 3. The name of the author may not be used to endorse or promote products
14 derived from this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 /*! \file kmfilegen.cpp
29 \brief large file test program
35 #include <KM_fileio.h>
37 #include <openssl/aes.h>
43 static const char* PROGRAM_NAME = "kmfilegen"; // program name for messages
44 const ui32_t RNG_KEY_SIZE = 16;
45 const ui32_t RNG_KEY_SIZE_BITS = 128;
46 const ui32_t RNG_BLOCK_SIZE = 16;
53 //------------------------------------------------------------------------------------------
55 // command line option parser class
57 // Increment the iterator, test for an additional non-option command line argument.
58 // Causes the caller to return if there are no remaining arguments or if the next
59 // argument begins with '-'.
60 #define TEST_EXTRA_ARG(i,c) if ( ++i >= argc || argv[(i)][0] == '-' ) \
62 fprintf(stderr, "Argument not found for option -%c.\n", (c)); \
68 banner(FILE* stream = stdout)
72 Copyright (c) 2005-2009 John Hurst\n\
73 %s is part of the asdcplib DCP tools package.\n\
74 asdcplib may be copied only under the terms of the license found at\n\
75 the top of every file in the asdcplib distribution kit.\n\n\
76 Specify the -h (help) option for further information about %s\n\n",
77 PROGRAM_NAME, Kumu::Version(), PROGRAM_NAME, PROGRAM_NAME);
83 usage(FILE* stream = stdout)
86 USAGE: %s [-c <file-size>] [-v] <output-file>\n\
88 %s [-o <fwd|rev|rand>] [-v] <input-file>\n\
90 %s [-w <output-file>] [-v] <input-file>\n\
94 -c <file-size> - Create test file containing <file-size> megabytes of data\n\
95 -h | -help - Show help\n\
96 -o <fwd|rev|rand> - Specify order used when validating a file.\n\
97 One of fwd|rev|rand, default is rand\n\
98 -v - Verbose. Prints informative messages to stderr\n\
99 -V - Show version information\n\
100 -w <output-file> - Read-Validate-Write - file is written to <output-file>\n\
101 (sequential read only)\n\
103 NOTES: o There is no option grouping, all options must be distinct arguments.\n\
104 o All option arguments must be separated from the option by whitespace.\n\
105 \n", PROGRAM_NAME, PROGRAM_NAME, PROGRAM_NAME, PROGRAM_NAME);
121 bool error_flag; // true if the given options are in error or not complete
122 const char* order; // one of fwd|rev|rand
123 bool verbose_flag; // true if the verbose option was selected
124 bool version_flag; // true if the version display option was selected
125 bool help_flag; // true if the help display option was selected
126 std::string filename; // filename to be processed
127 std::string write_filename; // filename to write with val_write_flag
129 MajorMode_t mode; // MajorMode selector
132 CommandOptions(int argc, const char** argv) :
133 error_flag(true), order(""), verbose_flag(false),
134 version_flag(false), help_flag(false),
135 chunk_count(0), mode(MMT_VALIDATE)
139 for ( int i = 1; i < argc; i++ )
142 if ( (strcmp( argv[i], "-help") == 0) )
148 if ( argv[i][0] == '-' && isalpha(argv[i][1]) && argv[i][2] == 0 )
150 switch ( argv[i][1] )
154 TEST_EXTRA_ARG(i, 'c');
155 chunk_count = Kumu::xabs(strtol(argv[i], 0, 10));
158 case 'V': version_flag = true; break;
159 case 'h': help_flag = true; break;
160 case 'v': verbose_flag = true; break;
163 TEST_EXTRA_ARG(i, 'o');
166 if ( strcmp(order, "fwd" ) != 0
167 && strcmp(order, "rev" ) != 0
168 && strcmp(order, "rand" ) != 0 )
170 fprintf(stderr, "Unexpected order token: %s, expecting fwd|rev|rand\n", order);
177 mode = MMT_VAL_WRITE;
178 TEST_EXTRA_ARG(i, 'w');
179 write_filename = argv[i];
183 fprintf(stderr, "Unrecognized option: %s\n", argv[i]);
189 if (argv[i][0] != '-' )
191 if ( ! filename.empty() )
193 fprintf(stderr, "Extra filename found: %s\n", argv[i]);
201 fprintf(stderr, "Unrecognized option: %s\n", argv[i]);
207 if ( help_flag || version_flag )
210 if ( filename.empty() )
212 fprintf(stderr, "Filename required.\n");
216 if ( mode != MMT_VALIDATE && strcmp(order, "") != 0 )
218 fprintf(stderr, "-o option not valid with -c or -w options.\n");
222 if ( strcmp(order, "") == 0 )
225 if ( filename == write_filename )
227 fprintf(stderr, "Output and input files must be different.\n");
235 //------------------------------------------------------------------------------------------
243 byte_t m_key[RNG_KEY_SIZE];
244 byte_t m_preamble[8];
248 KM_NO_COPY_CONSTRUCT(CTR_Setup);
254 inline ui32_t Nonce() { return KM_i32_LE(m_nonce); }
255 inline ui32_t WriteSize() { return ( sizeof(m_key) + sizeof(m_preamble)
256 + sizeof(m_nonce) + sizeof(m_ctr) ); }
259 void SetupWrite(byte_t* buf)
262 s_RNG.FillRandom(m_key, WriteSize());
264 m_nonce = KM_i32_LE(s_Nonce--);
265 m_ctr &= KM_i32_LE(0x7fffffff); // make sure we have 2GB headroom
266 memcpy(buf, m_key, WriteSize());
267 AES_set_encrypt_key(m_key, RNG_KEY_SIZE_BITS, &m_Context);
271 void SetupRead(const byte_t* buf)
274 memcpy(m_key, buf, WriteSize());
275 AES_set_encrypt_key(m_key, RNG_KEY_SIZE_BITS, &m_Context);
279 void FillRandom(byte_t* buf, ui32_t buf_len)
281 ui32_t gen_count = 0;
282 while ( gen_count + RNG_BLOCK_SIZE <= buf_len )
284 AES_encrypt(m_preamble, buf + gen_count, &m_Context);
285 m_ctr = KM_i32_LE(KM_i32_LE(m_ctr) + 1);
286 gen_count += RNG_BLOCK_SIZE;
293 CreateLargeFile(CommandOptions& Options)
295 ui32_t write_total = 0;
296 ui32_t write_count = 0;
300 FB.Capacity(Megabyte);
301 assert(FB.Capacity() == Megabyte);
303 fprintf(stderr, "Writing %u chunks:\n", Options.chunk_count);
304 s_Nonce = Options.chunk_count;
305 Result_t result = Writer.OpenWrite(Options.filename);
307 while ( KM_SUCCESS(result) && write_total < Options.chunk_count )
309 if ( KM_SUCCESS(result))
312 CTR.SetupWrite(FB.Data());
313 CTR.FillRandom(FB.Data() + CTR.WriteSize(), Megabyte - CTR.WriteSize());
314 result = Writer.Write(FB.RoData(), Megabyte, &write_count);
315 assert(write_count == Megabyte);
316 fprintf(stderr, "\r%8u ", ++write_total);
327 validate_chunk(ByteString& FB, ByteString& CB, ui32_t* nonce_value)
331 CTR.SetupRead(FB.RoData());
333 CTR.FillRandom(CB.Data() + CTR.WriteSize(),
334 Megabyte - CTR.WriteSize());
336 if ( memcmp(FB.RoData() + CTR.WriteSize(),
337 CB.RoData() + CTR.WriteSize(),
338 Megabyte - CTR.WriteSize()) != 0 )
340 fprintf(stderr, "Check data mismatched in chunk\n");
344 *nonce_value = CTR.Nonce();
353 Kumu::fpos_t position;
358 randomize_list(read_list_t* read_list, ui32_t check_total)
360 static ui32_t tmp_ints[4];
361 static ui32_t seq = 0;
363 for ( ui32_t j = 0; j < check_total; j++ )
369 s_RNG.FillRandom((byte_t*)tmp_ints, 16);
371 ui32_t i = tmp_ints[seq++] % (check_total - 1);
376 read_list_t t = read_list[i];
377 read_list[i] = read_list[j];
384 ReadValidateWriteLargeFile(CommandOptions& Options)
386 assert(!Options.write_filename.empty());
387 ui32_t check_total = 0;
388 ui32_t write_total = 0;
389 ui32_t read_count = 0;
390 ui32_t write_count = 0;
393 ByteString FB, CB; // Frame Buffer and Check Buffer
396 FB.Capacity(Megabyte);
397 assert(FB.Capacity() == Megabyte);
398 CB.Capacity(Megabyte);
399 assert(CB.Capacity() == Megabyte);
401 Result_t result = Reader.OpenRead(Options.filename);
403 if ( KM_SUCCESS(result) )
404 result = Writer.OpenWrite(Options.write_filename);
406 // read the first chunk and get set up
407 while ( KM_SUCCESS(result) )
409 result = Reader.Read(FB.Data(), Megabyte, &read_count);
411 if ( KM_SUCCESS(result) )
413 if ( read_count < Megabyte )
415 fprintf(stderr, "Read() returned short buffer: %u\n", read_count);
416 result = RESULT_FAIL;
419 result = validate_chunk(FB, CB, &check_total);
421 if ( KM_SUCCESS(result) )
423 result = Writer.Write(FB.RoData(), Megabyte, &write_count);
424 assert(write_count == Megabyte);
425 fprintf(stderr, "\r%8u ", ++write_total);
428 else if ( result == RESULT_ENDOFFILE )
442 ValidateLargeFile(CommandOptions& Options)
444 ui32_t check_total = 0;
445 ui32_t read_count = 0;
446 ui32_t read_list_i = 0;
447 read_list_t* read_list = 0;
449 ByteString FB, CB; // Frame Buffer and Check Buffer
451 FB.Capacity(Megabyte);
452 assert(FB.Capacity() == Megabyte);
453 CB.Capacity(Megabyte);
454 assert(CB.Capacity() == Megabyte);
456 Result_t result = Reader.OpenRead(Options.filename);
458 // read the first chunk and get set up
459 if ( KM_SUCCESS(result) )
461 result = Reader.Read(FB.Data(), Megabyte, &read_count);
463 if ( read_count < Megabyte )
465 fprintf(stderr, "Read() returned short buffer: %u\n", read_count);
466 result = RESULT_FAIL;
468 else if ( KM_SUCCESS(result) )
469 result = validate_chunk(FB, CB, &check_total);
471 if ( KM_SUCCESS(result) )
473 fprintf(stderr, "Validating %u chunk%s in %s order:\n",
474 check_total, (check_total == 1 ? "" : "s"), Options.order);
475 assert(read_list == 0);
476 read_list = (read_list_t*)malloc(check_total * sizeof(read_list_t));
479 // Set up an index to the chunks. The chunks are written
480 // to the file in order of descending nonce value.
481 if ( strcmp(Options.order, "fwd") == 0 )
483 for ( ui32_t i = 0; i < check_total; i++ )
485 read_list[i].nonce = check_total - i;
486 Kumu::fpos_t ofst = check_total - read_list[i].nonce;
487 read_list[i].position = ofst * (Kumu::fpos_t)Megabyte;
492 for ( ui32_t i = 0; i < check_total; i++ )
494 read_list[i].nonce = i + 1;
495 Kumu::fpos_t ofst = check_total - read_list[i].nonce;
496 read_list[i].position = ofst * (Kumu::fpos_t)Megabyte;
499 if ( strcmp(Options.order, "rand") == 0 )
500 randomize_list(read_list, check_total); // this makes it random
505 if ( KM_SUCCESS(result) )
510 for ( read_list_i = 0;
511 read_list_i < check_total && KM_SUCCESS(result);
514 fprintf(stderr, "\r%8u [%8u] ", read_list_i+1, read_list[read_list_i].nonce);
515 result = Reader.Seek(read_list[read_list_i].position);
517 if ( KM_SUCCESS(result) )
518 result = Reader.Read(FB.Data(), Megabyte, &read_count);
520 if ( result == RESULT_ENDOFFILE )
523 else if ( read_count < Megabyte )
525 fprintf(stderr, "Read() returned short buffer: %u\n", read_count);
526 result = RESULT_FAIL;
528 else if ( KM_SUCCESS(result) )
530 result = validate_chunk(FB, CB, &nonce);
532 if ( nonce != read_list[read_list_i].nonce )
534 fprintf(stderr, "Nonce mismatch: expecting %u, got %u\n",
535 nonce, read_list[read_list_i].nonce);
545 if ( result == RESULT_ENDOFFILE )
547 if ( check_total == read_list_i )
551 fprintf(stderr, "Unexpected chunk count, got %u, wanted %u\n",
552 read_list_i, check_total);
553 result = RESULT_FAIL;
562 main(int argc, const char **argv)
564 Result_t result = RESULT_FAIL;
565 CommandOptions Options(argc, argv);
567 if ( Options.version_flag )
570 if ( Options.help_flag )
573 if ( Options.version_flag || Options.help_flag )
576 if ( Options.error_flag )
578 fprintf(stderr, "There was a problem. Type %s -h for help.\n", PROGRAM_NAME);
582 switch ( Options.mode )
586 result = CreateLargeFile(Options);
590 result = ValidateLargeFile(Options);
594 result = ReadValidateWriteLargeFile(Options);
598 if ( result != RESULT_OK )
600 fputs("Program stopped on error.\n", stderr);
602 if ( result != RESULT_FAIL )
604 fputs(result.Label(), stderr);