summaryrefslogtreecommitdiff
path: root/lib/backupstore/BackupStoreFileEncodeStream.cpp
blob: ccab807402aeabba677c585b738cda8d73ef690c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
// --------------------------------------------------------------------------
//
// File
//		Name:    BackupStoreFileEncodeStream.cpp
//		Purpose: Implement stream-based file encoding for the backup store
//		Created: 12/1/04
//
// --------------------------------------------------------------------------

#include "Box.h"

#include <string.h>

#include "BackgroundTask.h"
#include "BackupClientFileAttributes.h"
#include "BackupStoreConstants.h"
#include "BackupStoreException.h"
#include "BackupStoreFile.h"
#include "BackupStoreFileCryptVar.h"
#include "BackupStoreFileEncodeStream.h"
#include "BackupStoreFileWire.h"
#include "BackupStoreObjectMagic.h"
#include "BoxTime.h"
#include "FileStream.h"
#include "Random.h"
#include "RollingChecksum.h"

#include "MemLeakFindOn.h"

#include <cstring>

using namespace BackupStoreFileCryptVar;


// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::BackupStoreFileEncodeStream
//		Purpose: Constructor (opens file)
//		Created: 8/12/03
//
// --------------------------------------------------------------------------
BackupStoreFileEncodeStream::BackupStoreFileEncodeStream()
: mpRecipe(0),
  mpFile(0),
  mpLogging(0),
  mpRunStatusProvider(NULL),
  mpBackgroundTask(NULL),
  mStatus(Status_Header),
  mSendData(true),
  mTotalBlocks(0),
  mBytesToUpload(0),
  mBytesUploaded(0),
  mAbsoluteBlockNumber(-1),
  mInstructionNumber(-1),
  mNumBlocks(0),
  mCurrentBlock(-1),
  mCurrentBlockEncodedSize(0),
  mPositionInCurrentBlock(0),
  mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE),
  mLastBlockSize(0),
  mTotalBytesSent(0),
  mpRawBuffer(0),
  mAllocatedBufferSize(0),
  mEntryIVBase(0)
{
}

// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
//		Purpose: Destructor
//		Created: 8/12/03
//
// --------------------------------------------------------------------------
BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
{
	// Free buffers
	if(mpRawBuffer)
	{
		::free(mpRawBuffer);
		mpRawBuffer = 0;
	}
	
	// Close the file, which we might have open
	if(mpFile)
	{
		delete mpFile;
		mpFile = 0;
	}
	
	// Clear up logging stream
	if(mpLogging)
	{
		delete mpLogging;
		mpLogging = 0;
	}
	
	// Free the recipe
	if(mpRecipe != 0)
	{
		delete mpRecipe;
		mpRecipe = 0;
	}
}


// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *)
//		Purpose: Reads file information, and builds file header reading for sending.
//				 Takes ownership of the Recipe.
//		Created: 8/12/03
//
// --------------------------------------------------------------------------
void BackupStoreFileEncodeStream::Setup(const std::string& Filename,
	BackupStoreFileEncodeStream::Recipe *pRecipe,
	int64_t ContainerID, const BackupStoreFilename &rStoreFilename,
	int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger,
	RunStatusProvider* pRunStatusProvider,
	BackgroundTask* pBackgroundTask)
{
	// Pointer to a blank recipe which we might create
	BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0;

	try
	{
		// Get file attributes
		box_time_t modTime = 0;
		int64_t fileSize = 0;
		BackupClientFileAttributes attr;
		attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime,
			0 /* not interested in attr mod time */, &fileSize);
	
		// Might need to create a blank recipe...
		if(pRecipe == 0)
		{
			pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0);
			
			BackupStoreFileEncodeStream::RecipeInstruction instruction;
			instruction.mSpaceBefore = fileSize; // whole file
			instruction.mBlocks = 0; // no blocks
			instruction.mpStartBlock = 0; // no block
			pblankRecipe->push_back(instruction);		

			pRecipe = pblankRecipe;
		}
	
		// Tell caller?
		if(pModificationTime != 0)
		{
			*pModificationTime = modTime;
		}
		
		// Go through each instruction in the recipe and work out how many blocks
		// it will add, and the max clear size of these blocks
		int maxBlockClearSize = 0;
		for(uint64_t inst = 0; inst < pRecipe->size(); ++inst)
		{
			if((*pRecipe)[inst].mSpaceBefore > 0)
			{
				// Calculate the number of blocks the space before requires
				int64_t numBlocks;
				int32_t blockSize, lastBlockSize;
				CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize);
				// Add to accumlated total
				mTotalBlocks += numBlocks;
				mBytesToUpload += (*pRecipe)[inst].mSpaceBefore;
				// Update maximum clear size
				if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize;
				if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize;
			}
			
			// Add number of blocks copied from the previous file
			mTotalBlocks += (*pRecipe)[inst].mBlocks;
			
			// Check for bad things
			if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0))
			{
				THROW_EXCEPTION(BackupStoreException, Internal)
			}

			// Run through blocks to get the max clear size
			for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b)
			{
				if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize;
			}
		}
		
		// Send data? (symlinks don't have any data in them)
		mSendData = !attr.IsSymLink();

		// If not data is being sent, then the max clear block size is zero
		if(!mSendData)
		{
			maxBlockClearSize = 0;
		}
		
		// Header
		file_StreamFormat hdr;
		hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1);
		hdr.mNumBlocks = (mSendData)?(box_hton64(mTotalBlocks)):(0);
		hdr.mContainerID = box_hton64(ContainerID);
		hdr.mModificationTime = box_hton64(modTime);
		// add a bit to make it harder to tell what's going on -- try not to give away too much info about file size
		hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128);
		hdr.mOptions = 0;		// no options defined yet
		
		// Write header to stream
		mData.Write(&hdr, sizeof(hdr));
		
		// Write filename to stream
		rStoreFilename.WriteToStream(mData);
		
		// Write attributes to stream
		attr.WriteToStream(mData);
	
		// Allocate some buffers for writing data
		if(mSendData)
		{
			// Open the file
			mpFile = new FileStream(Filename);

			if (pLogger)
			{
				// Create logging stream
				mpLogging = new ReadLoggingStream(*mpFile,
					*pLogger);
			}
			else
			{
				// re-use FileStream instead
				mpLogging = mpFile;
				mpFile = NULL;
			}
		
			// Work out the largest possible block required for the encoded data
			mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize);
			
			// Then allocate two blocks of this size
			mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize);
			if(mpRawBuffer == 0)
			{
				throw std::bad_alloc();
			}
#ifndef BOX_RELEASE_BUILD
			// In debug builds, make sure that the reallocation code is exercised.
			mEncodedBuffer.Allocate(mAllocatedBufferSize / 4);
#else
			mEncodedBuffer.Allocate(mAllocatedBufferSize);
#endif
		}
		else
		{
			// Write an empty block index for the symlink
			file_BlockIndexHeader blkhdr;
			blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
			blkhdr.mOtherFileID = box_hton64(0);	// not other file ID
			blkhdr.mEntryIVBase = box_hton64(0);
			blkhdr.mNumBlocks = box_hton64(0);
			mData.Write(&blkhdr, sizeof(blkhdr));
		}
	
		// Ready for reading
		mData.SetForReading();
		
		// Update stats
		BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize;
		
		// Finally, store the pointer to the recipe, when we know exceptions won't occur
		mpRecipe = pRecipe;
	}
	catch(...)
	{
		// Clean up any blank recipe
		if(pblankRecipe != 0)
		{
			delete pblankRecipe;
			pblankRecipe = 0;
		}
		throw;
	}
	
	mpRunStatusProvider = pRunStatusProvider;
	mpBackgroundTask = pBackgroundTask;
}


// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &)
//		Purpose: Calculates the sizes of blocks in a section of the file
//		Created: 16/1/04
//
// --------------------------------------------------------------------------
void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut)
{
	// How many blocks, and how big?
	rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2;
	do
	{
		rBlockSizeOut *= 2;
		
		rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut;
		
	} while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER);
	
	// Last block size
	rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut);
	
	// Avoid small blocks?
	if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN
		&& rNumBlocksOut > 1)
	{
		// Add the small bit of data to the last block
		--rNumBlocksOut;
		rLastBlockSizeOut += rBlockSizeOut;
	}
	
	// checks!
	ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize);
	//TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut);
}



// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::Read(void *, int, int)
//		Purpose: As interface -- generates encoded file data on the fly from the raw file
//		Created: 8/12/03
//
// --------------------------------------------------------------------------
int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout)
{
	// Check there's something to do.
	if(mStatus == Status_Finished)
	{
		return 0;
	}
	
	if(mpRunStatusProvider && mpRunStatusProvider->StopRun())
	{
		THROW_EXCEPTION(BackupStoreException, SignalReceived);
	}

	if(mpBackgroundTask)
	{
		BackgroundTask::State state = (mpRecipe->at(0).mBlocks == 0)
			? BackgroundTask::Uploading_Full
			: BackgroundTask::Uploading_Patch;
		if(!mpBackgroundTask->RunBackgroundTask(state, mBytesUploaded,
			mBytesToUpload))
		{
			THROW_EXCEPTION(BackupStoreException,
				CancelledByBackgroundTask);
		}
	}

	int bytesToRead = NBytes;
	uint8_t *buffer = (uint8_t*)pBuffer;
	
	while(bytesToRead > 0 && mStatus != Status_Finished)
	{
		if(mStatus == Status_Header || mStatus == Status_BlockListing)
		{
			// Header or block listing phase -- send from the buffered stream
		
			// Send bytes from the data buffer
			int b = mData.Read(buffer, bytesToRead, Timeout);
			bytesToRead -= b;
			buffer += b;
			
			// Check to see if all the data has been used from this stream
			if(!mData.StreamDataLeft())
			{
				// Yes, move on to next phase (or finish, if there's no file data)
				if(!mSendData)
				{
					mStatus = Status_Finished;
				}
				else
				{
					// Reset the buffer so it can be used for the next phase
					mData.Reset();
		
					// Get buffer ready for index?
					if(mStatus == Status_Header)
					{
						// Just finished doing the stream header, create the block index header
						file_BlockIndexHeader blkhdr;
						blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
						ASSERT(mpRecipe != 0);
						blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID());
						blkhdr.mNumBlocks = box_hton64(mTotalBlocks);
						
						// Generate the IV base
						Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase));
						blkhdr.mEntryIVBase = box_hton64(mEntryIVBase);
						
						mData.Write(&blkhdr, sizeof(blkhdr));
					}
				
					++mStatus;
				}
			}
		}
		else if(mStatus == Status_Blocks)
		{
			// Block sending phase
			
			if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize)
			{
				// Next block!
				++mCurrentBlock;
				++mAbsoluteBlockNumber;
				if(mCurrentBlock >= mNumBlocks)
				{
					// Output extra blocks for this instruction and move forward in file
					if(mInstructionNumber >= 0)
					{
						SkipPreviousBlocksInInstruction();
					}
				
					// Is there another instruction to go?
					++mInstructionNumber;
					
					// Skip instructions which don't contain any data
					while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size())
						&& (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0)
					{
						SkipPreviousBlocksInInstruction();
						++mInstructionNumber;
					}
					
					if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size()))
					{
						// End of blocks, go to next phase
						++mStatus;
						
						// Set the data to reading so the index can be written
						mData.SetForReading();
					}
					else
					{
						// Get ready for this instruction
						SetForInstruction();
					}
				}

				// Can't use 'else' here as SetForInstruction() will change this
				if(mCurrentBlock < mNumBlocks)
				{
					EncodeCurrentBlock();
				}
			}
			
			// Send data from the current block (if there's data to send)
			if(mPositionInCurrentBlock < mCurrentBlockEncodedSize)
			{
				// How much data to put in the buffer?
				int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock;
				if(s > bytesToRead) s = bytesToRead;
				
				// Copy it in
				::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s);
				
				// Update variables
				bytesToRead -= s;
				buffer += s;
				mPositionInCurrentBlock += s;
			}
		}
		else
		{
			// Should never get here, as it'd be an invalid status
			ASSERT(false);
		}
	}
	
	// Add encoded size to stats
	BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead);
	mTotalBytesSent += (NBytes - bytesToRead);
	
	// Return size of data to caller
	return NBytes - bytesToRead;
}


// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction()
//		Purpose: Private. Stores the blocks of the old file referenced in the current
//				 instruction into the index and skips over the data in the file
//		Created: 16/1/04
//
// --------------------------------------------------------------------------
void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction()
{
	// Check something is necessary
	if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0)
	{
		return;
	}

	// Index of the first block in old file (being diffed from)
	int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock);
	
	int64_t sizeToSkip = 0;

	for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b)
	{
		// Update stats
		BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
	
		// Store the entry
		StoreBlockIndexEntry(0 - (firstIndex + b),
			(*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize,
			(*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum,
			(*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum);	

		// Increment the absolute block number -- kept encryption IV in sync
		++mAbsoluteBlockNumber;
		
		// Add the size of this block to the size to skip
		sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
	}
	
	// Move forward in the stream
	mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative);
}


// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::SetForInstruction()
//		Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe
//		Created: 16/1/04
//
// --------------------------------------------------------------------------
void BackupStoreFileEncodeStream::SetForInstruction()
{
	// Calculate block sizes
	CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize);
	
	// Set variables
	mCurrentBlock = 0;
	mCurrentBlockEncodedSize = 0;
	mPositionInCurrentBlock = 0;
}



// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::EncodeCurrentBlock()
//		Purpose: Private. Encodes the current block, and writes the block data to the index
//		Created: 8/12/03
//
// --------------------------------------------------------------------------
void BackupStoreFileEncodeStream::EncodeCurrentBlock()
{
	// How big is the block, raw?
	int blockRawSize = mBlockSize;
	if(mCurrentBlock == (mNumBlocks - 1))
	{
		blockRawSize = mLastBlockSize;
	}
	ASSERT(blockRawSize < mAllocatedBufferSize);

	// Check file open
	if(mpLogging == 0)
	{
		// File should be open, but isn't. So logical error.
		THROW_EXCEPTION(BackupStoreException, Internal)
	}
	
	// Read the data in
	if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize,
		0 /* not interested in size if failure */))
	{
		// TODO: Do something more intelligent, and abort
		// this upload because the file has changed.
		THROW_EXCEPTION(BackupStoreException,
			Temp_FileEncodeStreamDidntReadBuffer)
	}
	
	// Encode it
	mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer,
		blockRawSize, mEncodedBuffer);

	mBytesUploaded += blockRawSize;
	
	//TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize);
	
	// Create block listing data -- generate checksums
	RollingChecksum weakChecksum(mpRawBuffer, blockRawSize);
	MD5Digest strongChecksum;
	strongChecksum.Add(mpRawBuffer, blockRawSize);
	strongChecksum.Finish();

	// Add entry to the index
	StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize,
		weakChecksum.GetChecksum(), strongChecksum.DigestAsData());
	
	// Set vars to reading this block
	mPositionInCurrentBlock = 0;
}

// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *)
//		Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream.
//		Created: 16/1/04
//
// --------------------------------------------------------------------------
void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum)
{
	// First, the encrypted section
	file_BlockIndexEntryEnc entryEnc;
	entryEnc.mSize = htonl(ClearSize);
	entryEnc.mWeakChecksum = htonl(WeakChecksum);
	::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum));

	// Then the clear section
	file_BlockIndexEntry entry;
	entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex));
	
	// Then encrypt the encryted section
	// Generate the IV from the block number
	if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase))
	{
		THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements)
	}
	uint64_t iv = mEntryIVBase;
	iv += mAbsoluteBlockNumber;
	// Convert to network byte order before encrypting with it, so that restores work on
	// platforms with different endiannesses.
	iv = box_hton64(iv);
	sBlowfishEncryptBlockEntry.SetIV(&iv);

	// Encode the data
	int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc));
	if(encodedSize != sizeof(entry.mEnEnc))
	{
		THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
	}

	// Save to data block for sending at the end of the stream
	mData.Write(&entry, sizeof(entry));
}


// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::Write(const void *, int)
//		Purpose: As interface. Exceptions.
//		Created: 8/12/03
//
// --------------------------------------------------------------------------
void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes,
	int Timeout)
{
	THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream)
}

// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::StreamDataLeft()
//		Purpose: As interface -- end of stream reached?
//		Created: 8/12/03
//
// --------------------------------------------------------------------------
bool BackupStoreFileEncodeStream::StreamDataLeft()
{
	return (mStatus != Status_Finished);
}


// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::StreamClosed()
//		Purpose: As interface
//		Created: 8/12/03
//
// --------------------------------------------------------------------------
bool BackupStoreFileEncodeStream::StreamClosed()
{
	return true;
}


// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t)
//		Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted
//		Created: 15/1/04
//
// --------------------------------------------------------------------------
BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex,
		int64_t NumBlocksInIndex, int64_t OtherFileID)
	: mpBlockIndex(pBlockIndex),
	  mNumBlocksInIndex(NumBlocksInIndex),
	  mOtherFileID(OtherFileID)
{
	ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0))
}

// --------------------------------------------------------------------------
//
// Function
//		Name:    BackupStoreFileEncodeStream::Recipe::~Recipe()
//		Purpose: Destructor
//		Created: 15/1/04
//
// --------------------------------------------------------------------------
BackupStoreFileEncodeStream::Recipe::~Recipe()
{
	// Free the block index, if there is one
	if(mpBlockIndex != 0)
	{
		::free(mpBlockIndex);
	}
}