diff options
Diffstat (limited to 'lib/backupstore/BackupStoreFile.cpp')
| -rw-r--r-- | lib/backupstore/BackupStoreFile.cpp | 630 |
1 files changed, 511 insertions, 119 deletions
diff --git a/lib/backupstore/BackupStoreFile.cpp b/lib/backupstore/BackupStoreFile.cpp index 519305ff..99562685 100644 --- a/lib/backupstore/BackupStoreFile.cpp +++ b/lib/backupstore/BackupStoreFile.cpp @@ -22,29 +22,30 @@ #include <stdio.h> #endif +#include "BackupClientFileAttributes.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" #include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFileWire.h" #include "BackupStoreFilename.h" -#include "BackupStoreException.h" -#include "IOStream.h" -#include "Guards.h" -#include "FileModificationTime.h" -#include "FileStream.h" -#include "BackupClientFileAttributes.h" +#include "BackupStoreInfo.h" #include "BackupStoreObjectMagic.h" -#include "Compress.h" -#include "CipherContext.h" -#include "CipherBlowfish.h" #include "CipherAES.h" -#include "BackupStoreConstants.h" +#include "CipherBlowfish.h" +#include "CipherContext.h" #include "CollectInBufferStream.h" -#include "RollingChecksum.h" +#include "Compress.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "Guards.h" +#include "IOStream.h" +#include "Logging.h" #include "MD5Digest.h" -#include "ReadGatherStream.h" #include "Random.h" -#include "BackupStoreFileEncodeStream.h" -#include "Logging.h" +#include "ReadGatherStream.h" +#include "RollingChecksum.h" #include "MemLeakFindOn.h" @@ -78,7 +79,8 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) + RunStatusProvider* pRunStatusProvider, + BackgroundTask* pBackgroundTask) { // Create the stream std::auto_ptr<BackupStoreFileEncodeStream> stream( @@ -86,8 +88,9 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( // Do the initial setup stream->Setup(Filename, 0 /* no recipe, just encode */, ContainerID, - rStoreFilename, pModificationTime, pLogger, pRunStatusProvider); - + rStoreFilename, pModificationTime, pLogger, pRunStatusProvider, + pBackgroundTask); + // Return the stream for the caller return stream; } @@ -100,7 +103,15 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( // requirements. Doesn't verify that the data is intact // and can be decoded. Optionally returns the ID of the // file which it is diffed from, and the (original) -// container ID. +// container ID. This is more efficient than +// BackupStoreFile::VerifyStream() when the file data +// already exists on disk and we can Seek() around in +// it, but less efficient if we are reading the stream +// from the network and not intending to Write() it to +// a file first, so we need both unfortunately. +// TODO FIXME: use a modified VerifyStream() which +// repositions the file pointer and Close()s early to +// deduplicate this code. // Created: 2003/08/28 // // -------------------------------------------------------------------------- @@ -120,7 +131,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read header return false; } - + // Check magic number if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -130,7 +141,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro { return false; } - + // Get a filename, see if it loads OK try { @@ -142,7 +153,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // an error occured while reading it, so that's not good return false; } - + // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not try { @@ -163,10 +174,10 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Get current position in file -- the end of the header int64_t headerEnd = rFile.GetPosition(); - + // Get number of blocks int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - + // Calculate where the block index will be, check it's reasonable int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); if(blockIndexLoc < headerEnd) @@ -183,7 +194,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read block index header -- assume bad file return false; } - + // Check header if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -195,10 +206,10 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Bad header -- either magic value or number of blocks is wrong return false; } - + // Flag for recording whether a block is referenced from another file bool blockFromOtherFileReferenced = false; - + // Read the index, checking that the length values all make sense int64_t currentBlockStart = headerEnd; for(int64_t b = 0; b < numBlocks; ++b) @@ -210,7 +221,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read block index entry -- assume bad file return false; } - + // Check size and location int64_t blkSize = box_ntoh64(blk.mEncodedSize); if(blkSize <= 0) @@ -226,19 +237,20 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Encoded size makes the block run over the index return false; } - - // Move the current block start ot the end of this block + + // Move the current block start to the end of this block currentBlockStart += blkSize; } } - + // Check that there's no empty space if(currentBlockStart != blockIndexLoc) { return false; } - - // Check that if another block is references, then the ID is there, and if one isn't there is no ID. + + // Check that if another file is referenced, then the ID is there, and if one + // isn't then there is no ID. int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); if((otherID != 0 && blockFromOtherFileReferenced == false) || (otherID == 0 && blockFromOtherFileReferenced == true)) @@ -246,13 +258,13 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Doesn't look good! return false; } - + // Does the caller want the other ID? if(pDiffFromObjectIDOut) { *pDiffFromObjectIDOut = otherID; } - + // Does the caller want the container ID? if(pContainerIDOut) { @@ -263,6 +275,346 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro return true; } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyStream::Write() +// Purpose: Handles writes to the verifying stream. If the write +// completes the current unit, then verify it, copy it +// to mpCopyToStream if not NULL, and move on to the +// next unit, otherwise throw an exception. +// Created: 2015/08/07 +// +// -------------------------------------------------------------------------- + +void BackupStoreFile::VerifyStream::Write(const void *pBuffer, int NBytes, int Timeout) +{ + // Check that we haven't already written too much to the current unit + size_t BytesToAdd; + if(mState == State_Blocks) + { + // We don't know how many bytes to expect + ASSERT(mCurrentUnitSize == 0); + BytesToAdd = NBytes; + } + else + { + ASSERT(mCurrentUnitData.GetSize() < mCurrentUnitSize); + size_t BytesLeftInCurrentUnit = mCurrentUnitSize - + mCurrentUnitData.GetSize(); + // Add however many bytes are needed/available to the current unit's buffer. + BytesToAdd = std::min(BytesLeftInCurrentUnit, (size_t)NBytes); + } + + // We must make progress here, or we could have infinite recursion. + ASSERT(BytesToAdd > 0); + + CollectInBufferStream* pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Write(pBuffer, BytesToAdd, Timeout); + if(mpCopyToStream) + { + mpCopyToStream->Write(pBuffer, BytesToAdd, Timeout); + } + + pBuffer = (uint8_t *)pBuffer + BytesToAdd; + NBytes -= BytesToAdd; + mCurrentPosition += BytesToAdd; + + if(mState == State_Blocks) + { + // The block index follows the blocks themselves, but without seeing the + // index we don't know how big the blocks are. So we just have to keep + // reading, holding the last mBlockIndexSize bytes in two buffers, until + // we reach the end of the file (when Close() is called) when we can look + // back over those buffers and extract the block index from them. + if(pCurrentBuffer->GetSize() >= mBlockIndexSize) + { + // Time to swap buffers, and clear the one we're about to + // overwrite. + mCurrentBufferIsAlternate = !mCurrentBufferIsAlternate; + pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Reset(); + } + + // We don't want to move data into the finished buffer while we're in this + // state, and we don't need to call ourselves recursively, so just return. + return; + } + + ASSERT(mState != State_Blocks); + + // If the current unit is not complete, just return now. + if(mCurrentUnitData.GetSize() < mCurrentUnitSize) + { + return; + } + + ASSERT(mCurrentUnitData.GetSize() == mCurrentUnitSize); + mCurrentUnitData.SetForReading(); + CollectInBufferStream finished(mCurrentUnitData); + + // This should leave mCurrentUnitData empty in write phase + ASSERT(!mCurrentUnitData.StreamClosed()); + ASSERT(mCurrentUnitData.GetSize() == 0); + + // Advance automatically to next state (to reduce code duplication) if the current + // state is anything but State_Blocks. We remain in that state for more than one + // read (processing one block at a time), and exit it manually. + int oldState = mState; + if(mState != State_Blocks) + { + mState++; + } + + // Process and complete the current unit, depending what it is. + if(oldState == State_Header) + { + // Get the header... + file_StreamFormat hdr; + ASSERT(finished.GetSize() == sizeof(hdr)); + memcpy(&hdr, finished.GetBuffer(), sizeof(hdr)); + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid header magic in stream: expected " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_MAGIC_VALUE_V1) << + " or " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_MAGIC_VALUE_V0) << + " but found " << + BOX_FORMAT_HEX32(ntohl(hdr.mMagicValue))); + } + + mNumBlocks = box_ntoh64(hdr.mNumBlocks); + mBlockIndexSize = (mNumBlocks * sizeof(file_BlockIndexEntry)) + + sizeof(file_BlockIndexHeader); + mContainerID = box_ntoh64(hdr.mContainerID); + + ASSERT(mState == State_FilenameHeader); + mCurrentUnitSize = 2; + } + else if(oldState == State_FilenameHeader) + { + // Check that the encoding is an accepted value. + unsigned int encoding = + BACKUPSTOREFILENAME_GET_ENCODING( + (uint8_t *)finished.GetBuffer()); + if(encoding < BackupStoreFilename::Encoding_Min || + encoding > BackupStoreFilename::Encoding_Max) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid encoding in filename: " << encoding); + } + + ASSERT(mState == State_Filename); + mCurrentUnitSize = BACKUPSTOREFILENAME_GET_SIZE( + (uint8_t *)finished.GetBuffer()); + // Copy the first two bytes back into the new buffer, to be used by the + // completed filename. + finished.CopyStreamTo(mCurrentUnitData); + } + else if(oldState == State_Filename) + { + BackupStoreFilename fn; + fn.ReadFromStream(finished, IOStream::TimeOutInfinite); + ASSERT(mState == State_AttributesSize); + mCurrentUnitSize = sizeof(int32_t); + } + else if(oldState == State_AttributesSize) + { + ASSERT(mState == State_Attributes); + mCurrentUnitSize = ntohl(*(int32_t *)finished.GetBuffer()); + } + else if(oldState == State_Attributes) + { + // Skip the attributes -- because they're encrypted, the server can't tell + // whether they're OK or not. + ASSERT(mState == State_Blocks); + mBlockDataPosition = mCurrentPosition; + mCurrentUnitSize = 0; + } + else if(oldState == State_Blocks) + { + // The block index follows the blocks themselves, but without seeing the + // index we don't know how big the blocks are. So we just have to keep + // reading, holding the last mBlockIndexSize bytes in two buffers, until + // we reach the end of the file (when Close() is called) when we can look + // back over those buffers and extract the block index from them. + ASSERT(mState == State_Blocks); + if(pCurrentBuffer->GetSize() >= mBlockIndexSize) + { + // Time to swap buffers, and clear the one we're about to + // overwrite. + mCurrentBufferIsAlternate = !mCurrentBufferIsAlternate; + pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Reset(); + } + } + + if(NBytes > 0) + { + // Still some data to process, so call recursively to deal with it. + Write(pBuffer, NBytes, Timeout); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyStream::Write() +// Purpose: Handles closing the verifying stream, which tells us +// that there is no more incoming data, at which point +// we can extract the block index from the data most +// recently read, and verify it. +// Created: 2015/08/07 +// +// -------------------------------------------------------------------------- + + +void BackupStoreFile::VerifyStream::Close(bool CloseCopyStream) +{ + if(mpCopyToStream && CloseCopyStream) + { + mpCopyToStream->Close(); + } + + if(mState != State_Blocks) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Stream closed too early to be a valid BackupStoreFile: was in " + "state " << mState << " instead of " << State_Blocks); + } + + // Find the last mBlockIndexSize bytes. + CollectInBufferStream finished; + + CollectInBufferStream* pCurrentBuffer = mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData; + CollectInBufferStream* pPreviousBuffer = mCurrentBufferIsAlternate ? + &mCurrentUnitData : &mAlternateData; + + int64_t BytesFromCurrentBuffer = std::min(mBlockIndexSize, + (int64_t)pCurrentBuffer->GetSize()); + int64_t BytesFromPreviousBuffer = mBlockIndexSize - BytesFromCurrentBuffer; + + if(pPreviousBuffer->GetSize() < BytesFromPreviousBuffer) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Not enough bytes for block index: expected " << + mBlockIndexSize << " but had collected " << + (pPreviousBuffer->GetSize() + pCurrentBuffer->GetSize()) << + " at " << mCurrentPosition << " bytes into file"); + } + + size_t PreviousBufferOffset = pPreviousBuffer->GetSize() - + BytesFromPreviousBuffer; + size_t CurrentBufferOffset = pCurrentBuffer->GetSize() - + BytesFromCurrentBuffer; + + file_BlockIndexHeader blkhdr; + finished.Write((uint8_t *)pPreviousBuffer->GetBuffer() + PreviousBufferOffset, + BytesFromPreviousBuffer); + finished.Write((uint8_t *)pCurrentBuffer->GetBuffer() + CurrentBufferOffset, + BytesFromCurrentBuffer); + ASSERT(finished.GetSize() == mBlockIndexSize); + + // Load the block index header + memcpy(&blkhdr, finished.GetBuffer(), sizeof(blkhdr)); + + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index magic in stream: expected " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) << + " or " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0) << + " but found " << + BOX_FORMAT_HEX32(ntohl(blkhdr.mMagicValue))); + } + + if((int64_t)box_ntoh64(blkhdr.mNumBlocks) != mNumBlocks) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index size in stream: expected " << + BOX_FORMAT_OBJECTID(mNumBlocks) << + " but found " << + BOX_FORMAT_OBJECTID(box_ntoh64(blkhdr.mNumBlocks))); + } + + // Flag for recording whether a block is referenced from another file + mBlockFromOtherFileReferenced = false; + int64_t blockIndexLoc = mCurrentPosition - mBlockIndexSize; + + // Read the index, checking that the length values all make sense + int64_t currentBlockStart = mBlockDataPosition; + file_BlockIndexEntry* pBlk = (file_BlockIndexEntry *) + ((uint8_t *)finished.GetBuffer() + sizeof(blkhdr)); + for(int64_t b = 0; b < mNumBlocks; ++b) + { + // Check size and location + int64_t blkSize = box_ntoh64(pBlk[b].mEncodedSize); + if(blkSize <= 0) + { + // Mark that this file references another file + mBlockFromOtherFileReferenced = true; + } + else + { + // This block is actually in this file + if((currentBlockStart + blkSize) > blockIndexLoc) + { + // Encoded size makes the block run over the index + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index: entry " << b << " makes " + "total size of block data exceed the available " + "space"); + } + + // Move the current block start to the end of this block + currentBlockStart += blkSize; + } + } + + // Check that there's no empty space + if(currentBlockStart != blockIndexLoc) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index: total size of blocks does not match the " + "actual amount of block data in the stream: expected " << + (blockIndexLoc - mBlockDataPosition) << " but found " << + (currentBlockStart - mBlockDataPosition)); + } + + // Check that if another file is referenced, then the ID is there, and if one + // isn't then there is no ID. + int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); + if((otherID != 0 && mBlockFromOtherFileReferenced == false) + || (otherID == 0 && mBlockFromOtherFileReferenced == true)) + { + // Doesn't look good! + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index header: actual dependency status does not " + "match the file header: expected to depend on file object " << + BOX_FORMAT_OBJECTID(otherID)); + } + + mDiffFromObjectID = otherID; +} + + // -------------------------------------------------------------------------- // // Function @@ -279,7 +631,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile { THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) } - + // Try, delete output file if error try { @@ -288,7 +640,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile // Get the decoding stream std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr)); - + // Is it a symlink? if(!stream->IsSymLink()) { @@ -311,7 +663,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile // of the block index. I hope that reading an extra byte // doesn't hurt! // ASSERT(drained == 0); - + // Write the attributes try { @@ -348,10 +700,10 @@ std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream( { // Create stream std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout)); - + // Get it ready stream->Setup(pAlterativeAttr); - + // Return to caller return stream; } @@ -431,7 +783,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - bool inFileOrder = true; + bool inFileOrder = true; switch(ntohl(magic)) { #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -455,7 +807,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl default: THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // If not in file order, then the index list must be read now if(!inFileOrder) { @@ -484,7 +836,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - } + } // Check magic number if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 @@ -498,7 +850,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Get the filename mFilename.ReadFromStream(mrEncodedFile, mTimeout); - + // Get the attributes (either from stream, or supplied attributes) if(pAlterativeAttr != 0) { @@ -514,7 +866,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Read the attributes from the stream mAttributes.ReadFromStream(mrEncodedFile, mTimeout); } - + // If it is in file order, go and read the file attributes // Requires that the stream can seek if(inFileOrder) @@ -524,30 +876,30 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl { THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) } - + // Store current location (beginning of encoded blocks) int64_t endOfHeaderPos = mrEncodedFile.GetPosition(); - + // Work out where the index is int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - + // Seek to that position mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute); - + // Read the block index - ReadBlockIndex(false /* magic number still to be read */); - + ReadBlockIndex(false /* magic number still to be read */); + // Seek back to the end of header position, ready for reading the chunks mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute); } - + // Check view of blocks from block header and file header match if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks)) { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Need to allocate some memory for the two blocks for reading encoded data, and clear data if(mNumBlocks > 0) { @@ -560,11 +912,11 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Get the clear and encoded size int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize); ASSERT(encodedSize > 0); - + // Larger? if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize; } - + // Allocate those blocks! mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32); @@ -589,7 +941,7 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) { // Header file_BlockIndexHeader blkhdr; - + // Read it in -- way depends on how whether the magic number has already been read if(MagicAlreadyRead) { @@ -609,7 +961,7 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - + // Check magic value if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -620,26 +972,26 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } } - + // Get the number of blocks out of the header mNumBlocks = box_ntoh64(blkhdr.mNumBlocks); - + // Read the IV base mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase); - + // Load the block entries in? if(mNumBlocks > 0) { // How big is the index? int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks; - + // Allocate some memory mpBlockIndex = ::malloc(indexSize); if(mpBlockIndex == 0) { throw std::bad_alloc(); } - + // Read it in if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout)) { @@ -676,7 +1028,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) int bytesToRead = NBytes; uint8_t *output = (uint8_t*)pBuffer; - + while(bytesToRead > 0 && mCurrentBlock < mNumBlocks) { // Anything left in the current block? @@ -685,16 +1037,16 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // Copy data out of this buffer int s = mCurrentBlockClearSize - mPositionInCurrentBlock; if(s > bytesToRead) s = bytesToRead; // limit to requested data - + // Copy ::memcpy(output, mpClearData + mPositionInCurrentBlock, s); - + // Update positions output += s; mPositionInCurrentBlock += s; bytesToRead -= s; } - + // Need to get some more data? if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize) { @@ -705,7 +1057,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // Stop now! break; } - + // Get the size from the block index const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize); @@ -716,14 +1068,14 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // It needs to be combined with the previous version first. THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining) } - + // Load in next block if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout)) { // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - + // Decode the data mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize); @@ -734,7 +1086,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // platforms with different endiannesses. iv = box_hton64(iv); sBlowfishDecryptBlockEntry.SetIV(&iv); - + // Decrypt the encrypted section file_BlockIndexEntryEnc entryEnc; int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), @@ -779,7 +1131,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) #endif } - + // Check the digest MD5Digest md5; md5.Add(mpClearData, mCurrentBlockClearSize); @@ -788,12 +1140,12 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck) } - + // Set vars to say what's happening mPositionInCurrentBlock = 0; } } - + ASSERT(bytesToRead >= 0); ASSERT(bytesToRead <= NBytes); @@ -816,14 +1168,14 @@ bool BackupStoreFile::DecodedStream::IsSymLink() { return false; } - + // So the attributes think it is a symlink. // Consistency check... if(mNumBlocks != 0) { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + return true; } @@ -836,7 +1188,8 @@ bool BackupStoreFile::DecodedStream::IsSymLink() // Created: 9/12/03 // // -------------------------------------------------------------------------- -void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes) +void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes, + int Timeout) { THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream) } @@ -916,7 +1269,7 @@ void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength) sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); sAESDecrypt.Reset(); sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); - + // Set encryption to use this key, instead of the "default" blowfish key spEncrypt = &sAESEncrypt; sEncryptCipherType = HEADER_AES_ENCODING; @@ -961,9 +1314,9 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi { rOutput.Reallocate(256); } - + // Check alignment of the block - ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + ASSERT((((uint64_t)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); // Want to compress it? bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE); @@ -981,10 +1334,10 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi const void *iv = spEncrypt->SetRandomIV(ivLen); ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen); outOffset += ivLen; - + // Start encryption process spEncrypt->Begin(); - + #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \ { \ if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \ @@ -992,13 +1345,13 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \ } \ } - + // Encode the chunk if(compressChunk) { // buffer to compress into uint8_t buffer[2048]; - + // Set compressor with all the chunk as an input Compress<true> compress; compress.Input(Chunk, ChunkSize); @@ -1011,7 +1364,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi if(s > 0) { ENCODECHUNK_CHECK_SPACE(s) - outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); } else { @@ -1031,7 +1384,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi ENCODECHUNK_CHECK_SPACE(16) outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); } - + ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check return outOffset; @@ -1051,7 +1404,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize) { // Check alignment of the encoded block - ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + ASSERT((((uint64_t)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); // First check if(EncodedSize < 1) @@ -1060,7 +1413,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } const uint8_t *input = (uint8_t*)Encoded; - + // Get header, make checks, etc uint8_t header = input[0]; bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED; @@ -1069,7 +1422,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out { THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding) } - + #ifndef HAVE_OLD_SSL // Choose cipher CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt); @@ -1081,7 +1434,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } CipherContext &cipher(sBlowfishDecrypt); #endif - + // Check enough space for header, an IV and one byte of input int ivLen = cipher.GetIVLength(); if(EncodedSize < (1 + ivLen + 1)) @@ -1092,7 +1445,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out // Set IV in decrypt context, and start cipher.SetIV(input + 1); cipher.Begin(); - + // Setup vars for code int inOffset = 1 + ivLen; uint8_t *output = (uint8_t*)Output; @@ -1104,10 +1457,10 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out // Do things in chunks uint8_t buffer[2048]; int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer)); - + // Decompressor Compress<false> decompress; - + while(inOffset < EncodedSize) { // Decrypt a block @@ -1115,7 +1468,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl); inOffset += bl; - + // Decompress the decrypted data if(s > 0) { @@ -1126,7 +1479,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out os = decompress.Output(output + outOffset, OutputSize - outOffset); outOffset += os; } while(os > 0); - + // Check that there's space left in the output buffer -- there always should be if(outOffset >= OutputSize) { @@ -1134,7 +1487,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } } } - + // Get any compressed data remaining in the cipher context and compression int s = cipher.Final(buffer, sizeof(buffer)); decompress.Input(buffer, s); @@ -1157,7 +1510,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset); outOffset += cipher.Final(output + outOffset, OutputSize - outOffset); } - + return outOffset; } @@ -1209,10 +1562,10 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Get number of blocks int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - + // Calculate where the block index will be, check it's reasonable int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); int64_t blockIndexLoc = fileSize - blockIndexSize; @@ -1221,10 +1574,10 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr // Doesn't look good! THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Build a reordered stream std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership)); - + // Set it up... ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get())); int component = rreordered.AddComponent(pStream); @@ -1232,7 +1585,7 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc); // And then the rest of the file rreordered.AddBlock(component, blockIndexLoc, true, 0); - + return reordered; } @@ -1287,7 +1640,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, { in.reset(new FileStream(Filename)); } - + // Read header file_BlockIndexHeader hdr; if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) @@ -1313,17 +1666,17 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, // Get basic information int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase); - + //TODO: Verify that these sizes look reasonable - + // setup void *data = 0; int32_t dataSize = -1; bool matches = true; int64_t totalSizeInBlockIndex = 0; - + try - { + { for(int64_t b = 0; b < numBlocks; ++b) { // Read an entry from the stream @@ -1332,8 +1685,8 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, { // Couldn't read entry THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - + } + // Calculate IV for this entry uint64_t iv = entryIVBase; iv += b; @@ -1345,8 +1698,8 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, iv = box_swap64(iv); } #endif - sBlowfishDecryptBlockEntry.SetIV(&iv); - + sBlowfishDecryptBlockEntry.SetIV(&iv); + // Decrypt the encrypted section file_BlockIndexEntryEnc entryEnc; int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), @@ -1381,7 +1734,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } dataSize = blockClearSize + 128; } - + // Load in the block from the file, if it's not a symlink if(!sourceIsSymlink) { @@ -1403,7 +1756,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } } } - + // Keep on going regardless, to make sure the entire block index stream is read // -- must always be consistent about what happens with the stream. } @@ -1418,14 +1771,14 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } throw; } - + // free block if(data != 0) { ::free(data); data = 0; } - + // Check for data left over if it's not a symlink if(!sourceIsSymlink) { @@ -1436,13 +1789,13 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, matches = false; } } - + // Symlinks must have zero size on server if(sourceIsSymlink) { matches = (totalSizeInBlockIndex == 0); } - + return matches; } @@ -1522,10 +1875,10 @@ void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) } // Copy data ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize); - + // Free old BackupStoreFile::CodingChunkFree(mpBuffer); - + // Store new buffer mpBuffer = buffer; mBufferSize = NewSize; @@ -1554,5 +1907,44 @@ DiffTimer::DiffTimer() // // -------------------------------------------------------------------------- DiffTimer::~DiffTimer() -{ +{ +} + +// Shortcut interface +int64_t BackupStoreFile::QueryStoreFileDiff(BackupProtocolCallable& protocol, + const std::string& LocalFilename, int64_t DirectoryObjectID, + int64_t DiffFromFileID, int64_t AttributesHash, + const BackupStoreFilenameClear& StoreFilename, int Timeout, + DiffTimer *pDiffTimer, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) +{ + int64_t ModificationTime; + std::auto_ptr<BackupStoreFileEncodeStream> pStream; + + if(DiffFromFileID) + { + // Fetch the block index for this one + std::auto_ptr<BackupProtocolSuccess> getblockindex = + protocol.QueryGetBlockIndexByName(DirectoryObjectID, + StoreFilename); + ASSERT(getblockindex->GetObjectID() == DiffFromFileID); + std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream()); + + pStream = EncodeFileDiff(LocalFilename, + DirectoryObjectID, StoreFilename, DiffFromFileID, + *(blockIndexStream.get()), Timeout, pDiffTimer, + &ModificationTime, NULL // pIsCompletelyDifferent + ); + } + else + { + pStream = BackupStoreFile::EncodeFile(LocalFilename, + DirectoryObjectID, StoreFilename, &ModificationTime); + } + + std::auto_ptr<IOStream> upload(pStream.release()); + return protocol.QueryStoreFile(DirectoryObjectID, + ModificationTime, AttributesHash, DiffFromFileID, + StoreFilename, upload)->GetObjectID(); } + |