diff options
Diffstat (limited to 'lib/backupclient')
22 files changed, 0 insertions, 6157 deletions
diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp deleted file mode 100644 index 0d7df4d7..00000000 --- a/lib/backupclient/BackupClientFileAttributes.cpp +++ /dev/null @@ -1,1188 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientFileAttributes.cpp -// Purpose: Storage of file attributes -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - -#include <sys/types.h> -#include <sys/stat.h> -#include <errno.h> -#include <limits.h> - -#include <algorithm> -#include <cstring> -#include <new> -#include <vector> - -#ifdef HAVE_SYS_XATTR_H -#include <cerrno> -#include <sys/xattr.h> -#endif - -#include <cstring> - -#include "BackupClientFileAttributes.h" -#include "CommonException.h" -#include "FileModificationTime.h" -#include "BoxTimeToUnix.h" -#include "BackupStoreException.h" -#include "CipherContext.h" -#include "CipherBlowfish.h" -#include "MD5Digest.h" - -#include "MemLeakFindOn.h" - -// set packing to one byte -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "BeginStructPackForWire.h" -#else -BEGIN_STRUCTURE_PACKING_FOR_WIRE -#endif - -#define ATTRIBUTETYPE_GENERIC_UNIX 1 - -#define ATTRIBUTE_ENCODING_BLOWFISH 2 - -typedef struct -{ - int32_t AttributeType; - u_int32_t UID; - u_int32_t GID; - u_int64_t ModificationTime; - u_int64_t AttrModificationTime; - u_int32_t UserDefinedFlags; - u_int32_t FileGenerationNumber; - u_int16_t Mode; - // Symbolic link filename may follow - // Extended attribute (xattr) information may follow, format is: - // u_int32_t Size of extended attribute block (excluding this word) - // For each of NumberOfAttributes (sorted by AttributeName): - // u_int16_t AttributeNameLength - // char AttributeName[AttributeNameLength] - // u_int32_t AttributeValueLength - // unsigned char AttributeValue[AttributeValueLength] - // AttributeName is 0 terminated, AttributeValue is not (and may be binary data) -} attr_StreamFormat; - -// This has wire packing so it's compatible across platforms -// Use wider than necessary sizes, just to be careful. -typedef struct -{ - int32_t uid, gid, mode; - #ifdef WIN32 - int64_t fileCreationTime; - #endif -} attributeHashData; - -// Use default packing -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "EndStructPackForWire.h" -#else -END_STRUCTURE_PACKING_FOR_WIRE -#endif - - -#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256 - -// Hide private static variables from the rest of the world -// -- don't put them as static class variables to avoid openssl/evp.h being -// included all over the project. -namespace -{ - CipherContext sBlowfishEncrypt; - CipherContext sBlowfishDecrypt; - uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH]; - int sAttributeHashSecretLength = 0; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::BackupClientFileAttributes() -// Purpose: Default constructor -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -BackupClientFileAttributes::BackupClientFileAttributes() - : mpClearAttributes(0) -{ - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &) -// Purpose: Copy constructor -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy) - : StreamableMemBlock(rToCopy), // base class does the hard work - mpClearAttributes(0) -{ -} -BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy) - : StreamableMemBlock(rToCopy), // base class does the hard work - mpClearAttributes(0) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::~BackupClientFileAttributes() -// Purpose: Destructor -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -BackupClientFileAttributes::~BackupClientFileAttributes() -{ - if(mpClearAttributes) - { - delete mpClearAttributes; - mpClearAttributes = 0; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &) -// Purpose: Assignment operator -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr) -{ - StreamableMemBlock::Set(rAttr); - RemoveClear(); // make sure no decrypted version held - return *this; -} -// Assume users play nice -BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr) -{ - StreamableMemBlock::Set(rAttr); - RemoveClear(); // make sure no decrypted version held - return *this; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &) -// Purpose: Comparison operator -// Created: 2003/10/09 -// -// -------------------------------------------------------------------------- -bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const -{ - EnsureClearAvailable(); - rAttr.EnsureClearAvailable(); - - return mpClearAttributes->operator==(*rAttr.mpClearAttributes); -} -// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data. -/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const -{ - StreamableMemBlock *pDecoded = 0; - - try - { - EnsureClearAvailable(); - StreamableMemBlock *pDecoded = MakeClear(rAttr); - - // Compare using clear version - bool compared = mpClearAttributes->operator==(rAttr); - - // Delete temporary - delete pDecoded; - - return compared; - } - catch(...) - { - delete pDecoded; - throw; - } -}*/ - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool) -// Purpose: Compare, optionally ignoring the attribute -// modification time and/or modification time, and some -// data which is irrelevant in practise (eg file -// generation number) -// Created: 10/12/03 -// -// -------------------------------------------------------------------------- -bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, - bool IgnoreAttrModTime, bool IgnoreModTime) const -{ - EnsureClearAvailable(); - rAttr.EnsureClearAvailable(); - - // Check sizes are the same, as a first check - if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize()) - { - BOX_TRACE("Attribute Compare: Attributes objects are " - "different sizes, cannot compare them: local " << - mpClearAttributes->GetSize() << " bytes, remote " << - rAttr.mpClearAttributes->GetSize() << " bytes"); - return false; - } - - // Then check the elements of the two things - // Bytes are checked in network order, but this doesn't matter as we're only checking for equality. - attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); - attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer(); - - #define COMPARE(attribute, message) \ - if (a1->attribute != a2->attribute) \ - { \ - BOX_TRACE("Attribute Compare: " << message << " differ: " \ - "local " << ntoh(a1->attribute) << ", " \ - "remote " << ntoh(a2->attribute)); \ - return false; \ - } - COMPARE(AttributeType, "Attribute types"); - COMPARE(UID, "UIDs"); - COMPARE(GID, "GIDs"); - COMPARE(UserDefinedFlags, "User-defined flags"); - COMPARE(Mode, "Modes"); - - if(!IgnoreModTime) - { - uint64_t t1 = box_ntoh64(a1->ModificationTime); - uint64_t t2 = box_ntoh64(a2->ModificationTime); - time_t s1 = BoxTimeToSeconds(t1); - time_t s2 = BoxTimeToSeconds(t2); - if(s1 != s2) - { - BOX_TRACE("Attribute Compare: File modification " - "times differ: local " << - FormatTime(t1, true) << " (" << s1 << "), " - "remote " << - FormatTime(t2, true) << " (" << s2 << ")"); - return false; - } - } - - if(!IgnoreAttrModTime) - { - uint64_t t1 = box_ntoh64(a1->AttrModificationTime); - uint64_t t2 = box_ntoh64(a2->AttrModificationTime); - time_t s1 = BoxTimeToSeconds(t1); - time_t s2 = BoxTimeToSeconds(t2); - if(s1 != s2) - { - BOX_TRACE("Attribute Compare: Attribute modification " - "times differ: local " << - FormatTime(t1, true) << " (" << s1 << "), " - "remote " << - FormatTime(t2, true) << " (" << s2 << ")"); - return false; - } - } - - // Check symlink string? - unsigned int size = mpClearAttributes->GetSize(); - if(size > sizeof(attr_StreamFormat)) - { - // Symlink strings don't match. This also compares xattrs - int datalen = size - sizeof(attr_StreamFormat); - - if(::memcmp(a1 + 1, a2 + 1, datalen) != 0) - { - std::string s1((char *)(a1 + 1), datalen); - std::string s2((char *)(a2 + 1), datalen); - BOX_TRACE("Attribute Compare: Symbolic link target " - "or extended attributes differ: " - "local " << PrintEscapedBinaryData(s1) << ", " - "remote " << PrintEscapedBinaryData(s2)); - return false; - } - } - - // Passes all test, must be OK - return true; -} - - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::ReadAttributes( -// const char *Filename, bool ZeroModificationTimes, -// box_time_t *pModTime, box_time_t *pAttrModTime, -// int64_t *pFileSize, InodeRefType *pInodeNumber, -// bool *pHasMultipleLinks) -// Purpose: Read the attributes of the file, and store them -// ready for streaming. Optionally retrieve the -// modification time and attribute modification time. -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::ReadAttributes(const char *Filename, - bool ZeroModificationTimes, box_time_t *pModTime, - box_time_t *pAttrModTime, int64_t *pFileSize, - InodeRefType *pInodeNumber, bool *pHasMultipleLinks) -{ - StreamableMemBlock *pnewAttr = 0; - try - { - EMU_STRUCT_STAT st; - if(EMU_LSTAT(Filename, &st) != 0) - { - BOX_LOG_SYS_ERROR("Failed to stat file: '" << - Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError) - } - - // Modification times etc - if(pModTime) {*pModTime = FileModificationTime(st);} - if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);} - if(pFileSize) {*pFileSize = st.st_size;} - if(pInodeNumber) {*pInodeNumber = st.st_ino;} - if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);} - - pnewAttr = new StreamableMemBlock; - - FillAttributes(*pnewAttr, Filename, st, ZeroModificationTimes); - -#ifndef WIN32 - // Is it a link? - if((st.st_mode & S_IFMT) == S_IFLNK) - { - FillAttributesLink(*pnewAttr, Filename, st); - } -#endif - - FillExtendedAttr(*pnewAttr, Filename); - -#ifdef WIN32 - //this is to catch those problems with invalid time stamps stored... - //need to find out the reason why - but also a catch as well. - - attr_StreamFormat *pattr = - (attr_StreamFormat*)pnewAttr->GetBuffer(); - ASSERT(pattr != 0); - - // __time64_t winTime = BoxTimeToSeconds( - // pnewAttr->ModificationTime); - - u_int64_t modTime = box_ntoh64(pattr->ModificationTime); - box_time_t modSecs = BoxTimeToSeconds(modTime); - __time64_t winTime = modSecs; - - // _MAX__TIME64_T doesn't seem to be defined, but the code below - // will throw an assertion failure if we exceed it :-) - // Microsoft says dates up to the year 3000 are valid, which - // is a bit more than 15 * 2^32. Even that doesn't seem - // to be true (still aborts), but it can at least hold 2^32. - if (winTime >= 0x100000000LL || _gmtime64(&winTime) == 0) - { - BOX_ERROR("Invalid Modification Time caught for " - "file: '" << Filename << "'"); - pattr->ModificationTime = 0; - } - - modTime = box_ntoh64(pattr->AttrModificationTime); - modSecs = BoxTimeToSeconds(modTime); - winTime = modSecs; - - if (winTime > 0x100000000LL || _gmtime64(&winTime) == 0) - { - BOX_ERROR("Invalid Attribute Modification Time " - "caught for file: '" << Filename << "'"); - pattr->AttrModificationTime = 0; - } -#endif - - // Attributes ready. Encrypt into this block - EncryptAttr(*pnewAttr); - - // Store the new attributes - RemoveClear(); - mpClearAttributes = pnewAttr; - pnewAttr = 0; - } - catch(...) - { - // clean up - delete pnewAttr; - pnewAttr = 0; - throw; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::ReadAttributesLink() -// Purpose: Private function, handles standard attributes for all objects -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, EMU_STRUCT_STAT &st, bool ZeroModificationTimes) -{ - outputBlock.ResizeBlock(sizeof(attr_StreamFormat)); - attr_StreamFormat *pattr = (attr_StreamFormat*)outputBlock.GetBuffer(); - ASSERT(pattr != 0); - - // Fill in the entries - pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX); - pattr->UID = htonl(st.st_uid); - pattr->GID = htonl(st.st_gid); - if(ZeroModificationTimes) - { - pattr->ModificationTime = 0; - pattr->AttrModificationTime = 0; - } - else - { - pattr->ModificationTime = box_hton64(FileModificationTime(st)); - pattr->AttrModificationTime = box_hton64(FileAttrModificationTime(st)); - } - pattr->Mode = htons(st.st_mode); - -#ifndef HAVE_STRUCT_STAT_ST_FLAGS - pattr->UserDefinedFlags = 0; - pattr->FileGenerationNumber = 0; -#else - pattr->UserDefinedFlags = htonl(st.st_flags); - pattr->FileGenerationNumber = htonl(st.st_gen); -#endif -} -#ifndef WIN32 -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::ReadAttributesLink() -// Purpose: Private function, handles the case where a symbolic link is needed -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st) -{ - // Make sure we're only called for symbolic links - ASSERT((st.st_mode & S_IFMT) == S_IFLNK); - - // Get the filename the link is linked to - char linkedTo[PATH_MAX+4]; - int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX); - if(linkedToSize == -1) - { - BOX_LOG_SYS_ERROR("Failed to readlink '" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError); - } - - int oldSize = outputBlock.GetSize(); - outputBlock.ResizeBlock(oldSize+linkedToSize+1); - char* buffer = static_cast<char*>(outputBlock.GetBuffer()); - - // Add the path name for the symbolic link, and add 0 termination - std::memcpy(buffer+oldSize, linkedTo, linkedToSize); - buffer[oldSize+linkedToSize] = '\0'; -} -#endif - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::ReadExtendedAttr(const char *, unsigned char**) -// Purpose: Private function, read the extended attributes of the file into the block -// Created: 2005/06/12 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename) -{ -#ifdef HAVE_SYS_XATTR_H - int listBufferSize = 10000; - char* list = new char[listBufferSize]; - - try - { - // This returns an unordered list of attribute names, each 0 terminated, - // concatenated together - int listSize = ::llistxattr(Filename, list, listBufferSize); - - if(listSize>listBufferSize) - { - delete[] list, list = NULL; - list = new char[listSize]; - listSize = ::llistxattr(Filename, list, listSize); - } - - if(listSize>0) - { - // Extract list of attribute names so we can sort them - std::vector<std::string> attrKeys; - for(int i = 0; i<listSize; ++i) - { - std::string attrKey(list+i); - i += attrKey.size(); - attrKeys.push_back(attrKey); - } - sort(attrKeys.begin(), attrKeys.end()); - - // Make initial space in block - int xattrSize = outputBlock.GetSize(); - int xattrBufferSize = (xattrSize+listSize)>500 ? (xattrSize+listSize)*2 : 1000; - outputBlock.ResizeBlock(xattrBufferSize); - unsigned char* buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); - - // Leave space for attr block size later - int xattrBlockSizeOffset = xattrSize; - xattrSize += sizeof(u_int32_t); - - // Loop for each attribute - for(std::vector<std::string>::const_iterator attrKeyI = attrKeys.begin(); attrKeyI!=attrKeys.end(); ++attrKeyI) - { - std::string attrKey(*attrKeyI); - - if(xattrSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t)>static_cast<unsigned int>(xattrBufferSize)) - { - xattrBufferSize = (xattrBufferSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t))*2; - outputBlock.ResizeBlock(xattrBufferSize); - buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); - } - - // Store length and text for attibute name - u_int16_t keyLength = htons(attrKey.size()+1); - std::memcpy(buffer+xattrSize, &keyLength, sizeof(u_int16_t)); - xattrSize += sizeof(u_int16_t); - std::memcpy(buffer+xattrSize, attrKey.c_str(), attrKey.size()+1); - xattrSize += attrKey.size()+1; - - // Leave space for value size - int valueSizeOffset = xattrSize; - xattrSize += sizeof(u_int32_t); - - // Find size of attribute (must call with buffer and length 0 on some platforms, - // as -1 is returned if the data doesn't fit.) - int valueSize = ::lgetxattr(Filename, attrKey.c_str(), 0, 0); - if(valueSize<0) - { - BOX_LOG_SYS_ERROR("Failed to get " - "extended attribute size of " - "'" << Filename << "': " << - attrKey); - THROW_EXCEPTION(CommonException, OSFileError); - } - - // Resize block, if needed - if(xattrSize+valueSize>xattrBufferSize) - { - xattrBufferSize = (xattrBufferSize+valueSize)*2; - outputBlock.ResizeBlock(xattrBufferSize); - buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); - } - - // This gets the attribute value (may be text or binary), no termination - valueSize = ::lgetxattr(Filename, attrKey.c_str(), buffer+xattrSize, xattrBufferSize-xattrSize); - if(valueSize<0) - { - BOX_LOG_SYS_ERROR("Failed to get " - "extended attribute of " - "'" << Filename << "': " << - attrKey); - THROW_EXCEPTION(CommonException, OSFileError); - } - xattrSize += valueSize; - - // Fill in value size - u_int32_t valueLength = htonl(valueSize); - std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(u_int32_t)); - } - - // Fill in attribute block size - u_int32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(u_int32_t)); - std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(u_int32_t)); - - outputBlock.ResizeBlock(xattrSize); - } - else if(listSize<0) - { - if(errno == EOPNOTSUPP || errno == EACCES) - { - // fail silently - } - else if(errno == ERANGE) - { - BOX_ERROR("Failed to list extended " - "attributes of '" << Filename << "': " - "buffer too small, not backed up"); - } - else - { - BOX_LOG_SYS_ERROR("Failed to list extended " - "attributes of '" << Filename << "', " - "not backed up"); - THROW_EXCEPTION(CommonException, OSFileError); - } - } - } - catch(...) - { - delete[] list; - throw; - } - delete[] list; -#endif -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::GetModificationTimes() -// Purpose: Returns the modification time embedded in the -// attributes. -// Created: 2010/02/24 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::GetModificationTimes( - box_time_t *pModificationTime, - box_time_t *pAttrModificationTime) const -{ - // Got something loaded - if(GetSize() <= 0) - { - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - // Make sure there are clear attributes to use - EnsureClearAvailable(); - ASSERT(mpClearAttributes != 0); - - // Check if the decrypted attributes are small enough, and the type of attributes stored - if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) - { - THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); - } - int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); - ASSERT(type != 0); - if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) - { - // Don't know what to do with these - THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); - } - - // Check there is enough space for an attributes block - if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) - { - // Too small - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - // Get pointer to structure - attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); - - if(pModificationTime) - { - *pModificationTime = box_ntoh64(pattr->ModificationTime); - } - - if(pAttrModificationTime) - { - *pAttrModificationTime = box_ntoh64(pattr->AttrModificationTime); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::WriteAttributes(const char *) -// Purpose: Apply the stored attributes to the file -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::WriteAttributes(const char *Filename, - bool MakeUserWritable) const -{ - // Got something loaded - if(GetSize() <= 0) - { - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - // Make sure there are clear attributes to use - EnsureClearAvailable(); - ASSERT(mpClearAttributes != 0); - - // Check if the decrypted attributes are small enough, and the type of attributes stored - if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) - { - THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); - } - int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); - ASSERT(type != 0); - if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) - { - // Don't know what to do with these - THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); - } - - // Check there is enough space for an attributes block - if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) - { - // Too small - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - // Get pointer to structure - attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); - int xattrOffset = sizeof(attr_StreamFormat); - - // is it a symlink? - int16_t mode = ntohs(pattr->Mode); - if((mode & S_IFMT) == S_IFLNK) - { - // Check things are sensible - if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1) - { - // Too small - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - -#ifdef WIN32 - BOX_WARNING("Cannot create symbolic links on Windows: '" << - Filename << "'"); -#else - // Make a symlink, first deleting anything in the way - ::unlink(Filename); - if(::symlink((char*)(pattr + 1), Filename) != 0) - { - BOX_LOG_SYS_ERROR("Failed to symlink '" << Filename << - "' to '" << (char*)(pattr + 1) << "'"); - THROW_EXCEPTION(CommonException, OSFileError) - } -#endif - - xattrOffset += std::strlen(reinterpret_cast<char*>(pattr+1))+1; - } - - // If working as root, set user IDs - if(::geteuid() == 0) - { - #ifndef HAVE_LCHOWN - // only if not a link, can't set their owner on this platform - if((mode & S_IFMT) != S_IFLNK) - { - // Not a link, use normal chown - if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) - { - BOX_LOG_SYS_ERROR("Failed to change " - "owner of file " - "'" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError) - } - } - #else - // use the version which sets things on symlinks - if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) - { - BOX_LOG_SYS_ERROR("Failed to change owner of " - "symbolic link '" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError) - } - #endif - } - - if(static_cast<int>(xattrOffset+sizeof(u_int32_t))<=mpClearAttributes->GetSize()) - { - WriteExtendedAttr(Filename, xattrOffset); - } - - // Stop now if symlink, because otherwise it'll just be applied to the target - if((mode & S_IFMT) == S_IFLNK) - { - return; - } - - // Set modification time? - box_time_t modtime = box_ntoh64(pattr->ModificationTime); - if(modtime != 0) - { - // Work out times as timevals - struct timeval times[2]; - - #ifdef WIN32 - BoxTimeToTimeval(box_ntoh64(pattr->ModificationTime), - times[1]); - BoxTimeToTimeval(box_ntoh64(pattr->AttrModificationTime), - times[0]); - // Because stat() returns the creation time in the ctime - // field under Windows, and this gets saved in the - // AttrModificationTime field of the serialised attributes, - // we subvert the first parameter of emu_utimes() to allow - // it to be reset to the right value on the restored file. - #else - BoxTimeToTimeval(modtime, times[1]); - // Copy access time as well, why not, got to set it to something - times[0] = times[1]; - // Attr modification time will be changed anyway, - // nothing that can be done about it - #endif - - // Try to apply - if(::utimes(Filename, times) != 0) - { - BOX_LOG_SYS_WARNING("Failed to change times of " - "file '" << Filename << "' to ctime=" << - BOX_FORMAT_TIMESPEC(times[0]) << ", mtime=" << - BOX_FORMAT_TIMESPEC(times[1])); - } - } - - if (MakeUserWritable) - { - mode |= S_IRWXU; - } - - // Apply everything else... (allowable mode flags only) - // Mode must be done last (think setuid) - if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID - | S_ISGID | S_ISVTX)) != 0) - { - BOX_LOG_SYS_ERROR("Failed to change permissions of file " - "'" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError) - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::IsSymLink() -// Purpose: Do these attributes represent a symbolic link? -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -bool BackupClientFileAttributes::IsSymLink() const -{ - EnsureClearAvailable(); - - // Got the right kind of thing? - if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) - { - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - // Get the type of attributes stored - int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); - ASSERT(type != 0); - if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat)) - { - // Check link - attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); - return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK; - } - - return false; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::RemoveClear() -// Purpose: Private. Deletes any clear version of the attributes that may be held -// Created: 3/12/03 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::RemoveClear() const -{ - if(mpClearAttributes) - { - delete mpClearAttributes; - } - mpClearAttributes = 0; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::EnsureClearAvailable() -// Purpose: Private. Makes sure the clear version is available -// Created: 3/12/03 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::EnsureClearAvailable() const -{ - if(mpClearAttributes == 0) - { - mpClearAttributes = MakeClear(*this); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) -// Purpose: Private function, apply the stored extended attributes to the file -// Created: 2005/06/13 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) const -{ -#ifdef HAVE_SYS_XATTR_H - const char* buffer = static_cast<char*>(mpClearAttributes->GetBuffer()); - - u_int32_t xattrBlockLength = 0; - std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(u_int32_t)); - int xattrBlockSize = ntohl(xattrBlockLength); - xattrOffset += sizeof(u_int32_t); - - int xattrEnd = xattrOffset+xattrBlockSize; - if(xattrEnd>mpClearAttributes->GetSize()) - { - // Too small - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - while(xattrOffset<xattrEnd) - { - u_int16_t keyLength = 0; - std::memcpy(&keyLength, buffer+xattrOffset, sizeof(u_int16_t)); - int keySize = ntohs(keyLength); - xattrOffset += sizeof(u_int16_t); - - const char* key = buffer+xattrOffset; - xattrOffset += keySize; - - u_int32_t valueLength = 0; - std::memcpy(&valueLength, buffer+xattrOffset, sizeof(u_int32_t)); - int valueSize = ntohl(valueLength); - xattrOffset += sizeof(u_int32_t); - - // FIXME: Warn on EOPNOTSUPP - if(::lsetxattr(Filename, key, buffer+xattrOffset, valueSize, 0)!=0 && errno!=EOPNOTSUPP) - { - BOX_LOG_SYS_ERROR("Failed to set extended attributes " - "on file '" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError); - } - - xattrOffset += valueSize; - } - - ASSERT(xattrOffset==xattrEnd); -#endif -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &) -// Purpose: Static. Decrypts stored attributes. -// Created: 3/12/03 -// -// -------------------------------------------------------------------------- -StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted) -{ - // New block - StreamableMemBlock *pdecrypted = 0; - - try - { - // Check the block is big enough for IV and header - int ivSize = sBlowfishEncrypt.GetIVLength(); - if(rEncrypted.GetSize() <= (ivSize + 1)) - { - THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes); - } - - // How much space is needed for the output? - int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize); - - // Allocate it - pdecrypted = new StreamableMemBlock(maxDecryptedSize); - - // ptr to block - uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer(); - - // Check that the header has right type - if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH) - { - THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding); - } - - // Set IV - sBlowfishDecrypt.SetIV(encBlock + 1); - - // Decrypt - int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1)); - - // Resize block to fit - pdecrypted->ResizeBlock(decryptedSize); - } - catch(...) - { - delete pdecrypted; - pdecrypted = 0; - } - - return pdecrypted; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int) -// Purpose: Static. Sets the key to use for encryption and decryption. -// Created: 3/12/03 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength) -{ - // IVs set later - sBlowfishEncrypt.Reset(); - sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); - sBlowfishDecrypt.Reset(); - sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &) -// Purpose: Private. Encrypt the given attributes into this block. -// Created: 3/12/03 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt) -{ - // Free any existing block - FreeBlock(); - - // Work out the maximum amount of space we need - int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize()); - // And the size of the IV - int ivSize = sBlowfishEncrypt.GetIVLength(); - - // Allocate this space - AllocateBlock(maxEncryptedSize + ivSize + 1); - - // Store the encoding byte - uint8_t *block = (uint8_t*)GetBuffer(); - block[0] = ATTRIBUTE_ENCODING_BLOWFISH; - - // Generate and store an IV for this attribute block - int ivSize2 = 0; - const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2); - ASSERT(ivSize == ivSize2); - - // Copy into the encrypted block - ::memcpy(block + 1, iv, ivSize); - - // Do the transform - int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize()); - - // Resize this block - ResizeBlock(encrytedSize + ivSize + 1); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int) -// Purpose: Set the secret for the filename attribute hash -// Created: 25/4/04 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength) -{ - if(SecretLength > (int)sizeof(sAttributeHashSecret)) - { - SecretLength = sizeof(sAttributeHashSecret); - } - if(SecretLength < 0) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Copy - ::memcpy(sAttributeHashSecret, pSecret, SecretLength); - sAttributeHashSecretLength = SecretLength; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::GenerateAttributeHash( -// struct stat &, const std::string &, -// const std::string &) -// Purpose: Generate a 64 bit hash from the attributes, used to -// detect changes. Include filename in the hash, so -// that it changes from one file to another, so don't -// reveal identical attributes. -// Created: 25/4/04 -// -// -------------------------------------------------------------------------- -uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, - const std::string &filename, const std::string &leafname) -{ - if(sAttributeHashSecretLength == 0) - { - THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet) - } - - // Assemble stuff we're interested in - attributeHashData hashData; - memset(&hashData, 0, sizeof(hashData)); - // Use network byte order and large sizes to be cross platform - hashData.uid = htonl(st.st_uid); - hashData.gid = htonl(st.st_gid); - hashData.mode = htonl(st.st_mode); - - #ifdef WIN32 - // On Windows, the "file attribute modification time" is the - // file creation time, and we want to back this up, restore - // it and compare it. - // - // On other platforms, it's not very important and can't - // reliably be set to anything other than the current time. - hashData.fileCreationTime = box_hton64(st.st_ctime); - #endif - - StreamableMemBlock xattr; - FillExtendedAttr(xattr, filename.c_str()); - - // Create a MD5 hash of the data, filename, and secret - MD5Digest digest; - digest.Add(&hashData, sizeof(hashData)); - digest.Add(xattr.GetBuffer(), xattr.GetSize()); - digest.Add(leafname.c_str(), leafname.size()); - digest.Add(sAttributeHashSecret, sAttributeHashSecretLength); - digest.Finish(); - - // Return the first 64 bits of the hash - uint64_t result = *((uint64_t *)(digest.DigestAsData())); - return result; -} diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h deleted file mode 100644 index f9a0d883..00000000 --- a/lib/backupclient/BackupClientFileAttributes.h +++ /dev/null @@ -1,78 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientFileAttributes.h -// Purpose: Storage of file attributes -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCLIENTFILEATTRIBUTES__H -#define BACKUPCLIENTFILEATTRIBUTES__H - -#include <string> - -#include "StreamableMemBlock.h" -#include "BoxTime.h" - -EMU_STRUCT_STAT; // declaration - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupClientFileAttributes -// Purpose: Storage, streaming and application of file attributes -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -class BackupClientFileAttributes : public StreamableMemBlock -{ -public: - BackupClientFileAttributes(); - BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy); - BackupClientFileAttributes(const StreamableMemBlock &rToCopy); - ~BackupClientFileAttributes(); - BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr); - BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr); - bool operator==(const BackupClientFileAttributes &rAttr) const; -// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous? - - bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const; - - // Prevent access to base class members accidently - void Set(); - - void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false, - box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0, - InodeRefType *pInodeNumber = 0, bool *pHasMultipleLinks = 0); - void WriteAttributes(const char *Filename, - bool MakeUserWritable = false) const; - void GetModificationTimes(box_time_t *pModificationTime, - box_time_t *pAttrModificationTime) const; - - bool IsSymLink() const; - - static void SetBlowfishKey(const void *pKey, int KeyLength); - static void SetAttributeHashSecret(const void *pSecret, int SecretLength); - - static uint64_t GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname); - static void FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename); - -private: - static void FillAttributes(StreamableMemBlock &outputBlock, - const char *Filename, EMU_STRUCT_STAT &st, - bool ZeroModificationTimes); - static void FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st); - void WriteExtendedAttr(const char *Filename, int xattrOffset) const; - - void RemoveClear() const; - void EnsureClearAvailable() const; - static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted); - void EncryptAttr(const StreamableMemBlock &rToEncrypt); - -private: - mutable StreamableMemBlock *mpClearAttributes; -}; - -#endif // BACKUPCLIENTFILEATTRIBUTES__H - diff --git a/lib/backupclient/BackupStoreConstants.h b/lib/backupclient/BackupStoreConstants.h deleted file mode 100644 index 2c33fd8f..00000000 --- a/lib/backupclient/BackupStoreConstants.h +++ /dev/null @@ -1,44 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreContants.h -// Purpose: constants for the backup system -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTORECONSTANTS__H -#define BACKUPSTORECONSTANTS__H - -#define BACKUPSTORE_ROOT_DIRECTORY_ID 1 - -#define BACKUP_STORE_SERVER_VERSION 1 - -// Minimum size for a chunk to be compressed -#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256 - -// min and max sizes for blocks -#define BACKUP_FILE_MIN_BLOCK_SIZE 4096 -#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024) - -// Increase the block size if there are more than this number of blocks -#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096 - -// Avoid creating blocks smaller than this -#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128 - -// Maximum number of sizes to do an rsync-like scan for -#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 64 - -// When doing rsync scans, do not scan for blocks smaller than -#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 128 - -// A limit to stop diffing running out of control: If more than this -// times the number of blocks in the original index are found, stop -// looking. This stops really bad cases of diffing files containing -// all the same byte using huge amounts of memory and processor time. -// This is a multiple of the number of blocks in the diff from file. -#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096 - -#endif // BACKUPSTORECONSTANTS__H - diff --git a/lib/backupclient/BackupStoreDirectory.cpp b/lib/backupclient/BackupStoreDirectory.cpp deleted file mode 100644 index 0d06da34..00000000 --- a/lib/backupclient/BackupStoreDirectory.cpp +++ /dev/null @@ -1,568 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreDirectory.h -// Purpose: Representation of a backup directory -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <sys/types.h> - -#include "BackupStoreDirectory.h" -#include "IOStream.h" -#include "BackupStoreException.h" -#include "BackupStoreObjectMagic.h" - -#include "MemLeakFindOn.h" - -// set packing to one byte -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "BeginStructPackForWire.h" -#else -BEGIN_STRUCTURE_PACKING_FOR_WIRE -#endif - -typedef struct -{ - int32_t mMagicValue; // also the version number - int32_t mNumEntries; - int64_t mObjectID; // this object ID - int64_t mContainerID; // ID of container - uint64_t mAttributesModTime; - int32_t mOptionsPresent; // bit mask of optional sections / features present - // Then a StreamableMemBlock for attributes -} dir_StreamFormat; - -typedef enum -{ - Option_DependencyInfoPresent = 1 -} dir_StreamFormatOptions; - -typedef struct -{ - uint64_t mModificationTime; - int64_t mObjectID; - int64_t mSizeInBlocks; - uint64_t mAttributesHash; - int16_t mFlags; // order smaller items after bigger ones (for alignment) - // Then a BackupStoreFilename - // Then a StreamableMemBlock for attributes -} en_StreamFormat; - -typedef struct -{ - int64_t mDependsNewer; - int64_t mDependsOlder; -} en_StreamFormatDepends; - -// Use default packing -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "EndStructPackForWire.h" -#else -END_STRUCTURE_PACKING_FOR_WIRE -#endif - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::BackupStoreDirectory() -// Purpose: Constructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::BackupStoreDirectory() - : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0) -{ - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); -} - - -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t) -// Purpose: Constructor giving object and container IDs -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID) - : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::~BackupStoreDirectory() -// Purpose: Destructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::~BackupStoreDirectory() -{ - for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) - { - delete (*i); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int) -// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads. -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) -{ - // Get the header - dir_StreamFormat hdr; - if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Check magic value... - if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue)) - { - THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat) - } - - // Get data - mObjectID = box_ntoh64(hdr.mObjectID); - mContainerID = box_ntoh64(hdr.mContainerID); - mAttributesModTime = box_ntoh64(hdr.mAttributesModTime); - - // Options - int32_t options = ntohl(hdr.mOptionsPresent); - - // Get attributes - mAttributes.ReadFromStream(rStream, Timeout); - - // Decode count - int count = ntohl(hdr.mNumEntries); - - // Clear existing list - for(std::vector<Entry*>::iterator i = mEntries.begin(); - i != mEntries.end(); i++) - { - delete (*i); - } - mEntries.clear(); - - // Read them in! - for(int c = 0; c < count; ++c) - { - Entry *pen = new Entry; - try - { - // Read from stream - pen->ReadFromStream(rStream, Timeout); - - // Add to list - mEntries.push_back(pen); - } - catch(...) - { - delete pen; - throw; - } - } - - // Read in dependency info? - if(options & Option_DependencyInfoPresent) - { - // Read in extra dependency data - for(int c = 0; c < count; ++c) - { - mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout); - } - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool) -// Purpose: Writes a selection of entries to a stream -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const -{ - // Get count of entries - int32_t count = mEntries.size(); - if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING) - { - // Need to count the entries - count = 0; - Iterator i(*this); - while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0) - { - count++; - } - } - - // Check that sensible IDs have been set - ASSERT(mObjectID != 0); - ASSERT(mContainerID != 0); - - // Need dependency info? - bool dependencyInfoRequired = false; - if(StreamDependencyInfo) - { - Iterator i(*this); - Entry *pen = 0; - while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) - { - if(pen->HasDependencies()) - { - dependencyInfoRequired = true; - } - } - } - - // Options - int32_t options = 0; - if(dependencyInfoRequired) options |= Option_DependencyInfoPresent; - - // Build header - dir_StreamFormat hdr; - hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE); - hdr.mNumEntries = htonl(count); - hdr.mObjectID = box_hton64(mObjectID); - hdr.mContainerID = box_hton64(mContainerID); - hdr.mAttributesModTime = box_hton64(mAttributesModTime); - hdr.mOptionsPresent = htonl(options); - - // Write header - rStream.Write(&hdr, sizeof(hdr)); - - // Write the attributes? - if(StreamAttributes) - { - mAttributes.WriteToStream(rStream); - } - else - { - // Write a blank header instead - StreamableMemBlock::WriteEmptyBlockToStream(rStream); - } - - // Then write all the entries - Iterator i(*this); - Entry *pen = 0; - while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) - { - pen->WriteToStream(rStream); - } - - // Write dependency info? - if(dependencyInfoRequired) - { - Iterator i(*this); - Entry *pen = 0; - while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) - { - pen->WriteToStreamDependencyInfo(rStream); - } - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::AddEntry(const Entry &) -// Purpose: Adds entry to directory (no checking) -// Created: 2003/08/27 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy) -{ - Entry *pnew = new Entry(rEntryToCopy); - try - { - mEntries.push_back(pnew); - } - catch(...) - { - delete pnew; - throw; - } - - return pnew; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t) -// Purpose: Adds entry to directory (no checking) -// Created: 2003/08/27 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime) -{ - Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime); - try - { - mEntries.push_back(pnew); - } - catch(...) - { - delete pnew; - throw; - } - - return pnew; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::DeleteEntry(int64_t) -// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient) -// Created: 2003/08/27 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) -{ - for(std::vector<Entry*>::iterator i(mEntries.begin()); - i != mEntries.end(); ++i) - { - if((*i)->mObjectID == ObjectID) - { - // Delete - delete (*i); - // Remove from list - mEntries.erase(i); - // Done - return; - } - } - - // Not found - THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::FindEntryByID(int64_t) -// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist. -// Created: 12/11/03 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const -{ - for(std::vector<Entry*>::const_iterator i(mEntries.begin()); - i != mEntries.end(); ++i) - { - if((*i)->mObjectID == ObjectID) - { - // Found - return (*i); - } - } - - // Not found - return 0; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::Entry() -// Purpose: Constructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry::Entry() - : mModificationTime(0), - mObjectID(0), - mSizeInBlocks(0), - mFlags(0), - mAttributesHash(0), - mMinMarkNumber(0), - mMarkNumber(0), - mDependsNewer(0), - mDependsOlder(0) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::~Entry() -// Purpose: Destructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry::~Entry() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::Entry(const Entry &) -// Purpose: Copy constructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) - : mName(rToCopy.mName), - mModificationTime(rToCopy.mModificationTime), - mObjectID(rToCopy.mObjectID), - mSizeInBlocks(rToCopy.mSizeInBlocks), - mFlags(rToCopy.mFlags), - mAttributesHash(rToCopy.mAttributesHash), - mAttributes(rToCopy.mAttributes), - mMinMarkNumber(rToCopy.mMinMarkNumber), - mMarkNumber(rToCopy.mMarkNumber), - mDependsNewer(rToCopy.mDependsNewer), - mDependsOlder(rToCopy.mDependsOlder) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t) -// Purpose: Constructor from values -// Created: 2003/08/27 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) - : mName(rName), - mModificationTime(ModificationTime), - mObjectID(ObjectID), - mSizeInBlocks(SizeInBlocks), - mFlags(Flags), - mAttributesHash(AttributesHash), - mMinMarkNumber(0), - mMarkNumber(0), - mDependsNewer(0), - mDependsOlder(0) -{ -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int) -// Purpose: Read an entry from a stream -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) -{ - // Grab the raw bytes from the stream which compose the header - en_StreamFormat entry; - if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Do reading first before modifying the variables, to be more exception safe - - // Get the filename - BackupStoreFilename name; - name.ReadFromStream(rStream, Timeout); - - // Get the attributes - mAttributes.ReadFromStream(rStream, Timeout); - - // Store the rest of the bits - mModificationTime = box_ntoh64(entry.mModificationTime); - mObjectID = box_ntoh64(entry.mObjectID); - mSizeInBlocks = box_ntoh64(entry.mSizeInBlocks); - mAttributesHash = box_ntoh64(entry.mAttributesHash); - mFlags = ntohs(entry.mFlags); - mName = name; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &) -// Purpose: Writes the entry to a stream -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const -{ - // Build a structure - en_StreamFormat entry; - entry.mModificationTime = box_hton64(mModificationTime); - entry.mObjectID = box_hton64(mObjectID); - entry.mSizeInBlocks = box_hton64(mSizeInBlocks); - entry.mAttributesHash = box_hton64(mAttributesHash); - entry.mFlags = htons(mFlags); - - // Write it - rStream.Write(&entry, sizeof(entry)); - - // Write the filename - mName.WriteToStream(rStream); - - // Write any attributes - mAttributes.WriteToStream(rStream); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int) -// Purpose: Read the optional dependency info from a stream -// Created: 13/7/04 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout) -{ - // Grab the raw bytes from the stream which compose the header - en_StreamFormatDepends depends; - if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Store the data - mDependsNewer = box_ntoh64(depends.mDependsNewer); - mDependsOlder = box_ntoh64(depends.mDependsOlder); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &) -// Purpose: Write the optional dependency info to a stream -// Created: 13/7/04 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const -{ - // Build structure - en_StreamFormatDepends depends; - depends.mDependsNewer = box_hton64(mDependsNewer); - depends.mDependsOlder = box_hton64(mDependsOlder); - // Write - rStream.Write(&depends, sizeof(depends)); -} - - - diff --git a/lib/backupclient/BackupStoreDirectory.h b/lib/backupclient/BackupStoreDirectory.h deleted file mode 100644 index 0dfe6422..00000000 --- a/lib/backupclient/BackupStoreDirectory.h +++ /dev/null @@ -1,285 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreDirectory.h -// Purpose: Representation of a backup directory -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREDIRECTORY__H -#define BACKUPSTOREDIRECTORY__H - -#include <string> -#include <vector> - -#include "BackupStoreFilenameClear.h" -#include "StreamableMemBlock.h" -#include "BoxTime.h" - -class IOStream; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupStoreDirectory -// Purpose: In memory representation of a directory -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -class BackupStoreDirectory -{ -public: - BackupStoreDirectory(); - BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID); -private: - // Copying not allowed - BackupStoreDirectory(const BackupStoreDirectory &rToCopy); -public: - ~BackupStoreDirectory(); - - class Entry - { - public: - friend class BackupStoreDirectory; - - Entry(); - ~Entry(); - Entry(const Entry &rToCopy); - Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash); - - void ReadFromStream(IOStream &rStream, int Timeout); - void WriteToStream(IOStream &rStream) const; - - const BackupStoreFilename &GetName() const {return mName;} - box_time_t GetModificationTime() const {return mModificationTime;} - int64_t GetObjectID() const {return mObjectID;} - int64_t GetSizeInBlocks() const {return mSizeInBlocks;} - int16_t GetFlags() const {return mFlags;} - void AddFlags(int16_t Flags) {mFlags |= Flags;} - void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;} - - // Some things can be changed - void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;} - void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;} - - // Attributes - bool HasAttributes() const {return !mAttributes.IsEmpty();} - void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;} - const StreamableMemBlock &GetAttributes() const {return mAttributes;} - uint64_t GetAttributesHash() const {return mAttributesHash;} - - // Marks - // The lowest mark number a version of a file of this name has ever had - uint32_t GetMinMarkNumber() const {return mMinMarkNumber;} - // The mark number on this file - uint32_t GetMarkNumber() const {return mMarkNumber;} - - // Make sure these flags are synced with those in backupprocotol.txt - // ListDirectory command - enum - { - Flags_INCLUDE_EVERYTHING = -1, - Flags_EXCLUDE_NOTHING = 0, - Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below! - Flags_File = 1, - Flags_Dir = 2, - Flags_Deleted = 4, - Flags_OldVersion = 8, - Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion - }; - // characters for textual listing of files -- see bbackupquery/BackupQueries - #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR" - - // convenience methods - bool inline IsDir() - { - return GetFlags() & Flags_Dir; - } - bool inline IsFile() - { - return GetFlags() & Flags_File; - } - bool inline IsOld() - { - return GetFlags() & Flags_OldVersion; - } - bool inline IsDeleted() - { - return GetFlags() & Flags_Deleted; - } - bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) - { - return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet)) - && ((mFlags & FlagsNotToBeSet) == 0); - }; - - // Get dependency info - // new version this depends on - int64_t GetDependsNewer() const {return mDependsNewer;} - void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;} - // older version which depends on this - int64_t GetDependsOlder() const {return mDependsOlder;} - void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;} - - // Dependency info saving - bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;} - void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout); - void WriteToStreamDependencyInfo(IOStream &rStream) const; - - private: - BackupStoreFilename mName; - box_time_t mModificationTime; - int64_t mObjectID; - int64_t mSizeInBlocks; - int16_t mFlags; - uint64_t mAttributesHash; - StreamableMemBlock mAttributes; - uint32_t mMinMarkNumber; - uint32_t mMarkNumber; - - uint64_t mDependsNewer; // new version this depends on - uint64_t mDependsOlder; // older version which depends on this - }; - - void ReadFromStream(IOStream &rStream, int Timeout); - void WriteToStream(IOStream &rStream, - int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, - int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING, - bool StreamAttributes = true, bool StreamDependencyInfo = true) const; - - Entry *AddEntry(const Entry &rEntryToCopy); - Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime); - void DeleteEntry(int64_t ObjectID); - Entry *FindEntryByID(int64_t ObjectID) const; - - int64_t GetObjectID() const {return mObjectID;} - int64_t GetContainerID() const {return mContainerID;} - - // Need to be able to update the container ID when moving objects - void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;} - - // Purely for use of server -- not serialised into streams - int64_t GetRevisionID() const {return mRevisionID;} - void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;} - - unsigned int GetNumberOfEntries() const {return mEntries.size();} - - // User info -- not serialised into streams - int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;} - void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;} - - // Attributes - bool HasAttributes() const {return !mAttributes.IsEmpty();} - void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;} - const StreamableMemBlock &GetAttributes() const {return mAttributes;} - box_time_t GetAttributesModTime() const {return mAttributesModTime;} - - class Iterator - { - public: - Iterator(const BackupStoreDirectory &rDir) - : mrDir(rDir), i(rDir.mEntries.begin()) - { - } - - BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) - { - // Skip over things which don't match the required flags - while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) - { - ++i; - } - // Not the last one? - if(i == mrDir.mEntries.end()) - { - return 0; - } - // Return entry, and increment - return (*(i++)); - } - - // WARNING: This function is really very inefficient. - // Only use when you want to look up ONE filename, not in a loop looking up lots. - // In a looping situation, cache the decrypted filenames in another memory structure. - BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) - { - // Skip over things which don't match the required flags or filename - while( (i != mrDir.mEntries.end()) - && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) - || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) ) - { - ++i; - } - // Not the last one? - if(i == mrDir.mEntries.end()) - { - return 0; - } - // Return entry, and increment - return (*(i++)); - } - - private: - const BackupStoreDirectory &mrDir; - std::vector<Entry*>::const_iterator i; - }; - - friend class Iterator; - - class ReverseIterator - { - public: - ReverseIterator(const BackupStoreDirectory &rDir) - : mrDir(rDir), i(rDir.mEntries.rbegin()) - { - } - - BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) - { - // Skip over things which don't match the required flags - while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) - { - ++i; - } - // Not the last one? - if(i == mrDir.mEntries.rend()) - { - return 0; - } - // Return entry, and increment - return (*(i++)); - } - - private: - const BackupStoreDirectory &mrDir; - std::vector<Entry*>::const_reverse_iterator i; - }; - - friend class ReverseIterator; - - // For recovery of the store - // Implemented in BackupStoreCheck2.cpp - bool CheckAndFix(); - void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags); - bool NameInUse(const BackupStoreFilename &rName); - // Don't use these functions in normal code! - - // For testing - void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;} - - // Debug and diagonistics - void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere - -private: - int64_t mRevisionID; - int64_t mObjectID; - int64_t mContainerID; - std::vector<Entry*> mEntries; - box_time_t mAttributesModTime; - StreamableMemBlock mAttributes; - int64_t mUserInfo1; -}; - -#endif // BACKUPSTOREDIRECTORY__H - diff --git a/lib/backupclient/BackupStoreException.h b/lib/backupclient/BackupStoreException.h deleted file mode 100644 index 981dfa60..00000000 --- a/lib/backupclient/BackupStoreException.h +++ /dev/null @@ -1,17 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreException.h -// Purpose: Exception -// Created: 2003/07/08 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREEXCEPTION__H -#define BACKUPSTOREEXCEPTION__H - -// Compatibility -#include "autogen_BackupStoreException.h" - -#endif // BACKUPSTOREEXCEPTION__H - diff --git a/lib/backupclient/BackupStoreException.txt b/lib/backupclient/BackupStoreException.txt deleted file mode 100644 index ece772c0..00000000 --- a/lib/backupclient/BackupStoreException.txt +++ /dev/null @@ -1,72 +0,0 @@ -EXCEPTION BackupStore 4 - -Internal 0 -BadAccountDatabaseFile 1 -AccountDatabaseNoSuchEntry 2 -InvalidBackupStoreFilename 3 -UnknownFilenameEncoding 4 -CouldntReadEntireStructureFromStream 5 -BadDirectoryFormat 6 -CouldNotFindEntryInDirectory 7 -OutputFileAlreadyExists 8 -OSFileError 9 -StreamDoesntHaveRequiredFeatures 10 -BadBackupStoreFile 11 -CouldNotLoadStoreInfo 12 -BadStoreInfoOnLoad 13 -StoreInfoIsReadOnly 14 -StoreInfoDirNotInList 15 -StoreInfoBlockDeltaMakesValueNegative 16 -DirectoryHasBeenDeleted 17 -StoreInfoNotInitialised 18 -StoreInfoAlreadyLoaded 19 -StoreInfoNotLoaded 20 -ReadFileFromStreamTimedOut 21 -FileWrongSizeAfterBeingStored 22 -AddedFileDoesNotVerify 23 -StoreInfoForWrongAccount 24 -ContextIsReadOnly 25 -AttributesNotLoaded 26 -AttributesNotUnderstood 27 -WrongServerVersion 28 # client side -ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account. -NameAlreadyExistsInDirectory 30 -BerkelyDBFailure 31 # client side -InodeMapIsReadOnly 32 # client side -InodeMapNotOpen 33 # client side -FilenameEncryptionKeyNotKnown 34 -FilenameEncryptionNoKeyForSpecifiedMethod 35 -FilenameEncryptionNotSetup 36 -CouldntLoadClientKeyMaterial 37 -BadEncryptedAttributes 38 -EncryptedAttributesHaveUnknownEncoding 39 -OutputSizeTooSmallForChunk 40 -BadEncodedChunk 41 -NotEnoughSpaceToDecodeChunk 42 -ChunkHasUnknownEncoding 43 -ChunkContainsBadCompressedData 44 -CantWriteToEncodedFileStream 45 -Temp_FileEncodeStreamDidntReadBuffer 46 -CantWriteToDecodedFileStream 47 -WhenDecodingExpectedToReadButCouldnt 48 -BackupStoreFileFailedIntegrityCheck 49 -ThereIsNoDataInASymLink 50 -IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51 -BlockEntryEncodingDidntGiveExpectedLength 52 -CouldNotFindUnusedIDDuringAllocation 53 -AddedFileExceedsStorageLimit 54 -CannotDiffAnIncompleteStoreFile 55 -CannotDecodeDiffedFilesWithoutCombining 56 -FailedToReadBlockOnCombine 57 -OnCombineFromFileIsIncomplete 58 -BadNotifySysadminEventCode 59 -InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60 -CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server? -AttributeHashSecretNotSet 62 -AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES -SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection. -IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related -DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory -PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. -UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. -MultiplyReferencedObject 69 Attempted to modify an object with multiple references, should be uncloned first diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp deleted file mode 100644 index bd62b7ba..00000000 --- a/lib/backupclient/BackupStoreFile.cpp +++ /dev/null @@ -1,1559 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFile.cpp -// Purpose: Utils for manipulating files -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - -#include <sys/stat.h> -#include <string.h> -#include <new> -#include <string.h> - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - #include <stdio.h> -#endif - -#include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreFileCryptVar.h" -#include "BackupStoreFilename.h" -#include "BackupStoreException.h" -#include "IOStream.h" -#include "Guards.h" -#include "FileModificationTime.h" -#include "FileStream.h" -#include "BackupClientFileAttributes.h" -#include "BackupStoreObjectMagic.h" -#include "Compress.h" -#include "CipherContext.h" -#include "CipherBlowfish.h" -#include "CipherAES.h" -#include "BackupStoreConstants.h" -#include "CollectInBufferStream.h" -#include "RollingChecksum.h" -#include "MD5Digest.h" -#include "ReadGatherStream.h" -#include "Random.h" -#include "BackupStoreFileEncodeStream.h" -#include "Logging.h" - -#include "MemLeakFindOn.h" - -using namespace BackupStoreFileCryptVar; - -// How big a buffer to use for copying files -#define COPY_BUFFER_SIZE (8*1024) - -// Statistics -BackupStoreFileStats BackupStoreFile::msStats = {0,0,0}; - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - bool sWarnedAboutBackwardsCompatiblity = false; -#endif - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &) -// Purpose: Encode a file into something for storing on file server. -// Requires a real filename so full info can be stored. -// -// Returns a stream. Most of the work is done by the stream -// when data is actually requested -- the file will be held -// open until the stream is deleted or the file finished. -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -std::auto_ptr<IOStream> BackupStoreFile::EncodeFile( - const char *Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, - int64_t *pModificationTime, - ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) -{ - // Create the stream - std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); - - // Do the initial setup - ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, - 0 /* no recipe, just encode */, - ContainerID, rStoreFilename, pModificationTime, pLogger, - pRunStatusProvider); - - // Return the stream for the caller - return stream; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &) -// Purpose: Verify that an encoded file meets the format -// requirements. Doesn't verify that the data is intact -// and can be decoded. Optionally returns the ID of the -// file which it is diffed from, and the (original) -// container ID. -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut) -{ - // Get the size of the file - int64_t fileSize = rFile.BytesLeftToRead(); - if(fileSize == IOStream::SizeOfStreamUnknown) - { - THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) - } - - // Get the header... - file_StreamFormat hdr; - if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) - { - // Couldn't read header - return false; - } - - // Check magic number - if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif - ) - { - return false; - } - - // Get a filename, see if it loads OK - try - { - BackupStoreFilename fn; - fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); - } - catch(...) - { - // an error occured while reading it, so that's not good - return false; - } - - // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not - try - { - int32_t size_s; - if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) - { - THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) - } - int size = ntohl(size_s); - // Skip forward the size - rFile.Seek(size, IOStream::SeekType_Relative); - } - catch(...) - { - // an error occured while reading it, so that's not good - return false; - } - - // Get current position in file -- the end of the header - int64_t headerEnd = rFile.GetPosition(); - - // Get number of blocks - int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - - // Calculate where the block index will be, check it's reasonable - int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - if(blockIndexLoc < headerEnd) - { - // Not enough space left for the block index, let alone the blocks themselves - return false; - } - - // Load the block index header - rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute); - file_BlockIndexHeader blkhdr; - if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */)) - { - // Couldn't read block index header -- assume bad file - return false; - } - - // Check header - if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 -#endif - ) - || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != numBlocks) - { - // Bad header -- either magic value or number of blocks is wrong - return false; - } - - // Flag for recording whether a block is referenced from another file - bool blockFromOtherFileReferenced = false; - - // Read the index, checking that the length values all make sense - int64_t currentBlockStart = headerEnd; - for(int64_t b = 0; b < numBlocks; ++b) - { - // Read block entry - file_BlockIndexEntry blk; - if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */)) - { - // Couldn't read block index entry -- assume bad file - return false; - } - - // Check size and location - int64_t blkSize = box_ntoh64(blk.mEncodedSize); - if(blkSize <= 0) - { - // Mark that this file references another file - blockFromOtherFileReferenced = true; - } - else - { - // This block is actually in this file - if((currentBlockStart + blkSize) > blockIndexLoc) - { - // Encoded size makes the block run over the index - return false; - } - - // Move the current block start ot the end of this block - currentBlockStart += blkSize; - } - } - - // Check that there's no empty space - if(currentBlockStart != blockIndexLoc) - { - return false; - } - - // Check that if another block is references, then the ID is there, and if one isn't there is no ID. - int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); - if((otherID != 0 && blockFromOtherFileReferenced == false) - || (otherID == 0 && blockFromOtherFileReferenced == true)) - { - // Doesn't look good! - return false; - } - - // Does the caller want the other ID? - if(pDiffFromObjectIDOut) - { - *pDiffFromObjectIDOut = otherID; - } - - // Does the caller want the container ID? - if(pContainerIDOut) - { - *pContainerIDOut = box_ntoh64(hdr.mContainerID); - } - - // Passes all tests - return true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodeFile(IOStream &, const char *) -// Purpose: Decode a file. Will set file attributes. File must not exist. -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) -{ - // Does file exist? - EMU_STRUCT_STAT st; - if(EMU_STAT(DecodedFilename, &st) == 0) - { - THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) - } - - // Try, delete output file if error - try - { - // Make a stream for outputting this file - FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL); - - // Get the decoding stream - std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr)); - - // Is it a symlink? - if(!stream->IsSymLink()) - { - // Copy it out to the file - stream->CopyStreamTo(out); - } - - out.Close(); - - // The stream might have uncertain size, in which case - // we need to drain it to get the - // Protocol::ProtocolStreamHeader_EndOfStream byte - // out of our connection stream. - char buffer[1]; - int drained = rEncodedFile.Read(buffer, 1); - - // The Read will return 0 if we are actually at the end - // of the stream, but some tests decode files directly, - // in which case we are actually positioned at the start - // of the block index. I hope that reading an extra byte - // doesn't hurt! - // ASSERT(drained == 0); - - // Write the attributes - try - { - stream->GetAttributes().WriteAttributes(DecodedFilename); - } - catch (std::exception& e) - { - BOX_WARNING("Failed to restore attributes on " << - DecodedFilename << ": " << e.what()); - } - } - catch(...) - { - ::unlink(DecodedFilename); - throw; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *) -// Purpose: Return a stream which will decode the encrypted file data on the fly. -// Accepts streams in block index first, or main header first, order. In the latter case, -// the stream must be Seek()able. -// -// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow -// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation. -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) -{ - // Create stream - std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout)); - - // Get it ready - stream->Setup(pAlterativeAttr); - - // Return to caller - return stream; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int) -// Purpose: Constructor -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout) - : mrEncodedFile(rEncodedFile), - mTimeout(Timeout), - mNumBlocks(0), - mpBlockIndex(0), - mpEncodedData(0), - mpClearData(0), - mClearDataSize(0), - mCurrentBlock(-1), - mCurrentBlockClearSize(0), - mPositionInCurrentBlock(0), - mEntryIVBase(42) // different to default value in the encoded stream! -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - , mIsOldVersion(false) -#endif -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::~DecodedStream() -// Purpose: Desctructor -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -BackupStoreFile::DecodedStream::~DecodedStream() -{ - // Free any allocated memory - if(mpBlockIndex) - { - ::free(mpBlockIndex); - } - if(mpEncodedData) - { - BackupStoreFile::CodingChunkFree(mpEncodedData); - } - if(mpClearData) - { - ::free(mpClearData); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *) -// Purpose: Get the stream ready to decode -- reads in headers -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr) -{ - // Get the size of the file - int64_t fileSize = mrEncodedFile.BytesLeftToRead(); - - // Get the magic number to work out which order the stream is in - int32_t magic; - if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read magic value - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - - bool inFileOrder = true; - switch(ntohl(magic)) - { -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - case OBJECTMAGIC_FILE_MAGIC_VALUE_V0: - mIsOldVersion = true; - // control flows on -#endif - case OBJECTMAGIC_FILE_MAGIC_VALUE_V1: - inFileOrder = true; - break; - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0: - mIsOldVersion = true; - // control flows on -#endif - case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1: - inFileOrder = false; - break; - - default: - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // If not in file order, then the index list must be read now - if(!inFileOrder) - { - ReadBlockIndex(true /* have already read and verified the magic number */); - } - - // Get header - file_StreamFormat hdr; - if(inFileOrder) - { - // Read the header, without the magic number - if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic), - 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - // Put in magic number - hdr.mMagicValue = magic; - } - else - { - // Not in file order, so need to read the full header - if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - } - - // Check magic number - if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif - ) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Get the filename - mFilename.ReadFromStream(mrEncodedFile, mTimeout); - - // Get the attributes (either from stream, or supplied attributes) - if(pAlterativeAttr != 0) - { - // Read dummy attributes - BackupClientFileAttributes attr; - attr.ReadFromStream(mrEncodedFile, mTimeout); - - // Set to supplied attributes - mAttributes = *pAlterativeAttr; - } - else - { - // Read the attributes from the stream - mAttributes.ReadFromStream(mrEncodedFile, mTimeout); - } - - // If it is in file order, go and read the file attributes - // Requires that the stream can seek - if(inFileOrder) - { - // Make sure the file size is known - if(fileSize == IOStream::SizeOfStreamUnknown) - { - THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) - } - - // Store current location (beginning of encoded blocks) - int64_t endOfHeaderPos = mrEncodedFile.GetPosition(); - - // Work out where the index is - int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - - // Seek to that position - mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute); - - // Read the block index - ReadBlockIndex(false /* magic number still to be read */); - - // Seek back to the end of header position, ready for reading the chunks - mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute); - } - - // Check view of blocks from block header and file header match - if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks)) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Need to allocate some memory for the two blocks for reading encoded data, and clear data - if(mNumBlocks > 0) - { - // Find the maximum encoded data size - int32_t maxEncodedDataSize = 0; - const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; - ASSERT(entry != 0); - for(int64_t e = 0; e < mNumBlocks; e++) - { - // Get the clear and encoded size - int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize); - ASSERT(encodedSize > 0); - - // Larger? - if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize; - } - - // Allocate those blocks! - mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32); - - // Allocate the block for the clear data, using the hint from the header. - // If this is wrong, things will exception neatly later on, so it can't be used - // to do anything more than cause an error on downloading. - mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32; - mpClearData = (uint8_t*)::malloc(mClearDataSize); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool) -// Purpose: Read the block index from the stream, and store in internal buffer (minus header) -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) -{ - // Header - file_BlockIndexHeader blkhdr; - - // Read it in -- way depends on how whether the magic number has already been read - if(MagicAlreadyRead) - { - // Read the header, without the magic number - if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue), - 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - } - else - { - // Magic not already read, so need to read the full header - if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - - // Check magic value - if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 -#endif - ) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - } - - // Get the number of blocks out of the header - mNumBlocks = box_ntoh64(blkhdr.mNumBlocks); - - // Read the IV base - mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase); - - // Load the block entries in? - if(mNumBlocks > 0) - { - // How big is the index? - int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks; - - // Allocate some memory - mpBlockIndex = ::malloc(indexSize); - if(mpBlockIndex == 0) - { - throw std::bad_alloc(); - } - - // Read it in - if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::Read(void *, int, int) -// Purpose: As interface. Reads decrpyted data. -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) -{ - // Symlinks don't have data. So can't read it. Not even zero bytes. - if(IsSymLink()) - { - // Don't allow reading in this case - THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink); - } - - // Already finished? - if(mCurrentBlock >= mNumBlocks) - { - // At end of stream, nothing to do - return 0; - } - - int bytesToRead = NBytes; - uint8_t *output = (uint8_t*)pBuffer; - - while(bytesToRead > 0 && mCurrentBlock < mNumBlocks) - { - // Anything left in the current block? - if(mPositionInCurrentBlock < mCurrentBlockClearSize) - { - // Copy data out of this buffer - int s = mCurrentBlockClearSize - mPositionInCurrentBlock; - if(s > bytesToRead) s = bytesToRead; // limit to requested data - - // Copy - ::memcpy(output, mpClearData + mPositionInCurrentBlock, s); - - // Update positions - output += s; - mPositionInCurrentBlock += s; - bytesToRead -= s; - } - - // Need to get some more data? - if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize) - { - // Number of next block - ++mCurrentBlock; - if(mCurrentBlock >= mNumBlocks) - { - // Stop now! - break; - } - - // Get the size from the block index - const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; - int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize); - if(encodedSize <= 0) - { - // The caller is attempting to decode a file which is the direct result of a diff - // operation, and so does not contain all the data. - // It needs to be combined with the previous version first. - THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining) - } - - // Load in next block - if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - - // Decode the data - mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize); - - // Calculate IV for this entry - uint64_t iv = mEntryIVBase; - iv += mCurrentBlock; - // Convert to network byte order before encrypting with it, so that restores work on - // platforms with different endiannesses. - iv = box_hton64(iv); - sBlowfishDecryptBlockEntry.SetIV(&iv); - - // Decrypt the encrypted section - file_BlockIndexEntryEnc entryEnc; - int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), - entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); - if(sectionSize != sizeof(entryEnc)) - { - THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) - } - - // Make sure this is the right size - if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) - { -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - if(!mIsOldVersion) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the - // IV for the encrypted section. Try again, with the thing the other way round - iv = box_swap64(iv); - sBlowfishDecryptBlockEntry.SetIV(&iv); - int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), - entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); - if(sectionSize != sizeof(entryEnc)) - { - THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) - } - if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - else - { - // Warn and log this issue - if(!sWarnedAboutBackwardsCompatiblity) - { - BOX_WARNING("WARNING: Decoded one or more files using backwards compatibility mode for block index."); - sWarnedAboutBackwardsCompatiblity = true; - } - } -#else - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) -#endif - } - - // Check the digest - MD5Digest md5; - md5.Add(mpClearData, mCurrentBlockClearSize); - md5.Finish(); - if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum)) - { - THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck) - } - - // Set vars to say what's happening - mPositionInCurrentBlock = 0; - } - } - - ASSERT(bytesToRead >= 0); - ASSERT(bytesToRead <= NBytes); - - return NBytes - bytesToRead; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::IsSymLink() -// Purpose: Is the unencoded file actually a symlink? -// Created: 10/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFile::DecodedStream::IsSymLink() -{ - // First, check in with the attributes - if(!mAttributes.IsSymLink()) - { - return false; - } - - // So the attributes think it is a symlink. - // Consistency check... - if(mNumBlocks != 0) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - return true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::Write(const void *, int) -// Purpose: As interface. Throws exception, as you can't write to this stream. -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes) -{ - THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream) -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::StreamDataLeft() -// Purpose: As interface. Any data left? -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFile::DecodedStream::StreamDataLeft() -{ - return mCurrentBlock < mNumBlocks; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::StreamClosed() -// Purpose: As interface. Always returns true, no writing allowed. -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFile::DecodedStream::StreamClosed() -{ - // Can't write to this stream! - return true; -} - - - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::SetBlowfishKey(const void *, int) -// Purpose: Static. Sets the key to use for encryption and decryption. -// Created: 7/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength) -{ - // IVs set later - sBlowfishEncrypt.Reset(); - sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); - sBlowfishDecrypt.Reset(); - sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); - - sBlowfishEncryptBlockEntry.Reset(); - sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); - sBlowfishEncryptBlockEntry.UsePadding(false); - sBlowfishDecryptBlockEntry.Reset(); - sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); - sBlowfishDecryptBlockEntry.UsePadding(false); -} - - -#ifndef HAVE_OLD_SSL -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::SetAESKey(const void *, int) -// Purpose: Sets the AES key to use for file data encryption. Will select AES as -// the cipher to use when encrypting. -// Created: 27/4/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength) -{ - // Setup context - sAESEncrypt.Reset(); - sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); - sAESDecrypt.Reset(); - sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); - - // Set encryption to use this key, instead of the "default" blowfish key - spEncrypt = &sAESEncrypt; - sEncryptCipherType = HEADER_AES_ENCODING; -} -#endif - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int) -// Purpose: The maximum output size of a block, given the chunk size -// Created: 7/12/03 -// -// -------------------------------------------------------------------------- -int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize) -{ - // Calculate... the maximum size of output by first the largest it could be after compression, - // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck - // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing - // an problem where a rather non-compressable file didn't fit in a block buffer.) - return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1 - + sBlowfishEncrypt.GetIVLength() + 128; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &) -// Purpose: Encodes a chunk (encryption, possible compressed beforehand) -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput) -{ - ASSERT(spEncrypt != 0); - - // Check there's some space in the output block - if(rOutput.mBufferSize < 256) - { - rOutput.Reallocate(256); - } - - // Check alignment of the block - ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); - - // Want to compress it? - bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE); - - // Build header - uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT; - if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED; - - // Store header - rOutput.mpBuffer[0] = header; - int outOffset = 1; - - // Setup cipher, and store the IV - int ivLen = 0; - const void *iv = spEncrypt->SetRandomIV(ivLen); - ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen); - outOffset += ivLen; - - // Start encryption process - spEncrypt->Begin(); - - #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \ - { \ - if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \ - { \ - rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \ - } \ - } - - // Encode the chunk - if(compressChunk) - { - // buffer to compress into - uint8_t buffer[2048]; - - // Set compressor with all the chunk as an input - Compress<true> compress; - compress.Input(Chunk, ChunkSize); - compress.FinishInput(); - - // Get and encrypt output - while(!compress.OutputHasFinished()) - { - int s = compress.Output(buffer, sizeof(buffer)); - if(s > 0) - { - ENCODECHUNK_CHECK_SPACE(s) - outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); - } - else - { - // Should never happen, as we put all the input in in one go. - // So if this happens, it means there's a logical problem somewhere - THROW_EXCEPTION(BackupStoreException, Internal) - } - } - ENCODECHUNK_CHECK_SPACE(16) - outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); - } - else - { - // Straight encryption - ENCODECHUNK_CHECK_SPACE(ChunkSize) - outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize); - ENCODECHUNK_CHECK_SPACE(16) - outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); - } - - ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check - - return outOffset; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int) -// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find -// the extra output buffer size needed before calling. -// See notes in EncodeChunk() for notes re alignment of the -// encoded data. -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize) -{ - // Check alignment of the encoded block - ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); - - // First check - if(EncodedSize < 1) - { - THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) - } - - const uint8_t *input = (uint8_t*)Encoded; - - // Get header, make checks, etc - uint8_t header = input[0]; - bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED; - uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT); - if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING) - { - THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding) - } - -#ifndef HAVE_OLD_SSL - // Choose cipher - CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt); -#else - // AES not supported with this version of OpenSSL - if(encodingType == HEADER_AES_ENCODING) - { - THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL) - } - CipherContext &cipher(sBlowfishDecrypt); -#endif - - // Check enough space for header, an IV and one byte of input - int ivLen = cipher.GetIVLength(); - if(EncodedSize < (1 + ivLen + 1)) - { - THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) - } - - // Set IV in decrypt context, and start - cipher.SetIV(input + 1); - cipher.Begin(); - - // Setup vars for code - int inOffset = 1 + ivLen; - uint8_t *output = (uint8_t*)Output; - int outOffset = 0; - - // Do action - if(chunkCompressed) - { - // Do things in chunks - uint8_t buffer[2048]; - int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer)); - - // Decompressor - Compress<false> decompress; - - while(inOffset < EncodedSize) - { - // Decrypt a block - int bl = inputBlockLen; - if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long - int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl); - inOffset += bl; - - // Decompress the decrypted data - if(s > 0) - { - decompress.Input(buffer, s); - int os = 0; - do - { - os = decompress.Output(output + outOffset, OutputSize - outOffset); - outOffset += os; - } while(os > 0); - - // Check that there's space left in the output buffer -- there always should be - if(outOffset >= OutputSize) - { - THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) - } - } - } - - // Get any compressed data remaining in the cipher context and compression - int s = cipher.Final(buffer, sizeof(buffer)); - decompress.Input(buffer, s); - decompress.FinishInput(); - while(!decompress.OutputHasFinished()) - { - int os = decompress.Output(output + outOffset, OutputSize - outOffset); - outOffset += os; - - // Check that there's space left in the output buffer -- there always should be - if(outOffset >= OutputSize) - { - THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) - } - } - } - else - { - // Easy decryption - outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset); - outOffset += cipher.Final(output + outOffset, OutputSize - outOffset); - } - - return outOffset; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool) -// Purpose: Returns a stream which gives a Stream order version of the encoded file. -// If TakeOwnership == true, then the input stream will be deleted when the -// returned stream is deleted. -// The input stream must be seekable. -// Created: 10/12/03 -// -// -------------------------------------------------------------------------- -std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership) -{ - ASSERT(pStream != 0); - - // Get the size of the file - int64_t fileSize = pStream->BytesLeftToRead(); - if(fileSize == IOStream::SizeOfStreamUnknown) - { - THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) - } - - // Read the header - int bytesRead = 0; - file_StreamFormat hdr; - bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead); - - // Seek backwards to put the file pointer back where it was before we started this - pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative); - - // Check we got a block - if(!readBlock) - { - // Couldn't read header -- assume file bad - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Check magic number - if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif - ) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Get number of blocks - int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - - // Calculate where the block index will be, check it's reasonable - int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - int64_t blockIndexLoc = fileSize - blockIndexSize; - if(blockIndexLoc < 0) - { - // Doesn't look good! - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Build a reordered stream - std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership)); - - // Set it up... - ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get())); - int component = rreordered.AddComponent(pStream); - // Send out the block index - rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc); - // And then the rest of the file - rreordered.AddBlock(component, blockIndexLoc, true, 0); - - return reordered; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::ResetStats() -// Purpose: Reset the gathered statistics -// Created: 20/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::ResetStats() -{ - msStats.mBytesInEncodedFiles = 0; - msStats.mBytesAlreadyOnServer = 0; - msStats.mTotalFileStreamSize = 0; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &) -// Purpose: Compares the contents of a file against the checksums contained in the -// block index. Returns true if the checksums match, meaning the file is -// extremely likely to match the original. Will always consume the entire index. -// Created: 21/1/04 -// -// -------------------------------------------------------------------------- -bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout) -{ - // is it a symlink? - bool sourceIsSymlink = false; - { - EMU_STRUCT_STAT st; - if(EMU_LSTAT(Filename, &st) == -1) - { - THROW_EXCEPTION(CommonException, OSFileError) - } - if((st.st_mode & S_IFMT) == S_IFLNK) - { - sourceIsSymlink = true; - } - } - - // Open file, if it's not a symlink - std::auto_ptr<FileStream> in; - if(!sourceIsSymlink) - { - in.reset(new FileStream(Filename)); - } - - // Read header - file_BlockIndexHeader hdr; - if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Check magic - if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0) -#endif - ) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0); -#endif - - // Get basic information - int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase); - - //TODO: Verify that these sizes look reasonable - - // setup - void *data = 0; - int32_t dataSize = -1; - bool matches = true; - int64_t totalSizeInBlockIndex = 0; - - try - { - for(int64_t b = 0; b < numBlocks; ++b) - { - // Read an entry from the stream - file_BlockIndexEntry entry; - if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) - { - // Couldn't read entry - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Calculate IV for this entry - uint64_t iv = entryIVBase; - iv += b; - iv = box_hton64(iv); -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - if(isOldVersion) - { - // Reverse the IV for compatibility - iv = box_swap64(iv); - } -#endif - sBlowfishDecryptBlockEntry.SetIV(&iv); - - // Decrypt the encrypted section - file_BlockIndexEntryEnc entryEnc; - int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), - entry.mEnEnc, sizeof(entry.mEnEnc)); - if(sectionSize != sizeof(entryEnc)) - { - THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) - } - - // Size of block - int32_t blockClearSize = ntohl(entryEnc.mSize); - if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024)) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - totalSizeInBlockIndex += blockClearSize; - - // Make sure there's enough memory allocated to load the block in - if(dataSize < blockClearSize) - { - // Too small, free the block if it's already allocated - if(data != 0) - { - ::free(data); - data = 0; - } - // Allocate a block - data = ::malloc(blockClearSize + 128); - if(data == 0) - { - throw std::bad_alloc(); - } - dataSize = blockClearSize + 128; - } - - // Load in the block from the file, if it's not a symlink - if(!sourceIsSymlink) - { - if(in->Read(data, blockClearSize) != blockClearSize) - { - // Not enough data left in the file, can't possibly match - matches = false; - } - else - { - // Check the checksum - MD5Digest md5; - md5.Add(data, blockClearSize); - md5.Finish(); - if(!md5.DigestMatches(entryEnc.mStrongChecksum)) - { - // Checksum didn't match - matches = false; - } - } - } - - // Keep on going regardless, to make sure the entire block index stream is read - // -- must always be consistent about what happens with the stream. - } - } - catch(...) - { - // clean up in case of errors - if(data != 0) - { - ::free(data); - data = 0; - } - throw; - } - - // free block - if(data != 0) - { - ::free(data); - data = 0; - } - - // Check for data left over if it's not a symlink - if(!sourceIsSymlink) - { - // Anything left to read in the file? - if(in->BytesLeftToRead() != 0) - { - // File has extra data at the end - matches = false; - } - } - - // Symlinks must have zero size on server - if(sourceIsSymlink) - { - matches = (totalSizeInBlockIndex == 0); - } - - return matches; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer() -// Purpose: Constructor -// Created: 25/11/04 -// -// -------------------------------------------------------------------------- -BackupStoreFile::EncodingBuffer::EncodingBuffer() - : mpBuffer(0), - mBufferSize(0) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer() -// Purpose: Destructor -// Created: 25/11/04 -// -// -------------------------------------------------------------------------- -BackupStoreFile::EncodingBuffer::~EncodingBuffer() -{ - if(mpBuffer != 0) - { - BackupStoreFile::CodingChunkFree(mpBuffer); - mpBuffer = 0; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodingBuffer::Allocate(int) -// Purpose: Do initial allocation of block -// Created: 25/11/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::EncodingBuffer::Allocate(int Size) -{ - ASSERT(mpBuffer == 0); - uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size); - if(buffer == 0) - { - throw std::bad_alloc(); - } - mpBuffer = buffer; - mBufferSize = Size; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodingBuffer::Reallocate(int) -// Purpose: Reallocate the block. Try not to call this, it has to copy -// the entire contents as the block can't be reallocated straight. -// Created: 25/11/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) -{ - BOX_TRACE("Reallocating EncodingBuffer from " << mBufferSize << - " to " << NewSize); - ASSERT(mpBuffer != 0); - uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize); - if(buffer == 0) - { - throw std::bad_alloc(); - } - // Copy data - ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize); - - // Free old - BackupStoreFile::CodingChunkFree(mpBuffer); - - // Store new buffer - mpBuffer = buffer; - mBufferSize = NewSize; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: DiffTimer::DiffTimer(); -// Purpose: Constructor -// Created: 2005/02/01 -// -// -------------------------------------------------------------------------- -DiffTimer::DiffTimer() -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: DiffTimer::DiffTimer(); -// Purpose: Destructor -// Created: 2005/02/01 -// -// -------------------------------------------------------------------------- -DiffTimer::~DiffTimer() -{ -} diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h deleted file mode 100644 index f5bc1924..00000000 --- a/lib/backupclient/BackupStoreFile.h +++ /dev/null @@ -1,231 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFile.h -// Purpose: Utils for manipulating files -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILE__H -#define BACKUPSTOREFILE__H - -#include <cstdlib> -#include <memory> -#include <cstdlib> - -#include "BackupClientFileAttributes.h" -#include "BackupStoreFilename.h" -#include "IOStream.h" -#include "ReadLoggingStream.h" -#include "RunStatusProvider.h" - -typedef struct -{ - int64_t mBytesInEncodedFiles; - int64_t mBytesAlreadyOnServer; - int64_t mTotalFileStreamSize; -} BackupStoreFileStats; - -// Uncomment to disable backwards compatibility -//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - - -// Output buffer to EncodeChunk and input data to DecodeChunk must -// have specific alignment, see function comments. -#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16 -#define BACKUPSTOREFILE_CODING_OFFSET 15 - -// Have some memory allocation commands, note closing "Off" at end of file. -#include "MemLeakFindOn.h" - -// -------------------------------------------------------------------------- -// -// Class -// Name: DiffTimer -// Purpose: Interface for classes that can keep track of diffing time, -// and send SSL keepalive messages -// Created: 2006/01/19 -// -// -------------------------------------------------------------------------- -class DiffTimer -{ -public: - DiffTimer(); - virtual ~DiffTimer(); -public: - virtual void DoKeepAlive() = 0; - virtual int GetMaximumDiffingTime() = 0; - virtual bool IsManaged() = 0; -}; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupStoreFile -// Purpose: Class to hold together utils for manipulating files. -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -class BackupStoreFile -{ -public: - class DecodedStream : public IOStream - { - friend class BackupStoreFile; - private: - DecodedStream(IOStream &rEncodedFile, int Timeout); - DecodedStream(const DecodedStream &); // not allowed - DecodedStream &operator=(const DecodedStream &); // not allowed - public: - ~DecodedStream(); - - // Stream functions - virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); - virtual bool StreamDataLeft(); - virtual bool StreamClosed(); - - // Accessor functions - const BackupClientFileAttributes &GetAttributes() {return mAttributes;} - const BackupStoreFilename &GetFilename() {return mFilename;} - int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests - - bool IsSymLink(); - - private: - void Setup(const BackupClientFileAttributes *pAlterativeAttr); - void ReadBlockIndex(bool MagicAlreadyRead); - - private: - IOStream &mrEncodedFile; - int mTimeout; - BackupClientFileAttributes mAttributes; - BackupStoreFilename mFilename; - int64_t mNumBlocks; - void *mpBlockIndex; - uint8_t *mpEncodedData; - uint8_t *mpClearData; - int mClearDataSize; - int mCurrentBlock; - int mCurrentBlockClearSize; - int mPositionInCurrentBlock; - uint64_t mEntryIVBase; -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - bool mIsOldVersion; -#endif - }; - - - // Main interface - static std::auto_ptr<IOStream> EncodeFile - ( - const char *Filename, - int64_t ContainerID, const BackupStoreFilename &rStoreFilename, - int64_t *pModificationTime = 0, - ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL - ); - static std::auto_ptr<IOStream> EncodeFileDiff - ( - const char *Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, - int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, - int Timeout, - DiffTimer *pDiffTimer, - int64_t *pModificationTime = 0, - bool *pIsCompletelyDifferent = 0 - ); - static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); - static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); - static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); - static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0); - static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); - static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); - static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout); - static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false); - - // Stream manipulation - static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership); - static void MoveStreamPositionToBlockIndex(IOStream &rStream); - - // Crypto setup - static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength); -#ifndef HAVE_OLD_SSL - static void SetAESKey(const void *pKey, int KeyLength); -#endif - - // Allocation of properly aligning chunks for decoding and encoding chunks - inline static void *CodingChunkAlloc(int Size) - { - uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3)); - if(a == 0) return 0; - // Align to main block size - ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size - uint8_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE - - (uint8_t)(((unsigned long)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE); - uint8_t *b = (a + adjustment); - // Store adjustment - *b = adjustment; - // Return offset - return b + BACKUPSTOREFILE_CODING_OFFSET; - } - inline static void CodingChunkFree(void *Block) - { - // Check alignment is as expected - ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size - ASSERT((uint8_t)(((unsigned long)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); - uint8_t *a = (uint8_t*)Block; - a -= BACKUPSTOREFILE_CODING_OFFSET; - // Adjust downwards... - a -= *a; - free(a); - } - - static void DiffTimerExpired(); - - // Building blocks - class EncodingBuffer - { - public: - EncodingBuffer(); - ~EncodingBuffer(); - private: - // No copying - EncodingBuffer(const EncodingBuffer &); - EncodingBuffer &operator=(const EncodingBuffer &); - public: - void Allocate(int Size); - void Reallocate(int NewSize); - - uint8_t *mpBuffer; - int mBufferSize; - }; - static int MaxBlockSizeForChunkSize(int ChunkSize); - static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput); - - // Caller should know how big the output size is, but also allocate a bit more memory to cover various - // overheads allowed for in checks - static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize) - { - // Plenty big enough - return KnownChunkSize + 256; - } - static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize); - - // Statisitics, not designed to be completely reliable - static void ResetStats(); - static BackupStoreFileStats msStats; - - // For debug -#ifndef BOX_RELEASE_BUILD - static bool TraceDetailsOfDiffProcess; -#endif - - // For decoding encoded files - static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile); -}; - -#include "MemLeakFindOff.h" - -#endif // BACKUPSTOREFILE__H diff --git a/lib/backupclient/BackupStoreFileCryptVar.cpp b/lib/backupclient/BackupStoreFileCryptVar.cpp deleted file mode 100644 index e826de4e..00000000 --- a/lib/backupclient/BackupStoreFileCryptVar.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileCryptVar.cpp -// Purpose: Cryptographic keys for backup store files -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include "BackupStoreFileCryptVar.h" -#include "BackupStoreFileWire.h" - -#include "MemLeakFindOn.h" - -CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt; -CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt; - -#ifndef HAVE_OLD_SSL - CipherContext BackupStoreFileCryptVar::sAESEncrypt; - CipherContext BackupStoreFileCryptVar::sAESDecrypt; -#endif - -// Default to blowfish -CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt; -uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING; - -CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry; -CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry; - diff --git a/lib/backupclient/BackupStoreFileCryptVar.h b/lib/backupclient/BackupStoreFileCryptVar.h deleted file mode 100644 index 566813c8..00000000 --- a/lib/backupclient/BackupStoreFileCryptVar.h +++ /dev/null @@ -1,39 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileCryptVar.h -// Purpose: Cryptographic keys for backup store files -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILECRYPTVAR__H -#define BACKUPSTOREFILECRYPTVAR__H - -#include "CipherContext.h" - -// Hide private static variables from the rest of the world by putting them -// as static variables in a namespace. -// -- don't put them as static class variables to avoid openssl/evp.h being -// included all over the project. -namespace BackupStoreFileCryptVar -{ - // Keys for the main file data - extern CipherContext sBlowfishEncrypt; - extern CipherContext sBlowfishDecrypt; - // Use AES when available -#ifndef HAVE_OLD_SSL - extern CipherContext sAESEncrypt; - extern CipherContext sAESDecrypt; -#endif - // How encoding will be done - extern CipherContext *spEncrypt; - extern uint8_t sEncryptCipherType; - - // Keys for the block indicies - extern CipherContext sBlowfishEncryptBlockEntry; - extern CipherContext sBlowfishDecryptBlockEntry; -} - -#endif // BACKUPSTOREFILECRYPTVAR__H - diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp deleted file mode 100644 index e9d773f0..00000000 --- a/lib/backupclient/BackupStoreFileEncodeStream.cpp +++ /dev/null @@ -1,717 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileEncodeStream.cpp -// Purpose: Implement stream-based file encoding for the backup store -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <string.h> - -#include "BackupClientFileAttributes.h" -#include "BackupStoreConstants.h" -#include "BackupStoreException.h" -#include "BackupStoreFile.h" -#include "BackupStoreFileCryptVar.h" -#include "BackupStoreFileEncodeStream.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreObjectMagic.h" -#include "BoxTime.h" -#include "FileStream.h" -#include "Random.h" -#include "RollingChecksum.h" - -#include "MemLeakFindOn.h" - -#include <cstring> - -using namespace BackupStoreFileCryptVar; - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream -// Purpose: Constructor (opens file) -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() - : mpRecipe(0), - mpFile(0), - mpLogging(0), - mpRunStatusProvider(NULL), - mStatus(Status_Header), - mSendData(true), - mTotalBlocks(0), - mAbsoluteBlockNumber(-1), - mInstructionNumber(-1), - mNumBlocks(0), - mCurrentBlock(-1), - mCurrentBlockEncodedSize(0), - mPositionInCurrentBlock(0), - mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), - mLastBlockSize(0), - mTotalBytesSent(0), - mpRawBuffer(0), - mAllocatedBufferSize(0), - mEntryIVBase(0) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() -// Purpose: Destructor -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() -{ - // Free buffers - if(mpRawBuffer) - { - ::free(mpRawBuffer); - mpRawBuffer = 0; - } - - // Close the file, which we might have open - if(mpFile) - { - delete mpFile; - mpFile = 0; - } - - // Clear up logging stream - if(mpLogging) - { - delete mpLogging; - mpLogging = 0; - } - - // Free the recipe - if(mpRecipe != 0) - { - delete mpRecipe; - mpRecipe = 0; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *) -// Purpose: Reads file information, and builds file header reading for sending. -// Takes ownership of the Recipe. -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::Setup(const char *Filename, - BackupStoreFileEncodeStream::Recipe *pRecipe, - int64_t ContainerID, const BackupStoreFilename &rStoreFilename, - int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) -{ - // Pointer to a blank recipe which we might create - BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; - - try - { - // Get file attributes - box_time_t modTime = 0; - int64_t fileSize = 0; - BackupClientFileAttributes attr; - attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime, - 0 /* not interested in attr mod time */, &fileSize); - - // Might need to create a blank recipe... - if(pRecipe == 0) - { - pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); - - BackupStoreFileEncodeStream::RecipeInstruction instruction; - instruction.mSpaceBefore = fileSize; // whole file - instruction.mBlocks = 0; // no blocks - instruction.mpStartBlock = 0; // no block - pblankRecipe->push_back(instruction); - - pRecipe = pblankRecipe; - } - - // Tell caller? - if(pModificationTime != 0) - { - *pModificationTime = modTime; - } - - // Go through each instruction in the recipe and work out how many blocks - // it will add, and the max clear size of these blocks - int maxBlockClearSize = 0; - for(uint64_t inst = 0; inst < pRecipe->size(); ++inst) - { - if((*pRecipe)[inst].mSpaceBefore > 0) - { - // Calculate the number of blocks the space before requires - int64_t numBlocks; - int32_t blockSize, lastBlockSize; - CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize); - // Add to accumlated total - mTotalBlocks += numBlocks; - // Update maximum clear size - if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize; - if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize; - } - - // Add number of blocks copied from the previous file - mTotalBlocks += (*pRecipe)[inst].mBlocks; - - // Check for bad things - if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0)) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Run through blocks to get the max clear size - for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b) - { - if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize; - } - } - - // Send data? (symlinks don't have any data in them) - mSendData = !attr.IsSymLink(); - - // If not data is being sent, then the max clear block size is zero - if(!mSendData) - { - maxBlockClearSize = 0; - } - - // Header - file_StreamFormat hdr; - hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1); - hdr.mNumBlocks = (mSendData)?(box_hton64(mTotalBlocks)):(0); - hdr.mContainerID = box_hton64(ContainerID); - hdr.mModificationTime = box_hton64(modTime); - // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size - hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128); - hdr.mOptions = 0; // no options defined yet - - // Write header to stream - mData.Write(&hdr, sizeof(hdr)); - - // Write filename to stream - rStoreFilename.WriteToStream(mData); - - // Write attributes to stream - attr.WriteToStream(mData); - - // Allocate some buffers for writing data - if(mSendData) - { - // Open the file - mpFile = new FileStream(Filename); - - if (pLogger) - { - // Create logging stream - mpLogging = new ReadLoggingStream(*mpFile, - *pLogger); - } - else - { - // re-use FileStream instead - mpLogging = mpFile; - mpFile = NULL; - } - - // Work out the largest possible block required for the encoded data - mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); - - // Then allocate two blocks of this size - mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize); - if(mpRawBuffer == 0) - { - throw std::bad_alloc(); - } -#ifndef BOX_RELEASE_BUILD - // In debug builds, make sure that the reallocation code is exercised. - mEncodedBuffer.Allocate(mAllocatedBufferSize / 4); -#else - mEncodedBuffer.Allocate(mAllocatedBufferSize); -#endif - } - else - { - // Write an empty block index for the symlink - file_BlockIndexHeader blkhdr; - blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); - blkhdr.mOtherFileID = box_hton64(0); // not other file ID - blkhdr.mEntryIVBase = box_hton64(0); - blkhdr.mNumBlocks = box_hton64(0); - mData.Write(&blkhdr, sizeof(blkhdr)); - } - - // Ready for reading - mData.SetForReading(); - - // Update stats - BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize; - - // Finally, store the pointer to the recipe, when we know exceptions won't occur - mpRecipe = pRecipe; - } - catch(...) - { - // Clean up any blank recipe - if(pblankRecipe != 0) - { - delete pblankRecipe; - pblankRecipe = 0; - } - throw; - } - - mpRunStatusProvider = pRunStatusProvider; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &) -// Purpose: Calculates the sizes of blocks in a section of the file -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut) -{ - // How many blocks, and how big? - rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2; - do - { - rBlockSizeOut *= 2; - - rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut; - - } while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); - - // Last block size - rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut); - - // Avoid small blocks? - if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN - && rNumBlocksOut > 1) - { - // Add the small bit of data to the last block - --rNumBlocksOut; - rLastBlockSizeOut += rBlockSizeOut; - } - - // checks! - ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize); - //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut); -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::Read(void *, int, int) -// Purpose: As interface -- generates encoded file data on the fly from the raw file -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) -{ - // Check there's something to do. - if(mStatus == Status_Finished) - { - return 0; - } - - if(mpRunStatusProvider && mpRunStatusProvider->StopRun()) - { - THROW_EXCEPTION(BackupStoreException, SignalReceived); - } - - int bytesToRead = NBytes; - uint8_t *buffer = (uint8_t*)pBuffer; - - while(bytesToRead > 0 && mStatus != Status_Finished) - { - if(mStatus == Status_Header || mStatus == Status_BlockListing) - { - // Header or block listing phase -- send from the buffered stream - - // Send bytes from the data buffer - int b = mData.Read(buffer, bytesToRead, Timeout); - bytesToRead -= b; - buffer += b; - - // Check to see if all the data has been used from this stream - if(!mData.StreamDataLeft()) - { - // Yes, move on to next phase (or finish, if there's no file data) - if(!mSendData) - { - mStatus = Status_Finished; - } - else - { - // Reset the buffer so it can be used for the next phase - mData.Reset(); - - // Get buffer ready for index? - if(mStatus == Status_Header) - { - // Just finished doing the stream header, create the block index header - file_BlockIndexHeader blkhdr; - blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); - ASSERT(mpRecipe != 0); - blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID()); - blkhdr.mNumBlocks = box_hton64(mTotalBlocks); - - // Generate the IV base - Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase)); - blkhdr.mEntryIVBase = box_hton64(mEntryIVBase); - - mData.Write(&blkhdr, sizeof(blkhdr)); - } - - ++mStatus; - } - } - } - else if(mStatus == Status_Blocks) - { - // Block sending phase - - if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize) - { - // Next block! - ++mCurrentBlock; - ++mAbsoluteBlockNumber; - if(mCurrentBlock >= mNumBlocks) - { - // Output extra blocks for this instruction and move forward in file - if(mInstructionNumber >= 0) - { - SkipPreviousBlocksInInstruction(); - } - - // Is there another instruction to go? - ++mInstructionNumber; - - // Skip instructions which don't contain any data - while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size()) - && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0) - { - SkipPreviousBlocksInInstruction(); - ++mInstructionNumber; - } - - if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size())) - { - // End of blocks, go to next phase - ++mStatus; - - // Set the data to reading so the index can be written - mData.SetForReading(); - } - else - { - // Get ready for this instruction - SetForInstruction(); - } - } - - // Can't use 'else' here as SetForInstruction() will change this - if(mCurrentBlock < mNumBlocks) - { - EncodeCurrentBlock(); - } - } - - // Send data from the current block (if there's data to send) - if(mPositionInCurrentBlock < mCurrentBlockEncodedSize) - { - // How much data to put in the buffer? - int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock; - if(s > bytesToRead) s = bytesToRead; - - // Copy it in - ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s); - - // Update variables - bytesToRead -= s; - buffer += s; - mPositionInCurrentBlock += s; - } - } - else - { - // Should never get here, as it'd be an invalid status - ASSERT(false); - } - } - - // Add encoded size to stats - BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead); - mTotalBytesSent += (NBytes - bytesToRead); - - // Return size of data to caller - return NBytes - bytesToRead; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction() -// Purpose: Private. Stores the blocks of the old file referenced in the current -// instruction into the index and skips over the data in the file -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction() -{ - // Check something is necessary - if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0) - { - return; - } - - // Index of the first block in old file (being diffed from) - int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock); - - int64_t sizeToSkip = 0; - - for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b) - { - // Update stats - BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; - - // Store the entry - StoreBlockIndexEntry(0 - (firstIndex + b), - (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize, - (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum, - (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); - - // Increment the absolute block number -- kept encryption IV in sync - ++mAbsoluteBlockNumber; - - // Add the size of this block to the size to skip - sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; - } - - // Move forward in the stream - mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::SetForInstruction() -// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::SetForInstruction() -{ - // Calculate block sizes - CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize); - - // Set variables - mCurrentBlock = 0; - mCurrentBlockEncodedSize = 0; - mPositionInCurrentBlock = 0; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock() -// Purpose: Private. Encodes the current block, and writes the block data to the index -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::EncodeCurrentBlock() -{ - // How big is the block, raw? - int blockRawSize = mBlockSize; - if(mCurrentBlock == (mNumBlocks - 1)) - { - blockRawSize = mLastBlockSize; - } - ASSERT(blockRawSize < mAllocatedBufferSize); - - // Check file open - if(mpLogging == 0) - { - // File should be open, but isn't. So logical error. - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Read the data in - if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, - 0 /* not interested in size if failure */)) - { - // TODO: Do something more intelligent, and abort - // this upload because the file has changed. - THROW_EXCEPTION(BackupStoreException, - Temp_FileEncodeStreamDidntReadBuffer) - } - - // Encode it - mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, - blockRawSize, mEncodedBuffer); - - //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); - - // Create block listing data -- generate checksums - RollingChecksum weakChecksum(mpRawBuffer, blockRawSize); - MD5Digest strongChecksum; - strongChecksum.Add(mpRawBuffer, blockRawSize); - strongChecksum.Finish(); - - // Add entry to the index - StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, - weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); - - // Set vars to reading this block - mPositionInCurrentBlock = 0; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *) -// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream. -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum) -{ - // First, the encrypted section - file_BlockIndexEntryEnc entryEnc; - entryEnc.mSize = htonl(ClearSize); - entryEnc.mWeakChecksum = htonl(WeakChecksum); - ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum)); - - // Then the clear section - file_BlockIndexEntry entry; - entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex)); - - // Then encrypt the encryted section - // Generate the IV from the block number - if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase)) - { - THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements) - } - uint64_t iv = mEntryIVBase; - iv += mAbsoluteBlockNumber; - // Convert to network byte order before encrypting with it, so that restores work on - // platforms with different endiannesses. - iv = box_hton64(iv); - sBlowfishEncryptBlockEntry.SetIV(&iv); - - // Encode the data - int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc)); - if(encodedSize != sizeof(entry.mEnEnc)) - { - THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) - } - - // Save to data block for sending at the end of the stream - mData.Write(&entry, sizeof(entry)); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::Write(const void *, int) -// Purpose: As interface. Exceptions. -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes) -{ - THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream) -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::StreamDataLeft() -// Purpose: As interface -- end of stream reached? -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFileEncodeStream::StreamDataLeft() -{ - return (mStatus != Status_Finished); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::StreamClosed() -// Purpose: As interface -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFileEncodeStream::StreamClosed() -{ - return true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t) -// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted -// Created: 15/1/04 -// -// -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, - int64_t NumBlocksInIndex, int64_t OtherFileID) - : mpBlockIndex(pBlockIndex), - mNumBlocksInIndex(NumBlocksInIndex), - mOtherFileID(OtherFileID) -{ - ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::Recipe::~Recipe() -// Purpose: Destructor -// Created: 15/1/04 -// -// -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::Recipe::~Recipe() -{ - // Free the block index, if there is one - if(mpBlockIndex != 0) - { - ::free(mpBlockIndex); - } -} - - - - diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h deleted file mode 100644 index 023994af..00000000 --- a/lib/backupclient/BackupStoreFileEncodeStream.h +++ /dev/null @@ -1,137 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileEncodeStream.h -// Purpose: Implement stream-based file encoding for the backup store -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILEENCODESTREAM__H -#define BACKUPSTOREFILEENCODESTREAM__H - -#include <vector> - -#include "IOStream.h" -#include "BackupStoreFilename.h" -#include "CollectInBufferStream.h" -#include "MD5Digest.h" -#include "BackupStoreFile.h" -#include "ReadLoggingStream.h" -#include "RunStatusProvider.h" - -namespace BackupStoreFileCreation -{ - // Diffing and creation of files share some implementation details. - typedef struct _BlocksAvailableEntry - { - struct _BlocksAvailableEntry *mpNextInHashList; - int32_t mSize; // size in clear - uint32_t mWeakChecksum; // weak, rolling checksum - uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum - } BlocksAvailableEntry; - -} - - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupStoreFileEncodeStream -// Purpose: Encode a file into a stream -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -class BackupStoreFileEncodeStream : public IOStream -{ -public: - BackupStoreFileEncodeStream(); - ~BackupStoreFileEncodeStream(); - - typedef struct - { - int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go - int32_t mBlocks; // number of block to reuse, starting at this one - BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null - } RecipeInstruction; - - class Recipe : public std::vector<RecipeInstruction> - { - // NOTE: This class is rather tied in with the implementation of diffing. - public: - Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex, - int64_t OtherFileID = 0); - ~Recipe(); - - int64_t GetOtherFileID() {return mOtherFileID;} - int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock) - { - return pBlock - mpBlockIndex; - } - - private: - BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex; - int64_t mNumBlocksInIndex; - int64_t mOtherFileID; - }; - - void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, - int64_t *pModificationTime, - ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL); - - virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); - virtual bool StreamDataLeft(); - virtual bool StreamClosed(); - int64_t GetTotalBytesSent() { return mTotalBytesSent; } - -private: - enum - { - Status_Header = 0, - Status_Blocks = 1, - Status_BlockListing = 2, - Status_Finished = 3 - }; - -private: - void EncodeCurrentBlock(); - void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); - void SkipPreviousBlocksInInstruction(); - void SetForInstruction(); - void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum); - -private: - Recipe *mpRecipe; - IOStream *mpFile; // source file - CollectInBufferStream mData; // buffer for header and index entries - IOStream *mpLogging; - RunStatusProvider* mpRunStatusProvider; - int mStatus; - bool mSendData; // true if there's file data to send (ie not a symlink) - int64_t mTotalBlocks; // Total number of blocks in the file - int64_t mAbsoluteBlockNumber; // The absolute block number currently being output - // Instruction number - int64_t mInstructionNumber; - // All the below are within the current instruction - int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases - int64_t mCurrentBlock; - int32_t mCurrentBlockEncodedSize; - int32_t mPositionInCurrentBlock; // for reading out - int32_t mBlockSize; // Basic block size of most of the blocks in the file - int32_t mLastBlockSize; // the size (unencoded) of the last block in the file - int64_t mTotalBytesSent; - // Buffers - uint8_t *mpRawBuffer; // buffer for raw data - BackupStoreFile::EncodingBuffer mEncodedBuffer; - // buffer for encoded data - int32_t mAllocatedBufferSize; // size of above two allocated blocks - uint64_t mEntryIVBase; // base for block entry IV -}; - - - -#endif // BACKUPSTOREFILEENCODESTREAM__H - diff --git a/lib/backupclient/BackupStoreFileRevDiff.cpp b/lib/backupclient/BackupStoreFileRevDiff.cpp deleted file mode 100644 index 509eef61..00000000 --- a/lib/backupclient/BackupStoreFileRevDiff.cpp +++ /dev/null @@ -1,258 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileRevDiff.cpp -// Purpose: Reverse a patch, to build a new patch from new to old files -// Created: 12/7/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <new> -#include <stdlib.h> - -#include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreObjectMagic.h" -#include "BackupStoreException.h" -#include "BackupStoreConstants.h" -#include "BackupStoreFilename.h" - -#include "MemLeakFindOn.h" - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t) -// Purpose: Reverse a patch, to build a new patch from new to old files. Takes -// two independent copies to the From file, for efficiency. -// Created: 12/7/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent) -{ - // Read and copy the header from the from file to the out file -- beginnings of the patch - file_StreamFormat hdr; - if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - // Copy - rOut.Write(&hdr, sizeof(hdr)); - // Copy over filename and attributes - // BLOCK - { - BackupStoreFilename filename; - filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite); - filename.WriteToStream(rOut); - StreamableMemBlock attr; - attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite); - attr.WriteToStream(rOut); - } - - // Build an index of common blocks. - // For each block in the from file, we want to know it's index in the - // diff file. Allocate memory for this information. - int64_t fromNumBlocks = box_ntoh64(hdr.mNumBlocks); - int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t)); - if(pfromIndexInfo == 0) - { - throw std::bad_alloc(); - } - - // Buffer data - void *buffer = 0; - int bufferSize = 0; - - // flag - bool isCompletelyDifferent = true; - - try - { - // Initialise the index to be all 0, ie not filled in yet - for(int64_t i = 0; i < fromNumBlocks; ++i) - { - pfromIndexInfo[i] = 0; - } - - // Within the from file, skip to the index - MoveStreamPositionToBlockIndex(rDiff); - - // Read in header of index - file_BlockIndexHeader diffIdxHdr; - if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // And then read in each entry - int64_t diffNumBlocks = box_ntoh64(diffIdxHdr.mNumBlocks); - for(int64_t b = 0; b < diffNumBlocks; ++b) - { - file_BlockIndexEntry e; - if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Where's the block? - int64_t blockEn = box_ntoh64(e.mEncodedSize); - if(blockEn > 0) - { - // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file - } - else - { - // Block is in the original file, store which block it is in this file - int64_t fromIndex = 0 - blockEn; - if(fromIndex < 0 || fromIndex >= fromNumBlocks) - { - THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles) - } - - // Store information about where it is in the new file - // NOTE: This is slight different to how it'll be stored in the final index. - pfromIndexInfo[fromIndex] = -1 - b; - } - } - - // Open the index for the second copy of the from file - MoveStreamPositionToBlockIndex(rFrom2); - - // Read in header of index - file_BlockIndexHeader fromIdxHdr; - if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 - || box_ntoh64(fromIdxHdr.mOtherFileID) != 0) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // So, we can now start building the data in the file - int64_t filePosition = rFrom.GetPosition(); - for(int64_t b = 0; b < fromNumBlocks; ++b) - { - // Read entry from from index - file_BlockIndexEntry e; - if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Get size - int64_t blockSize = box_hton64(e.mEncodedSize); - if(blockSize < 0) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Copy this block? - if(pfromIndexInfo[b] == 0) - { - // Copy it, first move to file location - rFrom.Seek(filePosition, IOStream::SeekType_Absolute); - - // Make sure there's memory available to copy this - if(bufferSize < blockSize || buffer == 0) - { - // Free old block - if(buffer != 0) - { - ::free(buffer); - buffer = 0; - bufferSize = 0; - } - // Allocate new block - buffer = ::malloc(blockSize); - if(buffer == 0) - { - throw std::bad_alloc(); - } - bufferSize = blockSize; - } - ASSERT(bufferSize >= blockSize); - - // Copy the block - if(!rFrom.ReadFullBuffer(buffer, blockSize, 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - rOut.Write(buffer, blockSize); - - // Store the size - pfromIndexInfo[b] = blockSize; - } - else - { - // Block isn't needed, so it's not completely different - isCompletelyDifferent = false; - } - filePosition += blockSize; - } - - // Then write the index, modified header first - fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(box_hton64(ObjectIDOfFrom)); - rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr)); - - // Move to start of index entries - rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute); - - // Then copy modified entries - for(int64_t b = 0; b < fromNumBlocks; ++b) - { - // Read entry from from index - file_BlockIndexEntry e; - if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Modify... - int64_t s = pfromIndexInfo[b]; - // Adjust to reflect real block index (remember 0 has a different meaning here) - if(s < 0) ++s; - // Insert - e.mEncodedSize = box_hton64(s); - // Write - rOut.Write(&e, sizeof(e)); - } - } - catch(...) - { - ::free(pfromIndexInfo); - if(buffer != 0) - { - ::free(buffer); - } - throw; - } - - // Free memory used (oh for finally {} blocks) - ::free(pfromIndexInfo); - if(buffer != 0) - { - ::free(buffer); - } - - // return completely different flag - if(pIsCompletelyDifferent != 0) - { - *pIsCompletelyDifferent = isCompletelyDifferent; - } -} - - - diff --git a/lib/backupclient/BackupStoreFileWire.h b/lib/backupclient/BackupStoreFileWire.h deleted file mode 100644 index 49e94aa5..00000000 --- a/lib/backupclient/BackupStoreFileWire.h +++ /dev/null @@ -1,74 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileWire.h -// Purpose: On the wire / disc formats for backup store files -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILEWIRE__H -#define BACKUPSTOREFILEWIRE__H - -#include "MD5Digest.h" - -// set packing to one byte -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "BeginStructPackForWire.h" -#else -BEGIN_STRUCTURE_PACKING_FOR_WIRE -#endif - -typedef struct -{ - int32_t mMagicValue; // also the version number - int64_t mNumBlocks; // number of blocks contained in the file - int64_t mContainerID; - int64_t mModificationTime; - int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block - int32_t mOptions; // bitmask of options used - // Then a BackupStoreFilename - // Then a BackupClientFileAttributes -} file_StreamFormat; - -typedef struct -{ - int32_t mMagicValue; // different magic value - int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index - uint64_t mEntryIVBase; // base value for block IV - int64_t mNumBlocks; // repeat of value in file header -} file_BlockIndexHeader; - -typedef struct -{ - int32_t mSize; // size in clear - uint32_t mWeakChecksum; // weak, rolling checksum - uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum -} file_BlockIndexEntryEnc; - -typedef struct -{ - union - { - int64_t mEncodedSize; // size encoded, if > 0 - int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0 - }; - uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section -} file_BlockIndexEntry; - -// Use default packing -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "EndStructPackForWire.h" -#else -END_STRUCTURE_PACKING_FOR_WIRE -#endif - -// header for blocks of compressed data in files -#define HEADER_CHUNK_IS_COMPRESSED 1 // bit -#define HEADER_ENCODING_SHIFT 1 // shift value -#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7 -#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7 - - -#endif // BACKUPSTOREFILEWIRE__H - diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp deleted file mode 100644 index 72cd1acd..00000000 --- a/lib/backupclient/BackupStoreFilename.cpp +++ /dev/null @@ -1,281 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFilename.cpp -// Purpose: Filename for the backup store -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#include "Box.h" -#include "BackupStoreFilename.h" -#include "Protocol.h" -#include "BackupStoreException.h" -#include "IOStream.h" -#include "Guards.h" - -#include "MemLeakFindOn.h" - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::BackupStoreFilename() -// Purpose: Default constructor -- creates an invalid filename -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilename::BackupStoreFilename() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &) -// Purpose: Copy constructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy) - : mEncryptedName(rToCopy.mEncryptedName) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::~BackupStoreFilename() -// Purpose: Destructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilename::~BackupStoreFilename() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::CheckValid(bool) -// Purpose: Checks the encoded filename for validity -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const -{ - bool ok = true; - - if(mEncryptedName.size() < 2) - { - // Isn't long enough to have a header - ok = false; - } - else - { - // Check size is consistent - unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(this->mEncryptedName); - if(dsize != mEncryptedName.size()) - { - ok = false; - } - - // And encoding is an accepted value - unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName); - if(encoding < Encoding_Min || encoding > Encoding_Max) - { - ok = false; - } - } - - // Exception? - if(!ok && ExceptionIfInvalid) - { - THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename) - } - - return ok; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::ReadFromProtocol(Protocol &) -// Purpose: Reads the filename from the protocol object -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol) -{ - // Read the header - char hdr[2]; - rProtocol.Read(hdr, 2); - - // How big is it? - int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); - - // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us - std::string data; - rProtocol.Read(data, dsize - 2); - - // assign to this string, storing the header and the extra data - mEncryptedName.assign(hdr, 2); - mEncryptedName.append(data.c_str(), data.size()); - - // Check it - CheckValid(); - - // Alert derived classes - EncodedFilenameChanged(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::WriteToProtocol(Protocol &) -// Purpose: Writes the filename to the protocol object -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const -{ - CheckValid(); - - rProtocol.Write(mEncryptedName.c_str(), mEncryptedName.size()); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::ReadFromStream(IOStream &) -// Purpose: Reads the filename from a stream -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout) -{ - // Read the header - char hdr[2]; - if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // How big is it? - unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); - - // Assume most filenames are small - char buf[256]; - if(dsize < sizeof(buf)) - { - // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us - if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - // Copy in header - buf[0] = hdr[0]; buf[1] = hdr[1]; - - // assign to this string, storing the header and the extra data - mEncryptedName.assign(buf, dsize); - } - else - { - // Block of memory to hold it - MemoryBlockGuard<char*> dataB(dsize+2); - char *data = dataB; - - // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us - if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - // Copy in header - data[0] = hdr[0]; data[1] = hdr[1]; - - // assign to this string, storing the header and the extra data - mEncryptedName.assign(data, dsize); - } - - // Check it - CheckValid(); - - // Alert derived classes - EncodedFilenameChanged(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::WriteToStream(IOStream &) -// Purpose: Writes the filename to a stream -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::WriteToStream(IOStream &rStream) const -{ - CheckValid(); - - rStream.Write(mEncryptedName.c_str(), mEncryptedName.size()); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::EncodedFilenameChanged() -// Purpose: The encoded filename stored has changed -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::EncodedFilenameChanged() -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::IsEncrypted() -// Purpose: Returns true if the filename is stored using an encrypting encoding -// Created: 1/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFilename::IsEncrypted() const -{ - return BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName) != - Encoding_Clear; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::SetAsClearFilename(const char *) -// Purpose: Sets this object to be a valid filename, but with a -// filename in the clear. Used on the server to create -// filenames when there's no way of encrypting it. -// Created: 22/4/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::SetAsClearFilename(const char *Clear) -{ - // Make std::string from the clear name - std::string toEncode(Clear); - - // Make an encoded string - char hdr[2]; - BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear); - std::string encoded(hdr, 2); - encoded += toEncode; - ASSERT(encoded.size() == toEncode.size() + 2); - - // Store the encoded string - mEncryptedName.assign(encoded); - - // Stuff which must be done - EncodedFilenameChanged(); - CheckValid(false); -} - - - diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h deleted file mode 100644 index 80db9516..00000000 --- a/lib/backupclient/BackupStoreFilename.h +++ /dev/null @@ -1,107 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFilename.h -// Purpose: Filename for the backup store -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILENAME__H -#define BACKUPSTOREFILENAME__H - -#include <string> - -class Protocol; -class IOStream; - -// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE -// don't define this -- the problem of memory usage still appears without this. -// It's just that this class really showed up the problem. Instead, malloc allocation -// is globally defined in BoxPlatform.h, for troublesome libraries. - -#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE - // Use a malloc_allocated string, because the STL default allocators really screw up with - // memory allocation, particularly with this class. - // Makes a few things a bit messy and inefficient with conversions. - // Given up using this, and use global malloc allocation instead, but thought it - // worth leaving this code in just in case it's useful for the future. - typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base; - // If this is changed, change GetClearFilename() back to returning a reference. -#else - typedef std::string BackupStoreFilename_base; -#endif - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupStoreFilename -// Purpose: Filename for the backup store -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -class BackupStoreFilename /* : public BackupStoreFilename_base */ -{ -private: - std::string mEncryptedName; - -public: - BackupStoreFilename(); - BackupStoreFilename(const BackupStoreFilename &rToCopy); - virtual ~BackupStoreFilename(); - - bool CheckValid(bool ExceptionIfInvalid = true) const; - - void ReadFromProtocol(Protocol &rProtocol); - void WriteToProtocol(Protocol &rProtocol) const; - - void ReadFromStream(IOStream &rStream, int Timeout); - void WriteToStream(IOStream &rStream) const; - - void SetAsClearFilename(const char *Clear); - - // Check that it's encrypted - bool IsEncrypted() const; - - // These enumerated types belong in the base class so - // the CheckValid() function can make sure that the encoding - // is a valid encoding - enum - { - Encoding_Min = 1, - Encoding_Clear = 1, - Encoding_Blowfish = 2, - Encoding_Max = 2 - }; - - const std::string& GetEncodedFilename() const - { - return mEncryptedName; - } - - bool operator==(const BackupStoreFilename& rOther) const - { - return mEncryptedName == rOther.mEncryptedName; - } - - bool operator!=(const BackupStoreFilename& rOther) const - { - return mEncryptedName != rOther.mEncryptedName; - } - -protected: - virtual void EncodedFilenameChanged(); - void SetEncodedFilename(const std::string &rEncoded) - { - mEncryptedName = rEncoded; - } -}; - -// On the wire utilities for class and derived class -#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2) -#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3) - -#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;} - -#endif // BACKUPSTOREFILENAME__H - diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp deleted file mode 100644 index e529d8d3..00000000 --- a/lib/backupclient/BackupStoreFilenameClear.cpp +++ /dev/null @@ -1,335 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFilenameClear.cpp -// Purpose: BackupStoreFilenames in the clear -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#include "Box.h" -#include "BackupStoreFilenameClear.h" -#include "BackupStoreException.h" -#include "CipherContext.h" -#include "CipherBlowfish.h" -#include "Guards.h" -#include "Logging.h" - -#include "MemLeakFindOn.h" - -// Hide private variables from the rest of the world -namespace -{ - int sEncodeMethod = BackupStoreFilename::Encoding_Clear; - CipherContext sBlowfishEncrypt; - CipherContext sBlowfishDecrypt; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::BackupStoreFilenameClear() -// Purpose: Default constructor, creates an invalid filename -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilenameClear::BackupStoreFilenameClear() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &) -// Purpose: Creates a filename, encoding from the given string -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode) -{ - SetClearFilename(rToEncode); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &) -// Purpose: Copy constructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy) - : BackupStoreFilename(rToCopy), - mClearFilename(rToCopy.mClearFilename) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) -// Purpose: Copy from base class -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) - : BackupStoreFilename(rToCopy) -{ - // Will get a clear filename when it's required -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear() -// Purpose: Destructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilenameClear::~BackupStoreFilenameClear() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::GetClearFilename() -// Purpose: Get the unencoded filename -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE -const std::string BackupStoreFilenameClear::GetClearFilename() const -{ - MakeClearAvailable(); - // When modifying, remember to change back to reference return if at all possible - // -- returns an object rather than a reference to allow easy use with other code. - return std::string(mClearFilename.c_str(), mClearFilename.size()); -} -#else -const std::string &BackupStoreFilenameClear::GetClearFilename() const -{ - MakeClearAvailable(); - return mClearFilename; -} -#endif - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &) -// Purpose: Encode and make available the clear filename -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode) -{ - // Only allow Blowfish encodings - if(sEncodeMethod != Encoding_Blowfish) - { - THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup) - } - - // Make an encoded string with blowfish encryption - EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish); - - // Store the clear filename - mClearFilename.assign(rToEncode.c_str(), rToEncode.size()); - - // Make sure we did the right thing - if(!CheckValid(false)) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::MakeClearAvailable() -// Purpose: Private. Make sure the clear filename is available -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::MakeClearAvailable() const -{ - if(!mClearFilename.empty()) - return; // nothing to do - - // Check valid - CheckValid(); - - // Decode the header - int size = BACKUPSTOREFILENAME_GET_SIZE(GetEncodedFilename()); - int encoding = BACKUPSTOREFILENAME_GET_ENCODING(GetEncodedFilename()); - - // Decode based on encoding given in the header - switch(encoding) - { - case Encoding_Clear: - BOX_TRACE("**** BackupStoreFilename encoded with " - "Clear encoding ****"); - mClearFilename.assign(GetEncodedFilename().c_str() + 2, - size - 2); - break; - - case Encoding_Blowfish: - DecryptEncoded(sBlowfishDecrypt); - break; - - default: - THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding) - break; - } -} - - -// Buffer for encoding and decoding -- do this all in one single buffer to -// avoid lots of string allocation, which stuffs up memory usage. -// These static memory vars are, of course, not thread safe, but we don't use threads. -static int sEncDecBufferSize = 0; -static MemoryBlockGuard<uint8_t *> *spEncDecBuffer = 0; - -static void EnsureEncDecBufferSize(int BufSize) -{ - if(spEncDecBuffer == 0) - { -#ifndef WIN32 - BOX_TRACE("Allocating filename encoding/decoding buffer " - "with size " << BufSize); -#endif - spEncDecBuffer = new MemoryBlockGuard<uint8_t *>(BufSize); - MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer); - MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer); - sEncDecBufferSize = BufSize; - } - else - { - if(sEncDecBufferSize < BufSize) - { - BOX_TRACE("Reallocating filename encoding/decoding " - "buffer from " << sEncDecBufferSize << - " to " << BufSize); - spEncDecBuffer->Resize(BufSize); - sEncDecBufferSize = BufSize; - MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer); - } - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int) -// Purpose: Private. Assigns the encoded filename string, encrypting. -// Created: 1/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding) -{ - // Work out max size - int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4; - - // Make sure encode/decode buffer has enough space - EnsureEncDecBufferSize(maxOutSize); - - // Pointer to buffer - uint8_t *buffer = *spEncDecBuffer; - - // Encode -- do entire block in one go - int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size()); - // and add in header size - encSize += 2; - - // Adjust header - BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding); - - // Store the encoded string - SetEncodedFilename(std::string((char*)buffer, encSize)); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &) -// Purpose: Decrypt the encoded filename using the cipher context -// Created: 1/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const -{ - const std::string& rEncoded = GetEncodedFilename(); - - // Work out max size - int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rEncoded.size()) + 4; - - // Make sure encode/decode buffer has enough space - EnsureEncDecBufferSize(maxOutSize); - - // Pointer to buffer - uint8_t *buffer = *spEncDecBuffer; - - // Decrypt - const char *str = rEncoded.c_str() + 2; - int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, rEncoded.size() - 2); - - // Assign to this - mClearFilename.assign((char*)buffer, sizeOut); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::EncodedFilenameChanged() -// Purpose: The encoded filename stored has changed -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::EncodedFilenameChanged() -{ - BackupStoreFilename::EncodedFilenameChanged(); - - // Delete stored filename in clear - mClearFilename.erase(); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int) -// Purpose: Set the key used for Blowfish encryption of filenames -// Created: 1/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength) -{ - // Initialisation vector not used. Can't use a different vector for each filename as - // that would stop comparisions on the server working. - sBlowfishEncrypt.Reset(); - sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); - ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength); - sBlowfishEncrypt.SetIV(pIV); - sBlowfishDecrypt.Reset(); - sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); - ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength); - sBlowfishDecrypt.SetIV(pIV); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::SetEncodingMethod(int) -// Purpose: Set the encoding method used for filenames -// Created: 1/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::SetEncodingMethod(int Method) -{ - sEncodeMethod = Method; -} - - - diff --git a/lib/backupclient/BackupStoreFilenameClear.h b/lib/backupclient/BackupStoreFilenameClear.h deleted file mode 100644 index d4c45701..00000000 --- a/lib/backupclient/BackupStoreFilenameClear.h +++ /dev/null @@ -1,60 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFilenameClear.h -// Purpose: BackupStoreFilenames in the clear -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILENAMECLEAR__H -#define BACKUPSTOREFILENAMECLEAR__H - -#include "BackupStoreFilename.h" - -class CipherContext; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupStoreFilenameClear -// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -class BackupStoreFilenameClear : public BackupStoreFilename -{ -public: - BackupStoreFilenameClear(); - BackupStoreFilenameClear(const std::string &rToEncode); - BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy); - BackupStoreFilenameClear(const BackupStoreFilename &rToCopy); - virtual ~BackupStoreFilenameClear(); - - // Because we need to use a different allocator for this class to avoid - // nasty things happening, can't return this as a reference. Which is a - // pity. But probably not too bad. -#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE - const std::string GetClearFilename() const; -#else - const std::string &GetClearFilename() const; -#endif - void SetClearFilename(const std::string &rToEncode); - - // Setup for encryption of filenames - static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength); - static void SetEncodingMethod(int Method); - -protected: - void MakeClearAvailable() const; - virtual void EncodedFilenameChanged(); - void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding); - void DecryptEncoded(CipherContext &rCipherContext) const; - -private: - mutable BackupStoreFilename_base mClearFilename; -}; - -#endif // BACKUPSTOREFILENAMECLEAR__H - - diff --git a/lib/backupclient/BackupStoreObjectMagic.h b/lib/backupclient/BackupStoreObjectMagic.h deleted file mode 100644 index 7ee600a2..00000000 --- a/lib/backupclient/BackupStoreObjectMagic.h +++ /dev/null @@ -1,31 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreObjectMagic.h -// Purpose: Magic values for the start of objects in the backup store -// Created: 19/11/03 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREOBJECTMAGIC__H -#define BACKUPSTOREOBJECTMAGIC__H - -// Each of these values is the first 4 bytes of the object file. -// Remember to swap from network to host byte order. - -// Magic value for file streams -#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65 -// Do not use v0 in any new code! -#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45 - -// Magic for the block index at the file stream -- used to -// ensure streams are reordered as expected -#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478 -// Do not use v0 in any new code! -#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B - -// Magic value for directory streams -#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F - -#endif // BACKUPSTOREOBJECTMAGIC__H - diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra deleted file mode 100644 index df3319df..00000000 --- a/lib/backupclient/Makefile.extra +++ /dev/null @@ -1,16 +0,0 @@ - -MAKEPROTOCOL = ../../lib/server/makeprotocol.pl - -GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt - -# AUTOGEN SEEDING -autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt - $(_PERL) $(GEN_CMD_SRV) - - -MAKEEXCEPTION = ../../lib/common/makeexception.pl - -# AUTOGEN SEEDING -autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt - $(_PERL) $(MAKEEXCEPTION) BackupStoreException.txt - diff --git a/lib/backupclient/RunStatusProvider.h b/lib/backupclient/RunStatusProvider.h deleted file mode 100644 index 89f361ca..00000000 --- a/lib/backupclient/RunStatusProvider.h +++ /dev/null @@ -1,29 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: RunStatusProvider.h -// Purpose: Declares the RunStatusProvider interface. -// Created: 2008/08/14 -// -// -------------------------------------------------------------------------- - -#ifndef RUNSTATUSPROVIDER__H -#define RUNSTATUSPROVIDER__H - -// -------------------------------------------------------------------------- -// -// Class -// Name: RunStatusProvider -// Purpose: Provides a StopRun() method which returns true if -// the current backup should be halted. -// Created: 2005/11/15 -// -// -------------------------------------------------------------------------- -class RunStatusProvider -{ - public: - virtual ~RunStatusProvider() { } - virtual bool StopRun() = 0; -}; - -#endif // RUNSTATUSPROVIDER__H |