diff options
Diffstat (limited to 'lib/backupstore')
27 files changed, 9370 insertions, 0 deletions
diff --git a/lib/backupstore/BackupClientFileAttributes.cpp b/lib/backupstore/BackupClientFileAttributes.cpp new file mode 100644 index 00000000..0d7df4d7 --- /dev/null +++ b/lib/backupstore/BackupClientFileAttributes.cpp @@ -0,0 +1,1188 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientFileAttributes.cpp +// Purpose: Storage of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> +#include <limits.h> + +#include <algorithm> +#include <cstring> +#include <new> +#include <vector> + +#ifdef HAVE_SYS_XATTR_H +#include <cerrno> +#include <sys/xattr.h> +#endif + +#include <cstring> + +#include "BackupClientFileAttributes.h" +#include "CommonException.h" +#include "FileModificationTime.h" +#include "BoxTimeToUnix.h" +#include "BackupStoreException.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "MD5Digest.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +#define ATTRIBUTETYPE_GENERIC_UNIX 1 + +#define ATTRIBUTE_ENCODING_BLOWFISH 2 + +typedef struct +{ + int32_t AttributeType; + u_int32_t UID; + u_int32_t GID; + u_int64_t ModificationTime; + u_int64_t AttrModificationTime; + u_int32_t UserDefinedFlags; + u_int32_t FileGenerationNumber; + u_int16_t Mode; + // Symbolic link filename may follow + // Extended attribute (xattr) information may follow, format is: + // u_int32_t Size of extended attribute block (excluding this word) + // For each of NumberOfAttributes (sorted by AttributeName): + // u_int16_t AttributeNameLength + // char AttributeName[AttributeNameLength] + // u_int32_t AttributeValueLength + // unsigned char AttributeValue[AttributeValueLength] + // AttributeName is 0 terminated, AttributeValue is not (and may be binary data) +} attr_StreamFormat; + +// This has wire packing so it's compatible across platforms +// Use wider than necessary sizes, just to be careful. +typedef struct +{ + int32_t uid, gid, mode; + #ifdef WIN32 + int64_t fileCreationTime; + #endif +} attributeHashData; + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + + +#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256 + +// Hide private static variables from the rest of the world +// -- don't put them as static class variables to avoid openssl/evp.h being +// included all over the project. +namespace +{ + CipherContext sBlowfishEncrypt; + CipherContext sBlowfishDecrypt; + uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH]; + int sAttributeHashSecretLength = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::BackupClientFileAttributes() +// Purpose: Default constructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::BackupClientFileAttributes() + : mpClearAttributes(0) +{ + ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &) +// Purpose: Copy constructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy) + : StreamableMemBlock(rToCopy), // base class does the hard work + mpClearAttributes(0) +{ +} +BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy) + : StreamableMemBlock(rToCopy), // base class does the hard work + mpClearAttributes(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::~BackupClientFileAttributes() +// Purpose: Destructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::~BackupClientFileAttributes() +{ + if(mpClearAttributes) + { + delete mpClearAttributes; + mpClearAttributes = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &) +// Purpose: Assignment operator +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr) +{ + StreamableMemBlock::Set(rAttr); + RemoveClear(); // make sure no decrypted version held + return *this; +} +// Assume users play nice +BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr) +{ + StreamableMemBlock::Set(rAttr); + RemoveClear(); // make sure no decrypted version held + return *this; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &) +// Purpose: Comparison operator +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const +{ + EnsureClearAvailable(); + rAttr.EnsureClearAvailable(); + + return mpClearAttributes->operator==(*rAttr.mpClearAttributes); +} +// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data. +/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const +{ + StreamableMemBlock *pDecoded = 0; + + try + { + EnsureClearAvailable(); + StreamableMemBlock *pDecoded = MakeClear(rAttr); + + // Compare using clear version + bool compared = mpClearAttributes->operator==(rAttr); + + // Delete temporary + delete pDecoded; + + return compared; + } + catch(...) + { + delete pDecoded; + throw; + } +}*/ + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool) +// Purpose: Compare, optionally ignoring the attribute +// modification time and/or modification time, and some +// data which is irrelevant in practise (eg file +// generation number) +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, + bool IgnoreAttrModTime, bool IgnoreModTime) const +{ + EnsureClearAvailable(); + rAttr.EnsureClearAvailable(); + + // Check sizes are the same, as a first check + if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize()) + { + BOX_TRACE("Attribute Compare: Attributes objects are " + "different sizes, cannot compare them: local " << + mpClearAttributes->GetSize() << " bytes, remote " << + rAttr.mpClearAttributes->GetSize() << " bytes"); + return false; + } + + // Then check the elements of the two things + // Bytes are checked in network order, but this doesn't matter as we're only checking for equality. + attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer(); + + #define COMPARE(attribute, message) \ + if (a1->attribute != a2->attribute) \ + { \ + BOX_TRACE("Attribute Compare: " << message << " differ: " \ + "local " << ntoh(a1->attribute) << ", " \ + "remote " << ntoh(a2->attribute)); \ + return false; \ + } + COMPARE(AttributeType, "Attribute types"); + COMPARE(UID, "UIDs"); + COMPARE(GID, "GIDs"); + COMPARE(UserDefinedFlags, "User-defined flags"); + COMPARE(Mode, "Modes"); + + if(!IgnoreModTime) + { + uint64_t t1 = box_ntoh64(a1->ModificationTime); + uint64_t t2 = box_ntoh64(a2->ModificationTime); + time_t s1 = BoxTimeToSeconds(t1); + time_t s2 = BoxTimeToSeconds(t2); + if(s1 != s2) + { + BOX_TRACE("Attribute Compare: File modification " + "times differ: local " << + FormatTime(t1, true) << " (" << s1 << "), " + "remote " << + FormatTime(t2, true) << " (" << s2 << ")"); + return false; + } + } + + if(!IgnoreAttrModTime) + { + uint64_t t1 = box_ntoh64(a1->AttrModificationTime); + uint64_t t2 = box_ntoh64(a2->AttrModificationTime); + time_t s1 = BoxTimeToSeconds(t1); + time_t s2 = BoxTimeToSeconds(t2); + if(s1 != s2) + { + BOX_TRACE("Attribute Compare: Attribute modification " + "times differ: local " << + FormatTime(t1, true) << " (" << s1 << "), " + "remote " << + FormatTime(t2, true) << " (" << s2 << ")"); + return false; + } + } + + // Check symlink string? + unsigned int size = mpClearAttributes->GetSize(); + if(size > sizeof(attr_StreamFormat)) + { + // Symlink strings don't match. This also compares xattrs + int datalen = size - sizeof(attr_StreamFormat); + + if(::memcmp(a1 + 1, a2 + 1, datalen) != 0) + { + std::string s1((char *)(a1 + 1), datalen); + std::string s2((char *)(a2 + 1), datalen); + BOX_TRACE("Attribute Compare: Symbolic link target " + "or extended attributes differ: " + "local " << PrintEscapedBinaryData(s1) << ", " + "remote " << PrintEscapedBinaryData(s2)); + return false; + } + } + + // Passes all test, must be OK + return true; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadAttributes( +// const char *Filename, bool ZeroModificationTimes, +// box_time_t *pModTime, box_time_t *pAttrModTime, +// int64_t *pFileSize, InodeRefType *pInodeNumber, +// bool *pHasMultipleLinks) +// Purpose: Read the attributes of the file, and store them +// ready for streaming. Optionally retrieve the +// modification time and attribute modification time. +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::ReadAttributes(const char *Filename, + bool ZeroModificationTimes, box_time_t *pModTime, + box_time_t *pAttrModTime, int64_t *pFileSize, + InodeRefType *pInodeNumber, bool *pHasMultipleLinks) +{ + StreamableMemBlock *pnewAttr = 0; + try + { + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) != 0) + { + BOX_LOG_SYS_ERROR("Failed to stat file: '" << + Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Modification times etc + if(pModTime) {*pModTime = FileModificationTime(st);} + if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);} + if(pFileSize) {*pFileSize = st.st_size;} + if(pInodeNumber) {*pInodeNumber = st.st_ino;} + if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);} + + pnewAttr = new StreamableMemBlock; + + FillAttributes(*pnewAttr, Filename, st, ZeroModificationTimes); + +#ifndef WIN32 + // Is it a link? + if((st.st_mode & S_IFMT) == S_IFLNK) + { + FillAttributesLink(*pnewAttr, Filename, st); + } +#endif + + FillExtendedAttr(*pnewAttr, Filename); + +#ifdef WIN32 + //this is to catch those problems with invalid time stamps stored... + //need to find out the reason why - but also a catch as well. + + attr_StreamFormat *pattr = + (attr_StreamFormat*)pnewAttr->GetBuffer(); + ASSERT(pattr != 0); + + // __time64_t winTime = BoxTimeToSeconds( + // pnewAttr->ModificationTime); + + u_int64_t modTime = box_ntoh64(pattr->ModificationTime); + box_time_t modSecs = BoxTimeToSeconds(modTime); + __time64_t winTime = modSecs; + + // _MAX__TIME64_T doesn't seem to be defined, but the code below + // will throw an assertion failure if we exceed it :-) + // Microsoft says dates up to the year 3000 are valid, which + // is a bit more than 15 * 2^32. Even that doesn't seem + // to be true (still aborts), but it can at least hold 2^32. + if (winTime >= 0x100000000LL || _gmtime64(&winTime) == 0) + { + BOX_ERROR("Invalid Modification Time caught for " + "file: '" << Filename << "'"); + pattr->ModificationTime = 0; + } + + modTime = box_ntoh64(pattr->AttrModificationTime); + modSecs = BoxTimeToSeconds(modTime); + winTime = modSecs; + + if (winTime > 0x100000000LL || _gmtime64(&winTime) == 0) + { + BOX_ERROR("Invalid Attribute Modification Time " + "caught for file: '" << Filename << "'"); + pattr->AttrModificationTime = 0; + } +#endif + + // Attributes ready. Encrypt into this block + EncryptAttr(*pnewAttr); + + // Store the new attributes + RemoveClear(); + mpClearAttributes = pnewAttr; + pnewAttr = 0; + } + catch(...) + { + // clean up + delete pnewAttr; + pnewAttr = 0; + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadAttributesLink() +// Purpose: Private function, handles standard attributes for all objects +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, EMU_STRUCT_STAT &st, bool ZeroModificationTimes) +{ + outputBlock.ResizeBlock(sizeof(attr_StreamFormat)); + attr_StreamFormat *pattr = (attr_StreamFormat*)outputBlock.GetBuffer(); + ASSERT(pattr != 0); + + // Fill in the entries + pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX); + pattr->UID = htonl(st.st_uid); + pattr->GID = htonl(st.st_gid); + if(ZeroModificationTimes) + { + pattr->ModificationTime = 0; + pattr->AttrModificationTime = 0; + } + else + { + pattr->ModificationTime = box_hton64(FileModificationTime(st)); + pattr->AttrModificationTime = box_hton64(FileAttrModificationTime(st)); + } + pattr->Mode = htons(st.st_mode); + +#ifndef HAVE_STRUCT_STAT_ST_FLAGS + pattr->UserDefinedFlags = 0; + pattr->FileGenerationNumber = 0; +#else + pattr->UserDefinedFlags = htonl(st.st_flags); + pattr->FileGenerationNumber = htonl(st.st_gen); +#endif +} +#ifndef WIN32 +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadAttributesLink() +// Purpose: Private function, handles the case where a symbolic link is needed +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st) +{ + // Make sure we're only called for symbolic links + ASSERT((st.st_mode & S_IFMT) == S_IFLNK); + + // Get the filename the link is linked to + char linkedTo[PATH_MAX+4]; + int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX); + if(linkedToSize == -1) + { + BOX_LOG_SYS_ERROR("Failed to readlink '" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError); + } + + int oldSize = outputBlock.GetSize(); + outputBlock.ResizeBlock(oldSize+linkedToSize+1); + char* buffer = static_cast<char*>(outputBlock.GetBuffer()); + + // Add the path name for the symbolic link, and add 0 termination + std::memcpy(buffer+oldSize, linkedTo, linkedToSize); + buffer[oldSize+linkedToSize] = '\0'; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadExtendedAttr(const char *, unsigned char**) +// Purpose: Private function, read the extended attributes of the file into the block +// Created: 2005/06/12 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename) +{ +#ifdef HAVE_SYS_XATTR_H + int listBufferSize = 10000; + char* list = new char[listBufferSize]; + + try + { + // This returns an unordered list of attribute names, each 0 terminated, + // concatenated together + int listSize = ::llistxattr(Filename, list, listBufferSize); + + if(listSize>listBufferSize) + { + delete[] list, list = NULL; + list = new char[listSize]; + listSize = ::llistxattr(Filename, list, listSize); + } + + if(listSize>0) + { + // Extract list of attribute names so we can sort them + std::vector<std::string> attrKeys; + for(int i = 0; i<listSize; ++i) + { + std::string attrKey(list+i); + i += attrKey.size(); + attrKeys.push_back(attrKey); + } + sort(attrKeys.begin(), attrKeys.end()); + + // Make initial space in block + int xattrSize = outputBlock.GetSize(); + int xattrBufferSize = (xattrSize+listSize)>500 ? (xattrSize+listSize)*2 : 1000; + outputBlock.ResizeBlock(xattrBufferSize); + unsigned char* buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); + + // Leave space for attr block size later + int xattrBlockSizeOffset = xattrSize; + xattrSize += sizeof(u_int32_t); + + // Loop for each attribute + for(std::vector<std::string>::const_iterator attrKeyI = attrKeys.begin(); attrKeyI!=attrKeys.end(); ++attrKeyI) + { + std::string attrKey(*attrKeyI); + + if(xattrSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t)>static_cast<unsigned int>(xattrBufferSize)) + { + xattrBufferSize = (xattrBufferSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t))*2; + outputBlock.ResizeBlock(xattrBufferSize); + buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); + } + + // Store length and text for attibute name + u_int16_t keyLength = htons(attrKey.size()+1); + std::memcpy(buffer+xattrSize, &keyLength, sizeof(u_int16_t)); + xattrSize += sizeof(u_int16_t); + std::memcpy(buffer+xattrSize, attrKey.c_str(), attrKey.size()+1); + xattrSize += attrKey.size()+1; + + // Leave space for value size + int valueSizeOffset = xattrSize; + xattrSize += sizeof(u_int32_t); + + // Find size of attribute (must call with buffer and length 0 on some platforms, + // as -1 is returned if the data doesn't fit.) + int valueSize = ::lgetxattr(Filename, attrKey.c_str(), 0, 0); + if(valueSize<0) + { + BOX_LOG_SYS_ERROR("Failed to get " + "extended attribute size of " + "'" << Filename << "': " << + attrKey); + THROW_EXCEPTION(CommonException, OSFileError); + } + + // Resize block, if needed + if(xattrSize+valueSize>xattrBufferSize) + { + xattrBufferSize = (xattrBufferSize+valueSize)*2; + outputBlock.ResizeBlock(xattrBufferSize); + buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); + } + + // This gets the attribute value (may be text or binary), no termination + valueSize = ::lgetxattr(Filename, attrKey.c_str(), buffer+xattrSize, xattrBufferSize-xattrSize); + if(valueSize<0) + { + BOX_LOG_SYS_ERROR("Failed to get " + "extended attribute of " + "'" << Filename << "': " << + attrKey); + THROW_EXCEPTION(CommonException, OSFileError); + } + xattrSize += valueSize; + + // Fill in value size + u_int32_t valueLength = htonl(valueSize); + std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(u_int32_t)); + } + + // Fill in attribute block size + u_int32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(u_int32_t)); + std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(u_int32_t)); + + outputBlock.ResizeBlock(xattrSize); + } + else if(listSize<0) + { + if(errno == EOPNOTSUPP || errno == EACCES) + { + // fail silently + } + else if(errno == ERANGE) + { + BOX_ERROR("Failed to list extended " + "attributes of '" << Filename << "': " + "buffer too small, not backed up"); + } + else + { + BOX_LOG_SYS_ERROR("Failed to list extended " + "attributes of '" << Filename << "', " + "not backed up"); + THROW_EXCEPTION(CommonException, OSFileError); + } + } + } + catch(...) + { + delete[] list; + throw; + } + delete[] list; +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::GetModificationTimes() +// Purpose: Returns the modification time embedded in the +// attributes. +// Created: 2010/02/24 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::GetModificationTimes( + box_time_t *pModificationTime, + box_time_t *pAttrModificationTime) const +{ + // Got something loaded + if(GetSize() <= 0) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Make sure there are clear attributes to use + EnsureClearAvailable(); + ASSERT(mpClearAttributes != 0); + + // Check if the decrypted attributes are small enough, and the type of attributes stored + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) + { + // Don't know what to do with these + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + + // Check there is enough space for an attributes block + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get pointer to structure + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + + if(pModificationTime) + { + *pModificationTime = box_ntoh64(pattr->ModificationTime); + } + + if(pAttrModificationTime) + { + *pAttrModificationTime = box_ntoh64(pattr->AttrModificationTime); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::WriteAttributes(const char *) +// Purpose: Apply the stored attributes to the file +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::WriteAttributes(const char *Filename, + bool MakeUserWritable) const +{ + // Got something loaded + if(GetSize() <= 0) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Make sure there are clear attributes to use + EnsureClearAvailable(); + ASSERT(mpClearAttributes != 0); + + // Check if the decrypted attributes are small enough, and the type of attributes stored + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) + { + // Don't know what to do with these + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + + // Check there is enough space for an attributes block + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get pointer to structure + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + int xattrOffset = sizeof(attr_StreamFormat); + + // is it a symlink? + int16_t mode = ntohs(pattr->Mode); + if((mode & S_IFMT) == S_IFLNK) + { + // Check things are sensible + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + +#ifdef WIN32 + BOX_WARNING("Cannot create symbolic links on Windows: '" << + Filename << "'"); +#else + // Make a symlink, first deleting anything in the way + ::unlink(Filename); + if(::symlink((char*)(pattr + 1), Filename) != 0) + { + BOX_LOG_SYS_ERROR("Failed to symlink '" << Filename << + "' to '" << (char*)(pattr + 1) << "'"); + THROW_EXCEPTION(CommonException, OSFileError) + } +#endif + + xattrOffset += std::strlen(reinterpret_cast<char*>(pattr+1))+1; + } + + // If working as root, set user IDs + if(::geteuid() == 0) + { + #ifndef HAVE_LCHOWN + // only if not a link, can't set their owner on this platform + if((mode & S_IFMT) != S_IFLNK) + { + // Not a link, use normal chown + if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) + { + BOX_LOG_SYS_ERROR("Failed to change " + "owner of file " + "'" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError) + } + } + #else + // use the version which sets things on symlinks + if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) + { + BOX_LOG_SYS_ERROR("Failed to change owner of " + "symbolic link '" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError) + } + #endif + } + + if(static_cast<int>(xattrOffset+sizeof(u_int32_t))<=mpClearAttributes->GetSize()) + { + WriteExtendedAttr(Filename, xattrOffset); + } + + // Stop now if symlink, because otherwise it'll just be applied to the target + if((mode & S_IFMT) == S_IFLNK) + { + return; + } + + // Set modification time? + box_time_t modtime = box_ntoh64(pattr->ModificationTime); + if(modtime != 0) + { + // Work out times as timevals + struct timeval times[2]; + + #ifdef WIN32 + BoxTimeToTimeval(box_ntoh64(pattr->ModificationTime), + times[1]); + BoxTimeToTimeval(box_ntoh64(pattr->AttrModificationTime), + times[0]); + // Because stat() returns the creation time in the ctime + // field under Windows, and this gets saved in the + // AttrModificationTime field of the serialised attributes, + // we subvert the first parameter of emu_utimes() to allow + // it to be reset to the right value on the restored file. + #else + BoxTimeToTimeval(modtime, times[1]); + // Copy access time as well, why not, got to set it to something + times[0] = times[1]; + // Attr modification time will be changed anyway, + // nothing that can be done about it + #endif + + // Try to apply + if(::utimes(Filename, times) != 0) + { + BOX_LOG_SYS_WARNING("Failed to change times of " + "file '" << Filename << "' to ctime=" << + BOX_FORMAT_TIMESPEC(times[0]) << ", mtime=" << + BOX_FORMAT_TIMESPEC(times[1])); + } + } + + if (MakeUserWritable) + { + mode |= S_IRWXU; + } + + // Apply everything else... (allowable mode flags only) + // Mode must be done last (think setuid) + if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID + | S_ISGID | S_ISVTX)) != 0) + { + BOX_LOG_SYS_ERROR("Failed to change permissions of file " + "'" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::IsSymLink() +// Purpose: Do these attributes represent a symbolic link? +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::IsSymLink() const +{ + EnsureClearAvailable(); + + // Got the right kind of thing? + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get the type of attributes stored + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat)) + { + // Check link + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK; + } + + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::RemoveClear() +// Purpose: Private. Deletes any clear version of the attributes that may be held +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::RemoveClear() const +{ + if(mpClearAttributes) + { + delete mpClearAttributes; + } + mpClearAttributes = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::EnsureClearAvailable() +// Purpose: Private. Makes sure the clear version is available +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::EnsureClearAvailable() const +{ + if(mpClearAttributes == 0) + { + mpClearAttributes = MakeClear(*this); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) +// Purpose: Private function, apply the stored extended attributes to the file +// Created: 2005/06/13 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) const +{ +#ifdef HAVE_SYS_XATTR_H + const char* buffer = static_cast<char*>(mpClearAttributes->GetBuffer()); + + u_int32_t xattrBlockLength = 0; + std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(u_int32_t)); + int xattrBlockSize = ntohl(xattrBlockLength); + xattrOffset += sizeof(u_int32_t); + + int xattrEnd = xattrOffset+xattrBlockSize; + if(xattrEnd>mpClearAttributes->GetSize()) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + while(xattrOffset<xattrEnd) + { + u_int16_t keyLength = 0; + std::memcpy(&keyLength, buffer+xattrOffset, sizeof(u_int16_t)); + int keySize = ntohs(keyLength); + xattrOffset += sizeof(u_int16_t); + + const char* key = buffer+xattrOffset; + xattrOffset += keySize; + + u_int32_t valueLength = 0; + std::memcpy(&valueLength, buffer+xattrOffset, sizeof(u_int32_t)); + int valueSize = ntohl(valueLength); + xattrOffset += sizeof(u_int32_t); + + // FIXME: Warn on EOPNOTSUPP + if(::lsetxattr(Filename, key, buffer+xattrOffset, valueSize, 0)!=0 && errno!=EOPNOTSUPP) + { + BOX_LOG_SYS_ERROR("Failed to set extended attributes " + "on file '" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError); + } + + xattrOffset += valueSize; + } + + ASSERT(xattrOffset==xattrEnd); +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &) +// Purpose: Static. Decrypts stored attributes. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted) +{ + // New block + StreamableMemBlock *pdecrypted = 0; + + try + { + // Check the block is big enough for IV and header + int ivSize = sBlowfishEncrypt.GetIVLength(); + if(rEncrypted.GetSize() <= (ivSize + 1)) + { + THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes); + } + + // How much space is needed for the output? + int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize); + + // Allocate it + pdecrypted = new StreamableMemBlock(maxDecryptedSize); + + // ptr to block + uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer(); + + // Check that the header has right type + if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH) + { + THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding); + } + + // Set IV + sBlowfishDecrypt.SetIV(encBlock + 1); + + // Decrypt + int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1)); + + // Resize block to fit + pdecrypted->ResizeBlock(decryptedSize); + } + catch(...) + { + delete pdecrypted; + pdecrypted = 0; + } + + return pdecrypted; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int) +// Purpose: Static. Sets the key to use for encryption and decryption. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength) +{ + // IVs set later + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &) +// Purpose: Private. Encrypt the given attributes into this block. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt) +{ + // Free any existing block + FreeBlock(); + + // Work out the maximum amount of space we need + int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize()); + // And the size of the IV + int ivSize = sBlowfishEncrypt.GetIVLength(); + + // Allocate this space + AllocateBlock(maxEncryptedSize + ivSize + 1); + + // Store the encoding byte + uint8_t *block = (uint8_t*)GetBuffer(); + block[0] = ATTRIBUTE_ENCODING_BLOWFISH; + + // Generate and store an IV for this attribute block + int ivSize2 = 0; + const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2); + ASSERT(ivSize == ivSize2); + + // Copy into the encrypted block + ::memcpy(block + 1, iv, ivSize); + + // Do the transform + int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize()); + + // Resize this block + ResizeBlock(encrytedSize + ivSize + 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int) +// Purpose: Set the secret for the filename attribute hash +// Created: 25/4/04 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength) +{ + if(SecretLength > (int)sizeof(sAttributeHashSecret)) + { + SecretLength = sizeof(sAttributeHashSecret); + } + if(SecretLength < 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Copy + ::memcpy(sAttributeHashSecret, pSecret, SecretLength); + sAttributeHashSecretLength = SecretLength; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::GenerateAttributeHash( +// struct stat &, const std::string &, +// const std::string &) +// Purpose: Generate a 64 bit hash from the attributes, used to +// detect changes. Include filename in the hash, so +// that it changes from one file to another, so don't +// reveal identical attributes. +// Created: 25/4/04 +// +// -------------------------------------------------------------------------- +uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, + const std::string &filename, const std::string &leafname) +{ + if(sAttributeHashSecretLength == 0) + { + THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet) + } + + // Assemble stuff we're interested in + attributeHashData hashData; + memset(&hashData, 0, sizeof(hashData)); + // Use network byte order and large sizes to be cross platform + hashData.uid = htonl(st.st_uid); + hashData.gid = htonl(st.st_gid); + hashData.mode = htonl(st.st_mode); + + #ifdef WIN32 + // On Windows, the "file attribute modification time" is the + // file creation time, and we want to back this up, restore + // it and compare it. + // + // On other platforms, it's not very important and can't + // reliably be set to anything other than the current time. + hashData.fileCreationTime = box_hton64(st.st_ctime); + #endif + + StreamableMemBlock xattr; + FillExtendedAttr(xattr, filename.c_str()); + + // Create a MD5 hash of the data, filename, and secret + MD5Digest digest; + digest.Add(&hashData, sizeof(hashData)); + digest.Add(xattr.GetBuffer(), xattr.GetSize()); + digest.Add(leafname.c_str(), leafname.size()); + digest.Add(sAttributeHashSecret, sAttributeHashSecretLength); + digest.Finish(); + + // Return the first 64 bits of the hash + uint64_t result = *((uint64_t *)(digest.DigestAsData())); + return result; +} diff --git a/lib/backupstore/BackupClientFileAttributes.h b/lib/backupstore/BackupClientFileAttributes.h new file mode 100644 index 00000000..f9a0d883 --- /dev/null +++ b/lib/backupstore/BackupClientFileAttributes.h @@ -0,0 +1,78 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientFileAttributes.h +// Purpose: Storage of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTFILEATTRIBUTES__H +#define BACKUPCLIENTFILEATTRIBUTES__H + +#include <string> + +#include "StreamableMemBlock.h" +#include "BoxTime.h" + +EMU_STRUCT_STAT; // declaration + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientFileAttributes +// Purpose: Storage, streaming and application of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +class BackupClientFileAttributes : public StreamableMemBlock +{ +public: + BackupClientFileAttributes(); + BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy); + BackupClientFileAttributes(const StreamableMemBlock &rToCopy); + ~BackupClientFileAttributes(); + BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr); + BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr); + bool operator==(const BackupClientFileAttributes &rAttr) const; +// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous? + + bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const; + + // Prevent access to base class members accidently + void Set(); + + void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false, + box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0, + InodeRefType *pInodeNumber = 0, bool *pHasMultipleLinks = 0); + void WriteAttributes(const char *Filename, + bool MakeUserWritable = false) const; + void GetModificationTimes(box_time_t *pModificationTime, + box_time_t *pAttrModificationTime) const; + + bool IsSymLink() const; + + static void SetBlowfishKey(const void *pKey, int KeyLength); + static void SetAttributeHashSecret(const void *pSecret, int SecretLength); + + static uint64_t GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname); + static void FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename); + +private: + static void FillAttributes(StreamableMemBlock &outputBlock, + const char *Filename, EMU_STRUCT_STAT &st, + bool ZeroModificationTimes); + static void FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st); + void WriteExtendedAttr(const char *Filename, int xattrOffset) const; + + void RemoveClear() const; + void EnsureClearAvailable() const; + static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted); + void EncryptAttr(const StreamableMemBlock &rToEncrypt); + +private: + mutable StreamableMemBlock *mpClearAttributes; +}; + +#endif // BACKUPCLIENTFILEATTRIBUTES__H + diff --git a/lib/backupstore/BackupCommands.cpp b/lib/backupstore/BackupCommands.cpp new file mode 100644 index 00000000..34f813df --- /dev/null +++ b/lib/backupstore/BackupCommands.cpp @@ -0,0 +1,959 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupCommands.cpp +// Purpose: Implement commands for the Backup store protocol +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <set> +#include <sstream> + +#include "autogen_BackupProtocolServer.h" +#include "autogen_RaidFileException.h" +#include "BackupConstants.h" +#include "BackupStoreContext.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreInfo.h" +#include "BufferedStream.h" +#include "CollectInBufferStream.h" +#include "FileStream.h" +#include "InvisibleTempFileStream.h" +#include "RaidFileController.h" +#include "StreamableMemBlock.h" + +#include "MemLeakFindOn.h" + +#define PROTOCOL_ERROR(code) \ + std::auto_ptr<ProtocolObject>(new BackupProtocolServerError( \ + BackupProtocolServerError::ErrorType, \ + BackupProtocolServerError::code)); + +#define CHECK_PHASE(phase) \ + if(rContext.GetPhase() != BackupStoreContext::phase) \ + { \ + return PROTOCOL_ERROR(Err_NotInRightProtocolPhase); \ + } + +#define CHECK_WRITEABLE_SESSION \ + if(rContext.SessionIsReadOnly()) \ + { \ + return PROTOCOL_ERROR(Err_SessionReadOnly); \ + } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerVersion::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Return the current version, or an error if the requested version isn't allowed +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerVersion::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Version) + + // Correct version? + if(mVersion != BACKUP_STORE_SERVER_VERSION) + { + return PROTOCOL_ERROR(Err_WrongVersion); + } + + // Mark the next phase + rContext.SetPhase(BackupStoreContext::Phase_Login); + + // Return our version + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerVersion(BACKUP_STORE_SERVER_VERSION)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerLogin::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Return the current version, or an error if the requested version isn't allowed +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerLogin::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Login) + + // Check given client ID against the ID in the certificate certificate + // and that the client actually has an account on this machine + if(mClientID != rContext.GetClientID()) + { + BOX_WARNING("Failed login from client ID " << + BOX_FORMAT_ACCOUNT(mClientID) << + ": wrong certificate for this account"); + return PROTOCOL_ERROR(Err_BadLogin); + } + + if(!rContext.GetClientHasAccount()) + { + BOX_WARNING("Failed login from client ID " << + BOX_FORMAT_ACCOUNT(mClientID) << + ": no such account on this server"); + return PROTOCOL_ERROR(Err_BadLogin); + } + + // If we need to write, check that nothing else has got a write lock + if((mFlags & Flags_ReadOnly) != Flags_ReadOnly) + { + // See if the context will get the lock + if(!rContext.AttemptToGetWriteLock()) + { + BOX_WARNING("Failed to get write lock for Client ID " << + BOX_FORMAT_ACCOUNT(mClientID)); + return PROTOCOL_ERROR(Err_CannotLockStoreForWriting); + } + + // Debug: check we got the lock + ASSERT(!rContext.SessionIsReadOnly()); + } + + // Load the store info + rContext.LoadStoreInfo(); + + // Get the last client store marker + int64_t clientStoreMarker = rContext.GetClientStoreMarker(); + + // Mark the next phase + rContext.SetPhase(BackupStoreContext::Phase_Commands); + + // Log login + BOX_NOTICE("Login from Client ID " << + BOX_FORMAT_ACCOUNT(mClientID) << + " " << + (((mFlags & Flags_ReadOnly) != Flags_ReadOnly) + ?"Read/Write":"Read-only")); + + // Get the usage info for reporting to the client + int64_t blocksUsed = 0, blocksSoftLimit = 0, blocksHardLimit = 0; + rContext.GetStoreDiscUsageInfo(blocksUsed, blocksSoftLimit, blocksHardLimit); + + // Return success + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerLoginConfirmed(clientStoreMarker, blocksUsed, blocksSoftLimit, blocksHardLimit)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerFinished::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Marks end of conversation (Protocol framework handles this) +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + BOX_NOTICE("Session finished for Client ID " << + BOX_FORMAT_ACCOUNT(rContext.GetClientID())); + + // Let the context know about it + rContext.ReceivedFinishCommand(); + + // can be called in any phase + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerFinished); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerListDirectory::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Command to list a directory +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerListDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + + // Store the listing to a stream + std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream); + + try + { + // Ask the context for a directory + const BackupStoreDirectory &rdir( + rContext.GetDirectory(mObjectID)); + rdir.WriteToStream(*stream, mFlagsMustBeSet, + mFlagsNotToBeSet, mSendAttributes, + false /* never send dependency info to the client */); + } + catch (RaidFileException &e) + { + if (e.GetSubType() == RaidFileException::RaidFileDoesntExist) + { + return PROTOCOL_ERROR(Err_DoesNotExist); + } + throw; + } + + stream->SetForReading(); + + // Get the protocol to send the stream + rProtocol.SendStreamAfterCommand(stream.release()); + + return std::auto_ptr<ProtocolObject>( + new BackupProtocolServerSuccess(mObjectID)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerStoreFile::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Command to store a file on the server +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerStoreFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + std::auto_ptr<ProtocolObject> hookResult = + rContext.StartCommandHook(*this); + if(hookResult.get()) + { + return hookResult; + } + + // Check that the diff from file actually exists, if it's specified + if(mDiffFromFileID != 0) + { + if(!rContext.ObjectExists(mDiffFromFileID, + BackupStoreContext::ObjectExists_File)) + { + return PROTOCOL_ERROR(Err_DiffFromFileDoesNotExist); + } + } + + // A stream follows, which contains the file + std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream()); + + // Ask the context to store it + int64_t id = 0; + try + { + id = rContext.AddFile(*dirstream, mDirectoryObjectID, + mModificationTime, mAttributesHash, mDiffFromFileID, + mFilename, + true /* mark files with same name as old versions */); + } + catch(BackupStoreException &e) + { + if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify) + { + return PROTOCOL_ERROR(Err_FileDoesNotVerify); + } + else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit) + { + return PROTOCOL_ERROR(Err_StorageLimitExceeded); + } + else + { + throw; + } + } + + // Tell the caller what the file was + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(id)); +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerGetObject::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Command to get an arbitary object from the server +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerGetObject::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + + // Check the object exists + if(!rContext.ObjectExists(mObjectID)) + { + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(NoObject)); + } + + // Open the object + std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID)); + + // Stream it to the peer + rProtocol.SendStreamAfterCommand(object.release()); + + // Tell the caller what the file was + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerGetFile::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Command to get an file object from the server -- may have to do a bit of +// work to get the object. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerGetFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + + // Check the objects exist + if(!rContext.ObjectExists(mObjectID) + || !rContext.ObjectExists(mInDirectory)) + { + return PROTOCOL_ERROR(Err_DoesNotExist); + } + + // Get the directory it's in + const BackupStoreDirectory &rdir(rContext.GetDirectory(mInDirectory)); + + // Find the object within the directory + BackupStoreDirectory::Entry *pfileEntry = rdir.FindEntryByID(mObjectID); + if(pfileEntry == 0) + { + return PROTOCOL_ERROR(Err_DoesNotExistInDirectory); + } + + // The result + std::auto_ptr<IOStream> stream; + + // Does this depend on anything? + if(pfileEntry->GetDependsNewer() != 0) + { + // File exists, but is a patch from a new version. Generate the older version. + std::vector<int64_t> patchChain; + int64_t id = mObjectID; + BackupStoreDirectory::Entry *en = 0; + do + { + patchChain.push_back(id); + en = rdir.FindEntryByID(id); + if(en == 0) + { + BOX_ERROR("Object " << + BOX_FORMAT_OBJECTID(mObjectID) << + " in dir " << + BOX_FORMAT_OBJECTID(mInDirectory) << + " for account " << + BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << + " references object " << + BOX_FORMAT_OBJECTID(id) << + " which does not exist in dir"); + return PROTOCOL_ERROR(Err_PatchConsistencyError); + } + id = en->GetDependsNewer(); + } + while(en != 0 && id != 0); + + // OK! The last entry in the chain is the full file, the others are patches back from it. + // Open the last one, which is the current from file + std::auto_ptr<IOStream> from(rContext.OpenObject(patchChain[patchChain.size() - 1])); + + // Then, for each patch in the chain, do a combine + for(int p = ((int)patchChain.size()) - 2; p >= 0; --p) + { + // ID of patch + int64_t patchID = patchChain[p]; + + // Open it a couple of times + std::auto_ptr<IOStream> diff(rContext.OpenObject(patchID)); + std::auto_ptr<IOStream> diff2(rContext.OpenObject(patchID)); + + // Choose a temporary filename for the result of the combination + std::ostringstream fs; + fs << rContext.GetStoreRoot() << ".recombinetemp." << p; + std::string tempFn = + RaidFileController::DiscSetPathToFileSystemPath( + rContext.GetStoreDiscSet(), fs.str(), + p + 16); + + // Open the temporary file + std::auto_ptr<IOStream> combined; + try + { + { + // Write nastily to allow this to work with gcc 2.x + std::auto_ptr<IOStream> t( + new InvisibleTempFileStream( + tempFn.c_str(), + O_RDWR | O_CREAT | + O_EXCL | O_BINARY | + O_TRUNC)); + combined = t; + } + } + catch(...) + { + // Make sure it goes + ::unlink(tempFn.c_str()); + throw; + } + + // Do the combining + BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined); + + // Move to the beginning of the combined file + combined->Seek(0, IOStream::SeekType_Absolute); + + // Then shuffle round for the next go + if (from.get()) from->Close(); + from = combined; + } + + // Now, from contains a nice file to send to the client. Reorder it + { + // Write nastily to allow this to work with gcc 2.x + std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */)); + stream = t; + } + + // Release from file to avoid double deletion + from.release(); + } + else + { + // Simple case: file already exists on disc ready to go + + // Open the object + std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID)); + BufferedStream buf(*object); + + // Verify it + if(!BackupStoreFile::VerifyEncodedFileFormat(buf)) + { + return PROTOCOL_ERROR(Err_FileDoesNotVerify); + } + + // Reset stream -- seek to beginning + object->Seek(0, IOStream::SeekType_Absolute); + + // Reorder the stream/file into stream order + { + // Write nastily to allow this to work with gcc 2.x + std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(object.get(), true /* take ownership */)); + stream = t; + } + + // Object will be deleted when the stream is deleted, + // so can release the object auto_ptr here to avoid + // premature deletion + object.release(); + } + + // Stream the reordered stream to the peer + rProtocol.SendStreamAfterCommand(stream.get()); + + // Don't delete the stream here + stream.release(); + + // Tell the caller what the file was + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerCreateDirectory::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Create directory command +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerCreateDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Get the stream containing the attributes + std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); + // Collect the attributes -- do this now so no matter what the outcome, + // the data has been absorbed. + StreamableMemBlock attr; + attr.Set(*attrstream, rProtocol.GetTimeout()); + + // Check to see if the hard limit has been exceeded + if(rContext.HardLimitExceeded()) + { + // Won't allow creation if the limit has been exceeded + return PROTOCOL_ERROR(Err_StorageLimitExceeded); + } + + bool alreadyExists = false; + int64_t id = rContext.AddDirectory(mContainingDirectoryID, mDirectoryName, attr, mAttributesModTime, alreadyExists); + + if(alreadyExists) + { + return PROTOCOL_ERROR(Err_DirectoryAlreadyExists); + } + + // Tell the caller what the file was + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(id)); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerChangeDirAttributes::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Change attributes on directory +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerChangeDirAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Get the stream containing the attributes + std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); + // Collect the attributes -- do this now so no matter what the outcome, + // the data has been absorbed. + StreamableMemBlock attr; + attr.Set(*attrstream, rProtocol.GetTimeout()); + + // Get the context to do it's magic + rContext.ChangeDirAttributes(mObjectID, attr, mAttributesModTime); + + // Tell the caller what the file was + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerSetReplacementFileAttributes::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Change attributes on directory +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerSetReplacementFileAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Get the stream containing the attributes + std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); + // Collect the attributes -- do this now so no matter what the outcome, + // the data has been absorbed. + StreamableMemBlock attr; + attr.Set(*attrstream, rProtocol.GetTimeout()); + + // Get the context to do it's magic + int64_t objectID = 0; + if(!rContext.ChangeFileAttributes(mFilename, mInDirectory, attr, mAttributesHash, objectID)) + { + // Didn't exist + return PROTOCOL_ERROR(Err_DoesNotExist); + } + + // Tell the caller what the file was + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID)); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &, BackupStoreContext &) +// Purpose: Delete a file +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Context handles this + int64_t objectID = 0; + rContext.DeleteFile(mFilename, mInDirectory, objectID); + + // return the object ID or zero for not found + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerUndeleteFile::DoCommand( +// BackupProtocolServer &, BackupStoreContext &) +// Purpose: Undelete a file +// Created: 2008-09-12 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerUndeleteFile::DoCommand( + BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Context handles this + bool result = rContext.UndeleteFile(mObjectID, mInDirectory); + + // return the object ID or zero for not found + return std::auto_ptr<ProtocolObject>( + new BackupProtocolServerSuccess(result ? mObjectID : 0)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &, BackupStoreContext &) +// Purpose: Delete a directory +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Check it's not asking for the root directory to be deleted + if(mObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID) + { + return PROTOCOL_ERROR(Err_CannotDeleteRoot); + } + + // Context handles this + try + { + rContext.DeleteDirectory(mObjectID); + } + catch (BackupStoreException &e) + { + if(e.GetSubType() == BackupStoreException::MultiplyReferencedObject) + { + return PROTOCOL_ERROR(Err_MultiplyReferencedObject); + } + + throw; + } + + // return the object ID + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &, BackupStoreContext &) +// Purpose: Undelete a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Check it's not asking for the root directory to be deleted + if(mObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID) + { + return PROTOCOL_ERROR(Err_CannotDeleteRoot); + } + + // Context handles this + rContext.DeleteDirectory(mObjectID, true /* undelete */); + + // return the object ID + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &, BackupStoreContext &) +// Purpose: Command to set the client's store marker +// Created: 2003/10/29 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Set the marker + rContext.SetClientStoreMarker(mClientStoreMarker); + + // return store marker set + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mClientStoreMarker)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &, BackupStoreContext &) +// Purpose: Command to move an object from one directory to another +// Created: 2003/11/12 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Let context do this, but modify error reporting on exceptions... + try + { + rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory, + mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName, + (mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject); + } + catch(BackupStoreException &e) + { + if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory) + { + return PROTOCOL_ERROR(Err_DoesNotExist); + } + else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory) + { + return PROTOCOL_ERROR(Err_TargetNameExists); + } + else + { + throw; + } + } + + // Return the object ID + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &, BackupStoreContext &) +// Purpose: Command to find the name of an object +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + + // Create a stream for the list of filenames + std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream); + + // Object and directory IDs + int64_t objectID = mObjectID; + int64_t dirID = mContainingDirectoryID; + + // Data to return in the reply + int32_t numNameElements = 0; + int16_t objectFlags = 0; + int64_t modTime = 0; + uint64_t attrModHash = 0; + bool haveModTimes = false; + + do + { + // Check the directory really exists + if(!rContext.ObjectExists(dirID, BackupStoreContext::ObjectExists_Directory)) + { + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0)); + } + + // Load up the directory + const BackupStoreDirectory &rdir(rContext.GetDirectory(dirID)); + + // Find the element in this directory and store it's name + if(objectID != ObjectID_DirectoryOnly) + { + const BackupStoreDirectory::Entry *en = rdir.FindEntryByID(objectID); + + // If this can't be found, then there is a problem... tell the caller it can't be found + if(en == 0) + { + // Abort! + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0)); + } + + // Store flags? + if(objectFlags == 0) + { + objectFlags = en->GetFlags(); + } + + // Store modification times? + if(!haveModTimes) + { + modTime = en->GetModificationTime(); + attrModHash = en->GetAttributesHash(); + haveModTimes = true; + } + + // Store the name in the stream + en->GetName().WriteToStream(*stream); + + // Count of name elements + ++numNameElements; + } + + // Setup for next time round + objectID = dirID; + dirID = rdir.GetContainerID(); + + } while(objectID != 0 && objectID != BACKUPSTORE_ROOT_DIRECTORY_ID); + + // Stream to send? + if(numNameElements > 0) + { + // Get the stream ready to go + stream->SetForReading(); + // Tell the protocol to send the stream + rProtocol.SendStreamAfterCommand(stream.release()); + } + + // Make reply + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(numNameElements, modTime, attrModHash, objectFlags)); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &, BackupStoreContext &) +// Purpose: Get the block index from a file, by ID +// Created: 19/1/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + + // Open the file + std::auto_ptr<IOStream> stream(rContext.OpenObject(mObjectID)); + + // Move the file pointer to the block index + BackupStoreFile::MoveStreamPositionToBlockIndex(*stream); + + // Return the stream to the client + rProtocol.SendStreamAfterCommand(stream.release()); + + // Return the object ID + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &, BackupStoreContext &) +// Purpose: Get the block index from a file, by name within a directory +// Created: 19/1/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + + // Get the directory + const BackupStoreDirectory &dir(rContext.GetDirectory(mInDirectory)); + + // Find the latest object ID within it which has the same name + int64_t objectID = 0; + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0) + { + if(en->GetName() == mFilename) + { + // Store the ID, if it's a newer ID than the last one + if(en->GetObjectID() > objectID) + { + objectID = en->GetObjectID(); + } + } + } + + // Found anything? + if(objectID == 0) + { + // No... return a zero object ID + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(0)); + } + + // Open the file + std::auto_ptr<IOStream> stream(rContext.OpenObject(objectID)); + + // Move the file pointer to the block index + BackupStoreFile::MoveStreamPositionToBlockIndex(*stream); + + // Return the stream to the client + rProtocol.SendStreamAfterCommand(stream.release()); + + // Return the object ID + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &, BackupStoreContext &) +// Purpose: Return the amount of disc space used +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + + // Get store info from context + const BackupStoreInfo &rinfo(rContext.GetBackupStoreInfo()); + + // Find block size + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(rinfo.GetDiscSetNumber())); + + // Return info + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerAccountUsage( + rinfo.GetBlocksUsed(), + rinfo.GetBlocksInOldFiles(), + rinfo.GetBlocksInDeletedFiles(), + rinfo.GetBlocksInDirectories(), + rinfo.GetBlocksSoftLimit(), + rinfo.GetBlocksHardLimit(), + rdiscSet.GetBlockSize() + )); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &, BackupStoreContext &) +// Purpose: Return the amount of disc space used +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + + // + // NOOP + // + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerIsAlive()); +} diff --git a/lib/backupstore/BackupConstants.h b/lib/backupstore/BackupConstants.h new file mode 100644 index 00000000..19d06a15 --- /dev/null +++ b/lib/backupstore/BackupConstants.h @@ -0,0 +1,21 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupConstants.h +// Purpose: Constants for the backup server and client +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCONSTANTS__H +#define BACKUPCONSTANTS__H + +// 15 minutes to timeout (milliseconds) +#define BACKUP_STORE_TIMEOUT (15*60*1000) + +// Should the store daemon convert files to Raid immediately? +#define BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY true + +#endif // BACKUPCONSTANTS__H + + diff --git a/lib/backupstore/BackupStoreConstants.h b/lib/backupstore/BackupStoreConstants.h new file mode 100644 index 00000000..2c33fd8f --- /dev/null +++ b/lib/backupstore/BackupStoreConstants.h @@ -0,0 +1,44 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreContants.h +// Purpose: constants for the backup system +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTORECONSTANTS__H +#define BACKUPSTORECONSTANTS__H + +#define BACKUPSTORE_ROOT_DIRECTORY_ID 1 + +#define BACKUP_STORE_SERVER_VERSION 1 + +// Minimum size for a chunk to be compressed +#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256 + +// min and max sizes for blocks +#define BACKUP_FILE_MIN_BLOCK_SIZE 4096 +#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024) + +// Increase the block size if there are more than this number of blocks +#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096 + +// Avoid creating blocks smaller than this +#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128 + +// Maximum number of sizes to do an rsync-like scan for +#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 64 + +// When doing rsync scans, do not scan for blocks smaller than +#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 128 + +// A limit to stop diffing running out of control: If more than this +// times the number of blocks in the original index are found, stop +// looking. This stops really bad cases of diffing files containing +// all the same byte using huge amounts of memory and processor time. +// This is a multiple of the number of blocks in the diff from file. +#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096 + +#endif // BACKUPSTORECONSTANTS__H + diff --git a/lib/backupstore/BackupStoreContext.cpp b/lib/backupstore/BackupStoreContext.cpp new file mode 100644 index 00000000..a62655d3 --- /dev/null +++ b/lib/backupstore/BackupStoreContext.cpp @@ -0,0 +1,1808 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreContext.cpp +// Purpose: Context for backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> + +#include "BackupConstants.h" +#include "BackupStoreContext.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreInfo.h" +#include "BackupStoreObjectMagic.h" +#include "BufferedStream.h" +#include "BufferedWriteStream.h" +#include "FileStream.h" +#include "InvisibleTempFileStream.h" +#include "RaidFileController.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "StoreStructure.h" + +class BackupStoreDaemon; + +#include "MemLeakFindOn.h" + +// Maximum number of directories to keep in the cache +// When the cache is bigger than this, everything gets +// deleted. +#ifdef BOX_RELEASE_BUILD + #define MAX_CACHE_SIZE 32 +#else + #define MAX_CACHE_SIZE 2 +#endif + +// Allow the housekeeping process 4 seconds to release an account +#define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT 4 + +// Maximum amount of store info updates before it's actually saved to disc. +#define STORE_INFO_SAVE_DELAY 96 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::BackupStoreContext() +// Purpose: Constructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreContext::BackupStoreContext(int32_t ClientID, + HousekeepingInterface &rDaemon) + : mClientID(ClientID), + mrDaemon(rDaemon), + mProtocolPhase(Phase_START), + mClientHasAccount(false), + mStoreDiscSet(-1), + mReadOnly(true), + mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), + mpTestHook(NULL) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::~BackupStoreContext() +// Purpose: Destructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreContext::~BackupStoreContext() +{ + // Delete the objects in the cache + for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + { + delete (i->second); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::CleanUp() +// Purpose: Clean up after a connection +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::CleanUp() +{ + // Make sure the store info is saved, if it has been loaded, isn't read only and has been modified + if(mapStoreInfo.get() && !(mapStoreInfo->IsReadOnly()) && + mapStoreInfo->IsModified()) + { + mapStoreInfo->Save(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ReceivedFinishCommand() +// Purpose: Called when the finish command is received by the protocol +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::ReceivedFinishCommand() +{ + if(!mReadOnly && mapStoreInfo.get()) + { + // Save the store info, not delayed + SaveStoreInfo(false); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AttemptToGetWriteLock() +// Purpose: Attempt to get a write lock for the store, and if so, unset the read only flags +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::AttemptToGetWriteLock() +{ + // Make the filename of the write lock file + std::string writeLockFile; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); + + // Request the lock + bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); + + if(!gotLock) + { + // The housekeeping process might have the thing open -- ask it to stop + char msg[256]; + int msgLen = sprintf(msg, "r%x\n", mClientID); + // Send message + mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen); + + // Then try again a few times + int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; + do + { + ::sleep(1 /* second */); + --tries; + gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); + + } while(!gotLock && tries > 0); + } + + if(gotLock) + { + // Got the lock, mark as not read only + mReadOnly = false; + } + + return gotLock; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::LoadStoreInfo() +// Purpose: Load the store info from disc +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::LoadStoreInfo() +{ + if(mapStoreInfo.get() != 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded) + } + + // Load it up! + std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly)); + + // Check it + if(i->GetAccountID() != mClientID) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount) + } + + // Keep the pointer to it + mapStoreInfo = i; + + BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet); + + // try to load the reference count database + try + { + mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + } + catch(BoxException &e) + { + BOX_WARNING("Reference count database is missing or corrupted, " + "creating a new one, expect housekeeping to find and " + "fix problems with reference counts later."); + + BackupStoreRefCountDatabase::CreateForRegeneration(account); + mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::SaveStoreInfo(bool) +// Purpose: Potentially delayed saving of the store info +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::SaveStoreInfo(bool AllowDelay) +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Can delay saving it a little while? + if(AllowDelay) + { + --mSaveStoreInfoDelay; + if(mSaveStoreInfoDelay > 0) + { + return; + } + } + + // Want to save now + mapStoreInfo->Save(); + + // Set count for next delay + mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::MakeObjectFilename(int64_t, std::string &, bool) +// Purpose: Create the filename of an object in the store, optionally creating the +// containing directory if it doesn't already exist. +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists) +{ + // Delegate to utility function + StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetDirectoryInternal(int64_t) +// Purpose: Return a reference to a directory. Valid only until the +// next time a function which affects directories is called. +// Mainly this funciton, and creation of files. +// Private version of this, which returns non-const directories. +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) +{ + // Get the filename + std::string filename; + MakeObjectFilename(ObjectID, filename); + + // Already in cache? + std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID)); + if(item != mDirectoryCache.end()) + { + // Check the revision ID of the file -- does it need refreshing? + int64_t revID = 0; + if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID)) + { + THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) + } + + if(revID == item->second->GetRevisionID()) + { + // Looks good... return the cached object + BOX_TRACE("Returning object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " from cache, modtime = " << revID); + return *(item->second); + } + + BOX_TRACE("Refreshing object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " in cache, modtime changed from " << + item->second->GetRevisionID() << " to " << revID); + + // Delete this cached object + delete item->second; + mDirectoryCache.erase(item); + } + + // Need to load it up + + // First check to see if the cache is too big + if(mDirectoryCache.size() > MAX_CACHE_SIZE) + { + // Very simple. Just delete everything! + for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + { + delete (i->second); + } + mDirectoryCache.clear(); + } + + // Get a RaidFileRead to read it + int64_t revID = 0; + std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID)); + ASSERT(revID != 0); + + // New directory object + std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory); + + // Read it from the stream, then set it's revision ID + BufferedStream buf(*objectFile); + dir->ReadFromStream(buf, IOStream::TimeOutInfinite); + dir->SetRevisionID(revID); + + // Make sure the size of the directory is available for writing the dir back + int64_t dirSize = objectFile->GetDiscUsageInBlocks(); + ASSERT(dirSize > 0); + dir->SetUserInfo1_SizeInBlocks(dirSize); + + // Store in cache + BackupStoreDirectory *pdir = dir.release(); + try + { + mDirectoryCache[ObjectID] = pdir; + } + catch(...) + { + delete pdir; + throw; + } + + // Return it + return *pdir; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AllocateObjectID() +// Purpose: Allocate a new object ID, tolerant of failures to save store info +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::AllocateObjectID() +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + // Given that the store info may not be saved for STORE_INFO_SAVE_DELAY + // times after it has been updated, this is a reasonable number of times + // to try for finding an unused ID. + // (Sizes used in the store info are fixed by the housekeeping process) + int retryLimit = (STORE_INFO_SAVE_DELAY * 2); + + while(retryLimit > 0) + { + // Attempt to allocate an ID from the store + int64_t id = mapStoreInfo->AllocateObjectID(); + + // Generate filename + std::string filename; + MakeObjectFilename(id, filename); + // Check it doesn't exist + if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) + { + // Success! + return id; + } + + // Decrement retry count, and try again + --retryLimit; + + // Mark that the store info should be saved as soon as possible + mSaveStoreInfoDelay = 0; + + BOX_WARNING("When allocating object ID, found that " << + BOX_FORMAT_OBJECTID(id) << " is already in use"); + } + + THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AddFile(IOStream &, int64_t, +// int64_t, int64_t, const BackupStoreFilename &, bool) +// Purpose: Add a file to the store, from a given stream, into +// a specified directory. Returns object ID of the new +// file. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, + int64_t ModificationTime, int64_t AttributesHash, + int64_t DiffFromFileID, const BackupStoreFilename &rFilename, + bool MarkFileWithSameNameAsOldVersions) +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // This is going to be a bit complex to make sure it copes OK + // with things going wrong. + // The only thing which isn't safe is incrementing the object ID + // and keeping the blocks used entirely accurate -- but these + // aren't big problems if they go horribly wrong. The sizes will + // be corrected the next time the account has a housekeeping run, + // and the object ID allocation code is tolerant of missed IDs. + // (the info is written lazily, so these are necessary) + + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Allocate the next ID + int64_t id = AllocateObjectID(); + + // Stream the file to disc + std::string fn; + MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); + int64_t newObjectBlocksUsed = 0; + RaidFileWrite *ppreviousVerStoreFile = 0; + bool reversedDiffIsCompletelyDifferent = false; + int64_t oldVersionNewBlocksUsed = 0; + try + { + RaidFileWrite storeFile(mStoreDiscSet, fn); + storeFile.Open(false /* no overwriting */); + + // size adjustment from use of patch in old file + int64_t spaceSavedByConversionToPatch = 0; + + // Diff or full file? + if(DiffFromFileID == 0) + { + // A full file, just store to disc + if(!rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT)) + { + THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) + } + } + else + { + // Check that the diffed from ID actually exists in the directory + if(dir.FindEntryByID(DiffFromFileID) == 0) + { + THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory) + } + + // Diff file, needs to be recreated. + // Choose a temporary filename. + std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp", + 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */)); + + try + { + // Open it twice +#ifdef WIN32 + InvisibleTempFileStream diff(tempFn.c_str(), + O_RDWR | O_CREAT | O_BINARY); + InvisibleTempFileStream diff2(tempFn.c_str(), + O_RDWR | O_BINARY); +#else + FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); + FileStream diff2(tempFn.c_str(), O_RDONLY); + + // Unlink it immediately, so it definitely goes away + if(::unlink(tempFn.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } +#endif + + // Stream the incoming diff to this temporary file + if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) + { + THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) + } + + // Verify the diff + diff.Seek(0, IOStream::SeekType_Absolute); + if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) + { + THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) + } + + // Seek to beginning of diff file + diff.Seek(0, IOStream::SeekType_Absolute); + + // Filename of the old version + std::string oldVersionFilename; + MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */); + + // Reassemble that diff -- open previous file, and combine the patch and file + std::auto_ptr<RaidFileRead> from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); + BackupStoreFile::CombineFile(diff, diff2, *from, storeFile); + + // Then... reverse the patch back (open the from file again, and create a write file to overwrite it) + std::auto_ptr<RaidFileRead> from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); + ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename); + ppreviousVerStoreFile->Open(true /* allow overwriting */); + from->Seek(0, IOStream::SeekType_Absolute); + diff.Seek(0, IOStream::SeekType_Absolute); + BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile, + DiffFromFileID, &reversedDiffIsCompletelyDifferent); + + // Store disc space used + oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks(); + + // And make a space adjustment for the size calculation + spaceSavedByConversionToPatch = + from->GetDiscUsageInBlocks() - + oldVersionNewBlocksUsed; + + // Everything cleans up here... + } + catch(...) + { + // Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway + ::unlink(tempFn.c_str()); + throw; + } + } + + // Get the blocks used + newObjectBlocksUsed = storeFile.GetDiscUsageInBlocks(); + + // Exceeds the hard limit? + int64_t newBlocksUsed = mapStoreInfo->GetBlocksUsed() + + newObjectBlocksUsed - spaceSavedByConversionToPatch; + if(newBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) + { + THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) + // The store file will be deleted automatically by the RaidFile object + } + + // Commit the file + storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + } + catch(...) + { + // Delete any previous version store file + if(ppreviousVerStoreFile != 0) + { + delete ppreviousVerStoreFile; + ppreviousVerStoreFile = 0; + } + + throw; + } + + // Verify the file -- only necessary for non-diffed versions + // NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because + // in the non-diffed code path it's never allocated. + if(DiffFromFileID == 0) + { + std::auto_ptr<RaidFileRead> checkFile(RaidFileRead::Open(mStoreDiscSet, fn)); + if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile)) + { + // Error! Delete the file + RaidFileWrite del(mStoreDiscSet, fn); + del.Delete(); + + // Exception + THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) + } + } + + // Modify the directory -- first make all files with the same name + // marked as an old version + int64_t blocksInOldFiles = 0; + try + { + if(MarkFileWithSameNameAsOldVersions) + { + BackupStoreDirectory::Iterator i(dir); + + BackupStoreDirectory::Entry *e = 0; + while((e = i.Next()) != 0) + { + // First, check it's not an old version (cheaper comparison) + if(! e->IsOld()) + { + // Compare name + if(e->GetName() == rFilename) + { + // Check that it's definately not an old version + ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0); + // Set old version flag + e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion); + // Can safely do this, because we know we won't be here if it's already + // an old version + blocksInOldFiles += e->GetSizeInBlocks(); + } + } + } + } + + // Then the new entry + BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename, + ModificationTime, id, newObjectBlocksUsed, + BackupStoreDirectory::Entry::Flags_File, + AttributesHash); + + // Adjust for the patch back stuff? + if(DiffFromFileID != 0) + { + // Get old version entry + BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID); + ASSERT(poldEntry != 0); + + // Adjust dependency info of file? + if(!reversedDiffIsCompletelyDifferent) + { + poldEntry->SetDependsNewer(id); + pnewEntry->SetDependsOlder(DiffFromFileID); + } + + // Adjust size of old entry + int64_t oldSize = poldEntry->GetSizeInBlocks(); + poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); + + // And adjust blocks used count, for later adjustment + newObjectBlocksUsed += (oldVersionNewBlocksUsed - oldSize); + blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize); + } + + // Write the directory back to disc + SaveDirectory(dir, InDirectory); + + // Commit the old version's new patched version, now that the directory safely reflects + // the state of the files on disc. + if(ppreviousVerStoreFile != 0) + { + ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + delete ppreviousVerStoreFile; + ppreviousVerStoreFile = 0; + } + } + catch(...) + { + // Back out on adding that file + RaidFileWrite del(mStoreDiscSet, fn); + del.Delete(); + + // Remove this entry from the cache + RemoveDirectoryFromCache(InDirectory); + + // Delete any previous version store file + if(ppreviousVerStoreFile != 0) + { + delete ppreviousVerStoreFile; + ppreviousVerStoreFile = 0; + } + + // Don't worry about the incremented number in the store info + throw; + } + + // Check logic + ASSERT(ppreviousVerStoreFile == 0); + + // Modify the store info + + if(DiffFromFileID == 0) + { + mapStoreInfo->AdjustNumFiles(1); + } + else + { + mapStoreInfo->AdjustNumOldFiles(1); + } + + mapStoreInfo->ChangeBlocksUsed(newObjectBlocksUsed); + mapStoreInfo->ChangeBlocksInCurrentFiles(newObjectBlocksUsed - + blocksInOldFiles); + mapStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles); + + // Increment reference count on the new directory to one + mapRefCount->AddReference(id); + + // Save the store info -- can cope if this exceptions because infomation + // will be rebuilt by housekeeping, and ID allocation can recover. + SaveStoreInfo(false); + + // Return the ID to the caller + return id; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) +// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut) +{ + // Essential checks! + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Find the directory the file is in (will exception if it fails) + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Setup flags + bool fileExisted = false; + bool madeChanges = false; + rObjectIDOut = 0; // not found + + // Count of deleted blocks + int64_t blocksDel = 0; + + try + { + // Iterate through directory, only looking at files which haven't been deleted + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *e = 0; + while((e = i.Next(BackupStoreDirectory::Entry::Flags_File, + BackupStoreDirectory::Entry::Flags_Deleted)) != 0) + { + // Compare name + if(e->GetName() == rFilename) + { + // Check that it's definately not already deleted + ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0); + // Set deleted flag + e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); + // Mark as made a change + madeChanges = true; + // Can safely do this, because we know we won't be here if it's already + // an old version + blocksDel += e->GetSizeInBlocks(); + // Is this the last version? + if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) + { + // Yes. It's been found. + rObjectIDOut = e->GetObjectID(); + fileExisted = true; + } + } + } + + // Save changes? + if(madeChanges) + { + // Save the directory back + SaveDirectory(dir, InDirectory); + + // Modify the store info, and write + // It definitely wasn't an old or deleted version + mapStoreInfo->AdjustNumFiles(-1); + mapStoreInfo->AdjustNumDeletedFiles(1); + mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); + + SaveStoreInfo(false); + } + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } + + return fileExisted; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::UndeleteFile(int64_t, int64_t) +// Purpose: Undeletes a file, if it exists, returning true if +// the file existed. +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory) +{ + // Essential checks! + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Find the directory the file is in (will exception if it fails) + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Setup flags + bool fileExisted = false; + bool madeChanges = false; + + // Count of deleted blocks + int64_t blocksDel = 0; + + try + { + // Iterate through directory, only looking at files which have been deleted + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *e = 0; + while((e = i.Next(BackupStoreDirectory::Entry::Flags_File | + BackupStoreDirectory::Entry::Flags_Deleted, 0)) != 0) + { + // Compare name + if(e->GetObjectID() == ObjectID) + { + // Check that it's definitely already deleted + ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0); + // Clear deleted flag + e->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); + // Mark as made a change + madeChanges = true; + blocksDel -= e->GetSizeInBlocks(); + + // Is this the last version? + if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) + { + // Yes. It's been found. + fileExisted = true; + } + } + } + + // Save changes? + if(madeChanges) + { + // Save the directory back + SaveDirectory(dir, InDirectory); + + // Modify the store info, and write + mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); + + // Maybe postponed save of store info + SaveStoreInfo(); + } + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } + + return fileExisted; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::RemoveDirectoryFromCache(int64_t) +// Purpose: Remove directory from cache +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID) +{ + std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID)); + if(item != mDirectoryCache.end()) + { + // Delete this cached object + delete item->second; + // Erase the entry form the map + mDirectoryCache.erase(item); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &, int64_t) +// Purpose: Save directory back to disc, update time in cache +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID) +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(rDir.GetObjectID() != ObjectID) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + try + { + // Write to disc, adjust size in store info + std::string dirfn; + MakeObjectFilename(ObjectID, dirfn); + { + RaidFileWrite writeDir(mStoreDiscSet, dirfn); + writeDir.Open(true /* allow overwriting */); + + BufferedWriteStream buffer(writeDir); + rDir.WriteToStream(buffer); + buffer.Flush(); + + // get the disc usage (must do this before commiting it) + int64_t dirSize = writeDir.GetDiscUsageInBlocks(); + + // Commit directory + writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + + // Make sure the size of the directory is available for writing the dir back + ASSERT(dirSize > 0); + int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks(); + mapStoreInfo->ChangeBlocksUsed(sizeAdjustment); + mapStoreInfo->ChangeBlocksInDirectories(sizeAdjustment); + // Update size stored in directory + rDir.SetUserInfo1_SizeInBlocks(dirSize); + } + // Refresh revision ID in cache + { + int64_t revid = 0; + if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + rDir.SetRevisionID(revid); + } + } + catch(...) + { + // Remove it from the cache if anything went wrong + RemoveDirectoryFromCache(ObjectID); + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AddDirectory(int64_t, +// const BackupStoreFilename &, bool &) +// Purpose: Creates a directory (or just returns the ID of an +// existing one). rAlreadyExists set appropraitely. +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists) +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Flags as not already existing + rAlreadyExists = false; + + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Scan the directory for the name (only looking for directories which already exist) + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, + BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) // Ignore deleted and old directories + { + if(en->GetName() == rFilename) + { + // Already exists + rAlreadyExists = true; + return en->GetObjectID(); + } + } + } + + // Allocate the next ID + int64_t id = AllocateObjectID(); + + // Create an empty directory with the given attributes on disc + std::string fn; + MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); + { + BackupStoreDirectory emptyDir(id, InDirectory); + // add the atttribues + emptyDir.SetAttributes(Attributes, AttributesModTime); + + // Write... + RaidFileWrite dirFile(mStoreDiscSet, fn); + dirFile.Open(false /* no overwriting */); + emptyDir.WriteToStream(dirFile); + // Get disc usage, before it's commited + int64_t dirSize = dirFile.GetDiscUsageInBlocks(); + // Commit the file + dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + + // Make sure the size of the directory is added to the usage counts in the info + ASSERT(dirSize > 0); + mapStoreInfo->ChangeBlocksUsed(dirSize); + mapStoreInfo->ChangeBlocksInDirectories(dirSize); + // Not added to cache, so don't set the size in the directory + } + + // Then add it into the parent directory + try + { + dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */); + SaveDirectory(dir, InDirectory); + + // Increment reference count on the new directory to one + mapRefCount->AddReference(id); + } + catch(...) + { + // Back out on adding that directory + RaidFileWrite del(mStoreDiscSet, fn); + del.Delete(); + + // Remove this entry from the cache + RemoveDirectoryFromCache(InDirectory); + + // Don't worry about the incremented number in the store info + throw; + } + + // Save the store info (may not be postponed) + mapStoreInfo->AdjustNumDirectories(1); + SaveStoreInfo(false); + + // tell caller what the ID was + return id; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &, bool) +// Purpose: Recusively deletes a directory (or undeletes if Undelete = true) +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) +{ + // Essential checks! + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Containing directory + int64_t InDirectory = 0; + + // Count of blocks deleted + int64_t blocksDeleted = 0; + + try + { + // Get the directory that's to be deleted + { + // In block, because dir may not be valid after the delete directory call + BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); + + // Store the directory it's in for later + InDirectory = dir.GetContainerID(); + + // Depth first delete of contents + DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete); + } + + // Remove the entry from the directory it's in + ASSERT(InDirectory != 0); + BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory)); + + BackupStoreDirectory::Iterator i(parentDir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), + Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) + { + if(en->GetObjectID() == ObjectID) + { + // This is the one to delete + if(Undelete) + { + en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + else + { + en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + + // Save it + SaveDirectory(parentDir, InDirectory); + + // Done + break; + } + } + + // Update blocks deleted count + mapStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted)); + mapStoreInfo->AdjustNumDirectories(-1); + SaveStoreInfo(false); + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteDirectoryRecurse(BackupStoreDirectory &, int64_t) +// Purpose: Private. Deletes a directory depth-first recusively. +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete) +{ + try + { + // Does things carefully to avoid using a directory in the cache after recursive call + // because it may have been deleted. + + // Do sub directories + { + // Get the directory... + BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); + + // Then scan it for directories + std::vector<int64_t> subDirs; + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + if(Undelete) + { + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted, // deleted dirs + BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING)) != 0) + { + // Store the directory ID. + subDirs.push_back(en->GetObjectID()); + } + } + else + { + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir, // dirs only + BackupStoreDirectory::Entry::Flags_Deleted)) != 0) // but not deleted ones + { + // Store the directory ID. + subDirs.push_back(en->GetObjectID()); + } + } + + // Done with the directory for now. Recurse to sub directories + for(std::vector<int64_t>::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i) + { + DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete); + } + } + + // Then, delete the files. Will need to load the directory again because it might have + // been removed from the cache. + { + // Get the directory... + BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); + + // Changes made? + bool changesMade = false; + + // Run through files + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + + while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), + Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) + { + // Add/remove the deleted flags + if(Undelete) + { + en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + else + { + en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + + // Keep count of the deleted blocks + if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) + { + rBlocksDeletedOut += en->GetSizeInBlocks(); + } + + // Did something + changesMade = true; + } + + // Save the directory + if(changesMade) + { + SaveDirectory(dir, ObjectID); + } + } + } + catch(...) + { + RemoveDirectoryFromCache(ObjectID); + throw; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ChangeDirAttributes(int64_t, const StreamableMemBlock &, int64_t) +// Purpose: Change the attributes of a directory +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime) +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + try + { + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(Directory)); + + // Set attributes + dir.SetAttributes(Attributes, AttributesModTime); + + // Save back + SaveDirectory(dir, Directory); + } + catch(...) + { + RemoveDirectoryFromCache(Directory); + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ChangeFileAttributes(int64_t, int64_t, const StreamableMemBlock &, int64_t) +// Purpose: Sets the attributes on a directory entry. Returns true if the object existed, false if it didn't. +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut) +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + try + { + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Find the file entry + BackupStoreDirectory::Entry *en = 0; + // Iterate through current versions of files, only + BackupStoreDirectory::Iterator i(dir); + while((en = i.Next( + BackupStoreDirectory::Entry::Flags_File, + BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion) + ) != 0) + { + if(en->GetName() == rFilename) + { + // Set attributes + en->SetAttributes(Attributes, AttributesHash); + + // Tell caller the object ID + rObjectIDOut = en->GetObjectID(); + + // Done + break; + } + } + if(en == 0) + { + // Didn't find it + return false; + } + + // Save back + SaveDirectory(dir, InDirectory); + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } + + // Changed, everything OK + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ObjectExists(int64_t) +// Purpose: Test to see if an object of this ID exists in the store +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + // Note that we need to allow object IDs a little bit greater than the last one in the store info, + // because the store info may not have got saved in an error condition. Max greater ID is + // STORE_INFO_SAVE_DELAY in this case, *2 to be safe. + if(ObjectID <= 0 || ObjectID > (mapStoreInfo->GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2))) + { + // Obviously bad object ID + return false; + } + + // Test to see if it exists on the disc + std::string filename; + MakeObjectFilename(ObjectID, filename); + if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) + { + // RaidFile reports no file there + return false; + } + + // Do we need to be more specific? + if(MustBe != ObjectExists_Anything) + { + // Open the file + std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename)); + + // Read the first integer + u_int32_t magic; + if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */)) + { + // Failed to get any bytes, must have failed + return false; + } + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(MustBe == ObjectExists_File && ntohl(magic) == OBJECTMAGIC_FILE_MAGIC_VALUE_V0) + { + // Old version detected + return true; + } +#endif + + // Right one? + u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; + + // Check + if(ntohl(magic) != requiredMagic) + { + return false; + } + + // File is implicitly closed + } + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::OpenObject(int64_t) +// Purpose: Opens an object +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreContext::OpenObject(int64_t ObjectID) +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + // Attempt to open the file + std::string fn; + MakeObjectFilename(ObjectID, fn); + return std::auto_ptr<IOStream>(RaidFileRead::Open(mStoreDiscSet, fn).release()); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetClientStoreMarker() +// Purpose: Retrieve the client store marker +// Created: 2003/10/29 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::GetClientStoreMarker() +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + return mapStoreInfo->GetClientStoreMarker(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetStoreDiscUsageInfo(int64_t &, int64_t &, int64_t &) +// Purpose: Get disc usage info from store info +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit) +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + rBlocksUsed = mapStoreInfo->GetBlocksUsed(); + rBlocksSoftLimit = mapStoreInfo->GetBlocksSoftLimit(); + rBlocksHardLimit = mapStoreInfo->GetBlocksHardLimit(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::HardLimitExceeded() +// Purpose: Returns true if the hard limit has been exceeded +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::HardLimitExceeded() +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + return mapStoreInfo->GetBlocksUsed() > mapStoreInfo->GetBlocksHardLimit(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::SetClientStoreMarker(int64_t) +// Purpose: Sets the client store marker, and commits it to disc +// Created: 2003/10/29 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker) +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + mapStoreInfo->SetClientStoreMarker(ClientStoreMarker); + SaveStoreInfo(false /* don't delay saving this */); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::MoveObject(int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) +// Purpose: Move an object (and all objects with the same name) from one directory to another +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Should deleted files be excluded when checking for the existance of objects with the target name? + int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject) + ?(BackupStoreDirectory::Entry::Flags_Deleted) + :(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); + + // Special case if the directories are the same... + if(MoveFromDirectory == MoveToDirectory) + { + try + { + // Get the first directory + BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory)); + + // Find the file entry + BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); + + // Error if not found + if(en == 0) + { + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) + } + + // Check the new name doens't already exist (optionally ignoring deleted files) + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0) + { + if(c->GetName() == rNewFilename) + { + THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory) + } + } + } + + // Need to get all the entries with the same name? + if(MoveAllWithSameName) + { + // Iterate through the directory, copying all with matching names + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next()) != 0) + { + if(c->GetName() == en->GetName()) + { + // Rename this one + c->SetName(rNewFilename); + } + } + } + else + { + // Just copy this one + en->SetName(rNewFilename); + } + + // Save the directory back + SaveDirectory(dir, MoveFromDirectory); + } + catch(...) + { + RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same + throw; + } + + return; + } + + // Got to be careful how this is written, as we can't guarentte that if we have two + // directories open, the first won't be deleted as the second is opened. (cache) + + // List of entries to move + std::vector<BackupStoreDirectory::Entry *> moving; + + // list of directory IDs which need to have containing dir id changed + std::vector<int64_t> dirsToChangeContainingID; + + try + { + // First of all, get copies of the entries to move to the to directory. + + { + // Get the first directory + BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); + + // Find the file entry + BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID); + + // Error if not found + if(en == 0) + { + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) + } + + // Need to get all the entries with the same name? + if(MoveAllWithSameName) + { + // Iterate through the directory, copying all with matching names + BackupStoreDirectory::Iterator i(from); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next()) != 0) + { + if(c->GetName() == en->GetName()) + { + // Copy + moving.push_back(new BackupStoreDirectory::Entry(*c)); + + // Check for containing directory correction + if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID()); + } + } + ASSERT(!moving.empty()); + } + else + { + // Just copy this one + moving.push_back(new BackupStoreDirectory::Entry(*en)); + + // Check for containing directory correction + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID()); + } + } + + // Secondly, insert them into the to directory, and save it + + { + // To directory + BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); + + // Check the new name doens't already exist + { + BackupStoreDirectory::Iterator i(to); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0) + { + if(c->GetName() == rNewFilename) + { + THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory) + } + } + } + + // Copy the entries into it, changing the name as we go + for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) + { + BackupStoreDirectory::Entry *en = (*i); + en->SetName(rNewFilename); + to.AddEntry(*en); // adds copy + } + + // Save back + SaveDirectory(to, MoveToDirectory); + } + + // Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory + try + { + // Get directory + BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); + + // Delete each one + for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) + { + from.DeleteEntry((*i)->GetObjectID()); + } + + // Save back + SaveDirectory(from, MoveFromDirectory); + } + catch(...) + { + // UNDO modification to To directory + + // Get directory + BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); + + // Delete each one + for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) + { + to.DeleteEntry((*i)->GetObjectID()); + } + + // Save back + SaveDirectory(to, MoveToDirectory); + + // Throw the error + throw; + } + + // Finally... for all the directories we moved, modify their containing directory ID + for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) + { + // Load the directory + BackupStoreDirectory &change(GetDirectoryInternal(*i)); + + // Modify containing dir ID + change.SetContainerID(MoveToDirectory); + + // Save it back + SaveDirectory(change, *i); + } + } + catch(...) + { + // Make sure directories aren't in the cache, as they may have been modified + RemoveDirectoryFromCache(MoveToDirectory); + RemoveDirectoryFromCache(MoveFromDirectory); + for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) + { + RemoveDirectoryFromCache(*i); + } + + while(!moving.empty()) + { + delete moving.back(); + moving.pop_back(); + } + throw; + } + + // Clean up + while(!moving.empty()) + { + delete moving.back(); + moving.pop_back(); + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetBackupStoreInfo() +// Purpose: Return the backup store info object, exception if it isn't loaded +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +const BackupStoreInfo &BackupStoreContext::GetBackupStoreInfo() const +{ + if(mapStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + return *(mapStoreInfo.get()); +} + + diff --git a/lib/backupstore/BackupStoreContext.h b/lib/backupstore/BackupStoreContext.h new file mode 100644 index 00000000..44a05dd8 --- /dev/null +++ b/lib/backupstore/BackupStoreContext.h @@ -0,0 +1,186 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreContext.h +// Purpose: Context for backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCONTEXT__H +#define BACKUPCONTEXT__H + +#include <string> +#include <map> +#include <memory> + +#include "BackupStoreRefCountDatabase.h" +#include "NamedLock.h" +#include "ProtocolObject.h" +#include "Utils.h" + +class BackupStoreDirectory; +class BackupStoreFilename; +class BackupStoreDaemon; +class BackupStoreInfo; +class IOStream; +class BackupProtocolObject; +class StreamableMemBlock; + +class HousekeepingInterface +{ + public: + virtual ~HousekeepingInterface() { } + virtual void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreContext +// Purpose: Context for backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +class BackupStoreContext +{ +public: + BackupStoreContext(int32_t ClientID, HousekeepingInterface &rDaemon); + ~BackupStoreContext(); +private: + BackupStoreContext(const BackupStoreContext &rToCopy); +public: + + void ReceivedFinishCommand(); + void CleanUp(); + + int32_t GetClientID() {return mClientID;} + + enum + { + Phase_START = 0, + Phase_Version = 0, + Phase_Login = 1, + Phase_Commands = 2 + }; + + int GetPhase() const {return mProtocolPhase;} + void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;} + + // Read only locking + bool SessionIsReadOnly() {return mReadOnly;} + bool AttemptToGetWriteLock(); + + void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;} + bool GetClientHasAccount() const {return mClientHasAccount;} + const std::string &GetStoreRoot() const {return mStoreRoot;} + int GetStoreDiscSet() const {return mStoreDiscSet;} + + // Store info + void LoadStoreInfo(); + void SaveStoreInfo(bool AllowDelay = true); + const BackupStoreInfo &GetBackupStoreInfo() const; + + // Client marker + int64_t GetClientStoreMarker(); + void SetClientStoreMarker(int64_t ClientStoreMarker); + + // Usage information + void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit); + bool HardLimitExceeded(); + + // Reading directories + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupStoreContext::GetDirectory(int64_t) + // Purpose: Return a reference to a directory. Valid only until the + // next time a function which affects directories is called. + // Mainly this funciton, and creation of files. + // Created: 2003/09/02 + // + // -------------------------------------------------------------------------- + const BackupStoreDirectory &GetDirectory(int64_t ObjectID) + { + // External callers aren't allowed to change it -- this function + // merely turns the the returned directory const. + return GetDirectoryInternal(ObjectID); + } + + // Manipulating files/directories + int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions); + int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists); + void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime); + bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut); + bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut); + bool UndeleteFile(int64_t ObjectID, int64_t InDirectory); + void DeleteDirectory(int64_t ObjectID, bool Undelete = false); + void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject); + + // Manipulating objects + enum + { + ObjectExists_Anything = 0, + ObjectExists_File = 1, + ObjectExists_Directory = 2 + }; + bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything); + std::auto_ptr<IOStream> OpenObject(int64_t ObjectID); + + // Info + int32_t GetClientID() const {return mClientID;} + +private: + void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false); + BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID); + void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID); + void RemoveDirectoryFromCache(int64_t ObjectID); + void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete); + int64_t AllocateObjectID(); + + int32_t mClientID; + HousekeepingInterface &mrDaemon; + int mProtocolPhase; + bool mClientHasAccount; + std::string mStoreRoot; // has final directory separator + int mStoreDiscSet; + bool mReadOnly; + NamedLock mWriteLock; + int mSaveStoreInfoDelay; // how many times to delay saving the store info + + // Store info + std::auto_ptr<BackupStoreInfo> mapStoreInfo; + + // Refcount database + std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount; + + // Directory cache + std::map<int64_t, BackupStoreDirectory*> mDirectoryCache; + +public: + class TestHook + { + public: + virtual std::auto_ptr<ProtocolObject> StartCommand(BackupProtocolObject& + rCommand) = 0; + virtual ~TestHook() { } + }; + void SetTestHook(TestHook& rTestHook) + { + mpTestHook = &rTestHook; + } + std::auto_ptr<ProtocolObject> StartCommandHook(BackupProtocolObject& rCommand) + { + if(mpTestHook) + { + return mpTestHook->StartCommand(rCommand); + } + return std::auto_ptr<ProtocolObject>(); + } + +private: + TestHook* mpTestHook; +}; + +#endif // BACKUPCONTEXT__H + diff --git a/lib/backupstore/BackupStoreDirectory.cpp b/lib/backupstore/BackupStoreDirectory.cpp new file mode 100644 index 00000000..0d06da34 --- /dev/null +++ b/lib/backupstore/BackupStoreDirectory.cpp @@ -0,0 +1,568 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory.h +// Purpose: Representation of a backup directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> + +#include "BackupStoreDirectory.h" +#include "IOStream.h" +#include "BackupStoreException.h" +#include "BackupStoreObjectMagic.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mNumEntries; + int64_t mObjectID; // this object ID + int64_t mContainerID; // ID of container + uint64_t mAttributesModTime; + int32_t mOptionsPresent; // bit mask of optional sections / features present + // Then a StreamableMemBlock for attributes +} dir_StreamFormat; + +typedef enum +{ + Option_DependencyInfoPresent = 1 +} dir_StreamFormatOptions; + +typedef struct +{ + uint64_t mModificationTime; + int64_t mObjectID; + int64_t mSizeInBlocks; + uint64_t mAttributesHash; + int16_t mFlags; // order smaller items after bigger ones (for alignment) + // Then a BackupStoreFilename + // Then a StreamableMemBlock for attributes +} en_StreamFormat; + +typedef struct +{ + int64_t mDependsNewer; + int64_t mDependsOlder; +} en_StreamFormatDepends; + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::BackupStoreDirectory() +// Purpose: Constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::BackupStoreDirectory() + : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0) +{ + ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); +} + + +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t) +// Purpose: Constructor giving object and container IDs +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID) + : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::~BackupStoreDirectory() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::~BackupStoreDirectory() +{ + for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) + { + delete (*i); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int) +// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Get the header + dir_StreamFormat hdr; + if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic value... + if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue)) + { + THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat) + } + + // Get data + mObjectID = box_ntoh64(hdr.mObjectID); + mContainerID = box_ntoh64(hdr.mContainerID); + mAttributesModTime = box_ntoh64(hdr.mAttributesModTime); + + // Options + int32_t options = ntohl(hdr.mOptionsPresent); + + // Get attributes + mAttributes.ReadFromStream(rStream, Timeout); + + // Decode count + int count = ntohl(hdr.mNumEntries); + + // Clear existing list + for(std::vector<Entry*>::iterator i = mEntries.begin(); + i != mEntries.end(); i++) + { + delete (*i); + } + mEntries.clear(); + + // Read them in! + for(int c = 0; c < count; ++c) + { + Entry *pen = new Entry; + try + { + // Read from stream + pen->ReadFromStream(rStream, Timeout); + + // Add to list + mEntries.push_back(pen); + } + catch(...) + { + delete pen; + throw; + } + } + + // Read in dependency info? + if(options & Option_DependencyInfoPresent) + { + // Read in extra dependency data + for(int c = 0; c < count; ++c) + { + mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool) +// Purpose: Writes a selection of entries to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const +{ + // Get count of entries + int32_t count = mEntries.size(); + if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING) + { + // Need to count the entries + count = 0; + Iterator i(*this); + while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0) + { + count++; + } + } + + // Check that sensible IDs have been set + ASSERT(mObjectID != 0); + ASSERT(mContainerID != 0); + + // Need dependency info? + bool dependencyInfoRequired = false; + if(StreamDependencyInfo) + { + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + if(pen->HasDependencies()) + { + dependencyInfoRequired = true; + } + } + } + + // Options + int32_t options = 0; + if(dependencyInfoRequired) options |= Option_DependencyInfoPresent; + + // Build header + dir_StreamFormat hdr; + hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE); + hdr.mNumEntries = htonl(count); + hdr.mObjectID = box_hton64(mObjectID); + hdr.mContainerID = box_hton64(mContainerID); + hdr.mAttributesModTime = box_hton64(mAttributesModTime); + hdr.mOptionsPresent = htonl(options); + + // Write header + rStream.Write(&hdr, sizeof(hdr)); + + // Write the attributes? + if(StreamAttributes) + { + mAttributes.WriteToStream(rStream); + } + else + { + // Write a blank header instead + StreamableMemBlock::WriteEmptyBlockToStream(rStream); + } + + // Then write all the entries + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + pen->WriteToStream(rStream); + } + + // Write dependency info? + if(dependencyInfoRequired) + { + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + pen->WriteToStreamDependencyInfo(rStream); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddEntry(const Entry &) +// Purpose: Adds entry to directory (no checking) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy) +{ + Entry *pnew = new Entry(rEntryToCopy); + try + { + mEntries.push_back(pnew); + } + catch(...) + { + delete pnew; + throw; + } + + return pnew; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t) +// Purpose: Adds entry to directory (no checking) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime) +{ + Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime); + try + { + mEntries.push_back(pnew); + } + catch(...) + { + delete pnew; + throw; + } + + return pnew; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::DeleteEntry(int64_t) +// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) +{ + for(std::vector<Entry*>::iterator i(mEntries.begin()); + i != mEntries.end(); ++i) + { + if((*i)->mObjectID == ObjectID) + { + // Delete + delete (*i); + // Remove from list + mEntries.erase(i); + // Done + return; + } + } + + // Not found + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::FindEntryByID(int64_t) +// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist. +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const +{ + for(std::vector<Entry*>::const_iterator i(mEntries.begin()); + i != mEntries.end(); ++i) + { + if((*i)->mObjectID == ObjectID) + { + // Found + return (*i); + } + } + + // Not found + return 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry() +// Purpose: Constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry() + : mModificationTime(0), + mObjectID(0), + mSizeInBlocks(0), + mFlags(0), + mAttributesHash(0), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::~Entry() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::~Entry() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry(const Entry &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) + : mName(rToCopy.mName), + mModificationTime(rToCopy.mModificationTime), + mObjectID(rToCopy.mObjectID), + mSizeInBlocks(rToCopy.mSizeInBlocks), + mFlags(rToCopy.mFlags), + mAttributesHash(rToCopy.mAttributesHash), + mAttributes(rToCopy.mAttributes), + mMinMarkNumber(rToCopy.mMinMarkNumber), + mMarkNumber(rToCopy.mMarkNumber), + mDependsNewer(rToCopy.mDependsNewer), + mDependsOlder(rToCopy.mDependsOlder) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t) +// Purpose: Constructor from values +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) + : mName(rName), + mModificationTime(ModificationTime), + mObjectID(ObjectID), + mSizeInBlocks(SizeInBlocks), + mFlags(Flags), + mAttributesHash(AttributesHash), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int) +// Purpose: Read an entry from a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Grab the raw bytes from the stream which compose the header + en_StreamFormat entry; + if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Do reading first before modifying the variables, to be more exception safe + + // Get the filename + BackupStoreFilename name; + name.ReadFromStream(rStream, Timeout); + + // Get the attributes + mAttributes.ReadFromStream(rStream, Timeout); + + // Store the rest of the bits + mModificationTime = box_ntoh64(entry.mModificationTime); + mObjectID = box_ntoh64(entry.mObjectID); + mSizeInBlocks = box_ntoh64(entry.mSizeInBlocks); + mAttributesHash = box_ntoh64(entry.mAttributesHash); + mFlags = ntohs(entry.mFlags); + mName = name; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &) +// Purpose: Writes the entry to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const +{ + // Build a structure + en_StreamFormat entry; + entry.mModificationTime = box_hton64(mModificationTime); + entry.mObjectID = box_hton64(mObjectID); + entry.mSizeInBlocks = box_hton64(mSizeInBlocks); + entry.mAttributesHash = box_hton64(mAttributesHash); + entry.mFlags = htons(mFlags); + + // Write it + rStream.Write(&entry, sizeof(entry)); + + // Write the filename + mName.WriteToStream(rStream); + + // Write any attributes + mAttributes.WriteToStream(rStream); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int) +// Purpose: Read the optional dependency info from a stream +// Created: 13/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout) +{ + // Grab the raw bytes from the stream which compose the header + en_StreamFormatDepends depends; + if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Store the data + mDependsNewer = box_ntoh64(depends.mDependsNewer); + mDependsOlder = box_ntoh64(depends.mDependsOlder); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &) +// Purpose: Write the optional dependency info to a stream +// Created: 13/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const +{ + // Build structure + en_StreamFormatDepends depends; + depends.mDependsNewer = box_hton64(mDependsNewer); + depends.mDependsOlder = box_hton64(mDependsOlder); + // Write + rStream.Write(&depends, sizeof(depends)); +} + + + diff --git a/lib/backupstore/BackupStoreDirectory.h b/lib/backupstore/BackupStoreDirectory.h new file mode 100644 index 00000000..0dfe6422 --- /dev/null +++ b/lib/backupstore/BackupStoreDirectory.h @@ -0,0 +1,285 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory.h +// Purpose: Representation of a backup directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREDIRECTORY__H +#define BACKUPSTOREDIRECTORY__H + +#include <string> +#include <vector> + +#include "BackupStoreFilenameClear.h" +#include "StreamableMemBlock.h" +#include "BoxTime.h" + +class IOStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreDirectory +// Purpose: In memory representation of a directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreDirectory +{ +public: + BackupStoreDirectory(); + BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID); +private: + // Copying not allowed + BackupStoreDirectory(const BackupStoreDirectory &rToCopy); +public: + ~BackupStoreDirectory(); + + class Entry + { + public: + friend class BackupStoreDirectory; + + Entry(); + ~Entry(); + Entry(const Entry &rToCopy); + Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash); + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream) const; + + const BackupStoreFilename &GetName() const {return mName;} + box_time_t GetModificationTime() const {return mModificationTime;} + int64_t GetObjectID() const {return mObjectID;} + int64_t GetSizeInBlocks() const {return mSizeInBlocks;} + int16_t GetFlags() const {return mFlags;} + void AddFlags(int16_t Flags) {mFlags |= Flags;} + void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;} + + // Some things can be changed + void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;} + void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;} + + // Attributes + bool HasAttributes() const {return !mAttributes.IsEmpty();} + void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;} + const StreamableMemBlock &GetAttributes() const {return mAttributes;} + uint64_t GetAttributesHash() const {return mAttributesHash;} + + // Marks + // The lowest mark number a version of a file of this name has ever had + uint32_t GetMinMarkNumber() const {return mMinMarkNumber;} + // The mark number on this file + uint32_t GetMarkNumber() const {return mMarkNumber;} + + // Make sure these flags are synced with those in backupprocotol.txt + // ListDirectory command + enum + { + Flags_INCLUDE_EVERYTHING = -1, + Flags_EXCLUDE_NOTHING = 0, + Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below! + Flags_File = 1, + Flags_Dir = 2, + Flags_Deleted = 4, + Flags_OldVersion = 8, + Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion + }; + // characters for textual listing of files -- see bbackupquery/BackupQueries + #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR" + + // convenience methods + bool inline IsDir() + { + return GetFlags() & Flags_Dir; + } + bool inline IsFile() + { + return GetFlags() & Flags_File; + } + bool inline IsOld() + { + return GetFlags() & Flags_OldVersion; + } + bool inline IsDeleted() + { + return GetFlags() & Flags_Deleted; + } + bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) + { + return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet)) + && ((mFlags & FlagsNotToBeSet) == 0); + }; + + // Get dependency info + // new version this depends on + int64_t GetDependsNewer() const {return mDependsNewer;} + void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;} + // older version which depends on this + int64_t GetDependsOlder() const {return mDependsOlder;} + void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;} + + // Dependency info saving + bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;} + void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout); + void WriteToStreamDependencyInfo(IOStream &rStream) const; + + private: + BackupStoreFilename mName; + box_time_t mModificationTime; + int64_t mObjectID; + int64_t mSizeInBlocks; + int16_t mFlags; + uint64_t mAttributesHash; + StreamableMemBlock mAttributes; + uint32_t mMinMarkNumber; + uint32_t mMarkNumber; + + uint64_t mDependsNewer; // new version this depends on + uint64_t mDependsOlder; // older version which depends on this + }; + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream, + int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, + int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING, + bool StreamAttributes = true, bool StreamDependencyInfo = true) const; + + Entry *AddEntry(const Entry &rEntryToCopy); + Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime); + void DeleteEntry(int64_t ObjectID); + Entry *FindEntryByID(int64_t ObjectID) const; + + int64_t GetObjectID() const {return mObjectID;} + int64_t GetContainerID() const {return mContainerID;} + + // Need to be able to update the container ID when moving objects + void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;} + + // Purely for use of server -- not serialised into streams + int64_t GetRevisionID() const {return mRevisionID;} + void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;} + + unsigned int GetNumberOfEntries() const {return mEntries.size();} + + // User info -- not serialised into streams + int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;} + void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;} + + // Attributes + bool HasAttributes() const {return !mAttributes.IsEmpty();} + void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;} + const StreamableMemBlock &GetAttributes() const {return mAttributes;} + box_time_t GetAttributesModTime() const {return mAttributesModTime;} + + class Iterator + { + public: + Iterator(const BackupStoreDirectory &rDir) + : mrDir(rDir), i(rDir.mEntries.begin()) + { + } + + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags + while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.end()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + // WARNING: This function is really very inefficient. + // Only use when you want to look up ONE filename, not in a loop looking up lots. + // In a looping situation, cache the decrypted filenames in another memory structure. + BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags or filename + while( (i != mrDir.mEntries.end()) + && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) ) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.end()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + private: + const BackupStoreDirectory &mrDir; + std::vector<Entry*>::const_iterator i; + }; + + friend class Iterator; + + class ReverseIterator + { + public: + ReverseIterator(const BackupStoreDirectory &rDir) + : mrDir(rDir), i(rDir.mEntries.rbegin()) + { + } + + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags + while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.rend()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + private: + const BackupStoreDirectory &mrDir; + std::vector<Entry*>::const_reverse_iterator i; + }; + + friend class ReverseIterator; + + // For recovery of the store + // Implemented in BackupStoreCheck2.cpp + bool CheckAndFix(); + void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags); + bool NameInUse(const BackupStoreFilename &rName); + // Don't use these functions in normal code! + + // For testing + void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;} + + // Debug and diagonistics + void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere + +private: + int64_t mRevisionID; + int64_t mObjectID; + int64_t mContainerID; + std::vector<Entry*> mEntries; + box_time_t mAttributesModTime; + StreamableMemBlock mAttributes; + int64_t mUserInfo1; +}; + +#endif // BACKUPSTOREDIRECTORY__H + diff --git a/lib/backupstore/BackupStoreException.h b/lib/backupstore/BackupStoreException.h new file mode 100644 index 00000000..981dfa60 --- /dev/null +++ b/lib/backupstore/BackupStoreException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREEXCEPTION__H +#define BACKUPSTOREEXCEPTION__H + +// Compatibility +#include "autogen_BackupStoreException.h" + +#endif // BACKUPSTOREEXCEPTION__H + diff --git a/lib/backupstore/BackupStoreException.txt b/lib/backupstore/BackupStoreException.txt new file mode 100644 index 00000000..ece772c0 --- /dev/null +++ b/lib/backupstore/BackupStoreException.txt @@ -0,0 +1,72 @@ +EXCEPTION BackupStore 4 + +Internal 0 +BadAccountDatabaseFile 1 +AccountDatabaseNoSuchEntry 2 +InvalidBackupStoreFilename 3 +UnknownFilenameEncoding 4 +CouldntReadEntireStructureFromStream 5 +BadDirectoryFormat 6 +CouldNotFindEntryInDirectory 7 +OutputFileAlreadyExists 8 +OSFileError 9 +StreamDoesntHaveRequiredFeatures 10 +BadBackupStoreFile 11 +CouldNotLoadStoreInfo 12 +BadStoreInfoOnLoad 13 +StoreInfoIsReadOnly 14 +StoreInfoDirNotInList 15 +StoreInfoBlockDeltaMakesValueNegative 16 +DirectoryHasBeenDeleted 17 +StoreInfoNotInitialised 18 +StoreInfoAlreadyLoaded 19 +StoreInfoNotLoaded 20 +ReadFileFromStreamTimedOut 21 +FileWrongSizeAfterBeingStored 22 +AddedFileDoesNotVerify 23 +StoreInfoForWrongAccount 24 +ContextIsReadOnly 25 +AttributesNotLoaded 26 +AttributesNotUnderstood 27 +WrongServerVersion 28 # client side +ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account. +NameAlreadyExistsInDirectory 30 +BerkelyDBFailure 31 # client side +InodeMapIsReadOnly 32 # client side +InodeMapNotOpen 33 # client side +FilenameEncryptionKeyNotKnown 34 +FilenameEncryptionNoKeyForSpecifiedMethod 35 +FilenameEncryptionNotSetup 36 +CouldntLoadClientKeyMaterial 37 +BadEncryptedAttributes 38 +EncryptedAttributesHaveUnknownEncoding 39 +OutputSizeTooSmallForChunk 40 +BadEncodedChunk 41 +NotEnoughSpaceToDecodeChunk 42 +ChunkHasUnknownEncoding 43 +ChunkContainsBadCompressedData 44 +CantWriteToEncodedFileStream 45 +Temp_FileEncodeStreamDidntReadBuffer 46 +CantWriteToDecodedFileStream 47 +WhenDecodingExpectedToReadButCouldnt 48 +BackupStoreFileFailedIntegrityCheck 49 +ThereIsNoDataInASymLink 50 +IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51 +BlockEntryEncodingDidntGiveExpectedLength 52 +CouldNotFindUnusedIDDuringAllocation 53 +AddedFileExceedsStorageLimit 54 +CannotDiffAnIncompleteStoreFile 55 +CannotDecodeDiffedFilesWithoutCombining 56 +FailedToReadBlockOnCombine 57 +OnCombineFromFileIsIncomplete 58 +BadNotifySysadminEventCode 59 +InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60 +CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server? +AttributeHashSecretNotSet 62 +AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES +SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection. +IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related +DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory +PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. +UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. +MultiplyReferencedObject 69 Attempted to modify an object with multiple references, should be uncloned first diff --git a/lib/backupstore/BackupStoreFile.cpp b/lib/backupstore/BackupStoreFile.cpp new file mode 100644 index 00000000..bd62b7ba --- /dev/null +++ b/lib/backupstore/BackupStoreFile.cpp @@ -0,0 +1,1559 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFile.cpp +// Purpose: Utils for manipulating files +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <sys/stat.h> +#include <string.h> +#include <new> +#include <string.h> + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + #include <stdio.h> +#endif + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFilename.h" +#include "BackupStoreException.h" +#include "IOStream.h" +#include "Guards.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreObjectMagic.h" +#include "Compress.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "CipherAES.h" +#include "BackupStoreConstants.h" +#include "CollectInBufferStream.h" +#include "RollingChecksum.h" +#include "MD5Digest.h" +#include "ReadGatherStream.h" +#include "Random.h" +#include "BackupStoreFileEncodeStream.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +using namespace BackupStoreFileCryptVar; + +// How big a buffer to use for copying files +#define COPY_BUFFER_SIZE (8*1024) + +// Statistics +BackupStoreFileStats BackupStoreFile::msStats = {0,0,0}; + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool sWarnedAboutBackwardsCompatiblity = false; +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &) +// Purpose: Encode a file into something for storing on file server. +// Requires a real filename so full info can be stored. +// +// Returns a stream. Most of the work is done by the stream +// when data is actually requested -- the file will be held +// open until the stream is deleted or the file finished. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::EncodeFile( + const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, + ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) +{ + // Create the stream + std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); + + // Do the initial setup + ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, + 0 /* no recipe, just encode */, + ContainerID, rStoreFilename, pModificationTime, pLogger, + pRunStatusProvider); + + // Return the stream for the caller + return stream; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &) +// Purpose: Verify that an encoded file meets the format +// requirements. Doesn't verify that the data is intact +// and can be decoded. Optionally returns the ID of the +// file which it is diffed from, and the (original) +// container ID. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut) +{ + // Get the size of the file + int64_t fileSize = rFile.BytesLeftToRead(); + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Get the header... + file_StreamFormat hdr; + if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read header + return false; + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + return false; + } + + // Get a filename, see if it loads OK + try + { + BackupStoreFilename fn; + fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); + } + catch(...) + { + // an error occured while reading it, so that's not good + return false; + } + + // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not + try + { + int32_t size_s; + if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + int size = ntohl(size_s); + // Skip forward the size + rFile.Seek(size, IOStream::SeekType_Relative); + } + catch(...) + { + // an error occured while reading it, so that's not good + return false; + } + + // Get current position in file -- the end of the header + int64_t headerEnd = rFile.GetPosition(); + + // Get number of blocks + int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); + + // Calculate where the block index will be, check it's reasonable + int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + if(blockIndexLoc < headerEnd) + { + // Not enough space left for the block index, let alone the blocks themselves + return false; + } + + // Load the block index header + rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute); + file_BlockIndexHeader blkhdr; + if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read block index header -- assume bad file + return false; + } + + // Check header + if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != numBlocks) + { + // Bad header -- either magic value or number of blocks is wrong + return false; + } + + // Flag for recording whether a block is referenced from another file + bool blockFromOtherFileReferenced = false; + + // Read the index, checking that the length values all make sense + int64_t currentBlockStart = headerEnd; + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read block entry + file_BlockIndexEntry blk; + if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read block index entry -- assume bad file + return false; + } + + // Check size and location + int64_t blkSize = box_ntoh64(blk.mEncodedSize); + if(blkSize <= 0) + { + // Mark that this file references another file + blockFromOtherFileReferenced = true; + } + else + { + // This block is actually in this file + if((currentBlockStart + blkSize) > blockIndexLoc) + { + // Encoded size makes the block run over the index + return false; + } + + // Move the current block start ot the end of this block + currentBlockStart += blkSize; + } + } + + // Check that there's no empty space + if(currentBlockStart != blockIndexLoc) + { + return false; + } + + // Check that if another block is references, then the ID is there, and if one isn't there is no ID. + int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); + if((otherID != 0 && blockFromOtherFileReferenced == false) + || (otherID == 0 && blockFromOtherFileReferenced == true)) + { + // Doesn't look good! + return false; + } + + // Does the caller want the other ID? + if(pDiffFromObjectIDOut) + { + *pDiffFromObjectIDOut = otherID; + } + + // Does the caller want the container ID? + if(pContainerIDOut) + { + *pContainerIDOut = box_ntoh64(hdr.mContainerID); + } + + // Passes all tests + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeFile(IOStream &, const char *) +// Purpose: Decode a file. Will set file attributes. File must not exist. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) +{ + // Does file exist? + EMU_STRUCT_STAT st; + if(EMU_STAT(DecodedFilename, &st) == 0) + { + THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) + } + + // Try, delete output file if error + try + { + // Make a stream for outputting this file + FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL); + + // Get the decoding stream + std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr)); + + // Is it a symlink? + if(!stream->IsSymLink()) + { + // Copy it out to the file + stream->CopyStreamTo(out); + } + + out.Close(); + + // The stream might have uncertain size, in which case + // we need to drain it to get the + // Protocol::ProtocolStreamHeader_EndOfStream byte + // out of our connection stream. + char buffer[1]; + int drained = rEncodedFile.Read(buffer, 1); + + // The Read will return 0 if we are actually at the end + // of the stream, but some tests decode files directly, + // in which case we are actually positioned at the start + // of the block index. I hope that reading an extra byte + // doesn't hurt! + // ASSERT(drained == 0); + + // Write the attributes + try + { + stream->GetAttributes().WriteAttributes(DecodedFilename); + } + catch (std::exception& e) + { + BOX_WARNING("Failed to restore attributes on " << + DecodedFilename << ": " << e.what()); + } + } + catch(...) + { + ::unlink(DecodedFilename); + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *) +// Purpose: Return a stream which will decode the encrypted file data on the fly. +// Accepts streams in block index first, or main header first, order. In the latter case, +// the stream must be Seek()able. +// +// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow +// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) +{ + // Create stream + std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout)); + + // Get it ready + stream->Setup(pAlterativeAttr); + + // Return to caller + return stream; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int) +// Purpose: Constructor +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout) + : mrEncodedFile(rEncodedFile), + mTimeout(Timeout), + mNumBlocks(0), + mpBlockIndex(0), + mpEncodedData(0), + mpClearData(0), + mClearDataSize(0), + mCurrentBlock(-1), + mCurrentBlockClearSize(0), + mPositionInCurrentBlock(0), + mEntryIVBase(42) // different to default value in the encoded stream! +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + , mIsOldVersion(false) +#endif +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::~DecodedStream() +// Purpose: Desctructor +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFile::DecodedStream::~DecodedStream() +{ + // Free any allocated memory + if(mpBlockIndex) + { + ::free(mpBlockIndex); + } + if(mpEncodedData) + { + BackupStoreFile::CodingChunkFree(mpEncodedData); + } + if(mpClearData) + { + ::free(mpClearData); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *) +// Purpose: Get the stream ready to decode -- reads in headers +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr) +{ + // Get the size of the file + int64_t fileSize = mrEncodedFile.BytesLeftToRead(); + + // Get the magic number to work out which order the stream is in + int32_t magic; + if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read magic value + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + bool inFileOrder = true; + switch(ntohl(magic)) + { +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_MAGIC_VALUE_V0: + mIsOldVersion = true; + // control flows on +#endif + case OBJECTMAGIC_FILE_MAGIC_VALUE_V1: + inFileOrder = true; + break; + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0: + mIsOldVersion = true; + // control flows on +#endif + case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1: + inFileOrder = false; + break; + + default: + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // If not in file order, then the index list must be read now + if(!inFileOrder) + { + ReadBlockIndex(true /* have already read and verified the magic number */); + } + + // Get header + file_StreamFormat hdr; + if(inFileOrder) + { + // Read the header, without the magic number + if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic), + 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + // Put in magic number + hdr.mMagicValue = magic; + } + else + { + // Not in file order, so need to read the full header + if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Get the filename + mFilename.ReadFromStream(mrEncodedFile, mTimeout); + + // Get the attributes (either from stream, or supplied attributes) + if(pAlterativeAttr != 0) + { + // Read dummy attributes + BackupClientFileAttributes attr; + attr.ReadFromStream(mrEncodedFile, mTimeout); + + // Set to supplied attributes + mAttributes = *pAlterativeAttr; + } + else + { + // Read the attributes from the stream + mAttributes.ReadFromStream(mrEncodedFile, mTimeout); + } + + // If it is in file order, go and read the file attributes + // Requires that the stream can seek + if(inFileOrder) + { + // Make sure the file size is known + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Store current location (beginning of encoded blocks) + int64_t endOfHeaderPos = mrEncodedFile.GetPosition(); + + // Work out where the index is + int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); + int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + + // Seek to that position + mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute); + + // Read the block index + ReadBlockIndex(false /* magic number still to be read */); + + // Seek back to the end of header position, ready for reading the chunks + mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute); + } + + // Check view of blocks from block header and file header match + if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Need to allocate some memory for the two blocks for reading encoded data, and clear data + if(mNumBlocks > 0) + { + // Find the maximum encoded data size + int32_t maxEncodedDataSize = 0; + const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; + ASSERT(entry != 0); + for(int64_t e = 0; e < mNumBlocks; e++) + { + // Get the clear and encoded size + int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize); + ASSERT(encodedSize > 0); + + // Larger? + if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize; + } + + // Allocate those blocks! + mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32); + + // Allocate the block for the clear data, using the hint from the header. + // If this is wrong, things will exception neatly later on, so it can't be used + // to do anything more than cause an error on downloading. + mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32; + mpClearData = (uint8_t*)::malloc(mClearDataSize); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool) +// Purpose: Read the block index from the stream, and store in internal buffer (minus header) +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) +{ + // Header + file_BlockIndexHeader blkhdr; + + // Read it in -- way depends on how whether the magic number has already been read + if(MagicAlreadyRead) + { + // Read the header, without the magic number + if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue), + 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } + else + { + // Magic not already read, so need to read the full header + if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Check magic value + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + } + + // Get the number of blocks out of the header + mNumBlocks = box_ntoh64(blkhdr.mNumBlocks); + + // Read the IV base + mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase); + + // Load the block entries in? + if(mNumBlocks > 0) + { + // How big is the index? + int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks; + + // Allocate some memory + mpBlockIndex = ::malloc(indexSize); + if(mpBlockIndex == 0) + { + throw std::bad_alloc(); + } + + // Read it in + if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Read(void *, int, int) +// Purpose: As interface. Reads decrpyted data. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Symlinks don't have data. So can't read it. Not even zero bytes. + if(IsSymLink()) + { + // Don't allow reading in this case + THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink); + } + + // Already finished? + if(mCurrentBlock >= mNumBlocks) + { + // At end of stream, nothing to do + return 0; + } + + int bytesToRead = NBytes; + uint8_t *output = (uint8_t*)pBuffer; + + while(bytesToRead > 0 && mCurrentBlock < mNumBlocks) + { + // Anything left in the current block? + if(mPositionInCurrentBlock < mCurrentBlockClearSize) + { + // Copy data out of this buffer + int s = mCurrentBlockClearSize - mPositionInCurrentBlock; + if(s > bytesToRead) s = bytesToRead; // limit to requested data + + // Copy + ::memcpy(output, mpClearData + mPositionInCurrentBlock, s); + + // Update positions + output += s; + mPositionInCurrentBlock += s; + bytesToRead -= s; + } + + // Need to get some more data? + if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize) + { + // Number of next block + ++mCurrentBlock; + if(mCurrentBlock >= mNumBlocks) + { + // Stop now! + break; + } + + // Get the size from the block index + const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; + int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize); + if(encodedSize <= 0) + { + // The caller is attempting to decode a file which is the direct result of a diff + // operation, and so does not contain all the data. + // It needs to be combined with the previous version first. + THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining) + } + + // Load in next block + if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Decode the data + mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize); + + // Calculate IV for this entry + uint64_t iv = mEntryIVBase; + iv += mCurrentBlock; + // Convert to network byte order before encrypting with it, so that restores work on + // platforms with different endiannesses. + iv = box_hton64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Make sure this is the right size + if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) + { +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(!mIsOldVersion) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the + // IV for the encrypted section. Try again, with the thing the other way round + iv = box_swap64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + else + { + // Warn and log this issue + if(!sWarnedAboutBackwardsCompatiblity) + { + BOX_WARNING("WARNING: Decoded one or more files using backwards compatibility mode for block index."); + sWarnedAboutBackwardsCompatiblity = true; + } + } +#else + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) +#endif + } + + // Check the digest + MD5Digest md5; + md5.Add(mpClearData, mCurrentBlockClearSize); + md5.Finish(); + if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum)) + { + THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck) + } + + // Set vars to say what's happening + mPositionInCurrentBlock = 0; + } + } + + ASSERT(bytesToRead >= 0); + ASSERT(bytesToRead <= NBytes); + + return NBytes - bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::IsSymLink() +// Purpose: Is the unencoded file actually a symlink? +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::IsSymLink() +{ + // First, check in with the attributes + if(!mAttributes.IsSymLink()) + { + return false; + } + + // So the attributes think it is a symlink. + // Consistency check... + if(mNumBlocks != 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Write(const void *, int) +// Purpose: As interface. Throws exception, as you can't write to this stream. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::StreamDataLeft() +// Purpose: As interface. Any data left? +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::StreamDataLeft() +{ + return mCurrentBlock < mNumBlocks; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::StreamClosed() +// Purpose: As interface. Always returns true, no writing allowed. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::StreamClosed() +{ + // Can't write to this stream! + return true; +} + + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::SetBlowfishKey(const void *, int) +// Purpose: Static. Sets the key to use for encryption and decryption. +// Created: 7/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength) +{ + // IVs set later + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + + sBlowfishEncryptBlockEntry.Reset(); + sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); + sBlowfishEncryptBlockEntry.UsePadding(false); + sBlowfishDecryptBlockEntry.Reset(); + sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); + sBlowfishDecryptBlockEntry.UsePadding(false); +} + + +#ifndef HAVE_OLD_SSL +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::SetAESKey(const void *, int) +// Purpose: Sets the AES key to use for file data encryption. Will select AES as +// the cipher to use when encrypting. +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength) +{ + // Setup context + sAESEncrypt.Reset(); + sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); + sAESDecrypt.Reset(); + sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); + + // Set encryption to use this key, instead of the "default" blowfish key + spEncrypt = &sAESEncrypt; + sEncryptCipherType = HEADER_AES_ENCODING; +} +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int) +// Purpose: The maximum output size of a block, given the chunk size +// Created: 7/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize) +{ + // Calculate... the maximum size of output by first the largest it could be after compression, + // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck + // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing + // an problem where a rather non-compressable file didn't fit in a block buffer.) + return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1 + + sBlowfishEncrypt.GetIVLength() + 128; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &) +// Purpose: Encodes a chunk (encryption, possible compressed beforehand) +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput) +{ + ASSERT(spEncrypt != 0); + + // Check there's some space in the output block + if(rOutput.mBufferSize < 256) + { + rOutput.Reallocate(256); + } + + // Check alignment of the block + ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + + // Want to compress it? + bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE); + + // Build header + uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT; + if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED; + + // Store header + rOutput.mpBuffer[0] = header; + int outOffset = 1; + + // Setup cipher, and store the IV + int ivLen = 0; + const void *iv = spEncrypt->SetRandomIV(ivLen); + ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen); + outOffset += ivLen; + + // Start encryption process + spEncrypt->Begin(); + + #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \ + { \ + if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \ + { \ + rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \ + } \ + } + + // Encode the chunk + if(compressChunk) + { + // buffer to compress into + uint8_t buffer[2048]; + + // Set compressor with all the chunk as an input + Compress<true> compress; + compress.Input(Chunk, ChunkSize); + compress.FinishInput(); + + // Get and encrypt output + while(!compress.OutputHasFinished()) + { + int s = compress.Output(buffer, sizeof(buffer)); + if(s > 0) + { + ENCODECHUNK_CHECK_SPACE(s) + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); + } + else + { + // Should never happen, as we put all the input in in one go. + // So if this happens, it means there's a logical problem somewhere + THROW_EXCEPTION(BackupStoreException, Internal) + } + } + ENCODECHUNK_CHECK_SPACE(16) + outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); + } + else + { + // Straight encryption + ENCODECHUNK_CHECK_SPACE(ChunkSize) + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize); + ENCODECHUNK_CHECK_SPACE(16) + outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); + } + + ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check + + return outOffset; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int) +// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find +// the extra output buffer size needed before calling. +// See notes in EncodeChunk() for notes re alignment of the +// encoded data. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize) +{ + // Check alignment of the encoded block + ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + + // First check + if(EncodedSize < 1) + { + THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) + } + + const uint8_t *input = (uint8_t*)Encoded; + + // Get header, make checks, etc + uint8_t header = input[0]; + bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED; + uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT); + if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING) + { + THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding) + } + +#ifndef HAVE_OLD_SSL + // Choose cipher + CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt); +#else + // AES not supported with this version of OpenSSL + if(encodingType == HEADER_AES_ENCODING) + { + THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL) + } + CipherContext &cipher(sBlowfishDecrypt); +#endif + + // Check enough space for header, an IV and one byte of input + int ivLen = cipher.GetIVLength(); + if(EncodedSize < (1 + ivLen + 1)) + { + THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) + } + + // Set IV in decrypt context, and start + cipher.SetIV(input + 1); + cipher.Begin(); + + // Setup vars for code + int inOffset = 1 + ivLen; + uint8_t *output = (uint8_t*)Output; + int outOffset = 0; + + // Do action + if(chunkCompressed) + { + // Do things in chunks + uint8_t buffer[2048]; + int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer)); + + // Decompressor + Compress<false> decompress; + + while(inOffset < EncodedSize) + { + // Decrypt a block + int bl = inputBlockLen; + if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long + int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl); + inOffset += bl; + + // Decompress the decrypted data + if(s > 0) + { + decompress.Input(buffer, s); + int os = 0; + do + { + os = decompress.Output(output + outOffset, OutputSize - outOffset); + outOffset += os; + } while(os > 0); + + // Check that there's space left in the output buffer -- there always should be + if(outOffset >= OutputSize) + { + THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) + } + } + } + + // Get any compressed data remaining in the cipher context and compression + int s = cipher.Final(buffer, sizeof(buffer)); + decompress.Input(buffer, s); + decompress.FinishInput(); + while(!decompress.OutputHasFinished()) + { + int os = decompress.Output(output + outOffset, OutputSize - outOffset); + outOffset += os; + + // Check that there's space left in the output buffer -- there always should be + if(outOffset >= OutputSize) + { + THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) + } + } + } + else + { + // Easy decryption + outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset); + outOffset += cipher.Final(output + outOffset, OutputSize - outOffset); + } + + return outOffset; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool) +// Purpose: Returns a stream which gives a Stream order version of the encoded file. +// If TakeOwnership == true, then the input stream will be deleted when the +// returned stream is deleted. +// The input stream must be seekable. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership) +{ + ASSERT(pStream != 0); + + // Get the size of the file + int64_t fileSize = pStream->BytesLeftToRead(); + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Read the header + int bytesRead = 0; + file_StreamFormat hdr; + bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead); + + // Seek backwards to put the file pointer back where it was before we started this + pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative); + + // Check we got a block + if(!readBlock) + { + // Couldn't read header -- assume file bad + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Get number of blocks + int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); + + // Calculate where the block index will be, check it's reasonable + int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + int64_t blockIndexLoc = fileSize - blockIndexSize; + if(blockIndexLoc < 0) + { + // Doesn't look good! + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Build a reordered stream + std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership)); + + // Set it up... + ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get())); + int component = rreordered.AddComponent(pStream); + // Send out the block index + rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc); + // And then the rest of the file + rreordered.AddBlock(component, blockIndexLoc, true, 0); + + return reordered; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ResetStats() +// Purpose: Reset the gathered statistics +// Created: 20/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::ResetStats() +{ + msStats.mBytesInEncodedFiles = 0; + msStats.mBytesAlreadyOnServer = 0; + msStats.mTotalFileStreamSize = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &) +// Purpose: Compares the contents of a file against the checksums contained in the +// block index. Returns true if the checksums match, meaning the file is +// extremely likely to match the original. Will always consume the entire index. +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout) +{ + // is it a symlink? + bool sourceIsSymlink = false; + { + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + if((st.st_mode & S_IFMT) == S_IFLNK) + { + sourceIsSymlink = true; + } + } + + // Open file, if it's not a symlink + std::auto_ptr<FileStream> in; + if(!sourceIsSymlink) + { + in.reset(new FileStream(Filename)); + } + + // Read header + file_BlockIndexHeader hdr; + if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic + if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0) +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0); +#endif + + // Get basic information + int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); + uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase); + + //TODO: Verify that these sizes look reasonable + + // setup + void *data = 0; + int32_t dataSize = -1; + bool matches = true; + int64_t totalSizeInBlockIndex = 0; + + try + { + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read an entry from the stream + file_BlockIndexEntry entry; + if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read entry + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Calculate IV for this entry + uint64_t iv = entryIVBase; + iv += b; + iv = box_hton64(iv); +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(isOldVersion) + { + // Reverse the IV for compatibility + iv = box_swap64(iv); + } +#endif + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry.mEnEnc, sizeof(entry.mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Size of block + int32_t blockClearSize = ntohl(entryEnc.mSize); + if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + totalSizeInBlockIndex += blockClearSize; + + // Make sure there's enough memory allocated to load the block in + if(dataSize < blockClearSize) + { + // Too small, free the block if it's already allocated + if(data != 0) + { + ::free(data); + data = 0; + } + // Allocate a block + data = ::malloc(blockClearSize + 128); + if(data == 0) + { + throw std::bad_alloc(); + } + dataSize = blockClearSize + 128; + } + + // Load in the block from the file, if it's not a symlink + if(!sourceIsSymlink) + { + if(in->Read(data, blockClearSize) != blockClearSize) + { + // Not enough data left in the file, can't possibly match + matches = false; + } + else + { + // Check the checksum + MD5Digest md5; + md5.Add(data, blockClearSize); + md5.Finish(); + if(!md5.DigestMatches(entryEnc.mStrongChecksum)) + { + // Checksum didn't match + matches = false; + } + } + } + + // Keep on going regardless, to make sure the entire block index stream is read + // -- must always be consistent about what happens with the stream. + } + } + catch(...) + { + // clean up in case of errors + if(data != 0) + { + ::free(data); + data = 0; + } + throw; + } + + // free block + if(data != 0) + { + ::free(data); + data = 0; + } + + // Check for data left over if it's not a symlink + if(!sourceIsSymlink) + { + // Anything left to read in the file? + if(in->BytesLeftToRead() != 0) + { + // File has extra data at the end + matches = false; + } + } + + // Symlinks must have zero size on server + if(sourceIsSymlink) + { + matches = (totalSizeInBlockIndex == 0); + } + + return matches; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer() +// Purpose: Constructor +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +BackupStoreFile::EncodingBuffer::EncodingBuffer() + : mpBuffer(0), + mBufferSize(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer() +// Purpose: Destructor +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +BackupStoreFile::EncodingBuffer::~EncodingBuffer() +{ + if(mpBuffer != 0) + { + BackupStoreFile::CodingChunkFree(mpBuffer); + mpBuffer = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::Allocate(int) +// Purpose: Do initial allocation of block +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::EncodingBuffer::Allocate(int Size) +{ + ASSERT(mpBuffer == 0); + uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size); + if(buffer == 0) + { + throw std::bad_alloc(); + } + mpBuffer = buffer; + mBufferSize = Size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::Reallocate(int) +// Purpose: Reallocate the block. Try not to call this, it has to copy +// the entire contents as the block can't be reallocated straight. +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) +{ + BOX_TRACE("Reallocating EncodingBuffer from " << mBufferSize << + " to " << NewSize); + ASSERT(mpBuffer != 0); + uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + // Copy data + ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize); + + // Free old + BackupStoreFile::CodingChunkFree(mpBuffer); + + // Store new buffer + mpBuffer = buffer; + mBufferSize = NewSize; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: DiffTimer::DiffTimer(); +// Purpose: Constructor +// Created: 2005/02/01 +// +// -------------------------------------------------------------------------- +DiffTimer::DiffTimer() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: DiffTimer::DiffTimer(); +// Purpose: Destructor +// Created: 2005/02/01 +// +// -------------------------------------------------------------------------- +DiffTimer::~DiffTimer() +{ +} diff --git a/lib/backupstore/BackupStoreFile.h b/lib/backupstore/BackupStoreFile.h new file mode 100644 index 00000000..f5bc1924 --- /dev/null +++ b/lib/backupstore/BackupStoreFile.h @@ -0,0 +1,231 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFile.h +// Purpose: Utils for manipulating files +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILE__H +#define BACKUPSTOREFILE__H + +#include <cstdlib> +#include <memory> +#include <cstdlib> + +#include "BackupClientFileAttributes.h" +#include "BackupStoreFilename.h" +#include "IOStream.h" +#include "ReadLoggingStream.h" +#include "RunStatusProvider.h" + +typedef struct +{ + int64_t mBytesInEncodedFiles; + int64_t mBytesAlreadyOnServer; + int64_t mTotalFileStreamSize; +} BackupStoreFileStats; + +// Uncomment to disable backwards compatibility +//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + + +// Output buffer to EncodeChunk and input data to DecodeChunk must +// have specific alignment, see function comments. +#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16 +#define BACKUPSTOREFILE_CODING_OFFSET 15 + +// Have some memory allocation commands, note closing "Off" at end of file. +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: DiffTimer +// Purpose: Interface for classes that can keep track of diffing time, +// and send SSL keepalive messages +// Created: 2006/01/19 +// +// -------------------------------------------------------------------------- +class DiffTimer +{ +public: + DiffTimer(); + virtual ~DiffTimer(); +public: + virtual void DoKeepAlive() = 0; + virtual int GetMaximumDiffingTime() = 0; + virtual bool IsManaged() = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFile +// Purpose: Class to hold together utils for manipulating files. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +class BackupStoreFile +{ +public: + class DecodedStream : public IOStream + { + friend class BackupStoreFile; + private: + DecodedStream(IOStream &rEncodedFile, int Timeout); + DecodedStream(const DecodedStream &); // not allowed + DecodedStream &operator=(const DecodedStream &); // not allowed + public: + ~DecodedStream(); + + // Stream functions + virtual int Read(void *pBuffer, int NBytes, int Timeout); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + // Accessor functions + const BackupClientFileAttributes &GetAttributes() {return mAttributes;} + const BackupStoreFilename &GetFilename() {return mFilename;} + int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests + + bool IsSymLink(); + + private: + void Setup(const BackupClientFileAttributes *pAlterativeAttr); + void ReadBlockIndex(bool MagicAlreadyRead); + + private: + IOStream &mrEncodedFile; + int mTimeout; + BackupClientFileAttributes mAttributes; + BackupStoreFilename mFilename; + int64_t mNumBlocks; + void *mpBlockIndex; + uint8_t *mpEncodedData; + uint8_t *mpClearData; + int mClearDataSize; + int mCurrentBlock; + int mCurrentBlockClearSize; + int mPositionInCurrentBlock; + uint64_t mEntryIVBase; +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool mIsOldVersion; +#endif + }; + + + // Main interface + static std::auto_ptr<IOStream> EncodeFile + ( + const char *Filename, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime = 0, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL + ); + static std::auto_ptr<IOStream> EncodeFileDiff + ( + const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, + int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, + int Timeout, + DiffTimer *pDiffTimer, + int64_t *pModificationTime = 0, + bool *pIsCompletelyDifferent = 0 + ); + static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); + static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); + static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); + static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0); + static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); + static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); + static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout); + static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false); + + // Stream manipulation + static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership); + static void MoveStreamPositionToBlockIndex(IOStream &rStream); + + // Crypto setup + static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength); +#ifndef HAVE_OLD_SSL + static void SetAESKey(const void *pKey, int KeyLength); +#endif + + // Allocation of properly aligning chunks for decoding and encoding chunks + inline static void *CodingChunkAlloc(int Size) + { + uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3)); + if(a == 0) return 0; + // Align to main block size + ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size + uint8_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE + - (uint8_t)(((unsigned long)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE); + uint8_t *b = (a + adjustment); + // Store adjustment + *b = adjustment; + // Return offset + return b + BACKUPSTOREFILE_CODING_OFFSET; + } + inline static void CodingChunkFree(void *Block) + { + // Check alignment is as expected + ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size + ASSERT((uint8_t)(((unsigned long)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + uint8_t *a = (uint8_t*)Block; + a -= BACKUPSTOREFILE_CODING_OFFSET; + // Adjust downwards... + a -= *a; + free(a); + } + + static void DiffTimerExpired(); + + // Building blocks + class EncodingBuffer + { + public: + EncodingBuffer(); + ~EncodingBuffer(); + private: + // No copying + EncodingBuffer(const EncodingBuffer &); + EncodingBuffer &operator=(const EncodingBuffer &); + public: + void Allocate(int Size); + void Reallocate(int NewSize); + + uint8_t *mpBuffer; + int mBufferSize; + }; + static int MaxBlockSizeForChunkSize(int ChunkSize); + static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput); + + // Caller should know how big the output size is, but also allocate a bit more memory to cover various + // overheads allowed for in checks + static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize) + { + // Plenty big enough + return KnownChunkSize + 256; + } + static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize); + + // Statisitics, not designed to be completely reliable + static void ResetStats(); + static BackupStoreFileStats msStats; + + // For debug +#ifndef BOX_RELEASE_BUILD + static bool TraceDetailsOfDiffProcess; +#endif + + // For decoding encoded files + static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile); +}; + +#include "MemLeakFindOff.h" + +#endif // BACKUPSTOREFILE__H diff --git a/lib/backupstore/BackupStoreFileCryptVar.cpp b/lib/backupstore/BackupStoreFileCryptVar.cpp new file mode 100644 index 00000000..e826de4e --- /dev/null +++ b/lib/backupstore/BackupStoreFileCryptVar.cpp @@ -0,0 +1,31 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCryptVar.cpp +// Purpose: Cryptographic keys for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileWire.h" + +#include "MemLeakFindOn.h" + +CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt; +CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt; + +#ifndef HAVE_OLD_SSL + CipherContext BackupStoreFileCryptVar::sAESEncrypt; + CipherContext BackupStoreFileCryptVar::sAESDecrypt; +#endif + +// Default to blowfish +CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt; +uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING; + +CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry; +CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry; + diff --git a/lib/backupstore/BackupStoreFileCryptVar.h b/lib/backupstore/BackupStoreFileCryptVar.h new file mode 100644 index 00000000..566813c8 --- /dev/null +++ b/lib/backupstore/BackupStoreFileCryptVar.h @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCryptVar.h +// Purpose: Cryptographic keys for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILECRYPTVAR__H +#define BACKUPSTOREFILECRYPTVAR__H + +#include "CipherContext.h" + +// Hide private static variables from the rest of the world by putting them +// as static variables in a namespace. +// -- don't put them as static class variables to avoid openssl/evp.h being +// included all over the project. +namespace BackupStoreFileCryptVar +{ + // Keys for the main file data + extern CipherContext sBlowfishEncrypt; + extern CipherContext sBlowfishDecrypt; + // Use AES when available +#ifndef HAVE_OLD_SSL + extern CipherContext sAESEncrypt; + extern CipherContext sAESDecrypt; +#endif + // How encoding will be done + extern CipherContext *spEncrypt; + extern uint8_t sEncryptCipherType; + + // Keys for the block indicies + extern CipherContext sBlowfishEncryptBlockEntry; + extern CipherContext sBlowfishDecryptBlockEntry; +} + +#endif // BACKUPSTOREFILECRYPTVAR__H + diff --git a/lib/backupstore/BackupStoreFileEncodeStream.cpp b/lib/backupstore/BackupStoreFileEncodeStream.cpp new file mode 100644 index 00000000..e9d773f0 --- /dev/null +++ b/lib/backupstore/BackupStoreFileEncodeStream.cpp @@ -0,0 +1,717 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileEncodeStream.cpp +// Purpose: Implement stream-based file encoding for the backup store +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "BackupClientFileAttributes.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BoxTime.h" +#include "FileStream.h" +#include "Random.h" +#include "RollingChecksum.h" + +#include "MemLeakFindOn.h" + +#include <cstring> + +using namespace BackupStoreFileCryptVar; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream +// Purpose: Constructor (opens file) +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() + : mpRecipe(0), + mpFile(0), + mpLogging(0), + mpRunStatusProvider(NULL), + mStatus(Status_Header), + mSendData(true), + mTotalBlocks(0), + mAbsoluteBlockNumber(-1), + mInstructionNumber(-1), + mNumBlocks(0), + mCurrentBlock(-1), + mCurrentBlockEncodedSize(0), + mPositionInCurrentBlock(0), + mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), + mLastBlockSize(0), + mTotalBytesSent(0), + mpRawBuffer(0), + mAllocatedBufferSize(0), + mEntryIVBase(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() +// Purpose: Destructor +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() +{ + // Free buffers + if(mpRawBuffer) + { + ::free(mpRawBuffer); + mpRawBuffer = 0; + } + + // Close the file, which we might have open + if(mpFile) + { + delete mpFile; + mpFile = 0; + } + + // Clear up logging stream + if(mpLogging) + { + delete mpLogging; + mpLogging = 0; + } + + // Free the recipe + if(mpRecipe != 0) + { + delete mpRecipe; + mpRecipe = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *) +// Purpose: Reads file information, and builds file header reading for sending. +// Takes ownership of the Recipe. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::Setup(const char *Filename, + BackupStoreFileEncodeStream::Recipe *pRecipe, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) +{ + // Pointer to a blank recipe which we might create + BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; + + try + { + // Get file attributes + box_time_t modTime = 0; + int64_t fileSize = 0; + BackupClientFileAttributes attr; + attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime, + 0 /* not interested in attr mod time */, &fileSize); + + // Might need to create a blank recipe... + if(pRecipe == 0) + { + pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); + + BackupStoreFileEncodeStream::RecipeInstruction instruction; + instruction.mSpaceBefore = fileSize; // whole file + instruction.mBlocks = 0; // no blocks + instruction.mpStartBlock = 0; // no block + pblankRecipe->push_back(instruction); + + pRecipe = pblankRecipe; + } + + // Tell caller? + if(pModificationTime != 0) + { + *pModificationTime = modTime; + } + + // Go through each instruction in the recipe and work out how many blocks + // it will add, and the max clear size of these blocks + int maxBlockClearSize = 0; + for(uint64_t inst = 0; inst < pRecipe->size(); ++inst) + { + if((*pRecipe)[inst].mSpaceBefore > 0) + { + // Calculate the number of blocks the space before requires + int64_t numBlocks; + int32_t blockSize, lastBlockSize; + CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize); + // Add to accumlated total + mTotalBlocks += numBlocks; + // Update maximum clear size + if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize; + if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize; + } + + // Add number of blocks copied from the previous file + mTotalBlocks += (*pRecipe)[inst].mBlocks; + + // Check for bad things + if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Run through blocks to get the max clear size + for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b) + { + if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize; + } + } + + // Send data? (symlinks don't have any data in them) + mSendData = !attr.IsSymLink(); + + // If not data is being sent, then the max clear block size is zero + if(!mSendData) + { + maxBlockClearSize = 0; + } + + // Header + file_StreamFormat hdr; + hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1); + hdr.mNumBlocks = (mSendData)?(box_hton64(mTotalBlocks)):(0); + hdr.mContainerID = box_hton64(ContainerID); + hdr.mModificationTime = box_hton64(modTime); + // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size + hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128); + hdr.mOptions = 0; // no options defined yet + + // Write header to stream + mData.Write(&hdr, sizeof(hdr)); + + // Write filename to stream + rStoreFilename.WriteToStream(mData); + + // Write attributes to stream + attr.WriteToStream(mData); + + // Allocate some buffers for writing data + if(mSendData) + { + // Open the file + mpFile = new FileStream(Filename); + + if (pLogger) + { + // Create logging stream + mpLogging = new ReadLoggingStream(*mpFile, + *pLogger); + } + else + { + // re-use FileStream instead + mpLogging = mpFile; + mpFile = NULL; + } + + // Work out the largest possible block required for the encoded data + mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); + + // Then allocate two blocks of this size + mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize); + if(mpRawBuffer == 0) + { + throw std::bad_alloc(); + } +#ifndef BOX_RELEASE_BUILD + // In debug builds, make sure that the reallocation code is exercised. + mEncodedBuffer.Allocate(mAllocatedBufferSize / 4); +#else + mEncodedBuffer.Allocate(mAllocatedBufferSize); +#endif + } + else + { + // Write an empty block index for the symlink + file_BlockIndexHeader blkhdr; + blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); + blkhdr.mOtherFileID = box_hton64(0); // not other file ID + blkhdr.mEntryIVBase = box_hton64(0); + blkhdr.mNumBlocks = box_hton64(0); + mData.Write(&blkhdr, sizeof(blkhdr)); + } + + // Ready for reading + mData.SetForReading(); + + // Update stats + BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize; + + // Finally, store the pointer to the recipe, when we know exceptions won't occur + mpRecipe = pRecipe; + } + catch(...) + { + // Clean up any blank recipe + if(pblankRecipe != 0) + { + delete pblankRecipe; + pblankRecipe = 0; + } + throw; + } + + mpRunStatusProvider = pRunStatusProvider; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &) +// Purpose: Calculates the sizes of blocks in a section of the file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut) +{ + // How many blocks, and how big? + rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2; + do + { + rBlockSizeOut *= 2; + + rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut; + + } while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); + + // Last block size + rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut); + + // Avoid small blocks? + if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN + && rNumBlocksOut > 1) + { + // Add the small bit of data to the last block + --rNumBlocksOut; + rLastBlockSizeOut += rBlockSizeOut; + } + + // checks! + ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize); + //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Read(void *, int, int) +// Purpose: As interface -- generates encoded file data on the fly from the raw file +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Check there's something to do. + if(mStatus == Status_Finished) + { + return 0; + } + + if(mpRunStatusProvider && mpRunStatusProvider->StopRun()) + { + THROW_EXCEPTION(BackupStoreException, SignalReceived); + } + + int bytesToRead = NBytes; + uint8_t *buffer = (uint8_t*)pBuffer; + + while(bytesToRead > 0 && mStatus != Status_Finished) + { + if(mStatus == Status_Header || mStatus == Status_BlockListing) + { + // Header or block listing phase -- send from the buffered stream + + // Send bytes from the data buffer + int b = mData.Read(buffer, bytesToRead, Timeout); + bytesToRead -= b; + buffer += b; + + // Check to see if all the data has been used from this stream + if(!mData.StreamDataLeft()) + { + // Yes, move on to next phase (or finish, if there's no file data) + if(!mSendData) + { + mStatus = Status_Finished; + } + else + { + // Reset the buffer so it can be used for the next phase + mData.Reset(); + + // Get buffer ready for index? + if(mStatus == Status_Header) + { + // Just finished doing the stream header, create the block index header + file_BlockIndexHeader blkhdr; + blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); + ASSERT(mpRecipe != 0); + blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID()); + blkhdr.mNumBlocks = box_hton64(mTotalBlocks); + + // Generate the IV base + Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase)); + blkhdr.mEntryIVBase = box_hton64(mEntryIVBase); + + mData.Write(&blkhdr, sizeof(blkhdr)); + } + + ++mStatus; + } + } + } + else if(mStatus == Status_Blocks) + { + // Block sending phase + + if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize) + { + // Next block! + ++mCurrentBlock; + ++mAbsoluteBlockNumber; + if(mCurrentBlock >= mNumBlocks) + { + // Output extra blocks for this instruction and move forward in file + if(mInstructionNumber >= 0) + { + SkipPreviousBlocksInInstruction(); + } + + // Is there another instruction to go? + ++mInstructionNumber; + + // Skip instructions which don't contain any data + while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size()) + && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0) + { + SkipPreviousBlocksInInstruction(); + ++mInstructionNumber; + } + + if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size())) + { + // End of blocks, go to next phase + ++mStatus; + + // Set the data to reading so the index can be written + mData.SetForReading(); + } + else + { + // Get ready for this instruction + SetForInstruction(); + } + } + + // Can't use 'else' here as SetForInstruction() will change this + if(mCurrentBlock < mNumBlocks) + { + EncodeCurrentBlock(); + } + } + + // Send data from the current block (if there's data to send) + if(mPositionInCurrentBlock < mCurrentBlockEncodedSize) + { + // How much data to put in the buffer? + int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock; + if(s > bytesToRead) s = bytesToRead; + + // Copy it in + ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s); + + // Update variables + bytesToRead -= s; + buffer += s; + mPositionInCurrentBlock += s; + } + } + else + { + // Should never get here, as it'd be an invalid status + ASSERT(false); + } + } + + // Add encoded size to stats + BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead); + mTotalBytesSent += (NBytes - bytesToRead); + + // Return size of data to caller + return NBytes - bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction() +// Purpose: Private. Stores the blocks of the old file referenced in the current +// instruction into the index and skips over the data in the file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction() +{ + // Check something is necessary + if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0) + { + return; + } + + // Index of the first block in old file (being diffed from) + int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock); + + int64_t sizeToSkip = 0; + + for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b) + { + // Update stats + BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; + + // Store the entry + StoreBlockIndexEntry(0 - (firstIndex + b), + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize, + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum, + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); + + // Increment the absolute block number -- kept encryption IV in sync + ++mAbsoluteBlockNumber; + + // Add the size of this block to the size to skip + sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; + } + + // Move forward in the stream + mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::SetForInstruction() +// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::SetForInstruction() +{ + // Calculate block sizes + CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize); + + // Set variables + mCurrentBlock = 0; + mCurrentBlockEncodedSize = 0; + mPositionInCurrentBlock = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock() +// Purpose: Private. Encodes the current block, and writes the block data to the index +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::EncodeCurrentBlock() +{ + // How big is the block, raw? + int blockRawSize = mBlockSize; + if(mCurrentBlock == (mNumBlocks - 1)) + { + blockRawSize = mLastBlockSize; + } + ASSERT(blockRawSize < mAllocatedBufferSize); + + // Check file open + if(mpLogging == 0) + { + // File should be open, but isn't. So logical error. + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Read the data in + if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, + 0 /* not interested in size if failure */)) + { + // TODO: Do something more intelligent, and abort + // this upload because the file has changed. + THROW_EXCEPTION(BackupStoreException, + Temp_FileEncodeStreamDidntReadBuffer) + } + + // Encode it + mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, + blockRawSize, mEncodedBuffer); + + //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); + + // Create block listing data -- generate checksums + RollingChecksum weakChecksum(mpRawBuffer, blockRawSize); + MD5Digest strongChecksum; + strongChecksum.Add(mpRawBuffer, blockRawSize); + strongChecksum.Finish(); + + // Add entry to the index + StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, + weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); + + // Set vars to reading this block + mPositionInCurrentBlock = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *) +// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream. +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum) +{ + // First, the encrypted section + file_BlockIndexEntryEnc entryEnc; + entryEnc.mSize = htonl(ClearSize); + entryEnc.mWeakChecksum = htonl(WeakChecksum); + ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum)); + + // Then the clear section + file_BlockIndexEntry entry; + entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex)); + + // Then encrypt the encryted section + // Generate the IV from the block number + if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase)) + { + THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements) + } + uint64_t iv = mEntryIVBase; + iv += mAbsoluteBlockNumber; + // Convert to network byte order before encrypting with it, so that restores work on + // platforms with different endiannesses. + iv = box_hton64(iv); + sBlowfishEncryptBlockEntry.SetIV(&iv); + + // Encode the data + int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc)); + if(encodedSize != sizeof(entry.mEnEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Save to data block for sending at the end of the stream + mData.Write(&entry, sizeof(entry)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Write(const void *, int) +// Purpose: As interface. Exceptions. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StreamDataLeft() +// Purpose: As interface -- end of stream reached? +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFileEncodeStream::StreamDataLeft() +{ + return (mStatus != Status_Finished); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StreamClosed() +// Purpose: As interface +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFileEncodeStream::StreamClosed() +{ + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t) +// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, + int64_t NumBlocksInIndex, int64_t OtherFileID) + : mpBlockIndex(pBlockIndex), + mNumBlocksInIndex(NumBlocksInIndex), + mOtherFileID(OtherFileID) +{ + ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Recipe::~Recipe() +// Purpose: Destructor +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::Recipe::~Recipe() +{ + // Free the block index, if there is one + if(mpBlockIndex != 0) + { + ::free(mpBlockIndex); + } +} + + + + diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h new file mode 100644 index 00000000..023994af --- /dev/null +++ b/lib/backupstore/BackupStoreFileEncodeStream.h @@ -0,0 +1,137 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileEncodeStream.h +// Purpose: Implement stream-based file encoding for the backup store +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILEENCODESTREAM__H +#define BACKUPSTOREFILEENCODESTREAM__H + +#include <vector> + +#include "IOStream.h" +#include "BackupStoreFilename.h" +#include "CollectInBufferStream.h" +#include "MD5Digest.h" +#include "BackupStoreFile.h" +#include "ReadLoggingStream.h" +#include "RunStatusProvider.h" + +namespace BackupStoreFileCreation +{ + // Diffing and creation of files share some implementation details. + typedef struct _BlocksAvailableEntry + { + struct _BlocksAvailableEntry *mpNextInHashList; + int32_t mSize; // size in clear + uint32_t mWeakChecksum; // weak, rolling checksum + uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum + } BlocksAvailableEntry; + +} + + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFileEncodeStream +// Purpose: Encode a file into a stream +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +class BackupStoreFileEncodeStream : public IOStream +{ +public: + BackupStoreFileEncodeStream(); + ~BackupStoreFileEncodeStream(); + + typedef struct + { + int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go + int32_t mBlocks; // number of block to reuse, starting at this one + BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null + } RecipeInstruction; + + class Recipe : public std::vector<RecipeInstruction> + { + // NOTE: This class is rather tied in with the implementation of diffing. + public: + Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex, + int64_t OtherFileID = 0); + ~Recipe(); + + int64_t GetOtherFileID() {return mOtherFileID;} + int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock) + { + return pBlock - mpBlockIndex; + } + + private: + BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex; + int64_t mNumBlocksInIndex; + int64_t mOtherFileID; + }; + + void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL); + + virtual int Read(void *pBuffer, int NBytes, int Timeout); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + int64_t GetTotalBytesSent() { return mTotalBytesSent; } + +private: + enum + { + Status_Header = 0, + Status_Blocks = 1, + Status_BlockListing = 2, + Status_Finished = 3 + }; + +private: + void EncodeCurrentBlock(); + void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); + void SkipPreviousBlocksInInstruction(); + void SetForInstruction(); + void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum); + +private: + Recipe *mpRecipe; + IOStream *mpFile; // source file + CollectInBufferStream mData; // buffer for header and index entries + IOStream *mpLogging; + RunStatusProvider* mpRunStatusProvider; + int mStatus; + bool mSendData; // true if there's file data to send (ie not a symlink) + int64_t mTotalBlocks; // Total number of blocks in the file + int64_t mAbsoluteBlockNumber; // The absolute block number currently being output + // Instruction number + int64_t mInstructionNumber; + // All the below are within the current instruction + int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases + int64_t mCurrentBlock; + int32_t mCurrentBlockEncodedSize; + int32_t mPositionInCurrentBlock; // for reading out + int32_t mBlockSize; // Basic block size of most of the blocks in the file + int32_t mLastBlockSize; // the size (unencoded) of the last block in the file + int64_t mTotalBytesSent; + // Buffers + uint8_t *mpRawBuffer; // buffer for raw data + BackupStoreFile::EncodingBuffer mEncodedBuffer; + // buffer for encoded data + int32_t mAllocatedBufferSize; // size of above two allocated blocks + uint64_t mEntryIVBase; // base for block entry IV +}; + + + +#endif // BACKUPSTOREFILEENCODESTREAM__H + diff --git a/lib/backupstore/BackupStoreFileRevDiff.cpp b/lib/backupstore/BackupStoreFileRevDiff.cpp new file mode 100644 index 00000000..509eef61 --- /dev/null +++ b/lib/backupstore/BackupStoreFileRevDiff.cpp @@ -0,0 +1,258 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileRevDiff.cpp +// Purpose: Reverse a patch, to build a new patch from new to old files +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <stdlib.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t) +// Purpose: Reverse a patch, to build a new patch from new to old files. Takes +// two independent copies to the From file, for efficiency. +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent) +{ + // Read and copy the header from the from file to the out file -- beginnings of the patch + file_StreamFormat hdr; + if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Copy + rOut.Write(&hdr, sizeof(hdr)); + // Copy over filename and attributes + // BLOCK + { + BackupStoreFilename filename; + filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + filename.WriteToStream(rOut); + StreamableMemBlock attr; + attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + attr.WriteToStream(rOut); + } + + // Build an index of common blocks. + // For each block in the from file, we want to know it's index in the + // diff file. Allocate memory for this information. + int64_t fromNumBlocks = box_ntoh64(hdr.mNumBlocks); + int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t)); + if(pfromIndexInfo == 0) + { + throw std::bad_alloc(); + } + + // Buffer data + void *buffer = 0; + int bufferSize = 0; + + // flag + bool isCompletelyDifferent = true; + + try + { + // Initialise the index to be all 0, ie not filled in yet + for(int64_t i = 0; i < fromNumBlocks; ++i) + { + pfromIndexInfo[i] = 0; + } + + // Within the from file, skip to the index + MoveStreamPositionToBlockIndex(rDiff); + + // Read in header of index + file_BlockIndexHeader diffIdxHdr; + if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // And then read in each entry + int64_t diffNumBlocks = box_ntoh64(diffIdxHdr.mNumBlocks); + for(int64_t b = 0; b < diffNumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Where's the block? + int64_t blockEn = box_ntoh64(e.mEncodedSize); + if(blockEn > 0) + { + // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file + } + else + { + // Block is in the original file, store which block it is in this file + int64_t fromIndex = 0 - blockEn; + if(fromIndex < 0 || fromIndex >= fromNumBlocks) + { + THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles) + } + + // Store information about where it is in the new file + // NOTE: This is slight different to how it'll be stored in the final index. + pfromIndexInfo[fromIndex] = -1 - b; + } + } + + // Open the index for the second copy of the from file + MoveStreamPositionToBlockIndex(rFrom2); + + // Read in header of index + file_BlockIndexHeader fromIdxHdr; + if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || box_ntoh64(fromIdxHdr.mOtherFileID) != 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // So, we can now start building the data in the file + int64_t filePosition = rFrom.GetPosition(); + for(int64_t b = 0; b < fromNumBlocks; ++b) + { + // Read entry from from index + file_BlockIndexEntry e; + if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Get size + int64_t blockSize = box_hton64(e.mEncodedSize); + if(blockSize < 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Copy this block? + if(pfromIndexInfo[b] == 0) + { + // Copy it, first move to file location + rFrom.Seek(filePosition, IOStream::SeekType_Absolute); + + // Make sure there's memory available to copy this + if(bufferSize < blockSize || buffer == 0) + { + // Free old block + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + bufferSize = 0; + } + // Allocate new block + buffer = ::malloc(blockSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + bufferSize = blockSize; + } + ASSERT(bufferSize >= blockSize); + + // Copy the block + if(!rFrom.ReadFullBuffer(buffer, blockSize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + rOut.Write(buffer, blockSize); + + // Store the size + pfromIndexInfo[b] = blockSize; + } + else + { + // Block isn't needed, so it's not completely different + isCompletelyDifferent = false; + } + filePosition += blockSize; + } + + // Then write the index, modified header first + fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(box_hton64(ObjectIDOfFrom)); + rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr)); + + // Move to start of index entries + rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute); + + // Then copy modified entries + for(int64_t b = 0; b < fromNumBlocks; ++b) + { + // Read entry from from index + file_BlockIndexEntry e; + if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Modify... + int64_t s = pfromIndexInfo[b]; + // Adjust to reflect real block index (remember 0 has a different meaning here) + if(s < 0) ++s; + // Insert + e.mEncodedSize = box_hton64(s); + // Write + rOut.Write(&e, sizeof(e)); + } + } + catch(...) + { + ::free(pfromIndexInfo); + if(buffer != 0) + { + ::free(buffer); + } + throw; + } + + // Free memory used (oh for finally {} blocks) + ::free(pfromIndexInfo); + if(buffer != 0) + { + ::free(buffer); + } + + // return completely different flag + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = isCompletelyDifferent; + } +} + + + diff --git a/lib/backupstore/BackupStoreFileWire.h b/lib/backupstore/BackupStoreFileWire.h new file mode 100644 index 00000000..49e94aa5 --- /dev/null +++ b/lib/backupstore/BackupStoreFileWire.h @@ -0,0 +1,74 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileWire.h +// Purpose: On the wire / disc formats for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILEWIRE__H +#define BACKUPSTOREFILEWIRE__H + +#include "MD5Digest.h" + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + int32_t mMagicValue; // also the version number + int64_t mNumBlocks; // number of blocks contained in the file + int64_t mContainerID; + int64_t mModificationTime; + int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block + int32_t mOptions; // bitmask of options used + // Then a BackupStoreFilename + // Then a BackupClientFileAttributes +} file_StreamFormat; + +typedef struct +{ + int32_t mMagicValue; // different magic value + int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index + uint64_t mEntryIVBase; // base value for block IV + int64_t mNumBlocks; // repeat of value in file header +} file_BlockIndexHeader; + +typedef struct +{ + int32_t mSize; // size in clear + uint32_t mWeakChecksum; // weak, rolling checksum + uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum +} file_BlockIndexEntryEnc; + +typedef struct +{ + union + { + int64_t mEncodedSize; // size encoded, if > 0 + int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0 + }; + uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section +} file_BlockIndexEntry; + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +// header for blocks of compressed data in files +#define HEADER_CHUNK_IS_COMPRESSED 1 // bit +#define HEADER_ENCODING_SHIFT 1 // shift value +#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7 +#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7 + + +#endif // BACKUPSTOREFILEWIRE__H + diff --git a/lib/backupstore/BackupStoreFilename.cpp b/lib/backupstore/BackupStoreFilename.cpp new file mode 100644 index 00000000..72cd1acd --- /dev/null +++ b/lib/backupstore/BackupStoreFilename.cpp @@ -0,0 +1,281 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilename.cpp +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreFilename.h" +#include "Protocol.h" +#include "BackupStoreException.h" +#include "IOStream.h" +#include "Guards.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::BackupStoreFilename() +// Purpose: Default constructor -- creates an invalid filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::BackupStoreFilename() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy) + : mEncryptedName(rToCopy.mEncryptedName) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::~BackupStoreFilename() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::~BackupStoreFilename() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::CheckValid(bool) +// Purpose: Checks the encoded filename for validity +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const +{ + bool ok = true; + + if(mEncryptedName.size() < 2) + { + // Isn't long enough to have a header + ok = false; + } + else + { + // Check size is consistent + unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(this->mEncryptedName); + if(dsize != mEncryptedName.size()) + { + ok = false; + } + + // And encoding is an accepted value + unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName); + if(encoding < Encoding_Min || encoding > Encoding_Max) + { + ok = false; + } + } + + // Exception? + if(!ok && ExceptionIfInvalid) + { + THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename) + } + + return ok; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::ReadFromProtocol(Protocol &) +// Purpose: Reads the filename from the protocol object +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol) +{ + // Read the header + char hdr[2]; + rProtocol.Read(hdr, 2); + + // How big is it? + int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); + + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + std::string data; + rProtocol.Read(data, dsize - 2); + + // assign to this string, storing the header and the extra data + mEncryptedName.assign(hdr, 2); + mEncryptedName.append(data.c_str(), data.size()); + + // Check it + CheckValid(); + + // Alert derived classes + EncodedFilenameChanged(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::WriteToProtocol(Protocol &) +// Purpose: Writes the filename to the protocol object +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const +{ + CheckValid(); + + rProtocol.Write(mEncryptedName.c_str(), mEncryptedName.size()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::ReadFromStream(IOStream &) +// Purpose: Reads the filename from a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Read the header + char hdr[2]; + if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // How big is it? + unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); + + // Assume most filenames are small + char buf[256]; + if(dsize < sizeof(buf)) + { + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + // Copy in header + buf[0] = hdr[0]; buf[1] = hdr[1]; + + // assign to this string, storing the header and the extra data + mEncryptedName.assign(buf, dsize); + } + else + { + // Block of memory to hold it + MemoryBlockGuard<char*> dataB(dsize+2); + char *data = dataB; + + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + // Copy in header + data[0] = hdr[0]; data[1] = hdr[1]; + + // assign to this string, storing the header and the extra data + mEncryptedName.assign(data, dsize); + } + + // Check it + CheckValid(); + + // Alert derived classes + EncodedFilenameChanged(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::WriteToStream(IOStream &) +// Purpose: Writes the filename to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::WriteToStream(IOStream &rStream) const +{ + CheckValid(); + + rStream.Write(mEncryptedName.c_str(), mEncryptedName.size()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::EncodedFilenameChanged() +// Purpose: The encoded filename stored has changed +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::EncodedFilenameChanged() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::IsEncrypted() +// Purpose: Returns true if the filename is stored using an encrypting encoding +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFilename::IsEncrypted() const +{ + return BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName) != + Encoding_Clear; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::SetAsClearFilename(const char *) +// Purpose: Sets this object to be a valid filename, but with a +// filename in the clear. Used on the server to create +// filenames when there's no way of encrypting it. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::SetAsClearFilename(const char *Clear) +{ + // Make std::string from the clear name + std::string toEncode(Clear); + + // Make an encoded string + char hdr[2]; + BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear); + std::string encoded(hdr, 2); + encoded += toEncode; + ASSERT(encoded.size() == toEncode.size() + 2); + + // Store the encoded string + mEncryptedName.assign(encoded); + + // Stuff which must be done + EncodedFilenameChanged(); + CheckValid(false); +} + + + diff --git a/lib/backupstore/BackupStoreFilename.h b/lib/backupstore/BackupStoreFilename.h new file mode 100644 index 00000000..80db9516 --- /dev/null +++ b/lib/backupstore/BackupStoreFilename.h @@ -0,0 +1,107 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilename.h +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILENAME__H +#define BACKUPSTOREFILENAME__H + +#include <string> + +class Protocol; +class IOStream; + +// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE +// don't define this -- the problem of memory usage still appears without this. +// It's just that this class really showed up the problem. Instead, malloc allocation +// is globally defined in BoxPlatform.h, for troublesome libraries. + +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE + // Use a malloc_allocated string, because the STL default allocators really screw up with + // memory allocation, particularly with this class. + // Makes a few things a bit messy and inefficient with conversions. + // Given up using this, and use global malloc allocation instead, but thought it + // worth leaving this code in just in case it's useful for the future. + typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base; + // If this is changed, change GetClearFilename() back to returning a reference. +#else + typedef std::string BackupStoreFilename_base; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFilename +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreFilename /* : public BackupStoreFilename_base */ +{ +private: + std::string mEncryptedName; + +public: + BackupStoreFilename(); + BackupStoreFilename(const BackupStoreFilename &rToCopy); + virtual ~BackupStoreFilename(); + + bool CheckValid(bool ExceptionIfInvalid = true) const; + + void ReadFromProtocol(Protocol &rProtocol); + void WriteToProtocol(Protocol &rProtocol) const; + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream) const; + + void SetAsClearFilename(const char *Clear); + + // Check that it's encrypted + bool IsEncrypted() const; + + // These enumerated types belong in the base class so + // the CheckValid() function can make sure that the encoding + // is a valid encoding + enum + { + Encoding_Min = 1, + Encoding_Clear = 1, + Encoding_Blowfish = 2, + Encoding_Max = 2 + }; + + const std::string& GetEncodedFilename() const + { + return mEncryptedName; + } + + bool operator==(const BackupStoreFilename& rOther) const + { + return mEncryptedName == rOther.mEncryptedName; + } + + bool operator!=(const BackupStoreFilename& rOther) const + { + return mEncryptedName != rOther.mEncryptedName; + } + +protected: + virtual void EncodedFilenameChanged(); + void SetEncodedFilename(const std::string &rEncoded) + { + mEncryptedName = rEncoded; + } +}; + +// On the wire utilities for class and derived class +#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2) +#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3) + +#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;} + +#endif // BACKUPSTOREFILENAME__H + diff --git a/lib/backupstore/BackupStoreFilenameClear.cpp b/lib/backupstore/BackupStoreFilenameClear.cpp new file mode 100644 index 00000000..e529d8d3 --- /dev/null +++ b/lib/backupstore/BackupStoreFilenameClear.cpp @@ -0,0 +1,335 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilenameClear.cpp +// Purpose: BackupStoreFilenames in the clear +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreFilenameClear.h" +#include "BackupStoreException.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "Guards.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +// Hide private variables from the rest of the world +namespace +{ + int sEncodeMethod = BackupStoreFilename::Encoding_Clear; + CipherContext sBlowfishEncrypt; + CipherContext sBlowfishDecrypt; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear() +// Purpose: Default constructor, creates an invalid filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &) +// Purpose: Creates a filename, encoding from the given string +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode) +{ + SetClearFilename(rToEncode); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy) + : BackupStoreFilename(rToCopy), + mClearFilename(rToCopy.mClearFilename) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) +// Purpose: Copy from base class +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) + : BackupStoreFilename(rToCopy) +{ + // Will get a clear filename when it's required +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::~BackupStoreFilenameClear() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::GetClearFilename() +// Purpose: Get the unencoded filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE +const std::string BackupStoreFilenameClear::GetClearFilename() const +{ + MakeClearAvailable(); + // When modifying, remember to change back to reference return if at all possible + // -- returns an object rather than a reference to allow easy use with other code. + return std::string(mClearFilename.c_str(), mClearFilename.size()); +} +#else +const std::string &BackupStoreFilenameClear::GetClearFilename() const +{ + MakeClearAvailable(); + return mClearFilename; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &) +// Purpose: Encode and make available the clear filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode) +{ + // Only allow Blowfish encodings + if(sEncodeMethod != Encoding_Blowfish) + { + THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup) + } + + // Make an encoded string with blowfish encryption + EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish); + + // Store the clear filename + mClearFilename.assign(rToEncode.c_str(), rToEncode.size()); + + // Make sure we did the right thing + if(!CheckValid(false)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::MakeClearAvailable() +// Purpose: Private. Make sure the clear filename is available +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::MakeClearAvailable() const +{ + if(!mClearFilename.empty()) + return; // nothing to do + + // Check valid + CheckValid(); + + // Decode the header + int size = BACKUPSTOREFILENAME_GET_SIZE(GetEncodedFilename()); + int encoding = BACKUPSTOREFILENAME_GET_ENCODING(GetEncodedFilename()); + + // Decode based on encoding given in the header + switch(encoding) + { + case Encoding_Clear: + BOX_TRACE("**** BackupStoreFilename encoded with " + "Clear encoding ****"); + mClearFilename.assign(GetEncodedFilename().c_str() + 2, + size - 2); + break; + + case Encoding_Blowfish: + DecryptEncoded(sBlowfishDecrypt); + break; + + default: + THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding) + break; + } +} + + +// Buffer for encoding and decoding -- do this all in one single buffer to +// avoid lots of string allocation, which stuffs up memory usage. +// These static memory vars are, of course, not thread safe, but we don't use threads. +static int sEncDecBufferSize = 0; +static MemoryBlockGuard<uint8_t *> *spEncDecBuffer = 0; + +static void EnsureEncDecBufferSize(int BufSize) +{ + if(spEncDecBuffer == 0) + { +#ifndef WIN32 + BOX_TRACE("Allocating filename encoding/decoding buffer " + "with size " << BufSize); +#endif + spEncDecBuffer = new MemoryBlockGuard<uint8_t *>(BufSize); + MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer); + MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer); + sEncDecBufferSize = BufSize; + } + else + { + if(sEncDecBufferSize < BufSize) + { + BOX_TRACE("Reallocating filename encoding/decoding " + "buffer from " << sEncDecBufferSize << + " to " << BufSize); + spEncDecBuffer->Resize(BufSize); + sEncDecBufferSize = BufSize; + MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int) +// Purpose: Private. Assigns the encoded filename string, encrypting. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding) +{ + // Work out max size + int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4; + + // Make sure encode/decode buffer has enough space + EnsureEncDecBufferSize(maxOutSize); + + // Pointer to buffer + uint8_t *buffer = *spEncDecBuffer; + + // Encode -- do entire block in one go + int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size()); + // and add in header size + encSize += 2; + + // Adjust header + BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding); + + // Store the encoded string + SetEncodedFilename(std::string((char*)buffer, encSize)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &) +// Purpose: Decrypt the encoded filename using the cipher context +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const +{ + const std::string& rEncoded = GetEncodedFilename(); + + // Work out max size + int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rEncoded.size()) + 4; + + // Make sure encode/decode buffer has enough space + EnsureEncDecBufferSize(maxOutSize); + + // Pointer to buffer + uint8_t *buffer = *spEncDecBuffer; + + // Decrypt + const char *str = rEncoded.c_str() + 2; + int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, rEncoded.size() - 2); + + // Assign to this + mClearFilename.assign((char*)buffer, sizeOut); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::EncodedFilenameChanged() +// Purpose: The encoded filename stored has changed +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::EncodedFilenameChanged() +{ + BackupStoreFilename::EncodedFilenameChanged(); + + // Delete stored filename in clear + mClearFilename.erase(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int) +// Purpose: Set the key used for Blowfish encryption of filenames +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength) +{ + // Initialisation vector not used. Can't use a different vector for each filename as + // that would stop comparisions on the server working. + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength); + sBlowfishEncrypt.SetIV(pIV); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength); + sBlowfishDecrypt.SetIV(pIV); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetEncodingMethod(int) +// Purpose: Set the encoding method used for filenames +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetEncodingMethod(int Method) +{ + sEncodeMethod = Method; +} + + + diff --git a/lib/backupstore/BackupStoreFilenameClear.h b/lib/backupstore/BackupStoreFilenameClear.h new file mode 100644 index 00000000..d4c45701 --- /dev/null +++ b/lib/backupstore/BackupStoreFilenameClear.h @@ -0,0 +1,60 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilenameClear.h +// Purpose: BackupStoreFilenames in the clear +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILENAMECLEAR__H +#define BACKUPSTOREFILENAMECLEAR__H + +#include "BackupStoreFilename.h" + +class CipherContext; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFilenameClear +// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreFilenameClear : public BackupStoreFilename +{ +public: + BackupStoreFilenameClear(); + BackupStoreFilenameClear(const std::string &rToEncode); + BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy); + BackupStoreFilenameClear(const BackupStoreFilename &rToCopy); + virtual ~BackupStoreFilenameClear(); + + // Because we need to use a different allocator for this class to avoid + // nasty things happening, can't return this as a reference. Which is a + // pity. But probably not too bad. +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE + const std::string GetClearFilename() const; +#else + const std::string &GetClearFilename() const; +#endif + void SetClearFilename(const std::string &rToEncode); + + // Setup for encryption of filenames + static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength); + static void SetEncodingMethod(int Method); + +protected: + void MakeClearAvailable() const; + virtual void EncodedFilenameChanged(); + void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding); + void DecryptEncoded(CipherContext &rCipherContext) const; + +private: + mutable BackupStoreFilename_base mClearFilename; +}; + +#endif // BACKUPSTOREFILENAMECLEAR__H + + diff --git a/lib/backupstore/BackupStoreObjectMagic.h b/lib/backupstore/BackupStoreObjectMagic.h new file mode 100644 index 00000000..7ee600a2 --- /dev/null +++ b/lib/backupstore/BackupStoreObjectMagic.h @@ -0,0 +1,31 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreObjectMagic.h +// Purpose: Magic values for the start of objects in the backup store +// Created: 19/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREOBJECTMAGIC__H +#define BACKUPSTOREOBJECTMAGIC__H + +// Each of these values is the first 4 bytes of the object file. +// Remember to swap from network to host byte order. + +// Magic value for file streams +#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65 +// Do not use v0 in any new code! +#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45 + +// Magic for the block index at the file stream -- used to +// ensure streams are reordered as expected +#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478 +// Do not use v0 in any new code! +#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B + +// Magic value for directory streams +#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F + +#endif // BACKUPSTOREOBJECTMAGIC__H + diff --git a/lib/backupstore/Makefile.extra b/lib/backupstore/Makefile.extra new file mode 100644 index 00000000..bc807fb6 --- /dev/null +++ b/lib/backupstore/Makefile.extra @@ -0,0 +1,20 @@ + +MAKEPROTOCOL = ../../lib/server/makeprotocol.pl + +GEN_CMD_CLI = $(MAKEPROTOCOL) Client backupprotocol.txt +GEN_CMD_SRV = $(MAKEPROTOCOL) Server backupprotocol.txt + +# AUTOGEN SEEDING +autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) backupprotocol.txt + $(_PERL) $(GEN_CMD_CLI) + +# AUTOGEN SEEDING +autogen_BackupProtocolServer.cpp autogen_BackupProtocolServer.h: $(MAKEPROTOCOL) backupprotocol.txt + $(_PERL) $(GEN_CMD_SRV) + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt + $(_PERL) $(MAKEEXCEPTION) BackupStoreException.txt + diff --git a/lib/backupstore/RunStatusProvider.h b/lib/backupstore/RunStatusProvider.h new file mode 100644 index 00000000..89f361ca --- /dev/null +++ b/lib/backupstore/RunStatusProvider.h @@ -0,0 +1,29 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RunStatusProvider.h +// Purpose: Declares the RunStatusProvider interface. +// Created: 2008/08/14 +// +// -------------------------------------------------------------------------- + +#ifndef RUNSTATUSPROVIDER__H +#define RUNSTATUSPROVIDER__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: RunStatusProvider +// Purpose: Provides a StopRun() method which returns true if +// the current backup should be halted. +// Created: 2005/11/15 +// +// -------------------------------------------------------------------------- +class RunStatusProvider +{ + public: + virtual ~RunStatusProvider() { } + virtual bool StopRun() = 0; +}; + +#endif // RUNSTATUSPROVIDER__H diff --git a/lib/backupstore/backupprotocol.txt b/lib/backupstore/backupprotocol.txt new file mode 100644 index 00000000..011458e8 --- /dev/null +++ b/lib/backupstore/backupprotocol.txt @@ -0,0 +1,235 @@ +# +# backup protocol definition +# + +Name Backup +IdentString Box-Backup:v=C +ServerContextClass BackupStoreContext BackupStoreContext.h + +ClientType Filename BackupStoreFilenameClear BackupStoreFilenameClear.h +ServerType Filename BackupStoreFilename BackupStoreFilename.h + +ImplementLog Server syslog +ImplementLog Client syslog +ImplementLog Client file + +LogTypeToText Client Filename \"%s\" VAR.GetClearFilename().c_str() + +BEGIN_OBJECTS + +# ------------------------------------------------------------------------------------- +# Session commands +# ------------------------------------------------------------------------------------- + +Error 0 IsError(Type,SubType) Reply + int32 Type + int32 SubType + CONSTANT ErrorType 1000 + CONSTANT Err_WrongVersion 1 + CONSTANT Err_NotInRightProtocolPhase 2 + CONSTANT Err_BadLogin 3 + CONSTANT Err_CannotLockStoreForWriting 4 + CONSTANT Err_SessionReadOnly 5 + CONSTANT Err_FileDoesNotVerify 6 + CONSTANT Err_DoesNotExist 7 + CONSTANT Err_DirectoryAlreadyExists 8 + CONSTANT Err_CannotDeleteRoot 9 + CONSTANT Err_TargetNameExists 10 + CONSTANT Err_StorageLimitExceeded 11 + CONSTANT Err_DiffFromFileDoesNotExist 12 + CONSTANT Err_DoesNotExistInDirectory 13 + CONSTANT Err_PatchConsistencyError 14 + CONSTANT Err_MultiplyReferencedObject 15 + +Version 1 Command(Version) Reply + int32 Version + + +Login 2 Command(LoginConfirmed) + int32 ClientID + int32 Flags + CONSTANT Flags_ReadOnly 1 + + +LoginConfirmed 3 Reply + int64 ClientStoreMarker + int64 BlocksUsed + int64 BlocksSoftLimit + int64 BlocksHardLimit + + +Finished 4 Command(Finished) Reply EndsConversation + + +# generic success object +Success 5 Reply + int64 ObjectID + + +SetClientStoreMarker 6 Command(Success) + int64 ClientStoreMarker + + +# ------------------------------------------------------------------------------------- +# Generic object commands +# ------------------------------------------------------------------------------------- + +GetObject 10 Command(Success) + int64 ObjectID + CONSTANT NoObject 0 + # reply has stream following, if ObjectID != NoObject + + +MoveObject 11 Command(Success) + int64 ObjectID + int64 MoveFromDirectory + int64 MoveToDirectory + int32 Flags + Filename NewFilename + + CONSTANT Flags_MoveAllWithSameName 1 + CONSTANT Flags_AllowMoveOverDeletedObject 2 + +# consider this an object command as, although it deals with directory entries, +# it's not specific to either a file or a directory + + +GetObjectName 12 Command(ObjectName) + int64 ObjectID + int64 ContainingDirectoryID + CONSTANT ObjectID_DirectoryOnly 0 + + # set ObjectID to ObjectID_DirectoryOnly to only get info on the directory + + +ObjectName 13 Reply + int32 NumNameElements + int64 ModificationTime + int64 AttributesHash + int16 Flags + # NumNameElements is zero if the object doesn't exist + CONSTANT NumNameElements_ObjectDoesntExist 0 + # a stream of Filename objects follows, if and only if NumNameElements > 0 + + +# ------------------------------------------------------------------------------------- +# Directory commands +# ------------------------------------------------------------------------------------- + +CreateDirectory 20 Command(Success) StreamWithCommand + int64 ContainingDirectoryID + int64 AttributesModTime + Filename DirectoryName + # stream following containing attributes + + +ListDirectory 21 Command(Success) + int64 ObjectID + int16 FlagsMustBeSet + int16 FlagsNotToBeSet + bool SendAttributes + # make sure these flags are synced with those in BackupStoreDirectory + CONSTANT Flags_INCLUDE_EVERYTHING -1 + CONSTANT Flags_EXCLUDE_NOTHING 0 + CONSTANT Flags_EXCLUDE_EVERYTHING 15 + CONSTANT Flags_File 1 + CONSTANT Flags_Dir 2 + CONSTANT Flags_Deleted 4 + CONSTANT Flags_OldVersion 8 + # make sure this is the same as in BackupStoreConstants.h + CONSTANT RootDirectory 1 + + # reply has stream following Success object, containing a stored BackupStoreDirectory + + +ChangeDirAttributes 22 Command(Success) StreamWithCommand + int64 ObjectID + int64 AttributesModTime + # stream following containing attributes + + +DeleteDirectory 23 Command(Success) + int64 ObjectID + +UndeleteDirectory 24 Command(Success) + int64 ObjectID + # may not have exactly the desired effect if files within in have been deleted before the directory was deleted. + + +# ------------------------------------------------------------------------------------- +# File commands +# ------------------------------------------------------------------------------------- + +StoreFile 30 Command(Success) StreamWithCommand + int64 DirectoryObjectID + int64 ModificationTime + int64 AttributesHash + int64 DiffFromFileID # 0 if the file is not a diff + Filename Filename + # then send a stream containing the encoded file + + +GetFile 31 Command(Success) + int64 InDirectory + int64 ObjectID + # error returned if not a file, or does not exist + # reply has stream following, containing an encoded file IN STREAM ORDER + # (use GetObject to get it in file order) + + +SetReplacementFileAttributes 32 Command(Success) StreamWithCommand + int64 InDirectory + int64 AttributesHash + Filename Filename + # stream follows containing attributes + + +DeleteFile 33 Command(Success) + int64 InDirectory + Filename Filename + # will return 0 if the object couldn't be found in the specified directory + + +GetBlockIndexByID 34 Command(Success) + int64 ObjectID + + # stream of the block index follows the reply + # returns an error if the object didn't exist + + +GetBlockIndexByName 35 Command(Success) + int64 InDirectory + Filename Filename + + # Success object contains the found ID -- or 0 if the entry wasn't found in the directory + # stream of the block index follows the reply if found ID != 0 + + +UndeleteFile 36 Command(Success) + int64 InDirectory + int64 ObjectID + # will return 0 if the object couldn't be found in the specified directory + + +# ------------------------------------------------------------------------------------- +# Information commands +# ------------------------------------------------------------------------------------- + +GetAccountUsage 40 Command(AccountUsage) + # no data members + +AccountUsage 41 Reply + int64 BlocksUsed + int64 BlocksInOldFiles + int64 BlocksInDeletedFiles + int64 BlocksInDirectories + int64 BlocksSoftLimit + int64 BlocksHardLimit + int32 BlockSize + +GetIsAlive 42 Command(IsAlive) + # no data members + +IsAlive 43 Reply + # no data members + |