diff options
author | Ben Summers <ben@fluffy.co.uk> | 2005-10-14 08:50:54 +0000 |
---|---|---|
committer | Ben Summers <ben@fluffy.co.uk> | 2005-10-14 08:50:54 +0000 |
commit | 99f8ce096bc5569adbfea1911dbcda24c28d8d8b (patch) | |
tree | 049c302161fea1f2f6223e1e8f3c40d9e8aadc8b /lib/backupclient |
Box Backup 0.09 with a few tweeks
Diffstat (limited to 'lib/backupclient')
34 files changed, 8622 insertions, 0 deletions
diff --git a/lib/backupclient/BackupClientCryptoKeys.cpp b/lib/backupclient/BackupClientCryptoKeys.cpp new file mode 100755 index 00000000..015dadd2 --- /dev/null +++ b/lib/backupclient/BackupClientCryptoKeys.cpp @@ -0,0 +1,67 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientCryptoKeys.cpp +// Purpose: function for setting up all the backup client keys +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "BackupClientCryptoKeys.h" +#include "FileStream.h" +#include "BackupStoreFilenameClear.h" +#include "BackupStoreException.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreFile.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientCryptoKeys_Setup(const char *) +// Purpose: Read in the key material file, and set keys to all the backup elements required. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename) +{ + // Read in the key material + unsigned char KeyMaterial[BACKUPCRYPTOKEYS_FILE_SIZE]; + + // Open the file + FileStream file(KeyMaterialFilename); + // Read in data + if(!file.ReadFullBuffer(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE, 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntLoadClientKeyMaterial) + } + + // Tell the filename how to encrypt + BackupStoreFilenameClear::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH, + KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH); + BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish); + + // Tell the attributes how to encrypt + BackupClientFileAttributes::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH); + // and the secret for hashing + BackupClientFileAttributes::SetAttributeHashSecret(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH); + + // Tell the files how to encrypt + BackupStoreFile::SetBlowfishKeys(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH, + KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH); +#ifndef PLATFORM_OLD_OPENSSL + // Use AES where available + BackupStoreFile::SetAESKey(KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH); +#endif + + // Wipe the key material from memory + ::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE); +} + + + diff --git a/lib/backupclient/BackupClientCryptoKeys.h b/lib/backupclient/BackupClientCryptoKeys.h new file mode 100755 index 00000000..5e3a7df2 --- /dev/null +++ b/lib/backupclient/BackupClientCryptoKeys.h @@ -0,0 +1,55 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientCryptoKeys.h +// Purpose: Format of crypto keys file, and function for setting everything up +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTCRYTOKEYS__H +#define BACKUPCLIENTCRYTOKEYS__H + + +// All keys are the maximum size that Blowfish supports. Since only the +// setup time is affected by key length (encryption same speed whatever) +// there is no disadvantage to using long keys as they are never +// transmitted and are static over long periods of time. + + +// All sizes in bytes. Some gaps deliberately left in the used material. + +// How long the key material file is expected to be +#define BACKUPCRYPTOKEYS_FILE_SIZE 1024 + +// key for encrypting filenames (448 bits) +#define BACKUPCRYPTOKEYS_FILENAME_KEY_START 0 +#define BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH 56 +#define BACKUPCRYPTOKEYS_FILENAME_IV_START (0 + BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH) +#define BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH 8 + +// key for encrypting attributes (448 bits) +#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START (BACKUPCRYPTOKEYS_FILENAME_KEY_START+64) +#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH 56 + +// Blowfish key for encrypting file data (448 bits (max blowfish key length)) +#define BACKUPCRYPTOKEYS_FILE_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START+64) +#define BACKUPCRYPTOKEYS_FILE_KEY_LENGTH 56 + +// key for encrypting file block index entries +#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START (BACKUPCRYPTOKEYS_FILE_KEY_START+64) +#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH 56 + +// Secret for hashing attributes +#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START (BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START+64) +#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH 128 + +// AES key for encrypting file data (256 bits (max AES key length)) +#define BACKUPCRYPTOKEYS_FILE_AES_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START+128) +#define BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH 32 + + +void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename); + +#endif // BACKUPCLIENTCRYTOKEYS__H + diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp new file mode 100755 index 00000000..4aa4edba --- /dev/null +++ b/lib/backupclient/BackupClientFileAttributes.cpp @@ -0,0 +1,773 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientFileAttributes.cpp +// Purpose: Storage of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <string.h> +#include <unistd.h> +#include <limits.h> + +#include "BackupClientFileAttributes.h" +#include "CommonException.h" +#include "FileModificationTime.h" +#include "BoxTimeToUnix.h" +#include "BackupStoreException.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "MD5Digest.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +#define ATTRIBUTETYPE_GENERIC_UNIX 1 + +#define ATTRIBUTE_ENCODING_BLOWFISH 2 + +typedef struct +{ + int32_t AttributeType; + u_int32_t UID; + u_int32_t GID; + u_int64_t ModificationTime; + u_int64_t AttrModificationTime; + u_int32_t UserDefinedFlags; + u_int32_t FileGenerationNumber; + u_int16_t Mode; + // Symbolic link filename may follow +} attr_StreamFormat; + +// This has wire packing so it's compatible across platforms +// Use wider than necessary sizes, just to be careful. +typedef struct +{ + int32_t uid, gid, mode; +} attributeHashData; + +// Use default packing +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + + +#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256 + +// Hide private static variables from the rest of the world +// -- don't put them as static class variables to avoid openssl/evp.h being +// included all over the project. +namespace +{ + CipherContext sBlowfishEncrypt; + CipherContext sBlowfishDecrypt; + uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH]; + int sAttributeHashSecretLength = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::BackupClientFileAttributes() +// Purpose: Default constructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::BackupClientFileAttributes() + : mpClearAttributes(0) +{ + ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &) +// Purpose: Copy constructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy) + : StreamableMemBlock(rToCopy), // base class does the hard work + mpClearAttributes(0) +{ +} +BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy) + : StreamableMemBlock(rToCopy), // base class does the hard work + mpClearAttributes(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::~BackupClientFileAttributes() +// Purpose: Destructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::~BackupClientFileAttributes() +{ + if(mpClearAttributes) + { + delete mpClearAttributes; + mpClearAttributes = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &) +// Purpose: Assignment operator +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr) +{ + StreamableMemBlock::Set(rAttr); + RemoveClear(); // make sure no decrypted version held + return *this; +} +// Assume users play nice +BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr) +{ + StreamableMemBlock::Set(rAttr); + RemoveClear(); // make sure no decrypted version held + return *this; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &) +// Purpose: Comparison operator +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const +{ + EnsureClearAvailable(); + rAttr.EnsureClearAvailable(); + + return mpClearAttributes->operator==(*rAttr.mpClearAttributes); +} +// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data. +/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const +{ + StreamableMemBlock *pDecoded = 0; + + try + { + EnsureClearAvailable(); + StreamableMemBlock *pDecoded = MakeClear(rAttr); + + // Compare using clear version + bool compared = mpClearAttributes->operator==(rAttr); + + // Delete temporary + delete pDecoded; + + return compared; + } + catch(...) + { + delete pDecoded; + throw; + } +}*/ + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool) +// Purpose: Compare, optionally ignoring the attribute modification time and/or modification time, and some data which is +// irrelevant in practise (eg file generation number) +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime, bool IgnoreModTime) const +{ + EnsureClearAvailable(); + rAttr.EnsureClearAvailable(); + + // Check sizes are the same, as a first check + if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize()) + { + return false; + } + + // Then check the elements of the two things + // Bytes are checked in network order, but this doesn't matter as we're only checking for equality. + attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer(); + + if(a1->AttributeType != a2->AttributeType + || a1->UID != a2->UID + || a1->GID != a2->GID + || a1->UserDefinedFlags != a2->UserDefinedFlags + || a1->Mode != a2->Mode) + { + return false; + } + + if(!IgnoreModTime) + { + if(a1->ModificationTime != a2->ModificationTime) + { + return false; + } + } + + if(!IgnoreAttrModTime) + { + if(a1->AttrModificationTime != a2->AttrModificationTime) + { + return false; + } + } + + // Check symlink string? + unsigned int size = mpClearAttributes->GetSize(); + if(size > sizeof(attr_StreamFormat)) + { + // Symlink strings don't match + if(::memcmp(a1 + 1, a2 + 1, size - sizeof(attr_StreamFormat)) != 0) + { + return false; + } + } + + // Passes all test, must be OK + return true; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadAttributes(const char *) +// Purpose: Read the attributes of the file, and store them ready for streaming. +// Optionally retrieve the modification time and attribute modification time. +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroModificationTimes, box_time_t *pModTime, + box_time_t *pAttrModTime, int64_t *pFileSize, ino_t *pInodeNumber, bool *pHasMultipleLinks) +{ + StreamableMemBlock *pnewAttr = 0; + try + { + struct stat st; + if(::lstat(Filename, &st) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Modification times etc + if(pModTime) {*pModTime = FileModificationTime(st);} + if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);} + if(pFileSize) {*pFileSize = st.st_size;} + if(pInodeNumber) {*pInodeNumber = st.st_ino;} + if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);} + + // Is it a link? + if((st.st_mode & S_IFMT) == S_IFLNK) + { + ReadAttributesLink(Filename, &st, ZeroModificationTimes); + return; + } + ASSERT((st.st_mode & S_IFMT) != S_IFLNK); + + // Now, can allocate the block + pnewAttr = new StreamableMemBlock(sizeof(attr_StreamFormat)); + + // Fill in the entries + attr_StreamFormat *pattr = (attr_StreamFormat*)pnewAttr->GetBuffer(); + +#define FILL_IN_ATTRIBUTES \ + ::memset(pattr, 0, sizeof(attr_StreamFormat)); \ + ASSERT(pattr != 0); \ + pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX); \ + pattr->UID = htonl(st.st_uid); \ + pattr->GID = htonl(st.st_gid); \ + if(ZeroModificationTimes) \ + { \ + pattr->ModificationTime = 0; \ + pattr->AttrModificationTime = 0; \ + } \ + else \ + { \ + pattr->ModificationTime = hton64(FileModificationTime(st)); \ + pattr->AttrModificationTime = hton64(FileAttrModificationTime(st)); \ + } \ + pattr->Mode = htons(st.st_mode); +#ifdef PLATFORM_stat_NO_st_flags + #define FILL_IN_ATTRIBUTES_2 \ + pattr->UserDefinedFlags = 0; \ + pattr->FileGenerationNumber = 0; +#else + #define FILL_IN_ATTRIBUTES_2 \ + pattr->UserDefinedFlags = htonl(st.st_flags); \ + pattr->FileGenerationNumber = htonl(st.st_gen); +#endif + + FILL_IN_ATTRIBUTES + FILL_IN_ATTRIBUTES_2 + + // Attributes ready. Encrypt into this block + EncryptAttr(*pnewAttr); + + // Store the new attributes + RemoveClear(); + mpClearAttributes = pnewAttr; + pnewAttr = 0; + } + catch(...) + { + // clean up + delete pnewAttr; + pnewAttr = 0; + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadAttributesLink() +// Purpose: Private function, handles the case where a symbolic link is needed +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::ReadAttributesLink(const char *Filename, void *pst, bool ZeroModificationTimes) +{ + StreamableMemBlock *pnewAttr = 0; + try + { + // Avoid needing to have struct stat available when including header file + struct stat &st = *((struct stat *)pst); + // Make sure we're only called for symbolic links + ASSERT((st.st_mode & S_IFMT) == S_IFLNK); + + // Get the filename the link is linked to + char linkedTo[PATH_MAX+4]; + int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX); + if(linkedToSize == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Now, can allocate the block + pnewAttr = new StreamableMemBlock(sizeof(attr_StreamFormat) + linkedToSize + 1); + + // Fill in the entries + attr_StreamFormat *pattr = (attr_StreamFormat*)pnewAttr->GetBuffer(); + + FILL_IN_ATTRIBUTES + FILL_IN_ATTRIBUTES_2 + + // Add the path name for the symbolic link + ::memcpy(pattr + 1, linkedTo, linkedToSize); + // Make sure it's neatly terminated + ((char*)(pattr + 1))[linkedToSize] = '\0'; + + // Attributes ready. Encrypt into this block + EncryptAttr(*pnewAttr); + + // Store the new attributes + RemoveClear(); + mpClearAttributes = pnewAttr; + pnewAttr = 0; + } + catch(...) + { + // clean up + delete pnewAttr; + pnewAttr = 0; + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::WriteAttributes(const char *) +// Purpose: Apply the stored attributes to the file +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::WriteAttributes(const char *Filename) const +{ + // Got something loaded + if(GetSize() <= 0) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Make sure there are clear attributes to use + EnsureClearAvailable(); + ASSERT(mpClearAttributes != 0); + + // Check if the decrypted attributes are small enough, and the type of attributes stored + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) + { + // Don't know what to do with these + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + + // Check there is enough space for an attributes block + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get pointer to structure + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + + // is it a symlink? + int16_t mode = ntohs(pattr->Mode); + if((mode & S_IFMT) == S_IFLNK) + { + // Check things are sensible + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Make a symlink, first deleting anything in the way + ::unlink(Filename); + if(::symlink((char*)(pattr + 1), Filename) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + + // If working as root, set user IDs + if(::geteuid() == 0) + { + #ifdef PLATFORM_LCHOWN_NOT_SUPPORTED + // only if not a link, can't set their owner on this platform + if((mode & S_IFMT) != S_IFLNK) + { + // Not a link, use normal chown + if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + #else + if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) // use the version which sets things on symlinks + { + THROW_EXCEPTION(CommonException, OSFileError) + } + #endif + } + + // Stop now if symlink, because otherwise it'll just be applied to the target + if((mode & S_IFMT) == S_IFLNK) + { + return; + } + + // Set modification time? + box_time_t modtime = ntoh64(pattr->ModificationTime); + if(modtime != 0) + { + // Work out times as timevals + struct timeval times[2]; + BoxTimeToTimeval(modtime, times[1]); + // Copy access time as well, why not, got to set it to something + times[0] = times[1]; + // Attr modification time will be changed anyway, nothing that can be done about it + + // Try to apply + if(::utimes(Filename, times) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + + // Apply everything else... (allowable mode flags only) + if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID | S_ISGID | S_ISVTX)) != 0) // mode must be done last (think setuid) + { + THROW_EXCEPTION(CommonException, OSFileError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::IsSymLink() +// Purpose: Do these attributes represent a symbolic link? +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::IsSymLink() const +{ + EnsureClearAvailable(); + + // Got the right kind of thing? + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get the type of attributes stored + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat)) + { + // Check link + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK; + } + + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::RemoveClear() +// Purpose: Private. Deletes any clear version of the attributes that may be held +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::RemoveClear() const +{ + if(mpClearAttributes) + { + delete mpClearAttributes; + } + mpClearAttributes = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::EnsureClearAvailable() +// Purpose: Private. Makes sure the clear version is available +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::EnsureClearAvailable() const +{ + if(mpClearAttributes == 0) + { + mpClearAttributes = MakeClear(*this); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &) +// Purpose: Static. Decrypts stored attributes. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted) +{ + // New block + StreamableMemBlock *pdecrypted = 0; + + try + { + // Check the block is big enough for IV and header + int ivSize = sBlowfishEncrypt.GetIVLength(); + if(rEncrypted.GetSize() <= (ivSize + 1)) + { + THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes); + } + + // How much space is needed for the output? + int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize); + + // Allocate it + pdecrypted = new StreamableMemBlock(maxDecryptedSize); + + // ptr to block + uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer(); + + // Check that the header has right type + if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH) + { + THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding); + } + + // Set IV + sBlowfishDecrypt.SetIV(encBlock + 1); + + // Decrypt + int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1)); + + // Resize block to fit + pdecrypted->ResizeBlock(decryptedSize); + } + catch(...) + { + delete pdecrypted; + pdecrypted = 0; + } + + return pdecrypted; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int) +// Purpose: Static. Sets the key to use for encryption and decryption. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength) +{ + // IVs set later + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &) +// Purpose: Private. Encrypt the given attributes into this block. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt) +{ + // Free any existing block + FreeBlock(); + + // Work out the maximum amount of space we need + int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize()); + // And the size of the IV + int ivSize = sBlowfishEncrypt.GetIVLength(); + + // Allocate this space + AllocateBlock(maxEncryptedSize + ivSize + 1); + + // Store the encoding byte + uint8_t *block = (uint8_t*)GetBuffer(); + block[0] = ATTRIBUTE_ENCODING_BLOWFISH; + + // Generate and store an IV for this attribute block + int ivSize2 = 0; + const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2); + ASSERT(ivSize == ivSize2); + + // Copy into the encrypted block + ::memcpy(block + 1, iv, ivSize); + + // Do the transform + int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize()); + + // Resize this block + ResizeBlock(encrytedSize + ivSize + 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int) +// Purpose: Set the secret for the filename attribute hash +// Created: 25/4/04 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength) +{ + if(SecretLength > (int)sizeof(sAttributeHashSecret)) + { + SecretLength = sizeof(sAttributeHashSecret); + } + if(SecretLength < 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Copy + ::memcpy(sAttributeHashSecret, pSecret, SecretLength); + sAttributeHashSecretLength = SecretLength; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::GenerateAttributeHash(struct stat &, const std::string &) +// Purpose: Generate a 64 bit hash from the attributes, used to detect changes. +// Include filename in the hash, so that it changes from one file to another, +// so don't reveal identical attributes. +// Created: 25/4/04 +// +// -------------------------------------------------------------------------- +uint64_t BackupClientFileAttributes::GenerateAttributeHash(struct stat &st, const std::string &rFilename) +{ + if(sAttributeHashSecretLength == 0) + { + THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet) + } + + // Assemble stuff we're interested in + attributeHashData hashData; + memset(&hashData, 0, sizeof(hashData)); + // Use network byte order and large sizes to be cross platform + hashData.uid = htonl(st.st_uid); + hashData.gid = htonl(st.st_gid); + hashData.mode = htonl(st.st_mode); + + // Create a MD5 hash of the data, filename, and secret + MD5Digest digest; + digest.Add(&hashData, sizeof(hashData)); + digest.Add(rFilename.c_str(), rFilename.size()); + digest.Add(sAttributeHashSecret, sAttributeHashSecretLength); + digest.Finish(); + + // Return the first 64 bits of the hash + uint64_t result = *((uint64_t *)(digest.DigestAsData())); + return result; +} + + + diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h new file mode 100755 index 00000000..dd930684 --- /dev/null +++ b/lib/backupclient/BackupClientFileAttributes.h @@ -0,0 +1,70 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientFileAttributes.h +// Purpose: Storage of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTFILEATTRIBUTES__H +#define BACKUPCLIENTFILEATTRIBUTES__H + +#include <string> + +#include "StreamableMemBlock.h" +#include "BoxTime.h" + +struct stat; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientFileAttributes +// Purpose: Storage, streaming and application of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +class BackupClientFileAttributes : public StreamableMemBlock +{ +public: + BackupClientFileAttributes(); + BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy); + BackupClientFileAttributes(const StreamableMemBlock &rToCopy); + ~BackupClientFileAttributes(); + BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr); + BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr); + bool operator==(const BackupClientFileAttributes &rAttr) const; +// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous? + + bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const; + + // Prevent access to base class members accidently + void Set(); + + void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false, + box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0, + ino_t *pInodeNumber = 0, bool *pHasMultipleLinks = 0); + void WriteAttributes(const char *Filename) const; + + bool IsSymLink() const; + + static void SetBlowfishKey(const void *pKey, int KeyLength); + static void SetAttributeHashSecret(const void *pSecret, int SecretLength); + + static uint64_t GenerateAttributeHash(struct stat &st, const std::string &rFilename); + +private: + void ReadAttributesLink(const char *Filename, void *pst, bool ZeroModificationTimes); + + void RemoveClear() const; + void EnsureClearAvailable() const; + static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted); + void EncryptAttr(const StreamableMemBlock &rToEncrypt); + +private: + mutable StreamableMemBlock *mpClearAttributes; +}; + +#endif // BACKUPCLIENTFILEATTRIBUTES__H + diff --git a/lib/backupclient/BackupClientMakeExcludeList.cpp b/lib/backupclient/BackupClientMakeExcludeList.cpp new file mode 100755 index 00000000..0615faaa --- /dev/null +++ b/lib/backupclient/BackupClientMakeExcludeList.cpp @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientMakeExcludeList.cpp +// Purpose: Makes exclude lists from bbbackupd config location entries +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "BackupClientMakeExcludeList.h" +#include "Configuration.h" +#include "ExcludeList.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientMakeExcludeList(const Configuration &, const char *, const char *) +// Purpose: Given a Configuration object corresponding to a bbackupd Location, and the +// two names of the keys for definite and regex entries, return a ExcludeList. +// Or 0 if it isn't required. +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName, + const char *AlwaysIncludeDefiniteName, const char *AlwaysIncludeRegexName) +{ + // Check that at least one of the entries exists + if(!rConfig.KeyExists(DefiniteName) && !rConfig.KeyExists(RegexName)) + { + // Neither exists -- return 0 as an Exclude list isn't required. + return 0; + } + + // Create the exclude list + ExcludeList *pexclude = new ExcludeList; + + try + { + // Definite names to add? + if(rConfig.KeyExists(DefiniteName)) + { + pexclude->AddDefiniteEntries(rConfig.GetKeyValue(DefiniteName)); + } + // Regular expressions to add? + if(rConfig.KeyExists(RegexName)) + { + pexclude->AddRegexEntries(rConfig.GetKeyValue(RegexName)); + } + + // Add a "always include" list? + if(AlwaysIncludeDefiniteName != 0 && AlwaysIncludeRegexName != 0) + { + // This will accept NULL as a valid argument, so safe to do this. + pexclude->SetAlwaysIncludeList( + BackupClientMakeExcludeList(rConfig, AlwaysIncludeDefiniteName, AlwaysIncludeRegexName) + ); + } + } + catch(...) + { + // Clean up + delete pexclude; + throw; + } + + return pexclude; +} + + + diff --git a/lib/backupclient/BackupClientMakeExcludeList.h b/lib/backupclient/BackupClientMakeExcludeList.h new file mode 100755 index 00000000..d5dd8af4 --- /dev/null +++ b/lib/backupclient/BackupClientMakeExcludeList.h @@ -0,0 +1,48 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientMakeExcludeList.h +// Purpose: Makes exclude lists from bbbackupd config location entries +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTMAKEEXCLUDELIST__H +#define BACKUPCLIENTMAKEEXCLUDELIST__H + +class ExcludeList; +class Configuration; + +ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName, + const char *AlwaysIncludeDefiniteName = 0, const char *AlwaysIncludeRegexName = 0); + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientMakeExcludeList_Files(const Configuration &) +// Purpose: Create a exclude list from config file entries for files. May return 0. +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +inline ExcludeList *BackupClientMakeExcludeList_Files(const Configuration &rConfig) +{ + return BackupClientMakeExcludeList(rConfig, "ExcludeFile", "ExcludeFilesRegex", "AlwaysIncludeFile", "AlwaysIncludeFilesRegex"); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientMakeExcludeList_Dirs(const Configuration &) +// Purpose: Create a exclude list from config file entries for directories. May return 0. +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +inline ExcludeList *BackupClientMakeExcludeList_Dirs(const Configuration &rConfig) +{ + return BackupClientMakeExcludeList(rConfig, "ExcludeDir", "ExcludeDirsRegex", "AlwaysIncludeDir", "AlwaysIncludeDirsRegex"); +} + + +#endif // BACKUPCLIENTMAKEEXCLUDELIST__H + diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp new file mode 100755 index 00000000..ebe1d365 --- /dev/null +++ b/lib/backupclient/BackupClientRestore.cpp @@ -0,0 +1,468 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientRestore.cpp +// Purpose: +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <string> +#include <set> +#include <unistd.h> +#include <limits.h> +#include <stdio.h> + +#include "BackupClientRestore.h" +#include "autogen_BackupProtocolClient.h" +#include "CommonException.h" +#include "BackupClientFileAttributes.h" +#include "IOStream.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreFile.h" +#include "CollectInBufferStream.h" +#include "FileStream.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +#define MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES (128*1024) + +class RestoreResumeInfo +{ +public: + // constructor + RestoreResumeInfo() + : mNextLevelID(0), + mpNextLevel(0) + { + } + + // destructor + ~RestoreResumeInfo() + { + delete mpNextLevel; + mpNextLevel = 0; + } + + // Get a next level object + RestoreResumeInfo &AddLevel(int64_t ID, const std::string &rLocalName) + { + ASSERT(mpNextLevel == 0 && mNextLevelID == 0); + mpNextLevel = new RestoreResumeInfo; + mNextLevelID = ID; + mNextLevelLocalName = rLocalName; + return *mpNextLevel; + } + + // Remove the next level info + void RemoveLevel() + { + ASSERT(mpNextLevel != 0 && mNextLevelID != 0); + delete mpNextLevel; + mpNextLevel = 0; + mNextLevelID = 0; + mNextLevelLocalName.erase(); + } + + void Save(const std::string &rFilename) const + { + // TODO: use proper buffered streams when they're done + // Build info in memory buffer + CollectInBufferStream write; + + // Save this level + SaveLevel(write); + + // Store in file + write.SetForReading(); + FileStream file(rFilename.c_str(), O_WRONLY | O_CREAT); + write.CopyStreamTo(file, IOStream::TimeOutInfinite, 8*1024 /* large buffer */); + } + + void SaveLevel(IOStream &rWrite) const + { + // Write the restored objects + int64_t numObjects = mRestoredObjects.size(); + rWrite.Write(&numObjects, sizeof(numObjects)); + for(std::set<int64_t>::const_iterator i(mRestoredObjects.begin()); i != mRestoredObjects.end(); ++i) + { + int64_t id = *i; + rWrite.Write(&id, sizeof(id)); + } + + // Next level? + if(mpNextLevel != 0) + { + // ID + rWrite.Write(&mNextLevelID, sizeof(mNextLevelID)); + // Name string + int32_t nsize = mNextLevelLocalName.size(); + rWrite.Write(&nsize, sizeof(nsize)); + rWrite.Write(mNextLevelLocalName.c_str(), nsize); + // And then the level itself + mpNextLevel->SaveLevel(rWrite); + } + else + { + // Just write a zero + int64_t zero = 0; + rWrite.Write(&zero, sizeof(zero)); + } + } + + // Not written to be efficient -- shouldn't be called very often. + bool Load(const std::string &rFilename) + { + // Delete and reset if necessary + if(mpNextLevel != 0) + { + RemoveLevel(); + } + + // Open file + FileStream file(rFilename.c_str()); + + // Load this level + return LoadLevel(file); + } + + #define CHECKED_READ(x, s) if(!rRead.ReadFullBuffer(x, s, 0)) {return false;} + bool LoadLevel(IOStream &rRead) + { + // Load the restored objects list + mRestoredObjects.clear(); + int64_t numObjects = 0; + CHECKED_READ(&numObjects, sizeof(numObjects)); + for(int64_t o = 0; o < numObjects; ++o) + { + int64_t id; + CHECKED_READ(&id, sizeof(id)); + mRestoredObjects.insert(id); + } + + // ID of next level? + int64_t nextID = 0; + CHECKED_READ(&nextID, sizeof(nextID)); + if(nextID != 0) + { + // Load the next level! + std::string name; + int32_t nsize = 0; + CHECKED_READ(&nsize, sizeof(nsize)); + char n[PATH_MAX]; + if(nsize > PATH_MAX) return false; + CHECKED_READ(n, nsize); + name.assign(n, nsize); + + // Create a new level + mpNextLevel = new RestoreResumeInfo; + mNextLevelID = nextID; + mNextLevelLocalName = name; + + // And ask it to read itself in + if(!mpNextLevel->LoadLevel(rRead)) + { + return false; + } + } + + return true; + } + + // List of objects at this level which have been done already + std::set<int64_t> mRestoredObjects; + // Next level ID + int64_t mNextLevelID; + // Pointer to next level + RestoreResumeInfo *mpNextLevel; + // Local filename of next level + std::string mNextLevelLocalName; +}; + +// parameters structure +typedef struct +{ + bool PrintDots; + bool RestoreDeleted; + std::string mRestoreResumeInfoFilename; + RestoreResumeInfo mResumeInfo; +} RestoreParams; + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientRestoreDir(BackupProtocolClient &, int64_t, const char *, bool) +// Purpose: Restore a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t DirectoryID, std::string &rLocalDirectoryName, + RestoreParams &Params, RestoreResumeInfo &rLevel) +{ + // If we're resuming... check that we haven't got a next level to look at + if(rLevel.mpNextLevel != 0) + { + // Recurse immediately + std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + rLevel.mNextLevelLocalName); + BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, localDirname, Params, *rLevel.mpNextLevel); + + // Add it to the list of done itmes + rLevel.mRestoredObjects.insert(rLevel.mNextLevelID); + + // Remove the level for the recursed directory + rLevel.RemoveLevel(); + } + + // Save the resumption information + Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + + // Create the local directory (if not already done) -- path and owner set later, just use restrictive owner mode + switch(ObjectExists(rLocalDirectoryName.c_str())) + { + case ObjectExists_Dir: + // Do nothing + break; + case ObjectExists_File: + { + // File exists with this name, which is fun. Get rid of it. + ::printf("WARNING: File present with name '%s', removing out of the way of restored directory. Use specific restore with ID to restore this object.", rLocalDirectoryName.c_str()); + if(::unlink(rLocalDirectoryName.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + TRACE1("In restore, directory name collision with file %s", rLocalDirectoryName.c_str()); + } + // follow through to... (no break) + case ObjectExists_NoObject: + if(::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + break; + default: + ASSERT(false); + break; + } + + // Fetch the directory listing from the server -- getting a list of files which is approparite to the restore type + rConnection.QueryListDirectory( + DirectoryID, + Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING), + BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)), + true /* want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(rConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, rConnection.GetTimeout()); + + // Apply attributes to the directory + const StreamableMemBlock &dirAttrBlock(dir.GetAttributes()); + BackupClientFileAttributes dirAttr(dirAttrBlock); + dirAttr.WriteAttributes(rLocalDirectoryName.c_str()); + + int64_t bytesWrittenSinceLastRestoreInfoSave = 0; + + // Process files + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0) + { + // Check ID hasn't already been done + if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end()) + { + // Local name + BackupStoreFilenameClear nm(en->GetName()); + std::string localFilename(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename()); + + // Unlink anything which already exists -- for resuming restores, we can't overwrite files already there. + ::unlink(localFilename.c_str()); + + // Request it from the store + rConnection.QueryGetFile(DirectoryID, en->GetObjectID()); + + // Stream containing encoded file + std::auto_ptr<IOStream> objectStream(rConnection.ReceiveStream()); + + // Decode the file -- need to do different things depending on whether + // the directory entry has additional attributes + if(en->HasAttributes()) + { + // Use these attributes + const StreamableMemBlock &storeAttr(en->GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout(), &attr); + } + else + { + // Use attributes stored in file + BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout()); + } + + // Progress display? + if(Params.PrintDots) + { + printf("."); + fflush(stdout); + } + + // Add it to the list of done itmes + rLevel.mRestoredObjects.insert(en->GetObjectID()); + + // Save restore info? + int64_t fileSize; + if(FileExists(localFilename.c_str(), &fileSize, true /* treat links as not existing */)) + { + // File exists... + bytesWrittenSinceLastRestoreInfoSave += fileSize; + + if(bytesWrittenSinceLastRestoreInfoSave > MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES) + { + // Save the restore info, in case it's needed later + Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + bytesWrittenSinceLastRestoreInfoSave = 0; + } + } + } + } + } + + // Make sure the restore info has been saved + if(bytesWrittenSinceLastRestoreInfoSave != 0) + { + // Save the restore info, in case it's needed later + Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + bytesWrittenSinceLastRestoreInfoSave = 0; + } + + + // Recuse to directories + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0) + { + // Check ID hasn't already been done + if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end()) + { + // Local name + BackupStoreFilenameClear nm(en->GetName()); + std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename()); + + // Add the level for the next entry + RestoreResumeInfo &rnextLevel(rLevel.AddLevel(en->GetObjectID(), nm.GetClearFilename())); + + // Recurse + BackupClientRestoreDir(rConnection, en->GetObjectID(), localDirname, Params, rnextLevel); + + // Remove the level for the above call + rLevel.RemoveLevel(); + + // Add it to the list of done itmes + rLevel.mRestoredObjects.insert(en->GetObjectID()); + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientRestore(BackupProtocolClient &, int64_t, const char *, bool, bool) +// Purpose: Restore a directory on the server to a local directory on the disc. +// +// The local directory must not already exist. +// +// If a restore is aborted for any reason, then it may be resumed if +// Resume == true. If Resume == false and resumption is possible, then +// Restore_ResumePossible is returned. +// +// Set RestoreDeleted to restore a deleted directory. This may not give the +// directory structure when it was deleted, because files may have been deleted +// within it before it was deleted. +// +// Returns Restore_TargetExists if the target directory exists, but +// there is no restore possible. (Won't attempt to overwrite things.) +// +// Returns Restore_Complete on success. (Exceptions on error.) +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName, + bool PrintDots, bool RestoreDeleted, bool UndeleteAfterRestoreDeleted, bool Resume) +{ + // Parameter block + RestoreParams params; + params.PrintDots = PrintDots; + params.RestoreDeleted = RestoreDeleted; + params.mRestoreResumeInfoFilename = LocalDirectoryName; + params.mRestoreResumeInfoFilename += ".boxbackupresume"; + + // Target exists? + int targetExistance = ObjectExists(LocalDirectoryName); + + // Does any resumption information exist? + bool doingResume = false; + if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && targetExistance == ObjectExists_Dir) + { + if(!Resume) + { + // Caller didn't specify that resume should be done, so refuse to do it + // but say why. + return Restore_ResumePossible; + } + + // Attempt to load the resume info file + if(!params.mResumeInfo.Load(params.mRestoreResumeInfoFilename)) + { + // failed -- bad file, so things have gone a bit wrong + return Restore_TargetExists; + } + + // Flag as doing resume so next check isn't actually performed + doingResume = true; + } + + // Does the directory already exist? + if(targetExistance != ObjectExists_NoObject && !doingResume) + { + // Don't do anything in this case! + return Restore_TargetExists; + } + + // Restore the directory + std::string localName(LocalDirectoryName); + BackupClientRestoreDir(rConnection, DirectoryID, localName, params, params.mResumeInfo); + + // Undelete the directory on the server? + if(RestoreDeleted && UndeleteAfterRestoreDeleted) + { + // Send the command + rConnection.QueryUndeleteDirectory(DirectoryID); + } + + // Finish progress display? + if(PrintDots) + { + printf("\n"); + fflush(stdout); + } + + // Delete the resume information file + ::unlink(params.mRestoreResumeInfoFilename.c_str()); + + return Restore_Complete; +} + + + + diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h new file mode 100755 index 00000000..4b382771 --- /dev/null +++ b/lib/backupclient/BackupClientRestore.h @@ -0,0 +1,26 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientRestore.h +// Purpose: Functions to restore files from a backup store +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSCLIENTRESTORE_H +#define BACKUPSCLIENTRESTORE__H + +class BackupProtocolClient; + +enum +{ + Restore_Complete = 0, + Restore_ResumePossible = 1, + Restore_TargetExists = 2 +}; + +int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName, + bool PrintDots = false, bool RestoreDeleted = false, bool UndeleteAfterRestoreDeleted = false, bool Resume = false); + +#endif // BACKUPSCLIENTRESTORE__H + diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp new file mode 100755 index 00000000..eec2ceaf --- /dev/null +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -0,0 +1,105 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemonConfigVerify.cpp +// Purpose: Configuration file definition for bbackupd +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupDaemonConfigVerify.h" +#include "Daemon.h" +#include "BoxPortsAndFiles.h" + +#include "MemLeakFindOn.h" + + +static const ConfigurationVerifyKey backuplocationkeys[] = +{ + {"ExcludeFile", 0, ConfigTest_MultiValueAllowed, 0}, + {"ExcludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0}, + {"ExcludeDir", 0, ConfigTest_MultiValueAllowed, 0}, + {"ExcludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0}, + {"AlwaysIncludeFile", 0, ConfigTest_MultiValueAllowed, 0}, + {"AlwaysIncludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0}, + {"AlwaysIncludeDir", 0, ConfigTest_MultiValueAllowed, 0}, + {"AlwaysIncludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0}, + {"Path", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} +}; + +static const ConfigurationVerify backuplocations[] = +{ + { + "*", + 0, + backuplocationkeys, + ConfigTest_LastEntry, + 0 + } +}; + +static const ConfigurationVerifyKey verifyserverkeys[] = +{ + DAEMON_VERIFY_SERVER_KEYS +}; + +static const ConfigurationVerify verifyserver[] = +{ + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists, + 0 + }, + { + "BackupLocations", + backuplocations, + 0, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } +}; + +static const ConfigurationVerifyKey verifyrootkeys[] = +{ + {"AccountNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + + {"UpdateStoreInterval", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"MinimumFileAge", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"MaxUploadWait", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"MaxFileTimeInFuture", "172800", ConfigTest_IsInt, 0}, // file is uploaded if the file is this much in the future (2 days default) + + {"AutomaticBackup", "yes", ConfigTest_IsBool, 0}, + + {"SyncAllowScript", 0, 0, 0}, // optional script to run to see if the sync should be started now + // return "now" if it's allowed, or a number of seconds if it's not + + {"MaximumDiffingTime", 0, ConfigTest_IsInt, 0}, + + {"FileTrackingSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"DiffingUploadSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"StoreHostname", 0, ConfigTest_Exists, 0}, + {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file + + {"CommandSocket", 0, 0, 0}, // not compulsory to have this + + {"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full + + {"CertificateFile", 0, ConfigTest_Exists, 0}, + {"PrivateKeyFile", 0, ConfigTest_Exists, 0}, + {"TrustedCAsFile", 0, ConfigTest_Exists, 0}, + {"KeysFile", 0, ConfigTest_Exists, 0}, + {"DataDirectory", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} +}; + +const ConfigurationVerify BackupDaemonConfigVerify = +{ + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 +}; + diff --git a/lib/backupclient/BackupDaemonConfigVerify.h b/lib/backupclient/BackupDaemonConfigVerify.h new file mode 100755 index 00000000..fefa703c --- /dev/null +++ b/lib/backupclient/BackupDaemonConfigVerify.h @@ -0,0 +1,18 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemonConfigVerify.h +// Purpose: Configuration file definition for bbackupd +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPDAEMONCONFIGVERIFY__H +#define BACKUPDAEMONCONFIGVERIFY__H + +#include "Configuration.h" + +extern const ConfigurationVerify BackupDaemonConfigVerify; + +#endif // BACKUPDAEMONCONFIGVERIFY__H + diff --git a/lib/backupclient/BackupStoreConstants.h b/lib/backupclient/BackupStoreConstants.h new file mode 100755 index 00000000..8254e918 --- /dev/null +++ b/lib/backupclient/BackupStoreConstants.h @@ -0,0 +1,53 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreContants.h +// Purpose: constants for the backup system +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTORECONSTANTS__H +#define BACKUPSTORECONSTANTS__H + +#define BACKUPSTORE_ROOT_DIRECTORY_ID 1 + +#define BACKUP_STORE_SERVER_VERSION 1 + +// Minimum size for a chunk to be compressed +#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256 + +// min and max sizes for blocks +#define BACKUP_FILE_MIN_BLOCK_SIZE 4096 +#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024) + +// Increase the block size if there are more than this number of blocks +#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096 + +// Avoid creating blocks smaller than this +#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128 + +// Maximum number of sizes to do an rsync-like scan for +#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 8 + +// When doing rsync scans, do not scan for blocks smaller than +#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 256 + +// A limit to stop diffing running out of control: If more than this +// times the number of blocks in the original index are found, stop +// looking. This stops really bad cases of diffing files containing +// all the same byte using huge amounts of memory and processor time. +// This is a multiple of the number of blocks in the diff from file. +#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096 + +// How many seconds to wait before deleting unused root directory entries? +#ifndef NDEBUG + // Debug: 30 seconds (easier to test) + #define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER 30 +#else + // Release: 2 days (plenty of time for sysadmins to notice, or change their mind) + #define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER 172800 +#endif + +#endif // BACKUPSTORECONSTANTS__H + diff --git a/lib/backupclient/BackupStoreDirectory.cpp b/lib/backupclient/BackupStoreDirectory.cpp new file mode 100755 index 00000000..31fed78d --- /dev/null +++ b/lib/backupclient/BackupStoreDirectory.cpp @@ -0,0 +1,563 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory.h +// Purpose: Representation of a backup directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> + +#include "BackupStoreDirectory.h" +#include "IOStream.h" +#include "BackupStoreException.h" +#include "BackupStoreObjectMagic.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mNumEntries; + int64_t mObjectID; // this object ID + int64_t mContainerID; // ID of container + uint64_t mAttributesModTime; + int32_t mOptionsPresent; // bit mask of optional sections / features present + // Then a StreamableMemBlock for attributes +} dir_StreamFormat; + +enum +{ + Option_DependencyInfoPresent = 1 +} dir_StreamFormatOptions; + +typedef struct +{ + uint64_t mModificationTime; + int64_t mObjectID; + int64_t mSizeInBlocks; + uint64_t mAttributesHash; + int16_t mFlags; // order smaller items after bigger ones (for alignment) + // Then a BackupStoreFilename + // Then a StreamableMemBlock for attributes +} en_StreamFormat; + +typedef struct +{ + int64_t mDependsNewer; + int64_t mDependsOlder; +} en_StreamFormatDepends; + +// Use default packing +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::BackupStoreDirectory() +// Purpose: Constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::BackupStoreDirectory() + : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0) +{ + ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); +} + + +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t) +// Purpose: Constructor giving object and container IDs +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID) + : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::~BackupStoreDirectory() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::~BackupStoreDirectory() +{ + for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) + { + delete (*i); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int) +// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Get the header + dir_StreamFormat hdr; + if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic value... + if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue)) + { + THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat) + } + + // Get data + mObjectID = ntoh64(hdr.mObjectID); + mContainerID = ntoh64(hdr.mContainerID); + mAttributesModTime = ntoh64(hdr.mAttributesModTime); + + // Options + int32_t options = ntohl(hdr.mOptionsPresent); + + // Get attributes + mAttributes.ReadFromStream(rStream, Timeout); + + // Decode count + int count = ntohl(hdr.mNumEntries); + + // Clear existing list + mEntries.clear(); + + // Read them in! + for(int c = 0; c < count; ++c) + { + Entry *pen = new Entry; + try + { + // Read from stream + pen->ReadFromStream(rStream, Timeout); + + // Add to list + mEntries.push_back(pen); + } + catch(...) + { + delete pen; + throw; + } + } + + // Read in dependency info? + if(options & Option_DependencyInfoPresent) + { + // Read in extra dependency data + for(int c = 0; c < count; ++c) + { + mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool) +// Purpose: Writes a selection of entries to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const +{ + // Get count of entries + int32_t count = mEntries.size(); + if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING) + { + // Need to count the entries + count = 0; + Iterator i(*this); + while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0) + { + count++; + } + } + + // Check that sensible IDs have been set + ASSERT(mObjectID != 0); + ASSERT(mContainerID != 0); + + // Need dependency info? + bool dependencyInfoRequired = false; + if(StreamDependencyInfo) + { + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + if(pen->HasDependencies()) + { + dependencyInfoRequired = true; + } + } + } + + // Options + int32_t options = 0; + if(dependencyInfoRequired) options |= Option_DependencyInfoPresent; + + // Build header + dir_StreamFormat hdr; + hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE); + hdr.mNumEntries = htonl(count); + hdr.mObjectID = hton64(mObjectID); + hdr.mContainerID = hton64(mContainerID); + hdr.mAttributesModTime = hton64(mAttributesModTime); + hdr.mOptionsPresent = htonl(options); + + // Write header + rStream.Write(&hdr, sizeof(hdr)); + + // Write the attributes? + if(StreamAttributes) + { + mAttributes.WriteToStream(rStream); + } + else + { + // Write a blank header instead + StreamableMemBlock::WriteEmptyBlockToStream(rStream); + } + + // Then write all the entries + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + pen->WriteToStream(rStream); + } + + // Write dependency info? + if(dependencyInfoRequired) + { + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + pen->WriteToStreamDependencyInfo(rStream); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddEntry(const Entry &) +// Purpose: Adds entry to directory (no checking) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy) +{ + Entry *pnew = new Entry(rEntryToCopy); + try + { + mEntries.push_back(pnew); + } + catch(...) + { + delete pnew; + throw; + } + + return pnew; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t) +// Purpose: Adds entry to directory (no checking) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime) +{ + Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime); + try + { + mEntries.push_back(pnew); + } + catch(...) + { + delete pnew; + throw; + } + + return pnew; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::DeleteEntry(int64_t) +// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) +{ + for(std::vector<Entry*>::iterator i(mEntries.begin()); + i != mEntries.end(); ++i) + { + if((*i)->mObjectID == ObjectID) + { + // Delete + delete (*i); + // Remove from list + mEntries.erase(i); + // Done + return; + } + } + + // Not found + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::FindEntryByID(int64_t) +// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist. +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const +{ + for(std::vector<Entry*>::const_iterator i(mEntries.begin()); + i != mEntries.end(); ++i) + { + if((*i)->mObjectID == ObjectID) + { + // Found + return (*i); + } + } + + // Not found + return 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry() +// Purpose: Constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry() + : mModificationTime(0), + mObjectID(0), + mSizeInBlocks(0), + mFlags(0), + mAttributesHash(0), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::~Entry() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::~Entry() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry(const Entry &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) + : mName(rToCopy.mName), + mModificationTime(rToCopy.mModificationTime), + mObjectID(rToCopy.mObjectID), + mSizeInBlocks(rToCopy.mSizeInBlocks), + mFlags(rToCopy.mFlags), + mAttributesHash(rToCopy.mAttributesHash), + mAttributes(rToCopy.mAttributes), + mMinMarkNumber(rToCopy.mMinMarkNumber), + mMarkNumber(rToCopy.mMarkNumber), + mDependsNewer(rToCopy.mDependsNewer), + mDependsOlder(rToCopy.mDependsOlder) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t) +// Purpose: Constructor from values +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) + : mName(rName), + mModificationTime(ModificationTime), + mObjectID(ObjectID), + mSizeInBlocks(SizeInBlocks), + mFlags(Flags), + mAttributesHash(AttributesHash), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int) +// Purpose: Read an entry from a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Grab the raw bytes from the stream which compose the header + en_StreamFormat entry; + if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Do reading first before modifying the variables, to be more exception safe + + // Get the filename + BackupStoreFilename name; + name.ReadFromStream(rStream, Timeout); + + // Get the attributes + mAttributes.ReadFromStream(rStream, Timeout); + + // Store the rest of the bits + mModificationTime = ntoh64(entry.mModificationTime); + mObjectID = ntoh64(entry.mObjectID); + mSizeInBlocks = ntoh64(entry.mSizeInBlocks); + mAttributesHash = ntoh64(entry.mAttributesHash); + mFlags = ntohs(entry.mFlags); + mName = name; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &) +// Purpose: Writes the entry to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const +{ + // Build a structure + en_StreamFormat entry; + entry.mModificationTime = hton64(mModificationTime); + entry.mObjectID = hton64(mObjectID); + entry.mSizeInBlocks = hton64(mSizeInBlocks); + entry.mAttributesHash = hton64(mAttributesHash); + entry.mFlags = htons(mFlags); + + // Write it + rStream.Write(&entry, sizeof(entry)); + + // Write the filename + mName.WriteToStream(rStream); + + // Write any attributes + mAttributes.WriteToStream(rStream); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int) +// Purpose: Read the optional dependency info from a stream +// Created: 13/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout) +{ + // Grab the raw bytes from the stream which compose the header + en_StreamFormatDepends depends; + if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Store the data + mDependsNewer = ntoh64(depends.mDependsNewer); + mDependsOlder = ntoh64(depends.mDependsOlder); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &) +// Purpose: Write the optional dependency info to a stream +// Created: 13/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const +{ + // Build structure + en_StreamFormatDepends depends; + depends.mDependsNewer = hton64(mDependsNewer); + depends.mDependsOlder = hton64(mDependsOlder); + // Write + rStream.Write(&depends, sizeof(depends)); +} + + + diff --git a/lib/backupclient/BackupStoreDirectory.h b/lib/backupclient/BackupStoreDirectory.h new file mode 100755 index 00000000..958eee81 --- /dev/null +++ b/lib/backupclient/BackupStoreDirectory.h @@ -0,0 +1,268 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory.h +// Purpose: Representation of a backup directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREDIRECTORY__H +#define BACKUPSTOREDIRECTORY__H + +#include <string> +#include <vector> + +#include "BackupStoreFilenameClear.h" +#include "StreamableMemBlock.h" +#include "BoxTime.h" + +class IOStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreDirectory +// Purpose: In memory representation of a directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreDirectory +{ +public: + BackupStoreDirectory(); + BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID); +private: + // Copying not allowed + BackupStoreDirectory(const BackupStoreDirectory &rToCopy); +public: + ~BackupStoreDirectory(); + + class Entry + { + public: + friend class BackupStoreDirectory; + + Entry(); + ~Entry(); + Entry(const Entry &rToCopy); + Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash); + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream) const; + + const BackupStoreFilename &GetName() const {return mName;} + box_time_t GetModificationTime() const {return mModificationTime;} + int64_t GetObjectID() const {return mObjectID;} + int64_t GetSizeInBlocks() const {return mSizeInBlocks;} + int16_t GetFlags() const {return mFlags;} + void AddFlags(int16_t Flags) {mFlags |= Flags;} + void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;} + + // Some things can be changed + void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;} + void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;} + + // Attributes + bool HasAttributes() const {return !mAttributes.IsEmpty();} + void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;} + const StreamableMemBlock &GetAttributes() const {return mAttributes;} + uint64_t GetAttributesHash() const {return mAttributesHash;} + + // Marks + // The lowest mark number a version of a file of this name has ever had + uint32_t GetMinMarkNumber() const {return mMinMarkNumber;} + // The mark number on this file + uint32_t GetMarkNumber() const {return mMarkNumber;} + + // Make sure these flags are synced with those in backupprocotol.txt + // ListDirectory command + enum + { + Flags_INCLUDE_EVERYTHING = -1, + Flags_EXCLUDE_NOTHING = 0, + Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below! + Flags_File = 1, + Flags_Dir = 2, + Flags_Deleted = 4, + Flags_OldVersion = 8, + Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion + }; + // characters for textual listing of files -- see bbackupquery/BackupQueries + #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR" + + bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) + { + return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet)) + && ((mFlags & FlagsNotToBeSet) == 0); + }; + + // Get dependency info + // new version this depends on + int64_t GetDependsNewer() const {return mDependsNewer;} + void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;} + // older version which depends on this + int64_t GetDependsOlder() const {return mDependsOlder;} + void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;} + + // Dependency info saving + bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;} + void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout); + void WriteToStreamDependencyInfo(IOStream &rStream) const; + + private: + BackupStoreFilename mName; + box_time_t mModificationTime; + int64_t mObjectID; + int64_t mSizeInBlocks; + int16_t mFlags; + uint64_t mAttributesHash; + StreamableMemBlock mAttributes; + uint32_t mMinMarkNumber; + uint32_t mMarkNumber; + + uint64_t mDependsNewer; // new version this depends on + uint64_t mDependsOlder; // older version which depends on this + }; + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream, + int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, + int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING, + bool StreamAttributes = true, bool StreamDependencyInfo = true) const; + + Entry *AddEntry(const Entry &rEntryToCopy); + Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime); + void DeleteEntry(int64_t ObjectID); + Entry *FindEntryByID(int64_t ObjectID) const; + + int64_t GetObjectID() const {return mObjectID;} + int64_t GetContainerID() const {return mContainerID;} + + // Need to be able to update the container ID when moving objects + void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;} + + // Purely for use of server -- not serialised into streams + int64_t GetRevisionID() const {return mRevisionID;} + void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;} + + unsigned int GetNumberOfEntries() const {return mEntries.size();} + + // User info -- not serialised into streams + int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;} + void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;} + + // Attributes + bool HasAttributes() const {return !mAttributes.IsEmpty();} + void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;} + const StreamableMemBlock &GetAttributes() const {return mAttributes;} + box_time_t GetAttributesModTime() const {return mAttributesModTime;} + + class Iterator + { + public: + Iterator(const BackupStoreDirectory &rDir) + : mrDir(rDir), i(rDir.mEntries.begin()) + { + } + + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags + while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.end()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + // WARNING: This function is really very inefficient. + // Only use when you want to look up ONE filename, not in a loop looking up lots. + // In a looping situation, cache the decrypted filenames in another memory structure. + BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags or filename + while( (i != mrDir.mEntries.end()) + && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) ) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.end()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + private: + const BackupStoreDirectory &mrDir; + std::vector<Entry*>::const_iterator i; + }; + + friend class Iterator; + + class ReverseIterator + { + public: + ReverseIterator(const BackupStoreDirectory &rDir) + : mrDir(rDir), i(rDir.mEntries.rbegin()) + { + } + + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags + while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.rend()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + private: + const BackupStoreDirectory &mrDir; + std::vector<Entry*>::const_reverse_iterator i; + }; + + friend class ReverseIterator; + + // For recovery of the store + // Implemented in BackupStoreCheck2.cpp + bool CheckAndFix(); + void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags); + bool NameInUse(const BackupStoreFilename &rName); + // Don't use these functions in normal code! + + // For testing + void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;} + + // Debug and diagonistics + void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere + +private: + int64_t mRevisionID; + int64_t mObjectID; + int64_t mContainerID; + std::vector<Entry*> mEntries; + box_time_t mAttributesModTime; + StreamableMemBlock mAttributes; + int64_t mUserInfo1; +}; + +#endif // BACKUPSTOREDIRECTORY__H + diff --git a/lib/backupclient/BackupStoreException.h b/lib/backupclient/BackupStoreException.h new file mode 100755 index 00000000..981dfa60 --- /dev/null +++ b/lib/backupclient/BackupStoreException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREEXCEPTION__H +#define BACKUPSTOREEXCEPTION__H + +// Compatibility +#include "autogen_BackupStoreException.h" + +#endif // BACKUPSTOREEXCEPTION__H + diff --git a/lib/backupclient/BackupStoreException.txt b/lib/backupclient/BackupStoreException.txt new file mode 100644 index 00000000..50c615b2 --- /dev/null +++ b/lib/backupclient/BackupStoreException.txt @@ -0,0 +1,70 @@ +EXCEPTION BackupStore 4 + +Internal 0 +BadAccountDatabaseFile 1 +AccountDatabaseNoSuchEntry 2 +InvalidBackupStoreFilename 3 +UnknownFilenameEncoding 4 +CouldntReadEntireStructureFromStream 5 +BadDirectoryFormat 6 +CouldNotFindEntryInDirectory 7 +OutputFileAlreadyExists 8 +OSFileError 9 +StreamDoesntHaveRequiredFeatures 10 +BadBackupStoreFile 11 +CouldNotLoadStoreInfo 12 +BadStoreInfoOnLoad 13 +StoreInfoIsReadOnly 14 +StoreInfoDirNotInList 15 +StoreInfoBlockDeltaMakesValueNegative 16 +DirectoryHasBeenDeleted 17 +StoreInfoNotInitialised 18 +StoreInfoAlreadyLoaded 19 +StoreInfoNotLoaded 20 +ReadFileFromStreamTimedOut 21 +FileWrongSizeAfterBeingStored 22 +AddedFileDoesNotVerify 23 +StoreInfoForWrongAccount 24 +ContextIsReadOnly 25 +AttributesNotLoaded 26 +AttributesNotUnderstood 27 +WrongServerVersion 28 # client side +ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account. +NameAlreadyExistsInDirectory 30 +BerkelyDBFailure 31 # client side +InodeMapIsReadOnly 32 # client side +InodeMapNotOpen 33 # client side +FilenameEncryptionKeyNotKnown 34 +FilenameEncryptionNoKeyForSpecifiedMethod 35 +FilenameEncryptionNotSetup 36 +CouldntLoadClientKeyMaterial 37 +BadEncryptedAttributes 38 +EncryptedAttributesHaveUnknownEncoding 39 +OutputSizeTooSmallForChunk 40 +BadEncodedChunk 41 +NotEnoughSpaceToDecodeChunk 42 +ChunkHasUnknownEncoding 43 +ChunkContainsBadCompressedData 44 +CantWriteToEncodedFileStream 45 +Temp_FileEncodeStreamDidntReadBuffer 46 +CantWriteToDecodedFileStream 47 +WhenDecodingExpectedToReadButCouldnt 48 +BackupStoreFileFailedIntegrityCheck 49 +ThereIsNoDataInASymLink 50 +IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51 +BlockEntryEncodingDidntGiveExpectedLength 52 +CouldNotFindUnusedIDDuringAllocation 53 +AddedFileExceedsStorageLimit 54 +CannotDiffAnIncompleteStoreFile 55 +CannotDecodeDiffedFilesWithoutCombining 56 +FailedToReadBlockOnCombine 57 +OnCombineFromFileIsIncomplete 58 +BadNotifySysadminEventCode 59 +InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60 +CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server? +AttributeHashSecretNotSet 62 +AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES +SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection. +IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related +DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory +PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp new file mode 100755 index 00000000..2ea877b4 --- /dev/null +++ b/lib/backupclient/BackupStoreFile.cpp @@ -0,0 +1,1499 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFile.cpp +// Purpose: Utils for manipulating files +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <unistd.h> +#include <sys/stat.h> +#include <string.h> +#include <new> +#include <string.h> +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + #include <syslog.h> + #include <stdio.h> +#endif + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFilename.h" +#include "BackupStoreException.h" +#include "IOStream.h" +#include "Guards.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreObjectMagic.h" +#include "Compress.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "CipherAES.h" +#include "BackupStoreConstants.h" +#include "CollectInBufferStream.h" +#include "RollingChecksum.h" +#include "MD5Digest.h" +#include "ReadGatherStream.h" +#include "Random.h" +#include "BackupStoreFileEncodeStream.h" + +#include "MemLeakFindOn.h" + +using namespace BackupStoreFileCryptVar; + +// How big a buffer to use for copying files +#define COPY_BUFFER_SIZE (8*1024) + +// Statistics +BackupStoreFileStats BackupStoreFile::msStats = {0,0,0}; + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool sWarnedAboutBackwardsCompatiblity = false; +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &) +// Purpose: Encode a file into something for storing on file server. +// Requires a real filename so full info can be stored. +// +// Returns a stream. Most of the work is done by the stream +// when data is actually requested -- the file will be held +// open until the stream is deleted or the file finished. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime) +{ + // Create the stream + std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); + + // Do the initial setup + ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, 0 /* no recipe, just encode */, + ContainerID, rStoreFilename, pModificationTime); + + // Return the stream for the caller + return stream; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &) +// Purpose: Verify that an encoded file meets the format requirements. +// Doesn't verify that the data is intact and can be decoded. +// Optionally returns the ID of the file which it is diffed from, +// and the (original) container ID. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut) +{ + // Get the size of the file + int64_t fileSize = rFile.BytesLeftToRead(); + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Get the header... + file_StreamFormat hdr; + if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read header + return false; + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + return false; + } + + // Get a filename, see if it loads OK + try + { + BackupStoreFilename fn; + fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); + } + catch(...) + { + // an error occured while reading it, so that's not good + return false; + } + + // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not + try + { + int32_t size_s; + if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + int size = ntohl(size_s); + // Skip forward the size + rFile.Seek(size, IOStream::SeekType_Relative); + } + catch(...) + { + // an error occured while reading it, so that's not good + return false; + } + + // Get current position in file -- the end of the header + int64_t headerEnd = rFile.GetPosition(); + + // Get number of blocks + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + + // Calculate where the block index will be, check it's reasonable + int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + if(blockIndexLoc < headerEnd) + { + // Not enough space left for the block index, let alone the blocks themselves + return false; + } + + // Load the block index header + rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute); + file_BlockIndexHeader blkhdr; + if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read block index header -- assume bad file + return false; + } + + // Check header + if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + || (int64_t)ntoh64(blkhdr.mNumBlocks) != numBlocks) + { + // Bad header -- either magic value or number of blocks is wrong + return false; + } + + // Flag for recording whether a block is referenced from another file + bool blockFromOtherFileReferenced = false; + + // Read the index, checking that the length values all make sense + int64_t currentBlockStart = headerEnd; + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read block entry + file_BlockIndexEntry blk; + if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read block index entry -- assume bad file + return false; + } + + // Check size and location + int64_t blkSize = ntoh64(blk.mEncodedSize); + if(blkSize <= 0) + { + // Mark that this file references another file + blockFromOtherFileReferenced = true; + } + else + { + // This block is actually in this file + if((currentBlockStart + blkSize) > blockIndexLoc) + { + // Encoded size makes the block run over the index + return false; + } + + // Move the current block start ot the end of this block + currentBlockStart += blkSize; + } + } + + // Check that there's no empty space + if(currentBlockStart != blockIndexLoc) + { + return false; + } + + // Check that if another block is references, then the ID is there, and if one isn't there is no ID. + int64_t otherID = ntoh64(blkhdr.mOtherFileID); + if((otherID != 0 && blockFromOtherFileReferenced == false) + || (otherID == 0 && blockFromOtherFileReferenced == true)) + { + // Doesn't look good! + return false; + } + + // Does the caller want the other ID? + if(pDiffFromObjectIDOut) + { + *pDiffFromObjectIDOut = otherID; + } + + // Does the caller want the container ID? + if(pContainerIDOut) + { + *pContainerIDOut = ntoh64(hdr.mContainerID); + } + + // Passes all tests + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeFile(IOStream &, const char *) +// Purpose: Decode a file. Will set file attributes. File must not exist. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) +{ + // Does file exist? + struct stat st; + if(::stat(DecodedFilename, &st) == 0) + { + THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) + } + + // Try, delete output file if error + try + { + // Make a stream for outputting this file + FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL); + + // Get the decoding stream + std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr)); + + // Is it a symlink? + if(!stream->IsSymLink()) + { + // Copy it out to the file + stream->CopyStreamTo(out); + } + + // Write the attributes + stream->GetAttributes().WriteAttributes(DecodedFilename); + } + catch(...) + { + ::unlink(DecodedFilename); + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *) +// Purpose: Return a stream which will decode the encrypted file data on the fly. +// Accepts streams in block index first, or main header first, order. In the latter case, +// the stream must be Seek()able. +// +// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow +// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) +{ + // Create stream + std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout)); + + // Get it ready + stream->Setup(pAlterativeAttr); + + // Return to caller + return stream; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int) +// Purpose: Constructor +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout) + : mrEncodedFile(rEncodedFile), + mTimeout(Timeout), + mNumBlocks(0), + mpBlockIndex(0), + mpEncodedData(0), + mpClearData(0), + mClearDataSize(0), + mCurrentBlock(-1), + mCurrentBlockClearSize(0), + mPositionInCurrentBlock(0), + mEntryIVBase(42) // different to default value in the encoded stream! +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + , mIsOldVersion(false) +#endif +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::~DecodedStream() +// Purpose: Desctructor +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFile::DecodedStream::~DecodedStream() +{ + // Free any allocated memory + if(mpBlockIndex) + { + ::free(mpBlockIndex); + } + if(mpEncodedData) + { + BackupStoreFile::CodingChunkFree(mpEncodedData); + } + if(mpClearData) + { + ::free(mpClearData); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *) +// Purpose: Get the stream ready to decode -- reads in headers +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr) +{ + // Get the size of the file + int64_t fileSize = mrEncodedFile.BytesLeftToRead(); + + // Get the magic number to work out which order the stream is in + int32_t magic; + if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read magic value + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + bool inFileOrder = true; + switch(ntohl(magic)) + { +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_MAGIC_VALUE_V0: + mIsOldVersion = true; + // control flows on +#endif + case OBJECTMAGIC_FILE_MAGIC_VALUE_V1: + inFileOrder = true; + break; + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0: + mIsOldVersion = true; + // control flows on +#endif + case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1: + inFileOrder = false; + break; + + default: + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // If not in file order, then the index list must be read now + if(!inFileOrder) + { + ReadBlockIndex(true /* have already read and verified the magic number */); + } + + // Get header + file_StreamFormat hdr; + if(inFileOrder) + { + // Read the header, without the magic number + if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic), + 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + // Put in magic number + hdr.mMagicValue = magic; + } + else + { + // Not in file order, so need to read the full header + if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Get the filename + mFilename.ReadFromStream(mrEncodedFile, mTimeout); + + // Get the attributes (either from stream, or supplied attributes) + if(pAlterativeAttr != 0) + { + // Read dummy attributes + BackupClientFileAttributes attr; + attr.ReadFromStream(mrEncodedFile, mTimeout); + + // Set to supplied attributes + mAttributes = *pAlterativeAttr; + } + else + { + // Read the attributes from the stream + mAttributes.ReadFromStream(mrEncodedFile, mTimeout); + } + + // If it is in file order, go and read the file attributes + // Requires that the stream can seek + if(inFileOrder) + { + // Make sure the file size is known + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Store current location (beginning of encoded blocks) + int64_t endOfHeaderPos = mrEncodedFile.GetPosition(); + + // Work out where the index is + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + + // Seek to that position + mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute); + + // Read the block index + ReadBlockIndex(false /* magic number still to be read */); + + // Seek back to the end of header position, ready for reading the chunks + mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute); + } + + // Check view of blocks from block header and file header match + if(mNumBlocks != (int64_t)ntoh64(hdr.mNumBlocks)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Need to allocate some memory for the two blocks for reading encoded data, and clear data + if(mNumBlocks > 0) + { + // Find the maximum encoded data size + int32_t maxEncodedDataSize = 0; + const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; + ASSERT(entry != 0); + for(int64_t e = 0; e < mNumBlocks; e++) + { + // Get the clear and encoded size + int32_t encodedSize = ntoh64(entry[e].mEncodedSize); + ASSERT(encodedSize > 0); + + // Larger? + if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize; + } + + // Allocate those blocks! + mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32); + + // Allocate the block for the clear data, using the hint from the header. + // If this is wrong, things will exception neatly later on, so it can't be used + // to do anything more than cause an error on downloading. + mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32; + mpClearData = (uint8_t*)::malloc(mClearDataSize); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool) +// Purpose: Read the block index from the stream, and store in internal buffer (minus header) +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) +{ + // Header + file_BlockIndexHeader blkhdr; + + // Read it in -- way depends on how whether the magic number has already been read + if(MagicAlreadyRead) + { + // Read the header, without the magic number + if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue), + 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } + else + { + // Magic not already read, so need to read the full header + if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Check magic value + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + } + + // Get the number of blocks out of the header + mNumBlocks = ntoh64(blkhdr.mNumBlocks); + + // Read the IV base + mEntryIVBase = ntoh64(blkhdr.mEntryIVBase); + + // Load the block entries in? + if(mNumBlocks > 0) + { + // How big is the index? + int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks; + + // Allocate some memory + mpBlockIndex = ::malloc(indexSize); + if(mpBlockIndex == 0) + { + throw std::bad_alloc(); + } + + // Read it in + if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Read(void *, int, int) +// Purpose: As interface. Reads decrpyted data. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Symlinks don't have data. So can't read it. Not even zero bytes. + if(IsSymLink()) + { + // Don't allow reading in this case + THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink); + } + + // Already finished? + if(mCurrentBlock >= mNumBlocks) + { + // At end of stream, nothing to do + return 0; + } + + int bytesToRead = NBytes; + uint8_t *output = (uint8_t*)pBuffer; + + while(bytesToRead > 0 && mCurrentBlock < mNumBlocks) + { + // Anything left in the current block? + if(mPositionInCurrentBlock < mCurrentBlockClearSize) + { + // Copy data out of this buffer + int s = mCurrentBlockClearSize - mPositionInCurrentBlock; + if(s > bytesToRead) s = bytesToRead; // limit to requested data + + // Copy + ::memcpy(output, mpClearData + mPositionInCurrentBlock, s); + + // Update positions + output += s; + mPositionInCurrentBlock += s; + bytesToRead -= s; + } + + // Need to get some more data? + if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize) + { + // Number of next block + ++mCurrentBlock; + if(mCurrentBlock >= mNumBlocks) + { + // Stop now! + break; + } + + // Get the size from the block index + const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; + int32_t encodedSize = ntoh64(entry[mCurrentBlock].mEncodedSize); + if(encodedSize <= 0) + { + // The caller is attempting to decode a file which is the direct result of a diff + // operation, and so does not contain all the data. + // It needs to be combined with the previous version first. + THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining) + } + + // Load in next block + if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Decode the data + mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize); + + // Calculate IV for this entry + uint64_t iv = mEntryIVBase; + iv += mCurrentBlock; + // Convert to network byte order before encrypting with it, so that restores work on + // platforms with different endiannesses. + iv = hton64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Make sure this is the right size + if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) + { +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(!mIsOldVersion) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the + // IV for the encrypted section. Try again, with the thing the other way round + iv = box_swap64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + else + { + // Warn and log this issue + if(!sWarnedAboutBackwardsCompatiblity) + { + ::printf("WARNING: Decoded one or more files using backwards compatibility mode for block index.\n"); + ::syslog(LOG_ERR, "WARNING: Decoded one or more files using backwards compatibility mode for block index.\n"); + sWarnedAboutBackwardsCompatiblity = true; + } + } +#else + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) +#endif + } + + // Check the digest + MD5Digest md5; + md5.Add(mpClearData, mCurrentBlockClearSize); + md5.Finish(); + if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum)) + { + THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck) + } + + // Set vars to say what's happening + mPositionInCurrentBlock = 0; + } + } + + ASSERT(bytesToRead >= 0); + ASSERT(bytesToRead <= NBytes); + + return NBytes - bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::IsSymLink() +// Purpose: Is the unencoded file actually a symlink? +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::IsSymLink() +{ + // First, check in with the attributes + if(!mAttributes.IsSymLink()) + { + return false; + } + + // So the attributes think it is a symlink. + // Consistency check... + if(mNumBlocks != 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Write(const void *, int) +// Purpose: As interface. Throws exception, as you can't write to this stream. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::StreamDataLeft() +// Purpose: As interface. Any data left? +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::StreamDataLeft() +{ + return mCurrentBlock < mNumBlocks; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::StreamClosed() +// Purpose: As interface. Always returns true, no writing allowed. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::StreamClosed() +{ + // Can't write to this stream! + return true; +} + + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::SetBlowfishKey(const void *, int) +// Purpose: Static. Sets the key to use for encryption and decryption. +// Created: 7/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength) +{ + // IVs set later + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + + sBlowfishEncryptBlockEntry.Reset(); + sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); + sBlowfishEncryptBlockEntry.UsePadding(false); + sBlowfishDecryptBlockEntry.Reset(); + sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); + sBlowfishDecryptBlockEntry.UsePadding(false); +} + + +#ifndef PLATFORM_OLD_OPENSSL +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::SetAESKey(const void *, int) +// Purpose: Sets the AES key to use for file data encryption. Will select AES as +// the cipher to use when encrypting. +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength) +{ + // Setup context + sAESEncrypt.Reset(); + sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); + sAESDecrypt.Reset(); + sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); + + // Set encryption to use this key, instead of the "default" blowfish key + spEncrypt = &sAESEncrypt; + sEncryptCipherType = HEADER_AES_ENCODING; +} +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int) +// Purpose: The maximum output size of a block, given the chunk size +// Created: 7/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize) +{ + // Calculate... the maximum size of output by first the largest it could be after compression, + // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck + // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing + // an problem where a rather non-compressable file didn't fit in a block buffer.) + return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1 + + sBlowfishEncrypt.GetIVLength() + 128; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &) +// Purpose: Encodes a chunk (encryption, possible compressed beforehand) +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput) +{ + ASSERT(spEncrypt != 0); + + // Check there's some space in the output block + if(rOutput.mBufferSize < 256) + { + rOutput.Reallocate(256); + } + + // Check alignment of the block + ASSERT((((uint32_t)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + + // Want to compress it? + bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE); + + // Build header + uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT; + if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED; + + // Store header + rOutput.mpBuffer[0] = header; + int outOffset = 1; + + // Setup cipher, and store the IV + int ivLen = 0; + const void *iv = spEncrypt->SetRandomIV(ivLen); + ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen); + outOffset += ivLen; + + // Start encryption process + spEncrypt->Begin(); + + #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \ + { \ + if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \ + { \ + rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \ + } \ + } + + // Encode the chunk + if(compressChunk) + { + // buffer to compress into + uint8_t buffer[2048]; + + // Set compressor with all the chunk as an input + Compress<true> compress; + compress.Input(Chunk, ChunkSize); + compress.FinishInput(); + + // Get and encrypt output + while(!compress.OutputHasFinished()) + { + int s = compress.Output(buffer, sizeof(buffer)); + if(s > 0) + { + ENCODECHUNK_CHECK_SPACE(s) + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); + } + else + { + // Should never happen, as we put all the input in in one go. + // So if this happens, it means there's a logical problem somewhere + THROW_EXCEPTION(BackupStoreException, Internal) + } + } + ENCODECHUNK_CHECK_SPACE(16) + outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); + } + else + { + // Straight encryption + ENCODECHUNK_CHECK_SPACE(ChunkSize) + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize); + ENCODECHUNK_CHECK_SPACE(16) + outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); + } + + ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check + + return outOffset; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int) +// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find +// the extra output buffer size needed before calling. +// See notes in EncodeChunk() for notes re alignment of the +// encoded data. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize) +{ + // Check alignment of the encoded block + ASSERT((((uint32_t)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + + // First check + if(EncodedSize < 1) + { + THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) + } + + const uint8_t *input = (uint8_t*)Encoded; + + // Get header, make checks, etc + uint8_t header = input[0]; + bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED; + uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT); + if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING) + { + THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding) + } + +#ifndef PLATFORM_OLD_OPENSSL + // Choose cipher + CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt); +#else + // AES not supported with this version of OpenSSL + if(encodingType == HEADER_AES_ENCODING) + { + THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL) + } + CipherContext &cipher(sBlowfishDecrypt); +#endif + + // Check enough space for header, an IV and one byte of input + int ivLen = cipher.GetIVLength(); + if(EncodedSize < (1 + ivLen + 1)) + { + THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) + } + + // Set IV in decrypt context, and start + cipher.SetIV(input + 1); + cipher.Begin(); + + // Setup vars for code + int inOffset = 1 + ivLen; + uint8_t *output = (uint8_t*)Output; + int outOffset = 0; + + // Do action + if(chunkCompressed) + { + // Do things in chunks + uint8_t buffer[2048]; + int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer)); + + // Decompressor + Compress<false> decompress; + + while(inOffset < EncodedSize) + { + // Decrypt a block + int bl = inputBlockLen; + if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long + int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl); + inOffset += bl; + + // Decompress the decrypted data + if(s > 0) + { + decompress.Input(buffer, s); + int os = 0; + do + { + os = decompress.Output(output + outOffset, OutputSize - outOffset); + outOffset += os; + } while(os > 0); + + // Check that there's space left in the output buffer -- there always should be + if(outOffset >= OutputSize) + { + THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) + } + } + } + + // Get any compressed data remaining in the cipher context and compression + int s = cipher.Final(buffer, sizeof(buffer)); + decompress.Input(buffer, s); + decompress.FinishInput(); + while(!decompress.OutputHasFinished()) + { + int os = decompress.Output(output + outOffset, OutputSize - outOffset); + outOffset += os; + + // Check that there's space left in the output buffer -- there always should be + if(outOffset >= OutputSize) + { + THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) + } + } + } + else + { + // Easy decryption + outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset); + outOffset += cipher.Final(output + outOffset, OutputSize - outOffset); + } + + return outOffset; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool) +// Purpose: Returns a stream which gives a Stream order version of the encoded file. +// If TakeOwnership == true, then the input stream will be deleted when the +// returned stream is deleted. +// The input stream must be seekable. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership) +{ + ASSERT(pStream != 0); + + // Get the size of the file + int64_t fileSize = pStream->BytesLeftToRead(); + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Read the header + int bytesRead = 0; + file_StreamFormat hdr; + bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead); + + // Seek backwards to put the file pointer back where it was before we started this + pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative); + + // Check we got a block + if(!readBlock) + { + // Couldn't read header -- assume file bad + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Get number of blocks + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + + // Calculate where the block index will be, check it's reasonable + int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + int64_t blockIndexLoc = fileSize - blockIndexSize; + if(blockIndexLoc < 0) + { + // Doesn't look good! + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Build a reordered stream + std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership)); + + // Set it up... + ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get())); + int component = rreordered.AddComponent(pStream); + // Send out the block index + rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc); + // And then the rest of the file + rreordered.AddBlock(component, blockIndexLoc, true, 0); + + return reordered; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ResetStats() +// Purpose: Reset the gathered statistics +// Created: 20/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::ResetStats() +{ + msStats.mBytesInEncodedFiles = 0; + msStats.mBytesAlreadyOnServer = 0; + msStats.mTotalFileStreamSize = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &) +// Purpose: Compares the contents of a file against the checksums contained in the +// block index. Returns true if the checksums match, meaning the file is +// extremely likely to match the original. Will always consume the entire index. +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout) +{ + // is it a symlink? + bool sourceIsSymlink = false; + { + struct stat st; + if(::lstat(Filename, &st) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + if((st.st_mode & S_IFMT) == S_IFLNK) + { + sourceIsSymlink = true; + } + } + + // Open file, if it's not a symlink + std::auto_ptr<FileStream> in; + if(!sourceIsSymlink) + { + in.reset(new FileStream(Filename)); + } + + // Read header + file_BlockIndexHeader hdr; + if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic + if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0) +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0); +#endif + + // Get basic information + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + uint64_t entryIVBase = ntoh64(hdr.mEntryIVBase); + + //TODO: Verify that these sizes look reasonable + + // setup + void *data = 0; + int32_t dataSize = -1; + bool matches = true; + int64_t totalSizeInBlockIndex = 0; + + try + { + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read an entry from the stream + file_BlockIndexEntry entry; + if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read entry + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Calculate IV for this entry + uint64_t iv = entryIVBase; + iv += b; + iv = hton64(iv); +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(isOldVersion) + { + // Reverse the IV for compatibility + iv = box_swap64(iv); + } +#endif + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry.mEnEnc, sizeof(entry.mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Size of block + int32_t blockClearSize = ntohl(entryEnc.mSize); + if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + totalSizeInBlockIndex += blockClearSize; + + // Make sure there's enough memory allocated to load the block in + if(dataSize < blockClearSize) + { + // Too small, free the block if it's already allocated + if(data != 0) + { + ::free(data); + data = 0; + } + // Allocate a block + data = ::malloc(blockClearSize + 128); + if(data == 0) + { + throw std::bad_alloc(); + } + dataSize = blockClearSize + 128; + } + + // Load in the block from the file, if it's not a symlink + if(!sourceIsSymlink) + { + if(in->Read(data, blockClearSize) != blockClearSize) + { + // Not enough data left in the file, can't possibly match + matches = false; + } + else + { + // Check the checksum + MD5Digest md5; + md5.Add(data, blockClearSize); + md5.Finish(); + if(!md5.DigestMatches(entryEnc.mStrongChecksum)) + { + // Checksum didn't match + matches = false; + } + } + } + + // Keep on going regardless, to make sure the entire block index stream is read + // -- must always be consistent about what happens with the stream. + } + } + catch(...) + { + // clean up in case of errors + if(data != 0) + { + ::free(data); + data = 0; + } + throw; + } + + // free block + if(data != 0) + { + ::free(data); + data = 0; + } + + // Check for data left over if it's not a symlink + if(!sourceIsSymlink) + { + // Anything left to read in the file? + if(in->BytesLeftToRead() != 0) + { + // File has extra data at the end + matches = false; + } + } + + // Symlinks must have zero size on server + if(sourceIsSymlink) + { + matches = (totalSizeInBlockIndex == 0); + } + + return matches; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer() +// Purpose: Constructor +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +BackupStoreFile::EncodingBuffer::EncodingBuffer() + : mpBuffer(0), + mBufferSize(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer() +// Purpose: Destructor +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +BackupStoreFile::EncodingBuffer::~EncodingBuffer() +{ + if(mpBuffer != 0) + { + BackupStoreFile::CodingChunkFree(mpBuffer); + mpBuffer = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::Allocate(int) +// Purpose: Do initial allocation of block +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::EncodingBuffer::Allocate(int Size) +{ + ASSERT(mpBuffer == 0); + uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size); + if(buffer == 0) + { + throw std::bad_alloc(); + } + mpBuffer = buffer; + mBufferSize = Size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::Reallocate(int) +// Purpose: Reallocate the block. Try not to call this, it has to copy +// the entire contents as the block can't be reallocated straight. +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) +{ + TRACE2("Reallocating EncodingBuffer from %d to %d\n", mBufferSize, NewSize); + ASSERT(mpBuffer != 0); + uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + // Copy data + ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize); + + // Free old + BackupStoreFile::CodingChunkFree(mpBuffer); + + // Store new buffer + mpBuffer = buffer; + mBufferSize = NewSize; +} + + diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h new file mode 100755 index 00000000..0274107b --- /dev/null +++ b/lib/backupclient/BackupStoreFile.h @@ -0,0 +1,194 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFile.h +// Purpose: Utils for manipulating files +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILE__H +#define BACKUPSTOREFILE__H + +#include "IOStream.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreFilename.h" + +#include <memory> + +typedef struct +{ + int64_t mBytesInEncodedFiles; + int64_t mBytesAlreadyOnServer; + int64_t mTotalFileStreamSize; +} BackupStoreFileStats; + +// Uncomment to disable backwards compatibility +//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + + +// Output buffer to EncodeChunk and input data to DecodeChunk must +// have specific alignment, see function comments. +#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16 +#define BACKUPSTOREFILE_CODING_OFFSET 15 + +// Have some memory allocation commands, note closing "Off" at end of file. +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFile +// Purpose: Class to hold together utils for maniplating files. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +class BackupStoreFile +{ +public: + class DecodedStream : public IOStream + { + friend class BackupStoreFile; + private: + DecodedStream(IOStream &rEncodedFile, int Timeout); + DecodedStream(const DecodedStream &); // not allowed + DecodedStream &operator=(const DecodedStream &); // not allowed + public: + ~DecodedStream(); + + // Stream functions + virtual int Read(void *pBuffer, int NBytes, int Timeout); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + // Accessor functions + const BackupClientFileAttributes &GetAttributes() {return mAttributes;} + const BackupStoreFilename &GetFilename() {return mFilename;} + int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests + + bool IsSymLink(); + + private: + void Setup(const BackupClientFileAttributes *pAlterativeAttr); + void ReadBlockIndex(bool MagicAlreadyRead); + + private: + IOStream &mrEncodedFile; + int mTimeout; + BackupClientFileAttributes mAttributes; + BackupStoreFilename mFilename; + int64_t mNumBlocks; + void *mpBlockIndex; + uint8_t *mpEncodedData; + uint8_t *mpClearData; + int mClearDataSize; + int mCurrentBlock; + int mCurrentBlockClearSize; + int mPositionInCurrentBlock; + uint64_t mEntryIVBase; +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool mIsOldVersion; +#endif + }; + + + // Main interface + static std::auto_ptr<IOStream> EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0); + static std::auto_ptr<IOStream> EncodeFileDiff(const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, + int Timeout, int64_t *pModificationTime = 0, bool *pIsCompletelyDifferent = 0); + static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); + static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); + static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); + static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0); + static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); + static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); + static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout); + static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false); + + // Stream manipulation + static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership); + static void MoveStreamPositionToBlockIndex(IOStream &rStream); + + // Crypto setup + static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength); +#ifndef PLATFORM_OLD_OPENSSL + static void SetAESKey(const void *pKey, int KeyLength); +#endif + + // Allocation of properly aligning chunks for decoding and encoding chunks + inline static void *CodingChunkAlloc(int Size) + { + uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3)); + if(a == 0) return 0; + // Align to main block size + ASSERT(sizeof(uint32_t) == sizeof(void*)); // make sure casting the right pointer size, will need to fix on platforms with 64 bit pointers + uint32_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE - (((uint32_t)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE); + uint8_t *b = (a + adjustment); + // Store adjustment + *b = (uint8_t)adjustment; + // Return offset + return b + BACKUPSTOREFILE_CODING_OFFSET; + } + inline static void CodingChunkFree(void *Block) + { + // Check alignment is as expected + ASSERT(sizeof(uint32_t) == sizeof(void*)); // make sure casting the right pointer size, will need to fix on platforms with 64 bit pointers + ASSERT((((uint32_t)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + uint8_t *a = (uint8_t*)Block; + a -= BACKUPSTOREFILE_CODING_OFFSET; + // Adjust downwards... + a -= *a; + free(a); + } + + // Limits + static void SetMaximumDiffingTime(int Seconds); + + // Building blocks + class EncodingBuffer + { + public: + EncodingBuffer(); + ~EncodingBuffer(); + private: + // No copying + EncodingBuffer(const EncodingBuffer &); + EncodingBuffer &operator=(const EncodingBuffer &); + public: + void Allocate(int Size); + void Reallocate(int NewSize); + + uint8_t *mpBuffer; + int mBufferSize; + }; + static int MaxBlockSizeForChunkSize(int ChunkSize); + static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput); + + // Caller should know how big the output size is, but also allocate a bit more memory to cover various + // overheads allowed for in checks + static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize) + { + // Plenty big enough + return KnownChunkSize + 256; + } + static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize); + + // Statisitics, not designed to be completely reliable + static void ResetStats(); + static BackupStoreFileStats msStats; + + // For debug +#ifndef NDEBUG + static bool TraceDetailsOfDiffProcess; +#endif + + // For decoding encoded files + static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile); +}; + +#include "MemLeakFindOff.h" + +#endif // BACKUPSTOREFILE__H + diff --git a/lib/backupclient/BackupStoreFileCmbDiff.cpp b/lib/backupclient/BackupStoreFileCmbDiff.cpp new file mode 100644 index 00000000..039b00eb --- /dev/null +++ b/lib/backupclient/BackupStoreFileCmbDiff.cpp @@ -0,0 +1,326 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCmbDiff.cpp +// Purpose: Combine two diffs together +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <stdlib.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CombineDiffs(IOStream &, IOStream &, IOStream &rOut) +// Purpose: Given two diffs, combine them into a single diff, to produce a diff +// which, combined with the original file, creates the result of applying +// rDiff, then rDiff2. Two opens of rDiff2 are required +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut) +{ + // Skip header of first diff, record where the data starts, and skip to the index + int64_t diff1DataStarts = 0; + { + // Read the header for the From file + file_StreamFormat diff1Hdr; + if(!rDiff1.ReadFullBuffer(&diff1Hdr, sizeof(diff1Hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diff1Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Skip over the filename and attributes of the From file + // BLOCK + { + BackupStoreFilename filename2; + filename2.ReadFromStream(rDiff1, IOStream::TimeOutInfinite); + int32_t size_s; + if(!rDiff1.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + int size = ntohl(size_s); + // Skip forward the size + rDiff1.Seek(size, IOStream::SeekType_Relative); + } + // Record position + diff1DataStarts = rDiff1.GetPosition(); + // Skip to index + rDiff1.Seek(0 - (((ntoh64(diff1Hdr.mNumBlocks)) * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + } + + // Read the index of the first diff + // Header first + file_BlockIndexHeader diff1IdxHdr; + if(!rDiff1.ReadFullBuffer(&diff1IdxHdr, sizeof(diff1IdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(diff1IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + int64_t diff1NumBlocks = ntoh64(diff1IdxHdr.mNumBlocks); + // Allocate some memory + int64_t *diff1BlockStartPositions = (int64_t*)::malloc((diff1NumBlocks + 1) * sizeof(int64_t)); + if(diff1BlockStartPositions == 0) + { + throw std::bad_alloc(); + } + + // Buffer data + void *buffer = 0; + int bufferSize = 0; + + try + { + // Then the entries: + // For each entry, want to know if it's in the file, and if so, how big it is. + // We'll store this as an array of file positions in the file, with an additioal + // entry on the end so that we can work out the length of the last block. + // If an entry isn't in the file, then store 0 - (position in other file). + int64_t diff1Position = diff1DataStarts; + for(int64_t b = 0; b < diff1NumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff1.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Where's the block? + int64_t blockEn = ntoh64(e.mEncodedSize); + if(blockEn <= 0) + { + // Just store the negated block number + diff1BlockStartPositions[b] = blockEn; + } + else + { + // Block is present in this file + diff1BlockStartPositions[b] = diff1Position; + diff1Position += blockEn; + } + } + + // Finish off the list, so the last entry can have it's size calcuated. + diff1BlockStartPositions[diff1NumBlocks] = diff1Position; + + // Now read the second diff's header, copying it to the out file + file_StreamFormat diff2Hdr; + if(!rDiff2.ReadFullBuffer(&diff2Hdr, sizeof(diff2Hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diff2Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Copy + rOut.Write(&diff2Hdr, sizeof(diff2Hdr)); + // Copy over filename and attributes + // BLOCK + { + BackupStoreFilename filename; + filename.ReadFromStream(rDiff2, IOStream::TimeOutInfinite); + filename.WriteToStream(rOut); + StreamableMemBlock attr; + attr.ReadFromStream(rDiff2, IOStream::TimeOutInfinite); + attr.WriteToStream(rOut); + } + + // Get to the index of rDiff2b, and read the header + MoveStreamPositionToBlockIndex(rDiff2b); + file_BlockIndexHeader diff2IdxHdr; + if(!rDiff2b.ReadFullBuffer(&diff2IdxHdr, sizeof(diff2IdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(diff2IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + int64_t diff2NumBlocks = ntoh64(diff2IdxHdr.mNumBlocks); + int64_t diff2IndexEntriesStart = rDiff2b.GetPosition(); + + // Then read all the entries + int64_t diff2FilePosition = rDiff2.GetPosition(); + for(int64_t b = 0; b < diff2NumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // What do to next about copying data + bool copyBlock = false; + int copySize = 0; + int64_t copyFrom = 0; + bool fromFileDiff1 = false; + + // Where's the block? + int64_t blockEn = ntoh64(e.mEncodedSize); + if(blockEn > 0) + { + // Block is present in this file -- copy to out + copyBlock = true; + copyFrom = diff2FilePosition; + copySize = (int)blockEn; + + // Move pointer onwards + diff2FilePosition += blockEn; + } + else + { + // Block isn't present here -- is it present in the old one? + int64_t blockIndex = 0 - blockEn; + if(blockIndex < 0 || blockIndex > diff1NumBlocks) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + if(diff1BlockStartPositions[blockIndex] > 0) + { + // Block is in the old diff file, copy it across + copyBlock = true; + copyFrom = diff1BlockStartPositions[blockIndex]; + int nb = blockIndex + 1; + while(diff1BlockStartPositions[nb] <= 0) + { + // This is safe, because the last entry will terminate it properly! + ++nb; + ASSERT(nb <= diff1NumBlocks); + } + copySize = diff1BlockStartPositions[nb] - copyFrom; + fromFileDiff1 = true; + } + } + //TRACE4("%d %d %lld %d\n", copyBlock, copySize, copyFrom, fromFileDiff1); + + // Copy data to the output file? + if(copyBlock) + { + // Allocate enough space + if(bufferSize < copySize || buffer == 0) + { + // Free old block + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + bufferSize = 0; + } + // Allocate new block + buffer = ::malloc(copySize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + bufferSize = copySize; + } + ASSERT(bufferSize >= copySize); + + // Load in the data + if(fromFileDiff1) + { + rDiff1.Seek(copyFrom, IOStream::SeekType_Absolute); + if(!rDiff1.ReadFullBuffer(buffer, copySize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + } + else + { + rDiff2.Seek(copyFrom, IOStream::SeekType_Absolute); + if(!rDiff2.ReadFullBuffer(buffer, copySize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + } + // Write out data + rOut.Write(buffer, copySize); + } + } + + // Write the modified header + diff2IdxHdr.mOtherFileID = diff1IdxHdr.mOtherFileID; + rOut.Write(&diff2IdxHdr, sizeof(diff2IdxHdr)); + + // Then we'll write out the index, reading the data again + rDiff2b.Seek(diff2IndexEntriesStart, IOStream::SeekType_Absolute); + for(int64_t b = 0; b < diff2NumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Where's the block? + int64_t blockEn = ntoh64(e.mEncodedSize); + + // If it's not in this file, it needs modification... + if(blockEn <= 0) + { + int64_t blockIndex = 0 - blockEn; + // In another file. Need to translate this against the other diff + if(diff1BlockStartPositions[blockIndex] > 0) + { + // Block is in the first diff file, stick in size + int nb = blockIndex + 1; + while(diff1BlockStartPositions[nb] <= 0) + { + // This is safe, because the last entry will terminate it properly! + ++nb; + ASSERT(nb <= diff1NumBlocks); + } + int64_t size = diff1BlockStartPositions[nb] - diff1BlockStartPositions[blockIndex]; + e.mEncodedSize = hton64(size); + } + else + { + // Block in the original file, use translated value + e.mEncodedSize = hton64(diff1BlockStartPositions[blockIndex]); + } + } + + // Write entry + rOut.Write(&e, sizeof(e)); + } + } + catch(...) + { + // clean up + ::free(diff1BlockStartPositions); + if(buffer != 0) + { + ::free(buffer); + } + throw; + } + + // Clean up allocated memory + ::free(diff1BlockStartPositions); + if(buffer != 0) + { + ::free(buffer); + } +} + diff --git a/lib/backupclient/BackupStoreFileCmbIdx.cpp b/lib/backupclient/BackupStoreFileCmbIdx.cpp new file mode 100644 index 00000000..253001c2 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCmbIdx.cpp @@ -0,0 +1,324 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCmbIdx.cpp +// Purpose: Combine indicies of a delta file and the file it's a diff from. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <string.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" + +#include "MemLeakFindOn.h" + +// Hide from outside world +namespace +{ + +class BSFCombinedIndexStream : public IOStream +{ +public: + BSFCombinedIndexStream(IOStream *pDiff); + ~BSFCombinedIndexStream(); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + virtual void Initialise(IOStream &rFrom); + +private: + IOStream *mpDiff; + bool mIsInitialised; + bool mHeaderWritten; + file_BlockIndexHeader mHeader; + int64_t mNumEntriesToGo; + int64_t mNumEntriesInFromFile; + int64_t *mFromBlockSizes; // NOTE: Entries in network byte order +}; + +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CombineFileIndices(IOStream &, IOStream &, bool) +// Purpose: Given a diff file and the file it's a diff from, return a stream from which +// can be read the index of the combined file, without actually combining them. +// The stream of the diff must have a lifetime greater than or equal to the +// lifetime of the returned stream object. The full "from" file stream +// only needs to exist during the actual function call. +// If you pass in dodgy files which aren't related, then you will either +// get an error or bad results. So don't do that. +// If DiffIsIndexOnly is true, then rDiff is assumed to be a stream positioned +// at the beginning of the block index. Similarly for FromIsIndexOnly. +// WARNING: Reads of the returned streams with buffer sizes less than 64 bytes +// will not return any data. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly, bool FromIsIndexOnly) +{ + // Reposition file pointers? + if(!DiffIsIndexOnly) + { + MoveStreamPositionToBlockIndex(rDiff); + } + if(!FromIsIndexOnly) + { + MoveStreamPositionToBlockIndex(rFrom); + } + + // Create object + std::auto_ptr<IOStream> stream(new BSFCombinedIndexStream(&rDiff)); + + // Initialise it + ((BSFCombinedIndexStream *)stream.get())->Initialise(rFrom); + + // And return the stream + return stream; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::BSFCombinedIndexStream() +// Purpose: Private class. Constructor. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +BSFCombinedIndexStream::BSFCombinedIndexStream(IOStream *pDiff) + : mpDiff(pDiff), + mIsInitialised(false), + mHeaderWritten(false), + mNumEntriesToGo(0), + mNumEntriesInFromFile(0), + mFromBlockSizes(0) +{ + ASSERT(mpDiff != 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::~BSFCombinedIndexStream() +// Purpose: Private class. Destructor. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +BSFCombinedIndexStream::~BSFCombinedIndexStream() +{ + if(mFromBlockSizes != 0) + { + ::free(mFromBlockSizes); + mFromBlockSizes = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::Initialise(IOStream &) +// Purpose: Private class. Initalise from the streams (diff passed in constructor). +// Both streams must have file pointer positioned at the block index. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +void BSFCombinedIndexStream::Initialise(IOStream &rFrom) +{ + // Paranoia is good. + if(mIsInitialised) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Look at the diff file: Read in the header + if(!mpDiff->ReadFullBuffer(&mHeader, sizeof(mHeader), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(mHeader.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Read relevant data. + mNumEntriesToGo = ntoh64(mHeader.mNumBlocks); + + // Adjust a bit to reflect the fact it's no longer a diff + mHeader.mOtherFileID = hton64(0); + + // Now look at the from file: Read header + file_BlockIndexHeader fromHdr; + if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Then... allocate memory for the list of sizes + mNumEntriesInFromFile = ntoh64(fromHdr.mNumBlocks); + mFromBlockSizes = (int64_t*)::malloc(mNumEntriesInFromFile * sizeof(int64_t)); + if(mFromBlockSizes == 0) + { + throw std::bad_alloc(); + } + + // And read them all in! + for(int64_t b = 0; b < mNumEntriesInFromFile; ++b) + { + file_BlockIndexEntry e; + if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check that the from file isn't a delta in itself + if(ntoh64(e.mEncodedSize) <= 0) + { + THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) + } + + // Store size (in network byte order) + mFromBlockSizes[b] = e.mEncodedSize; + } + + // Flag as initialised + mIsInitialised = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::Read(void *, int, int) +// Purpose: Private class. As interface. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Paranoia is good. + if(!mIsInitialised || mFromBlockSizes == 0 || mpDiff == 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + int written = 0; + + // Header output yet? + if(!mHeaderWritten) + { + // Enough space? + if(NBytes < (int)sizeof(mHeader)) return 0; + + // Copy in + ::memcpy(pBuffer, &mHeader, sizeof(mHeader)); + NBytes -= sizeof(mHeader); + written += sizeof(mHeader); + + // Flag it's done + mHeaderWritten = true; + } + + // How many entries can be written? + int entriesToWrite = NBytes / sizeof(file_BlockIndexEntry); + if(entriesToWrite > mNumEntriesToGo) + { + entriesToWrite = mNumEntriesToGo; + } + + // Setup ready to go + file_BlockIndexEntry *poutput = (file_BlockIndexEntry*)(((uint8_t*)pBuffer) + written); + + // Write entries + for(int b = 0; b < entriesToWrite; ++b) + { + if(!mpDiff->ReadFullBuffer(&(poutput[b]), sizeof(file_BlockIndexEntry), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Does this need adjusting? + int s = ntoh64(poutput[b].mEncodedSize); + if(s <= 0) + { + // A reference to a block in the from file + int block = 0 - s; + ASSERT(block >= 0); + if(block >= mNumEntriesInFromFile) + { + // That's not good, the block doesn't exist + THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) + } + + // Adjust the entry in the buffer + poutput[b].mEncodedSize = mFromBlockSizes[block]; // stored in network byte order, no translation necessary + } + } + + // Update written count + written += entriesToWrite * sizeof(file_BlockIndexEntry); + mNumEntriesToGo -= entriesToWrite; + + return written; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::Write(const void *, int) +// Purpose: Private class. As interface. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::StreamDataLeft() +// Purpose: Private class. As interface +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +bool BSFCombinedIndexStream::StreamDataLeft() +{ + return (!mHeaderWritten) || (mNumEntriesToGo > 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::StreamClosed() +// Purpose: Private class. As interface. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +bool BSFCombinedIndexStream::StreamClosed() +{ + return true; // doesn't do writing +} + diff --git a/lib/backupclient/BackupStoreFileCombine.cpp b/lib/backupclient/BackupStoreFileCombine.cpp new file mode 100755 index 00000000..562a32d9 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCombine.cpp @@ -0,0 +1,410 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCombine.cpp +// Purpose: File combining for BackupStoreFile +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" +#include "FileStream.h" + +#include "MemLeakFindOn.h" + +typedef struct +{ + int64_t mFilePosition; +} FromIndexEntry; + +static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries); +static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut); +static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut); + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CombineFile(IOStream &, IOStream &, IOStream &) +// Purpose: Where rDiff is a store file which is incomplete as a result of a +// diffing operation, rFrom is the file it is diffed from, and +// rOut is the stream in which to place the result, the old file +// and new file are combined into a file containing all the data. +// rDiff2 is the same file as rDiff, opened again to get two +// independent streams to the same file. +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut) +{ + // Read and copy the header. + file_StreamFormat hdr; + if(!rDiff.ReadFullBuffer(&hdr, sizeof(hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Copy + rOut.Write(&hdr, sizeof(hdr)); + // Copy over filename and attributes + // BLOCK + { + BackupStoreFilename filename; + filename.ReadFromStream(rDiff, IOStream::TimeOutInfinite); + filename.WriteToStream(rOut); + StreamableMemBlock attr; + attr.ReadFromStream(rDiff, IOStream::TimeOutInfinite); + attr.WriteToStream(rOut); + } + + // Read the header for the From file + file_StreamFormat fromHdr; + if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Skip over the filename and attributes of the From file + // BLOCK + { + BackupStoreFilename filename2; + filename2.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + int32_t size_s; + if(!rFrom.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + int size = ntohl(size_s); + // Skip forward the size + rFrom.Seek(size, IOStream::SeekType_Relative); + } + + // Allocate memory for the block index of the From file + int64_t fromNumBlocks = ntoh64(fromHdr.mNumBlocks); + // NOTE: An extra entry is required so that the length of the last block can be calculated + FromIndexEntry *pFromIndex = (FromIndexEntry*)::malloc((fromNumBlocks+1) * sizeof(FromIndexEntry)); + if(pFromIndex == 0) + { + throw std::bad_alloc(); + } + + try + { + // Load the index from the From file, calculating the offsets in the + // file as we go along, and enforce that everything should be present. + LoadFromIndex(rFrom, pFromIndex, fromNumBlocks); + + // Read in the block index of the Diff file in small chunks, and output data + // for each block, either from this file, or the other file. + int64_t diffNumBlocks = ntoh64(hdr.mNumBlocks); + CopyData(rDiff /* positioned at start of data */, rDiff2, diffNumBlocks, rFrom, pFromIndex, fromNumBlocks, rOut); + + // Read in the block index again, and output the new block index, simply + // filling in the sizes of blocks from the old file. + WriteNewIndex(rDiff, diffNumBlocks, pFromIndex, fromNumBlocks, rOut); + + // Free buffers + ::free(pFromIndex); + pFromIndex = 0; + } + catch(...) + { + // Clean up + if(pFromIndex != 0) + { + ::free(pFromIndex); + pFromIndex = 0; + } + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static LoadFromIndex(IOStream &, FromIndexEntry *, int64_t) +// Purpose: Static. Load the index from the From file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries) +{ + ASSERT(pIndex != 0); + ASSERT(NumEntries >= 0); + + // Get the starting point in the file + int64_t filePos = rFrom.GetPosition(); + + // Jump to the end of the file to read the index + rFrom.Seek(0 - ((NumEntries * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + + // Read block index header + file_BlockIndexHeader blkhdr; + if(!rFrom.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || (int64_t)ntoh64(blkhdr.mNumBlocks) != NumEntries) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // And then the block entries + for(int64_t b = 0; b < NumEntries; ++b) + { + // Read + file_BlockIndexEntry en; + if(!rFrom.ReadFullBuffer(&en, sizeof(en), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // Add to list + pIndex[b].mFilePosition = filePos; + + // Encoded size? + int64_t encodedSize = ntoh64(en.mEncodedSize); + // Check that the block is actually there + if(encodedSize <= 0) + { + THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) + } + + // Move file pointer on + filePos += encodedSize; + } + + // Store the position in the very last entry, so the size of the last entry can be calculated + pIndex[NumEntries].mFilePosition = filePos; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static CopyData(IOStream &, IOStream &, int64_t, IOStream &, FromIndexEntry *, int64_t, IOStream &) +// Purpose: Static. Copy data from the Diff and From file to the out file. +// rDiffData is at beginning of data. +// rDiffIndex at any position. +// rFrom is at any position. +// rOut is after the header, ready for data +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, + IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut) +{ + // Jump to the end of the diff file to read the index + rDiffIndex.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + + // Read block index header + file_BlockIndexHeader diffBlkhdr; + if(!rDiffIndex.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || (int64_t)ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Record where the From file is + int64_t fromPos = rFrom.GetPosition(); + + // Buffer data + void *buffer = 0; + int bufferSize = 0; + + try + { + // Read the blocks in! + for(int64_t b = 0; b < DiffNumBlocks; ++b) + { + // Read + file_BlockIndexEntry en; + if(!rDiffIndex.ReadFullBuffer(&en, sizeof(en), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // What's the size value stored in the entry + int64_t encodedSize = ntoh64(en.mEncodedSize); + + // How much data will be read? + int32_t blockSize = 0; + if(encodedSize > 0) + { + // The block is actually in the diff file + blockSize = encodedSize; + } + else + { + // It's in the from file. First, check to see if it's valid + int64_t blockIdx = (0 - encodedSize); + if(blockIdx > FromNumBlocks) + { + // References a block which doesn't actually exist + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Calculate size. This operation is safe because of the extra entry at the end + blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition; + } + ASSERT(blockSize > 0); + + // Make sure there's memory available to copy this + if(bufferSize < blockSize || buffer == 0) + { + // Free old block + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + bufferSize = 0; + } + // Allocate new block + buffer = ::malloc(blockSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + bufferSize = blockSize; + } + ASSERT(bufferSize >= blockSize); + + // Load in data from one of the files + if(encodedSize > 0) + { + // Load from diff file + if(!rDiffData.ReadFullBuffer(buffer, blockSize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + } + else + { + // Locate and read the data from the from file + int64_t blockIdx = (0 - encodedSize); + // Seek if necessary + if(fromPos != pFromIndex[blockIdx].mFilePosition) + { + rFrom.Seek(pFromIndex[blockIdx].mFilePosition, IOStream::SeekType_Absolute); + fromPos = pFromIndex[blockIdx].mFilePosition; + } + // Read + if(!rFrom.ReadFullBuffer(buffer, blockSize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // Update fromPos to current position + fromPos += blockSize; + } + + // Write data to out file + rOut.Write(buffer, blockSize); + } + + // Free buffer, if allocated + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + } + } + catch(...) + { + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + } + throw; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static WriteNewIndex(IOStream &, int64_t, FromIndexEntry *, int64_t, IOStream &) +// Purpose: Write the index to the out file, just copying from the diff file and +// adjusting the entries. +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut) +{ + // Jump to the end of the diff file to read the index + rDiff.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + + // Read block index header + file_BlockIndexHeader diffBlkhdr; + if(!rDiff.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || (int64_t)ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Write it out with a blanked out other file ID + diffBlkhdr.mOtherFileID = hton64(0); + rOut.Write(&diffBlkhdr, sizeof(diffBlkhdr)); + + // Rewrite the index + for(int64_t b = 0; b < DiffNumBlocks; ++b) + { + file_BlockIndexEntry en; + if(!rDiff.ReadFullBuffer(&en, sizeof(en), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // What's the size value stored in the entry + int64_t encodedSize = ntoh64(en.mEncodedSize); + + // Need to adjust it? + if(encodedSize <= 0) + { + // This actually refers to a block in the from file. So rewrite this. + int64_t blockIdx = (0 - encodedSize); + if(blockIdx > FromNumBlocks) + { + // References a block which doesn't actually exist + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Calculate size. This operation is safe because of the extra entry at the end + int32_t blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition; + // Then replace entry + en.mEncodedSize = hton64(((uint64_t)blockSize)); + } + + // Write entry + rOut.Write(&en, sizeof(en)); + } +} + + + + + diff --git a/lib/backupclient/BackupStoreFileCryptVar.cpp b/lib/backupclient/BackupStoreFileCryptVar.cpp new file mode 100755 index 00000000..eeed64e4 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCryptVar.cpp @@ -0,0 +1,31 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCryptVar.cpp +// Purpose: Cryptographic keys for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileWire.h" + +#include "MemLeakFindOn.h" + +CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt; +CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt; + +#ifndef PLATFORM_OLD_OPENSSL + CipherContext BackupStoreFileCryptVar::sAESEncrypt; + CipherContext BackupStoreFileCryptVar::sAESDecrypt; +#endif + +// Default to blowfish +CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt; +uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING; + +CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry; +CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry; + diff --git a/lib/backupclient/BackupStoreFileCryptVar.h b/lib/backupclient/BackupStoreFileCryptVar.h new file mode 100755 index 00000000..00a34f71 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCryptVar.h @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCryptVar.h +// Purpose: Cryptographic keys for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILECRYPTVAR__H +#define BACKUPSTOREFILECRYPTVAR__H + +#include "CipherContext.h" + +// Hide private static variables from the rest of the world by putting them +// as static variables in a namespace. +// -- don't put them as static class variables to avoid openssl/evp.h being +// included all over the project. +namespace BackupStoreFileCryptVar +{ + // Keys for the main file data + extern CipherContext sBlowfishEncrypt; + extern CipherContext sBlowfishDecrypt; + // Use AES when available +#ifndef PLATFORM_OLD_OPENSSL + extern CipherContext sAESEncrypt; + extern CipherContext sAESDecrypt; +#endif + // How encoding will be done + extern CipherContext *spEncrypt; + extern uint8_t sEncryptCipherType; + + // Keys for the block indicies + extern CipherContext sBlowfishEncryptBlockEntry; + extern CipherContext sBlowfishDecryptBlockEntry; +} + +#endif // BACKUPSTOREFILECRYPTVAR__H + diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp new file mode 100755 index 00000000..b72ce328 --- /dev/null +++ b/lib/backupclient/BackupStoreFileDiff.cpp @@ -0,0 +1,973 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileDiff.cpp +// Purpose: Functions relating to diffing BackupStoreFiles +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <map> +#include <signal.h> +#include <sys/time.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreConstants.h" +#include "FileStream.h" +#include "RollingChecksum.h" +#include "MD5Digest.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +using namespace BackupStoreFileCryptVar; +using namespace BackupStoreFileCreation; + +// By default, don't trace out details of the diff as we go along -- would fill up logs significantly. +// But it's useful for the test. +#ifndef NDEBUG + bool BackupStoreFile::TraceDetailsOfDiffProcess = false; +#endif + +static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis); +static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]); +static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]); +static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable); +static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, uint16_t Hash, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks); +static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile); + +// Avoid running on too long with diffs +static int sMaximumDiffTime = 10; // maximum time to spend diffing +static bool sDiffTimedOut = false; +static bool sSetTimerSignelHandler = false; +static void TimerSignalHandler(int signal); +static void StartDiffTimer(); + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::SetMaximumDiffingTime(int) +// Purpose: Sets the maximum time to spend diffing, in seconds. Time is +// process virutal time. +// Created: 19/3/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::SetMaximumDiffingTime(int Seconds) +{ + sMaximumDiffTime = Seconds; + TRACE1("Set maximum diffing time to %d seconds\n", Seconds); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &) +// Purpose: Move the file pointer in this stream to just before the block index. +// Assumes that the stream is at the beginning, seekable, and +// reading from the stream is OK. +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream) +{ + // Size of file + int64_t fileSize = rStream.BytesLeftToRead(); + + // Get header + file_StreamFormat hdr; + + // Read the header + if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Work out where the index is + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + int64_t blockHeaderPosFromEnd = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + + // Sanity check + if(blockHeaderPosFromEnd > static_cast<int64_t>(fileSize - sizeof(file_StreamFormat))) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Seek to that position + rStream.Seek(0 - blockHeaderPosFromEnd, IOStream::SeekType_End); + + // Done. Stream now in right position (as long as the file is formatted correctly) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *) +// Purpose: Similar to EncodeFile, but takes the object ID of the file it's +// diffing from, and the index of the blocks in a stream. It'll then +// calculate which blocks can be reused from that old file. +// The timeout is the timeout value for reading the diff block index. +// If pIsCompletelyDifferent != 0, it will be set to true if the +// the two files are completely different (do not share any block), false otherwise. +// +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff(const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, + int Timeout, int64_t *pModificationTime, bool *pIsCompletelyDifferent) +{ + // Is it a symlink? + { + struct stat st; + if(::lstat(Filename, &st) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + if((st.st_mode & S_IFLNK) == S_IFLNK) + { + // Don't do diffs for symlinks + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = true; + } + return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + } + } + + // Load in the blocks + BlocksAvailableEntry *pindex = 0; + int64_t blocksInIndex = 0; + bool canDiffFromThis = false; + LoadIndex(rDiffFromBlockIndex, DiffFromObjectID, &pindex, blocksInIndex, Timeout, canDiffFromThis); + //TRACE1("Diff: Blocks in index: %lld\n", blocksInIndex); + + if(!canDiffFromThis) + { + // Don't do diffing... + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = true; + } + return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + } + + // Pointer to recipe we're going to create + BackupStoreFileEncodeStream::Recipe *precipe = 0; + + // Start the timeout timer, so that the operation doesn't continue for ever + StartDiffTimer(); + + try + { + // Find which sizes should be scanned + int32_t sizesToScan[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]; + FindMostUsedSizes(pindex, blocksInIndex, sizesToScan); + + // Flag for reporting to the user + bool completelyDifferent; + + // BLOCK + { + // Search the file to find matching blocks + std::map<int64_t, int64_t> foundBlocks; // map of offset in file to index in block index + int64_t sizeOfInputFile = 0; + // BLOCK + { + FileStream file(Filename); + // Get size of file + sizeOfInputFile = file.BytesLeftToRead(); + // Find all those lovely matching blocks + SearchForMatchingBlocks(file, foundBlocks, pindex, blocksInIndex, sizesToScan); + + // Is it completely different? + completelyDifferent = (foundBlocks.size() == 0); + } + + // Create a recipe -- if the two files are completely different, don't put the from file ID in the recipe. + precipe = new BackupStoreFileEncodeStream::Recipe(pindex, blocksInIndex, completelyDifferent?(0):(DiffFromObjectID)); + BlocksAvailableEntry *pindexKeptRef = pindex; // we need this later, but must set pindex == 0 now, because of exceptions + pindex = 0; // Recipe now has ownership + + // Fill it in + GenerateRecipe(*precipe, pindexKeptRef, blocksInIndex, foundBlocks, sizeOfInputFile); + } + // foundBlocks no longer required + + // Create the stream + std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); + + // Do the initial setup + ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime); + precipe = 0; // Stream has taken ownership of this + + // Tell user about completely different status? + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = completelyDifferent; + } + + // Return the stream for the caller + return stream; + } + catch(...) + { + // cleanup + if(pindex != 0) + { + ::free(pindex); + pindex = 0; + } + if(precipe != 0) + { + delete precipe; + precipe = 0; + } + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static LoadIndex(IOStream &, int64_t, BlocksAvailableEntry **, int64_t, bool &) +// Purpose: Read in an index, and decrypt, and store in the in memory block format. +// rCanDiffFromThis is set to false if the version of the from file is too old. +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis) +{ + // Reset + rNumBlocksOut = 0; + rCanDiffFromThis = false; + + // Read header + file_BlockIndexHeader hdr; + if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + // Check against backwards comptaibility stuff + if(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)) + { + // Won't diff against old version + + // Absorb rest of stream + char buffer[2048]; + while(rBlockIndex.StreamDataLeft()) + { + rBlockIndex.Read(buffer, sizeof(buffer), 1000 /* 1 sec timeout */); + } + + // Tell caller + rCanDiffFromThis = false; + return; + } +#endif + + // Check magic + if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Check that we're not trying to diff against a file which references blocks from another file + if(((int64_t)ntoh64(hdr.mOtherFileID)) != 0) + { + THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile) + } + + // Mark as an acceptable diff. + rCanDiffFromThis = true; + + // Get basic information + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + uint64_t entryIVBase = ntoh64(hdr.mEntryIVBase); + + //TODO: Verify that these sizes look reasonable + + // Allocate space for the index + BlocksAvailableEntry *pindex = (BlocksAvailableEntry*)::malloc(sizeof(BlocksAvailableEntry) * numBlocks); + if(pindex == 0) + { + throw std::bad_alloc(); + } + + try + { + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read an entry from the stream + file_BlockIndexEntry entry; + if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read entry + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Calculate IV for this entry + uint64_t iv = entryIVBase; + iv += b; + // Network byte order + iv = hton64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry.mEnEnc, sizeof(entry.mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Check that we're not trying to diff against a file which references blocks from another file + if(((int64_t)ntoh64(entry.mEncodedSize)) <= 0) + { + THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile) + } + + // Store all the required information + pindex[b].mpNextInHashList = 0; // hash list not set up yet + pindex[b].mSize = ntohl(entryEnc.mSize); + pindex[b].mWeakChecksum = ntohl(entryEnc.mWeakChecksum); + ::memcpy(pindex[b].mStrongChecksum, entryEnc.mStrongChecksum, sizeof(pindex[b].mStrongChecksum)); + } + + // Store index pointer for called + ASSERT(ppIndex != 0); + *ppIndex = pindex; + + // Store number of blocks for caller + rNumBlocksOut = numBlocks; + + } + catch(...) + { + // clean up and send the exception along its way + ::free(pindex); + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static FindMostUsedSizes(BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +// Purpose: Finds the most commonly used block sizes in the index +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +{ + // Array for lengths + int64_t sizeCounts[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]; + + // Set arrays to lots of zeros (= unused entries) + for(int l = 0; l < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++l) + { + Sizes[l] = 0; + sizeCounts[l] = 0; + } + + // Array for collecting sizes + std::map<int32_t, int64_t> foundSizes; + + // Run through blocks and make a count of the entries + for(int64_t b = 0; b < NumBlocks; ++b) + { + // Only if the block size is bigger than the minimum size we'll scan for + if(pIndex[b].mSize > BACKUP_FILE_DIFF_MIN_BLOCK_SIZE) + { + // Find entry? + std::map<int32_t, int64_t>::const_iterator f(foundSizes.find(pIndex[b].mSize)); + if(f != foundSizes.end()) + { + // Increment existing entry + foundSizes[pIndex[b].mSize] = foundSizes[pIndex[b].mSize] + 1; + } + else + { + // New entry + foundSizes[pIndex[b].mSize] = 1; + } + } + } + + // Make the block sizes + for(std::map<int32_t, int64_t>::const_iterator i(foundSizes.begin()); i != foundSizes.end(); ++i) + { + // Find the position of the size in the array + for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t) + { + if(i->second > sizeCounts[t]) + { + // Then this size belong before this entry -- shuffle them up + for(int s = (BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1); s >= t; --s) + { + Sizes[s] = Sizes[s-1]; + sizeCounts[s] = sizeCounts[s-1]; + } + + // Insert this size + Sizes[t] = i->first; + sizeCounts[t] = i->second; + + // Shouldn't do any more searching + break; + } + } + } + + // trace the size table in debug builds +#ifndef NDEBUG + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t) + { + TRACE3("Diff block size %d: %d (count = %lld)\n", t, Sizes[t], sizeCounts[t]); + } + } +#endif +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +// Purpose: Find the matching blocks within the file. +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, + BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +{ + // Allocate the hash lookup table + BlocksAvailableEntry **phashTable = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024)); + + // Choose a size for the buffer, just a little bit more than the maximum block size + int32_t bufSize = Sizes[0]; + for(int z = 1; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z) + { + if(Sizes[z] > bufSize) bufSize = Sizes[z]; + } + bufSize += 4; + ASSERT(bufSize > Sizes[0]); + ASSERT(bufSize > 0); + if(bufSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // TODO: Use buffered file class + // Because we read in the file a scanned block size at a time, it is likely to be + // inefficient. Probably will be much better to use a buffering IOStream class which + // reads data in at the size of the filesystem block size. + + // Allocate the buffers. + uint8_t *pbuffer0 = (uint8_t *)::malloc(bufSize); + uint8_t *pbuffer1 = (uint8_t *)::malloc(bufSize); + try + { + // Check buffer allocation + if(pbuffer0 == 0 || pbuffer1 == 0 || phashTable == 0) + { + // If a buffer got alloocated, it will be cleaned up in the catch block + throw std::bad_alloc(); + } + + // Flag to abort the run, if too many blocks are found -- avoid using + // huge amounts of processor time when files contain many similar blocks. + bool abortSearch = false; + + // Search for each block size in turn + // NOTE: Do the smallest size first, so that the scheme for adding + // entries in the found list works as expected and replaces smallers block + // with larger blocks when it finds matches at the same offset in the file. + for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s) + { + ASSERT(Sizes[s] <= bufSize); + //TRACE2("Diff pass %d, for block size %d\n", s, Sizes[s]); + + // Check we haven't finished + if(Sizes[s] == 0) + { + // empty entry, try next size + continue; + } + + // Set up the hash table entries + SetupHashTable(pIndex, NumBlocks, Sizes[s], phashTable); + + // Shift file position to beginning + rFile.Seek(0, IOStream::SeekType_Absolute); + + // Read first block + if(rFile.Read(pbuffer0, Sizes[s]) != Sizes[s]) + { + // Size of file too short to match -- do next size + continue; + } + + // Setup block pointers + uint8_t *beginnings = pbuffer0; + uint8_t *endings = pbuffer1; + int offset = 0; + + // Calculate the first checksum, ready for rolling + RollingChecksum rolling(beginnings, Sizes[s]); + + // Then roll, until the file is exhausted + int64_t fileBlockNumber = 0; + int rollOverInitialBytes = 0; + while(true) + { + // Load in another block of data, and record how big it is + int bytesInEndings = rFile.Read(endings, Sizes[s]); + + // Skip any bytes from a previous matched block + while(rollOverInitialBytes > 0 && offset < bytesInEndings) + { + rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]); + ++offset; + --rollOverInitialBytes; + } + + while(offset < bytesInEndings) + { + // Put is current checksum in hash list? + uint16_t hash = rolling.GetComponentForHashing(); + if(phashTable[hash] != 0) + { + if(SecondStageMatch(phashTable[hash], hash, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks)) + { + // Block matched, roll the checksum forward to the next block without doing + // any more comparisons, because these are pointless (as any more matches will be ignored when + // the receipe is generated) and just take up valuable processor time. Edge cases are + // especially nasty, using huge amounts of time and memory. + int skip = Sizes[s]; + while(offset < bytesInEndings && skip > 0) + { + rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]); + ++offset; + --skip; + } + // Not all the bytes necessary will have been skipped, so get them + // skipped after the next block is loaded. + rollOverInitialBytes = skip; + + // End this loop, so the final byte isn't used again + break; + } + + if(static_cast<int64_t>(rFoundBlocks.size()) > (NumBlocks * BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE) + || sDiffTimedOut) + { + abortSearch = true; + break; + } + } + + // Roll checksum forward + rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]); + + // Increment offset + ++offset; + } + + if(abortSearch) break; + + // Finished? + if(bytesInEndings != Sizes[s]) + { + // No more data in file -- check the final block + // (Do a copy and paste of 5 lines of code instead of introducing a comparison for + // each byte of the file) + uint16_t hash = rolling.GetComponentForHashing(); + if(phashTable[hash] != 0) + { + SecondStageMatch(phashTable[hash], hash, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks); + } + + // finish + break; + } + + // Switch buffers, reset offset + beginnings = endings; + endings = (beginnings == pbuffer0)?(pbuffer1):(pbuffer0); // ie the other buffer + offset = 0; + + // And count the blocks which have been done + ++fileBlockNumber; + } + + if(abortSearch) break; + } + + // Free buffers and hash table + ::free(pbuffer1); + pbuffer1 = 0; + ::free(pbuffer0); + pbuffer0 = 0; + ::free(phashTable); + phashTable = 0; + } + catch(...) + { + // Cleanup and throw + if(pbuffer1 != 0) ::free(pbuffer1); + if(pbuffer0 != 0) ::free(pbuffer0); + if(phashTable != 0) ::free(phashTable); + throw; + } + +#ifndef NDEBUG + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + // Trace out the found blocks in debug mode + TRACE0("Diff: list of found blocks\n======== ======== ======== ========\n Offset BlkIdx Size Movement\n"); + for(std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); i != rFoundBlocks.end(); ++i) + { + int64_t orgLoc = 0; + for(int64_t b = 0; b < i->second; ++b) + { + orgLoc += pIndex[b].mSize; + } + TRACE4("%8lld %8lld %8lld %8lld\n", i->first, i->second, + (int64_t)(pIndex[i->second].mSize), i->first - orgLoc); + } + TRACE0("======== ======== ======== ========\n"); + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static SetupHashTable(BlocksAvailableEntry *, int64_t, in32_t, BlocksAvailableEntry **) +// Purpose: Set up the hash table ready for a scan +// Created: 14/1/04 +// +// -------------------------------------------------------------------------- +static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable) +{ + // Set all entries in the hash table to zero + ::memset(pHashTable, 0, (sizeof(BlocksAvailableEntry *) * (64*1024))); + + // Scan through the blocks, building the hash table + for(int64_t b = 0; b < NumBlocks; ++b) + { + // Only look at the required block size + if(pIndex[b].mSize == BlockSize) + { + // Get the value under which to hash this entry + uint16_t hash = RollingChecksum::ExtractHashingComponent(pIndex[b].mWeakChecksum); + + // Already present in table? + if(pHashTable[hash] != 0) + { + //TRACE1("Another hash entry for %d found\n", hash); + // Yes -- need to set the pointer in this entry to the current entry to build the linked list + pIndex[b].mpNextInHashList = pHashTable[hash]; + } + + // Put a pointer to this entry in the hash table + pHashTable[hash] = pIndex + b; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static bool SecondStageMatch(xxx) +// Purpose: When a match in the hash table is found, scan for second stage match using strong checksum. +// Created: 14/1/04 +// +// -------------------------------------------------------------------------- +static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, uint16_t Hash, uint8_t *pBeginnings, uint8_t *pEndings, + int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks) +{ + // Check parameters + ASSERT(pBeginnings != 0); + ASSERT(pEndings != 0); + ASSERT(Offset >= 0); + ASSERT(BlockSize > 0); + ASSERT(pFirstInHashList != 0); + ASSERT(pIndex != 0); + + // Calculate the strong MD5 digest for this block + MD5Digest strong; + // Add the data from the beginnings + strong.Add(pBeginnings + Offset, BlockSize - Offset); + // Add any data from the endings + if(Offset > 0) + { + strong.Add(pEndings, Offset); + } + strong.Finish(); + + // Then go through the entries in the hash list, comparing with the strong digest calculated + BlocksAvailableEntry *scan = pFirstInHashList; + //TRACE0("second stage match\n"); + while(scan != 0) + { + //TRACE3("scan size %d, block size %d, hash %d\n", scan->mSize, BlockSize, Hash); + ASSERT(scan->mSize == BlockSize); + ASSERT(RollingChecksum::ExtractHashingComponent(scan->mWeakChecksum) == Hash); + + // Compare? + if(strong.DigestMatches(scan->mStrongChecksum)) + { + //TRACE0("Match!\n"); + // Found! Add to list of found blocks... + int64_t fileOffset = (FileBlockNumber * BlockSize) + Offset; + int64_t blockIndex = (scan - pIndex); // pointer arthmitic is frowed apon. But most efficient way of doing it here -- alternative is to use more memory + + // Because we search for smallest blocks first, if there's an entry there + // we'll just be replacing it with a larger one, which is great news. + rFoundBlocks[fileOffset] = blockIndex; + + // No point in searching further, report success + return true; + } + + // Next + scan = scan->mpNextInHashList; + } + + // Not matched + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static GenerateRecipe(BackupStoreFileEncodeStream::Recipe &, BlocksAvailableEntry *, int64_t, std::map<int64_t, int64_t> &) +// Purpose: Fills in the recipe from the found block list +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, + int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile) +{ + // NOTE: This function could be a lot more sophisiticated. For example, if + // a small block overlaps a big block like this + // **** + // ******************************* + // then the small block will be used, not the big one. But it'd be better to + // just ignore the small block and keep the big one. However, some stats should + // be gathered about real world files before writing complex code which might + // go wrong. + + // Initialise a blank instruction + BackupStoreFileEncodeStream::RecipeInstruction instruction; + #define RESET_INSTRUCTION \ + instruction.mSpaceBefore = 0; \ + instruction.mBlocks = 0; \ + instruction.mpStartBlock = 0; + RESET_INSTRUCTION + + // First, a special case for when there are no found blocks + if(rFoundBlocks.size() == 0) + { + // No blocks, just a load of space + instruction.mSpaceBefore = SizeOfInputFile; + rRecipe.push_back(instruction); + + #ifndef NDEBUG + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + TRACE1("Diff: Default recipe generated, %lld bytes of file\n", SizeOfInputFile); + } + #endif + + // Don't do anything + return; + } + + // Current location + int64_t loc = 0; + + // Then iterate through the list, generating the recipe + std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); + ASSERT(i != rFoundBlocks.end()); // check logic + + // Counting for debug tracing +#ifndef NDEBUG + int64_t debug_NewBytesFound = 0; + int64_t debug_OldBlocksUsed = 0; +#endif + + for(; i != rFoundBlocks.end(); ++i) + { + // Remember... map is (position in file) -> (index of block in pIndex) + + if(i->first < loc) + { + // This block overlaps the last one + continue; + } + else if(i->first > loc) + { + // There's a gap between the end of the last thing and this block. + // If there's an instruction waiting, push it onto the list + if(instruction.mSpaceBefore != 0 || instruction.mpStartBlock != 0) + { + rRecipe.push_back(instruction); + } + // Start a new instruction, with the gap ready + RESET_INSTRUCTION + instruction.mSpaceBefore = i->first - loc; + // Move location forward to match + loc += instruction.mSpaceBefore; +#ifndef NDEBUG + debug_NewBytesFound += instruction.mSpaceBefore; +#endif + } + + // First, does the current instruction need pushing back, because this block is not + // sequential to the last one? + if(instruction.mpStartBlock != 0 && (pIndex + i->second) != (instruction.mpStartBlock + instruction.mBlocks)) + { + rRecipe.push_back(instruction); + RESET_INSTRUCTION + } + + // Add in this block + if(instruction.mpStartBlock == 0) + { + // This block starts a new instruction + instruction.mpStartBlock = pIndex + i->second; + instruction.mBlocks = 1; + } + else + { + // It continues the previous section of blocks + instruction.mBlocks += 1; + } + +#ifndef NDEBUG + debug_OldBlocksUsed++; +#endif + + // Move location forward + loc += pIndex[i->second].mSize; + } + + // Push the last instruction generated + rRecipe.push_back(instruction); + + // Is there any space left at the end which needs sending? + if(loc != SizeOfInputFile) + { + RESET_INSTRUCTION + instruction.mSpaceBefore = SizeOfInputFile - loc; +#ifndef NDEBUG + debug_NewBytesFound += instruction.mSpaceBefore; +#endif + rRecipe.push_back(instruction); + } + + // dump out the recipe +#ifndef NDEBUG + TRACE2("Diff: %lld new bytes found, %lld old blocks used\n", debug_NewBytesFound, debug_OldBlocksUsed); + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + TRACE1("Diff: Recipe generated (size %d)\n======== ========= ========\nSpace b4 FirstBlk NumBlks\n", rRecipe.size()); + { + for(unsigned int e = 0; e < rRecipe.size(); ++e) + { + char b[64]; + sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); + TRACE3("%8lld %s %8lld\n", rRecipe[e].mSpaceBefore, (rRecipe[e].mpStartBlock == 0)?" -":b, (int64_t)rRecipe[e].mBlocks); + } + } + TRACE0("======== ========= ========\n"); + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static TimerSignalHandler(int) +// Purpose: Signal handler +// Created: 19/3/04 +// +// -------------------------------------------------------------------------- +void TimerSignalHandler(int signal) +{ + sDiffTimedOut = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static StartDiffTimer() +// Purpose: Starts the diff timeout timer +// Created: 19/3/04 +// +// -------------------------------------------------------------------------- +void StartDiffTimer() +{ + // Set timer signal handler + if(!sSetTimerSignelHandler) + { + ::signal(SIGVTALRM, TimerSignalHandler); + sSetTimerSignelHandler = true; + } + + struct itimerval timeout; + // Don't want this to repeat + timeout.it_interval.tv_sec = 0; + timeout.it_interval.tv_usec = 0; + // Single timeout after the specified number of seconds + timeout.it_value.tv_sec = sMaximumDiffTime; + timeout.it_value.tv_usec = 0; + // Set timer + if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0) + { + TRACE0("WARNING: couldn't set diff timeout\n"); + } + + // Unset flag (last thing) + sDiffTimedOut = false; +} + + + diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp new file mode 100755 index 00000000..20e1fd80 --- /dev/null +++ b/lib/backupclient/BackupStoreFileEncodeStream.cpp @@ -0,0 +1,675 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileEncodeStream.cpp +// Purpose: Implement stream-based file encoding for the backup store +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BoxTime.h" +#include "BackupClientFileAttributes.h" +#include "FileStream.h" +#include "RollingChecksum.h" +#include "Random.h" + +#include "MemLeakFindOn.h" + +using namespace BackupStoreFileCryptVar; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream +// Purpose: Constructor (opens file) +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() + : mpRecipe(0), + mpFile(0), + mStatus(Status_Header), + mSendData(true), + mTotalBlocks(0), + mAbsoluteBlockNumber(-1), + mInstructionNumber(-1), + mNumBlocks(0), + mCurrentBlock(-1), + mCurrentBlockEncodedSize(0), + mPositionInCurrentBlock(0), + mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), + mLastBlockSize(0), + mpRawBuffer(0), + mAllocatedBufferSize(0), + mEntryIVBase(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() +// Purpose: Destructor +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() +{ + // Free buffers + if(mpRawBuffer) + { + ::free(mpRawBuffer); + mpRawBuffer = 0; + } + + // Close the file, which we might have open + if(mpFile) + { + delete mpFile; + mpFile = 0; + } + + // Free the recipe + if(mpRecipe != 0) + { + delete mpRecipe; + mpRecipe = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *) +// Purpose: Reads file information, and builds file header reading for sending. +// Takes ownership of the Recipe. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEncodeStream::Recipe *pRecipe, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime) +{ + // Pointer to a blank recipe which we might create + BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; + + try + { + // Get file attributes + box_time_t modTime = 0; + int64_t fileSize = 0; + BackupClientFileAttributes attr; + attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime, + 0 /* not interested in attr mod time */, &fileSize); + + // Might need to create a blank recipe... + if(pRecipe == 0) + { + pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); + + BackupStoreFileEncodeStream::RecipeInstruction instruction; + instruction.mSpaceBefore = fileSize; // whole file + instruction.mBlocks = 0; // no blocks + instruction.mpStartBlock = 0; // no block + pblankRecipe->push_back(instruction); + + pRecipe = pblankRecipe; + } + + // Tell caller? + if(pModificationTime != 0) + { + *pModificationTime = modTime; + } + + // Go through each instruction in the recipe and work out how many blocks + // it will add, and the max clear size of these blocks + int maxBlockClearSize = 0; + for(uint64_t inst = 0; inst < pRecipe->size(); ++inst) + { + if((*pRecipe)[inst].mSpaceBefore > 0) + { + // Calculate the number of blocks the space before requires + int64_t numBlocks; + int32_t blockSize, lastBlockSize; + CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize); + // Add to accumlated total + mTotalBlocks += numBlocks; + // Update maximum clear size + if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize; + if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize; + } + + // Add number of blocks copied from the previous file + mTotalBlocks += (*pRecipe)[inst].mBlocks; + + // Check for bad things + if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Run through blocks to get the max clear size + for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b) + { + if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize; + } + } + + // Send data? (symlinks don't have any data in them) + mSendData = !attr.IsSymLink(); + + // If not data is being sent, then the max clear block size is zero + if(!mSendData) + { + maxBlockClearSize = 0; + } + + // Header + file_StreamFormat hdr; + hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1); + hdr.mNumBlocks = (mSendData)?(hton64(mTotalBlocks)):(0); + hdr.mContainerID = hton64(ContainerID); + hdr.mModificationTime = hton64(modTime); + // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size + hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128); + hdr.mOptions = 0; // no options defined yet + + // Write header to stream + mData.Write(&hdr, sizeof(hdr)); + + // Write filename to stream + rStoreFilename.WriteToStream(mData); + + // Write attributes to stream + attr.WriteToStream(mData); + + // Allocate some buffers for writing data + if(mSendData) + { + // Open the file + mpFile = new FileStream(Filename); + + // Work out the largest possible block required for the encoded data + mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); + + // Then allocate two blocks of this size + mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize); + if(mpRawBuffer == 0) + { + throw std::bad_alloc(); + } +#ifndef NDEBUG + // In debug builds, make sure that the reallocation code is exercised. + mEncodedBuffer.Allocate(mAllocatedBufferSize / 4); +#else + mEncodedBuffer.Allocate(mAllocatedBufferSize); +#endif + } + else + { + // Write an empty block index for the symlink + file_BlockIndexHeader blkhdr; + blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); + blkhdr.mOtherFileID = hton64(0); // not other file ID + blkhdr.mEntryIVBase = hton64(0); + blkhdr.mNumBlocks = hton64(0); + mData.Write(&blkhdr, sizeof(blkhdr)); + } + + // Ready for reading + mData.SetForReading(); + + // Update stats + BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize; + + // Finally, store the pointer to the recipe, when we know exceptions won't occur + mpRecipe = pRecipe; + } + catch(...) + { + // Clean up any blank recipe + if(pblankRecipe != 0) + { + delete pblankRecipe; + pblankRecipe = 0; + } + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &) +// Purpose: Calculates the sizes of blocks in a section of the file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut) +{ + // How many blocks, and how big? + rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2; + do + { + rBlockSizeOut *= 2; + + rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut; + + } while(rBlockSizeOut <= BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); + + // Last block size + rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut); + + // Avoid small blocks? + if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN + && rNumBlocksOut > 1) + { + // Add the small bit of data to the last block + --rNumBlocksOut; + rLastBlockSizeOut += rBlockSizeOut; + } + + // checks! + ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize); + //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Read(void *, int, int) +// Purpose: As interface -- generates encoded file data on the fly from the raw file +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Check there's something to do. + if(mStatus == Status_Finished) + { + return 0; + } + + int bytesToRead = NBytes; + uint8_t *buffer = (uint8_t*)pBuffer; + + while(bytesToRead > 0 && mStatus != Status_Finished) + { + if(mStatus == Status_Header || mStatus == Status_BlockListing) + { + // Header or block listing phase -- send from the buffered stream + + // Send bytes from the data buffer + int b = mData.Read(buffer, bytesToRead, Timeout); + bytesToRead -= b; + buffer += b; + + // Check to see if all the data has been used from this stream + if(!mData.StreamDataLeft()) + { + // Yes, move on to next phase (or finish, if there's no file data) + if(!mSendData) + { + mStatus = Status_Finished; + } + else + { + // Reset the buffer so it can be used for the next phase + mData.Reset(); + + // Get buffer ready for index? + if(mStatus == Status_Header) + { + // Just finished doing the stream header, create the block index header + file_BlockIndexHeader blkhdr; + blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); + ASSERT(mpRecipe != 0); + blkhdr.mOtherFileID = hton64(mpRecipe->GetOtherFileID()); + blkhdr.mNumBlocks = hton64(mTotalBlocks); + + // Generate the IV base + Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase)); + blkhdr.mEntryIVBase = hton64(mEntryIVBase); + + mData.Write(&blkhdr, sizeof(blkhdr)); + } + + ++mStatus; + } + } + } + else if(mStatus == Status_Blocks) + { + // Block sending phase + + if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize) + { + // Next block! + ++mCurrentBlock; + ++mAbsoluteBlockNumber; + if(mCurrentBlock >= mNumBlocks) + { + // Output extra blocks for this instruction and move forward in file + if(mInstructionNumber >= 0) + { + SkipPreviousBlocksInInstruction(); + } + + // Is there another instruction to go? + ++mInstructionNumber; + + // Skip instructions which don't contain any data + while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size()) + && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0) + { + SkipPreviousBlocksInInstruction(); + ++mInstructionNumber; + } + + if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size())) + { + // End of blocks, go to next phase + ++mStatus; + + // Set the data to reading so the index can be written + mData.SetForReading(); + } + else + { + // Get ready for this instruction + SetForInstruction(); + } + } + + // Can't use 'else' here as SetForInstruction() will change this + if(mCurrentBlock < mNumBlocks) + { + EncodeCurrentBlock(); + } + } + + // Send data from the current block (if there's data to send) + if(mPositionInCurrentBlock < mCurrentBlockEncodedSize) + { + // How much data to put in the buffer? + int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock; + if(s > bytesToRead) s = bytesToRead; + + // Copy it in + ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s); + + // Update variables + bytesToRead -= s; + buffer += s; + mPositionInCurrentBlock += s; + } + } + else + { + // Should never get here, as it'd be an invalid status + ASSERT(false); + } + } + + // Add encoded size to stats + BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead); + + // Return size of data to caller + return NBytes - bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction() +// Purpose: Private. Stores the blocks of the old file referenced in the current +// instruction into the index and skips over the data in the file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction() +{ + // Check something is necessary + if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0) + { + return; + } + + // Index of the first block in old file (being diffed from) + int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock); + + int64_t sizeToSkip = 0; + + for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b) + { + // Update stats + BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; + + // Store the entry + StoreBlockIndexEntry(0 - (firstIndex + b), + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize, + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum, + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); + + // Increment the absolute block number -- kept encryption IV in sync + ++mAbsoluteBlockNumber; + + // Add the size of this block to the size to skip + sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; + } + + // Move forward in the stream + mpFile->Seek(sizeToSkip, IOStream::SeekType_Relative); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::SetForInstruction() +// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::SetForInstruction() +{ + // Calculate block sizes + CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize); + + // Set variables + mCurrentBlock = 0; + mCurrentBlockEncodedSize = 0; + mPositionInCurrentBlock = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock() +// Purpose: Private. Encodes the current block, and writes the block data to the index +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::EncodeCurrentBlock() +{ + // How big is the block, raw? + int blockRawSize = mBlockSize; + if(mCurrentBlock == (mNumBlocks - 1)) + { + blockRawSize = mLastBlockSize; + } + ASSERT(blockRawSize < mAllocatedBufferSize); + + // Check file open + if(mpFile == 0) + { + // File should be open, but isn't. So logical error. + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Read the data in + if(!mpFile->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */)) + { + // TODO: Do something more intelligent, and abort this upload because the file + // has changed + THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer) + } + + // Encode it + mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer); + + //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); + + // Create block listing data -- generate checksums + RollingChecksum weakChecksum(mpRawBuffer, blockRawSize); + MD5Digest strongChecksum; + strongChecksum.Add(mpRawBuffer, blockRawSize); + strongChecksum.Finish(); + + // Add entry to the index + StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); + + // Set vars to reading this block + mPositionInCurrentBlock = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *) +// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream. +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum) +{ + // First, the encrypted section + file_BlockIndexEntryEnc entryEnc; + entryEnc.mSize = htonl(ClearSize); + entryEnc.mWeakChecksum = htonl(WeakChecksum); + ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum)); + + // Then the clear section + file_BlockIndexEntry entry; + entry.mEncodedSize = hton64(((uint64_t)EncSizeOrBlkIndex)); + + // Then encrypt the encryted section + // Generate the IV from the block number + if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase)) + { + THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements) + } + uint64_t iv = mEntryIVBase; + iv += mAbsoluteBlockNumber; + // Convert to network byte order before encrypting with it, so that restores work on + // platforms with different endiannesses. + iv = hton64(iv); + sBlowfishEncryptBlockEntry.SetIV(&iv); + + // Encode the data + int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc)); + if(encodedSize != sizeof(entry.mEnEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Save to data block for sending at the end of the stream + mData.Write(&entry, sizeof(entry)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Write(const void *, int) +// Purpose: As interface. Exceptions. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StreamDataLeft() +// Purpose: As interface -- end of stream reached? +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFileEncodeStream::StreamDataLeft() +{ + return (mStatus != Status_Finished); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StreamClosed() +// Purpose: As interface +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFileEncodeStream::StreamClosed() +{ + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t) +// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, + int64_t NumBlocksInIndex, int64_t OtherFileID) + : mpBlockIndex(pBlockIndex), + mNumBlocksInIndex(NumBlocksInIndex), + mOtherFileID(OtherFileID) +{ + ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Recipe::~Recipe() +// Purpose: Destructor +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::Recipe::~Recipe() +{ + // Free the block index, if there is one + if(mpBlockIndex != 0) + { + ::free(mpBlockIndex); + } +} + + + + diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h new file mode 100755 index 00000000..1c748798 --- /dev/null +++ b/lib/backupclient/BackupStoreFileEncodeStream.h @@ -0,0 +1,127 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileEncodeStream.h +// Purpose: Implement stream-based file encoding for the backup store +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILEENCODESTREAM__H +#define BACKUPSTOREFILEENCODESTREAM__H + +#include <vector> + +#include "IOStream.h" +#include "BackupStoreFilename.h" +#include "CollectInBufferStream.h" +#include "MD5Digest.h" +#include "BackupStoreFile.h" + +namespace BackupStoreFileCreation +{ + // Diffing and creation of files share some implementation details. + typedef struct _BlocksAvailableEntry + { + struct _BlocksAvailableEntry *mpNextInHashList; + int32_t mSize; // size in clear + uint32_t mWeakChecksum; // weak, rolling checksum + uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum + } BlocksAvailableEntry; + +} + + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFileEncodeStream +// Purpose: Encode a file into a stream +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +class BackupStoreFileEncodeStream : public IOStream +{ +public: + BackupStoreFileEncodeStream(); + ~BackupStoreFileEncodeStream(); + + typedef struct + { + int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go + int32_t mBlocks; // number of block to reuse, starting at this one + BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null + } RecipeInstruction; + + class Recipe : public std::vector<RecipeInstruction> + { + // NOTE: This class is rather tied in with the implementation of diffing. + public: + Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex, + int64_t OtherFileID = 0); + ~Recipe(); + + int64_t GetOtherFileID() {return mOtherFileID;} + int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock) + { + return pBlock - mpBlockIndex; + } + + private: + BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex; + int64_t mNumBlocksInIndex; + int64_t mOtherFileID; + }; + + void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime); + + virtual int Read(void *pBuffer, int NBytes, int Timeout); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + enum + { + Status_Header = 0, + Status_Blocks = 1, + Status_BlockListing = 2, + Status_Finished = 3 + }; + +private: + void EncodeCurrentBlock(); + void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); + void SkipPreviousBlocksInInstruction(); + void SetForInstruction(); + void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum); + +private: + Recipe *mpRecipe; + IOStream *mpFile; // source file + CollectInBufferStream mData; // buffer for header and index entries + int mStatus; + bool mSendData; // true if there's file data to send (ie not a symlink) + int64_t mTotalBlocks; // Total number of blocks in the file + int64_t mAbsoluteBlockNumber; // The absolute block number currently being output + // Instruction number + int64_t mInstructionNumber; + // All the below are within the current instruction + int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases + int64_t mCurrentBlock; + int32_t mCurrentBlockEncodedSize; + int32_t mPositionInCurrentBlock; // for reading out + int32_t mBlockSize; // Basic block size of most of the blocks in the file + int32_t mLastBlockSize; // the size (unencoded) of the last block in the file + // Buffers + uint8_t *mpRawBuffer; // buffer for raw data + BackupStoreFile::EncodingBuffer mEncodedBuffer; + // buffer for encoded data + int32_t mAllocatedBufferSize; // size of above two allocated blocks + uint64_t mEntryIVBase; // base for block entry IV +}; + + + +#endif // BACKUPSTOREFILEENCODESTREAM__H + diff --git a/lib/backupclient/BackupStoreFileRevDiff.cpp b/lib/backupclient/BackupStoreFileRevDiff.cpp new file mode 100644 index 00000000..f1dc52d8 --- /dev/null +++ b/lib/backupclient/BackupStoreFileRevDiff.cpp @@ -0,0 +1,258 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileRevDiff.cpp +// Purpose: Reverse a patch, to build a new patch from new to old files +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <stdlib.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t) +// Purpose: Reverse a patch, to build a new patch from new to old files. Takes +// two independent copies to the From file, for efficiency. +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent) +{ + // Read and copy the header from the from file to the out file -- beginnings of the patch + file_StreamFormat hdr; + if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Copy + rOut.Write(&hdr, sizeof(hdr)); + // Copy over filename and attributes + // BLOCK + { + BackupStoreFilename filename; + filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + filename.WriteToStream(rOut); + StreamableMemBlock attr; + attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + attr.WriteToStream(rOut); + } + + // Build an index of common blocks. + // For each block in the from file, we want to know it's index in the + // diff file. Allocate memory for this information. + int64_t fromNumBlocks = ntoh64(hdr.mNumBlocks); + int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t)); + if(pfromIndexInfo == 0) + { + throw std::bad_alloc(); + } + + // Buffer data + void *buffer = 0; + int bufferSize = 0; + + // flag + bool isCompletelyDifferent = true; + + try + { + // Initialise the index to be all 0, ie not filled in yet + for(int64_t i = 0; i < fromNumBlocks; ++i) + { + pfromIndexInfo[i] = 0; + } + + // Within the from file, skip to the index + MoveStreamPositionToBlockIndex(rDiff); + + // Read in header of index + file_BlockIndexHeader diffIdxHdr; + if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // And then read in each entry + int64_t diffNumBlocks = ntoh64(diffIdxHdr.mNumBlocks); + for(int64_t b = 0; b < diffNumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Where's the block? + int64_t blockEn = ntoh64(e.mEncodedSize); + if(blockEn > 0) + { + // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file + } + else + { + // Block is in the original file, store which block it is in this file + int64_t fromIndex = 0 - blockEn; + if(fromIndex < 0 || fromIndex >= fromNumBlocks) + { + THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles) + } + + // Store information about where it is in the new file + // NOTE: This is slight different to how it'll be stored in the final index. + pfromIndexInfo[fromIndex] = -1 - b; + } + } + + // Open the index for the second copy of the from file + MoveStreamPositionToBlockIndex(rFrom2); + + // Read in header of index + file_BlockIndexHeader fromIdxHdr; + if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || ntoh64(fromIdxHdr.mOtherFileID) != 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // So, we can now start building the data in the file + int64_t filePosition = rFrom.GetPosition(); + for(int64_t b = 0; b < fromNumBlocks; ++b) + { + // Read entry from from index + file_BlockIndexEntry e; + if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Get size + int64_t blockSize = hton64(e.mEncodedSize); + if(blockSize < 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Copy this block? + if(pfromIndexInfo[b] == 0) + { + // Copy it, first move to file location + rFrom.Seek(filePosition, IOStream::SeekType_Absolute); + + // Make sure there's memory available to copy this + if(bufferSize < blockSize || buffer == 0) + { + // Free old block + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + bufferSize = 0; + } + // Allocate new block + buffer = ::malloc(blockSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + bufferSize = blockSize; + } + ASSERT(bufferSize >= blockSize); + + // Copy the block + if(!rFrom.ReadFullBuffer(buffer, blockSize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + rOut.Write(buffer, blockSize); + + // Store the size + pfromIndexInfo[b] = blockSize; + } + else + { + // Block isn't needed, so it's not completely different + isCompletelyDifferent = false; + } + filePosition += blockSize; + } + + // Then write the index, modified header first + fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(hton64(ObjectIDOfFrom)); + rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr)); + + // Move to start of index entries + rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute); + + // Then copy modified entries + for(int64_t b = 0; b < fromNumBlocks; ++b) + { + // Read entry from from index + file_BlockIndexEntry e; + if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Modify... + int64_t s = pfromIndexInfo[b]; + // Adjust to reflect real block index (remember 0 has a different meaning here) + if(s < 0) ++s; + // Insert + e.mEncodedSize = hton64(s); + // Write + rOut.Write(&e, sizeof(e)); + } + } + catch(...) + { + ::free(pfromIndexInfo); + if(buffer != 0) + { + ::free(buffer); + } + throw; + } + + // Free memory used (oh for finally {} blocks) + ::free(pfromIndexInfo); + if(buffer != 0) + { + ::free(buffer); + } + + // return completely different flag + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = isCompletelyDifferent; + } +} + + + diff --git a/lib/backupclient/BackupStoreFileWire.h b/lib/backupclient/BackupStoreFileWire.h new file mode 100755 index 00000000..5b1bc819 --- /dev/null +++ b/lib/backupclient/BackupStoreFileWire.h @@ -0,0 +1,74 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileWire.h +// Purpose: On the wire / disc formats for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILEWIRE__H +#define BACKUPSTOREFILEWIRE__H + +#include "MD5Digest.h" + +// set packing to one byte +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + int32_t mMagicValue; // also the version number + int64_t mNumBlocks; // number of blocks contained in the file + int64_t mContainerID; + int64_t mModificationTime; + int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block + int32_t mOptions; // bitmask of options used + // Then a BackupStoreFilename + // Then a BackupClientFileAttributes +} file_StreamFormat; + +typedef struct +{ + int32_t mMagicValue; // different magic value + int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index + uint64_t mEntryIVBase; // base value for block IV + int64_t mNumBlocks; // repeat of value in file header +} file_BlockIndexHeader; + +typedef struct +{ + int32_t mSize; // size in clear + uint32_t mWeakChecksum; // weak, rolling checksum + uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum +} file_BlockIndexEntryEnc; + +typedef struct +{ + union + { + int64_t mEncodedSize; // size encoded, if > 0 + int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0 + }; + uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section +} file_BlockIndexEntry; + +// Use default packing +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +// header for blocks of compressed data in files +#define HEADER_CHUNK_IS_COMPRESSED 1 // bit +#define HEADER_ENCODING_SHIFT 1 // shift value +#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7 +#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7 + + +#endif // BACKUPSTOREFILEWIRE__H + diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp new file mode 100755 index 00000000..fbfe3313 --- /dev/null +++ b/lib/backupclient/BackupStoreFilename.cpp @@ -0,0 +1,279 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilename.cpp +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreFilename.h" +#include "Protocol.h" +#include "BackupStoreException.h" +#include "IOStream.h" +#include "Guards.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::BackupStoreFilename() +// Purpose: Default constructor -- creates an invalid filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::BackupStoreFilename() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy) + : BackupStoreFilename_base(rToCopy) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::~BackupStoreFilename() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::~BackupStoreFilename() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::CheckValid(bool) +// Purpose: Checks the encoded filename for validity +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const +{ + bool ok = true; + + if(size() < 2) + { + // Isn't long enough to have a header + ok = false; + } + else + { + // Check size is consistent + unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(*this); + if(dsize != size()) + { + ok = false; + } + + // And encoding is an accepted value + unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this); + if(encoding < Encoding_Min || encoding > Encoding_Max) + { + ok = false; + } + } + + // Exception? + if(!ok && ExceptionIfInvalid) + { + THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename) + } + + return ok; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::ReadFromProtocol(Protocol &) +// Purpose: Reads the filename from the protocol object +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol) +{ + // Read the header + char hdr[2]; + rProtocol.Read(hdr, 2); + + // How big is it? + int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); + + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + std::string data; + rProtocol.Read(data, dsize - 2); + + // assign to this string, storing the header and the extra data + assign(hdr, 2); + append(data.c_str(), data.size()); + + // Check it + CheckValid(); + + // Alert derived classes + EncodedFilenameChanged(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::WriteToProtocol(Protocol &) +// Purpose: Writes the filename to the protocol object +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const +{ + CheckValid(); + + rProtocol.Write(c_str(), size()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::ReadFromStream(IOStream &) +// Purpose: Reads the filename from a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Read the header + char hdr[2]; + if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // How big is it? + unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); + + // Assume most filenames are small + char buf[256]; + if(dsize < sizeof(buf)) + { + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + // Copy in header + buf[0] = hdr[0]; buf[1] = hdr[1]; + + // assign to this string, storing the header and the extra data + assign(buf, dsize); + } + else + { + // Block of memory to hold it + MemoryBlockGuard<char*> dataB(dsize+2); + char *data = dataB; + + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + // Copy in header + data[0] = hdr[0]; data[1] = hdr[1]; + + // assign to this string, storing the header and the extra data + assign(data, dsize); + } + + // Check it + CheckValid(); + + // Alert derived classes + EncodedFilenameChanged(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::WriteToStream(IOStream &) +// Purpose: Writes the filename to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::WriteToStream(IOStream &rStream) const +{ + CheckValid(); + + rStream.Write(c_str(), size()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::EncodedFilenameChanged() +// Purpose: The encoded filename stored has changed +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::EncodedFilenameChanged() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::IsEncrypted() +// Purpose: Returns true if the filename is stored using an encrypting encoding +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFilename::IsEncrypted() const +{ + return BACKUPSTOREFILENAME_GET_ENCODING(*this) != Encoding_Clear; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::SetAsClearFilename(const char *) +// Purpose: Sets this object to be a valid filename, but with a filename in the clear. +// Used on the server to create filenames when there's no way of encrypting it. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::SetAsClearFilename(const char *Clear) +{ + // Make std::string from the clear name + std::string toEncode(Clear); + + // Make an encoded string + char hdr[2]; + BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear); + std::string encoded(hdr, 2); + encoded += toEncode; + ASSERT(encoded.size() == toEncode.size() + 2); + + // Store the encoded string + assign(encoded); + + // Stuff which must be done + EncodedFilenameChanged(); + CheckValid(false); +} + + + diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h new file mode 100755 index 00000000..4c951b6f --- /dev/null +++ b/lib/backupclient/BackupStoreFilename.h @@ -0,0 +1,85 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilename.h +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILENAME__H +#define BACKUPSTOREFILENAME__H + +#include <string> + +class Protocol; +class IOStream; + +// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE +// don't define this -- the problem of memory usage still appears without this. +// It's just that this class really showed up the problem. Instead, malloc allocation +// is globally defined in BoxPlatform.h, for troublesome libraries. + +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE + // Use a malloc_allocated string, because the STL default allocators really screw up with + // memory allocation, particularly with this class. + // Makes a few things a bit messy and inefficient with conversions. + // Given up using this, and use global malloc allocation instead, but thought it + // worth leaving this code in just in case it's useful for the future. + typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base; + // If this is changed, change GetClearFilename() back to returning a reference. +#else + typedef std::string BackupStoreFilename_base; +#endif // PLATFORM_HAVE_STL_MALLOC_ALLOC + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFilename +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreFilename : public BackupStoreFilename_base +{ +public: + BackupStoreFilename(); + BackupStoreFilename(const BackupStoreFilename &rToCopy); + virtual ~BackupStoreFilename(); + + bool CheckValid(bool ExceptionIfInvalid = true) const; + + void ReadFromProtocol(Protocol &rProtocol); + void WriteToProtocol(Protocol &rProtocol) const; + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream) const; + + void SetAsClearFilename(const char *Clear); + + // Check that it's encrypted + bool IsEncrypted() const; + + // These enumerated types belong in the base class so + // the CheckValid() function can make sure that the encoding + // is a valid encoding + enum + { + Encoding_Min = 1, + Encoding_Clear = 1, + Encoding_Blowfish = 2, + Encoding_Max = 2 + }; + +protected: + virtual void EncodedFilenameChanged(); +}; + +// On the wire utilities for class and derived class +#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2) +#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3) + +#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;} + +#endif // BACKUPSTOREFILENAME__H + diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp new file mode 100755 index 00000000..3069f612 --- /dev/null +++ b/lib/backupclient/BackupStoreFilenameClear.cpp @@ -0,0 +1,335 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilenameClear.cpp +// Purpose: BackupStoreFilenames in the clear +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreFilenameClear.h" +#include "BackupStoreException.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "Guards.h" + +#include "MemLeakFindOn.h" + +// Hide private variables from the rest of the world +namespace +{ + int sEncodeMethod = BackupStoreFilename::Encoding_Clear; + CipherContext sBlowfishEncrypt; + CipherContext sBlowfishDecrypt; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear() +// Purpose: Default constructor, creates an invalid filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &) +// Purpose: Creates a filename, encoding from the given string +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode) +{ + SetClearFilename(rToEncode); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy) + : BackupStoreFilename(rToCopy), + mClearFilename(rToCopy.mClearFilename) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) +// Purpose: Copy from base class +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) + : BackupStoreFilename(rToCopy) +{ + // Will get a clear filename when it's required +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::~BackupStoreFilenameClear() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::GetClearFilename() +// Purpose: Get the unencoded filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE +const std::string BackupStoreFilenameClear::GetClearFilename() const +{ + MakeClearAvailable(); + // When modifying, remember to change back to reference return if at all possible + // -- returns an object rather than a reference to allow easy use with other code. + return std::string(mClearFilename.c_str(), mClearFilename.size()); +} +#else +const std::string &BackupStoreFilenameClear::GetClearFilename() const +{ + MakeClearAvailable(); + return mClearFilename; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &) +// Purpose: Encode and make available the clear filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode) +{ + // Only allow Blowfish encodings + if(sEncodeMethod != Encoding_Blowfish) + { + THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup) + } + + // Make an encoded string with blowfish encryption + EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish); + + // Store the clear filename + mClearFilename.assign(rToEncode.c_str(), rToEncode.size()); + + // Make sure we did the right thing + if(!CheckValid(false)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::MakeClearAvailable() +// Purpose: Private. Make sure the clear filename is available +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::MakeClearAvailable() const +{ + if(!mClearFilename.empty()) + return; // nothing to do + + // Check valid + CheckValid(); + + // Decode the header + int size = BACKUPSTOREFILENAME_GET_SIZE(*this); + int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this); + + // Decode based on encoding given in the header + switch(encoding) + { + case Encoding_Clear: + TRACE0("**** BackupStoreFilename encoded with Clear encoding ****\n"); + mClearFilename.assign(c_str() + 2, size - 2); + break; + + case Encoding_Blowfish: + DecryptEncoded(sBlowfishDecrypt); + break; + + default: + THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding) + break; + } +} + + +// Buffer for encoding and decoding -- do this all in one single buffer to +// avoid lots of string allocation, which stuffs up memory usage. +// These static memory vars are, of course, not thread safe, but we don't use threads. +#ifndef NDEBUG +static int sEncDecBufferSize = 2; // small size for debug builds +#else +static int sEncDecBufferSize = 256; +#endif +static MemoryBlockGuard<uint8_t *> spEncDecBuffer(sEncDecBufferSize); + +// fudge to stop leak reporting +#ifdef BOX_MEMORY_LEAK_TESTING +namespace +{ + class leak_off + { + public: + leak_off() + { + MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer); + } + }; + leak_off dont_report_as_leak; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int) +// Purpose: Private. Assigns the encoded filename string, encrypting. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding) +{ + // Work out max size + int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4; + + // Make sure encode/decode buffer has enough space + if(sEncDecBufferSize < maxOutSize) + { + TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, maxOutSize); + spEncDecBuffer.Resize(maxOutSize); + sEncDecBufferSize = maxOutSize; + } + + // Pointer to buffer + uint8_t *buffer = spEncDecBuffer; + MEMLEAKFINDER_NOT_A_LEAK(buffer); + + // Encode -- do entire block in one go + int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size()); + // and add in header size + encSize += 2; + + // Adjust header + BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding); + + // Store the encoded string + assign((char*)buffer, encSize); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &) +// Purpose: Decrypt the encoded filename using the cipher context +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const +{ + // Work out max size + int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(size()) + 4; + + // Make sure encode/decode buffer has enough space + if(sEncDecBufferSize < maxOutSize) + { + TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, maxOutSize); + spEncDecBuffer.Resize(maxOutSize); + sEncDecBufferSize = maxOutSize; + } + + // Pointer to buffer + uint8_t *buffer = spEncDecBuffer; + MEMLEAKFINDER_NOT_A_LEAK(buffer); + + // Decrypt + const char *str = c_str() + 2; + int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, size() - 2); + + // Assign to this + mClearFilename.assign((char*)buffer, sizeOut); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::EncodedFilenameChanged() +// Purpose: The encoded filename stored has changed +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::EncodedFilenameChanged() +{ + BackupStoreFilename::EncodedFilenameChanged(); + + // Delete stored filename in clear + mClearFilename.erase(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int) +// Purpose: Set the key used for Blowfish encryption of filenames +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength) +{ + // Initialisation vector not used. Can't use a different vector for each filename as + // that would stop comparisions on the server working. + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength); + sBlowfishEncrypt.SetIV(pIV); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength); + sBlowfishDecrypt.SetIV(pIV); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetEncodingMethod(int) +// Purpose: Set the encoding method used for filenames +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetEncodingMethod(int Method) +{ + sEncodeMethod = Method; +} + + + diff --git a/lib/backupclient/BackupStoreFilenameClear.h b/lib/backupclient/BackupStoreFilenameClear.h new file mode 100755 index 00000000..d4c45701 --- /dev/null +++ b/lib/backupclient/BackupStoreFilenameClear.h @@ -0,0 +1,60 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilenameClear.h +// Purpose: BackupStoreFilenames in the clear +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILENAMECLEAR__H +#define BACKUPSTOREFILENAMECLEAR__H + +#include "BackupStoreFilename.h" + +class CipherContext; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFilenameClear +// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreFilenameClear : public BackupStoreFilename +{ +public: + BackupStoreFilenameClear(); + BackupStoreFilenameClear(const std::string &rToEncode); + BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy); + BackupStoreFilenameClear(const BackupStoreFilename &rToCopy); + virtual ~BackupStoreFilenameClear(); + + // Because we need to use a different allocator for this class to avoid + // nasty things happening, can't return this as a reference. Which is a + // pity. But probably not too bad. +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE + const std::string GetClearFilename() const; +#else + const std::string &GetClearFilename() const; +#endif + void SetClearFilename(const std::string &rToEncode); + + // Setup for encryption of filenames + static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength); + static void SetEncodingMethod(int Method); + +protected: + void MakeClearAvailable() const; + virtual void EncodedFilenameChanged(); + void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding); + void DecryptEncoded(CipherContext &rCipherContext) const; + +private: + mutable BackupStoreFilename_base mClearFilename; +}; + +#endif // BACKUPSTOREFILENAMECLEAR__H + + diff --git a/lib/backupclient/BackupStoreObjectDump.cpp b/lib/backupclient/BackupStoreObjectDump.cpp new file mode 100644 index 00000000..caa0e82f --- /dev/null +++ b/lib/backupclient/BackupStoreObjectDump.cpp @@ -0,0 +1,210 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreObjectDump.cpp +// Purpose: Implementations of dumping objects to stdout/TRACE +// Created: 3/5/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <stdarg.h> +#include <map> + +#include "BackupStoreDirectory.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreFilename.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreObjectMagic.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void OutputLine(FILE *, bool, const char *, ...) +// Purpose: Output a line for the object dumping, to file and/or trace... +// Created: 3/5/04 +// +// -------------------------------------------------------------------------- +static void OutputLine(FILE *file, bool ToTrace, const char *format, ...) +{ + char text[512]; + int r = 0; + va_list ap; + va_start(ap, format); + r = vsnprintf(text, sizeof(text), format, ap); + va_end(ap); + + if(file != 0) + { + ::fprintf(file, "%s", text); + } + if(ToTrace) + { + TRACE1("%s", text); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) +// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere) +// Dump the contents to a file, or trace. +// Created: 3/5/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) +{ + FILE *file = (FILE*)clibFileHandle; + + OutputLine(file, ToTrace, "Directory object.\nObject ID: %llx\nContainer ID: %llx\nNumber entries: %d\n"\ + "Attributes mod time: %llx\nAttributes size: %d\n", mObjectID, mContainerID, mEntries.size(), + mAttributesModTime, mAttributes.GetSize()); + + // So repeated filenames can be illustrated, even though they can't be decoded + std::map<BackupStoreFilename, int> nameNum; + int nameNumI = 0; + + // Dump items + OutputLine(file, ToTrace, "Items:\nID Size AttrHash AtSz NSz NIdx Flags\n"); + for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i) + { + // Choose file name index number for this file + std::map<BackupStoreFilename, int>::iterator nn(nameNum.find((*i)->GetName())); + int ni = nameNumI; + if(nn != nameNum.end()) + { + ni = nn->second; + } + else + { + nameNum[(*i)->GetName()] = nameNumI; + ++nameNumI; + } + + // Do dependencies + char depends[128]; + depends[0] = '\0'; + int depends_l = 0; + if((*i)->GetDependsNewer() != 0) + { + depends_l += ::sprintf(depends + depends_l, " depNew(%llx)", (*i)->GetDependsNewer()); + } + if((*i)->GetDependsOlder() != 0) + { + depends_l += ::sprintf(depends + depends_l, " depOld(%llx)", (*i)->GetDependsOlder()); + } + + // Output item + int16_t f = (*i)->GetFlags(); + OutputLine(file, ToTrace, "%06llx %4lld %016llx %4d %3d %4d%s%s%s%s%s%s\n", + (*i)->GetObjectID(), + (*i)->GetSizeInBlocks(), + (*i)->GetAttributesHash(), + (*i)->GetAttributes().GetSize(), + (*i)->GetName().size(), + ni, + ((f & BackupStoreDirectory::Entry::Flags_File)?" file":""), + ((f & BackupStoreDirectory::Entry::Flags_Dir)?" dir":""), + ((f & BackupStoreDirectory::Entry::Flags_Deleted)?" del":""), + ((f & BackupStoreDirectory::Entry::Flags_OldVersion)?" old":""), + ((f & BackupStoreDirectory::Entry::Flags_RemoveASAP)?" removeASAP":""), + depends); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DumpFile(void *, bool, IOStream &) +// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere) +// Dump the contents to a file, or trace. +// Created: 4/5/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile) +{ + FILE *file = (FILE*)clibFileHandle; + + // Read header + file_StreamFormat hdr; + if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), + 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Check and output header info + if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V0)) + { + OutputLine(file, ToTrace, "File header doesn't have the correct magic, aborting dump\n"); + return; + } + + OutputLine(file, ToTrace, "File object.\nContainer ID: %llx\nModification time: %llx\n"\ + "Max block clear size: %d\nOptions: %08x\nNum blocks: %d\n", ntoh64(hdr.mContainerID), + ntoh64(hdr.mModificationTime), ntohl(hdr.mMaxBlockClearSize), ntohl(hdr.mOptions), + ntoh64(hdr.mNumBlocks)); + + // Read the next two objects + BackupStoreFilename fn; + fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); + OutputLine(file, ToTrace, "Filename size: %d\n", fn.size()); + + BackupClientFileAttributes attr; + attr.ReadFromStream(rFile, IOStream::TimeOutInfinite); + OutputLine(file, ToTrace, "Attributes size: %d\n", attr.GetSize()); + + // Dump the blocks + rFile.Seek(0, IOStream::SeekType_Absolute); + BackupStoreFile::MoveStreamPositionToBlockIndex(rFile); + + // Read in header + file_BlockIndexHeader bhdr; + rFile.ReadFullBuffer(&bhdr, sizeof(bhdr), 0); + if(bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + && bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)) + { + OutputLine(file, ToTrace, "WARNING: Block header doesn't have the correct magic\n"); + } + // number of blocks + int64_t nblocks = ntoh64(bhdr.mNumBlocks); + OutputLine(file, ToTrace, "Other file ID (for block refs): %llx\nNum blocks (in blk hdr): %lld\n", + ntoh64(bhdr.mOtherFileID), nblocks); + + // Dump info about each block + OutputLine(file, ToTrace, "======== ===== ==========\n Index Where EncSz/Idx\n"); + int64_t nnew = 0, nold = 0; + for(int64_t b = 0; b < nblocks; ++b) + { + file_BlockIndexEntry en; + if(!rFile.ReadFullBuffer(&en, sizeof(en), 0)) + { + OutputLine(file, ToTrace, "Didn't manage to read block %lld from file\n", b); + continue; + } + int64_t s = ntoh64(en.mEncodedSize); + if(s > 0) + { + nnew++; + TRACE2("%8lld this s=%8lld\n", b, s); + } + else + { + nold++; + TRACE2("%8lld other i=%8lld\n", b, 0 - s); + } + } + TRACE0("======== ===== ==========\n"); +} + diff --git a/lib/backupclient/BackupStoreObjectMagic.h b/lib/backupclient/BackupStoreObjectMagic.h new file mode 100755 index 00000000..7ee600a2 --- /dev/null +++ b/lib/backupclient/BackupStoreObjectMagic.h @@ -0,0 +1,31 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreObjectMagic.h +// Purpose: Magic values for the start of objects in the backup store +// Created: 19/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREOBJECTMAGIC__H +#define BACKUPSTOREOBJECTMAGIC__H + +// Each of these values is the first 4 bytes of the object file. +// Remember to swap from network to host byte order. + +// Magic value for file streams +#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65 +// Do not use v0 in any new code! +#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45 + +// Magic for the block index at the file stream -- used to +// ensure streams are reordered as expected +#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478 +// Do not use v0 in any new code! +#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B + +// Magic value for directory streams +#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F + +#endif // BACKUPSTOREOBJECTMAGIC__H + diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra new file mode 100755 index 00000000..66203e3c --- /dev/null +++ b/lib/backupclient/Makefile.extra @@ -0,0 +1,16 @@ + +MAKEPROTOCOL = ../../lib/server/makeprotocol.pl + +GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt + +# AUTOGEN SEEDING +autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt + perl $(GEN_CMD_SRV) + + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt + perl $(MAKEEXCEPTION) BackupStoreException.txt + |