summaryrefslogtreecommitdiff
path: root/lib/backupclient
diff options
context:
space:
mode:
authorBen Summers <ben@fluffy.co.uk>2005-10-14 08:50:54 +0000
committerBen Summers <ben@fluffy.co.uk>2005-10-14 08:50:54 +0000
commit99f8ce096bc5569adbfea1911dbcda24c28d8d8b (patch)
tree049c302161fea1f2f6223e1e8f3c40d9e8aadc8b /lib/backupclient
Box Backup 0.09 with a few tweeks
Diffstat (limited to 'lib/backupclient')
-rwxr-xr-xlib/backupclient/BackupClientCryptoKeys.cpp67
-rwxr-xr-xlib/backupclient/BackupClientCryptoKeys.h55
-rwxr-xr-xlib/backupclient/BackupClientFileAttributes.cpp773
-rwxr-xr-xlib/backupclient/BackupClientFileAttributes.h70
-rwxr-xr-xlib/backupclient/BackupClientMakeExcludeList.cpp75
-rwxr-xr-xlib/backupclient/BackupClientMakeExcludeList.h48
-rwxr-xr-xlib/backupclient/BackupClientRestore.cpp468
-rwxr-xr-xlib/backupclient/BackupClientRestore.h26
-rwxr-xr-xlib/backupclient/BackupDaemonConfigVerify.cpp105
-rwxr-xr-xlib/backupclient/BackupDaemonConfigVerify.h18
-rwxr-xr-xlib/backupclient/BackupStoreConstants.h53
-rwxr-xr-xlib/backupclient/BackupStoreDirectory.cpp563
-rwxr-xr-xlib/backupclient/BackupStoreDirectory.h268
-rwxr-xr-xlib/backupclient/BackupStoreException.h17
-rw-r--r--lib/backupclient/BackupStoreException.txt70
-rwxr-xr-xlib/backupclient/BackupStoreFile.cpp1499
-rwxr-xr-xlib/backupclient/BackupStoreFile.h194
-rw-r--r--lib/backupclient/BackupStoreFileCmbDiff.cpp326
-rw-r--r--lib/backupclient/BackupStoreFileCmbIdx.cpp324
-rwxr-xr-xlib/backupclient/BackupStoreFileCombine.cpp410
-rwxr-xr-xlib/backupclient/BackupStoreFileCryptVar.cpp31
-rwxr-xr-xlib/backupclient/BackupStoreFileCryptVar.h39
-rwxr-xr-xlib/backupclient/BackupStoreFileDiff.cpp973
-rwxr-xr-xlib/backupclient/BackupStoreFileEncodeStream.cpp675
-rwxr-xr-xlib/backupclient/BackupStoreFileEncodeStream.h127
-rw-r--r--lib/backupclient/BackupStoreFileRevDiff.cpp258
-rwxr-xr-xlib/backupclient/BackupStoreFileWire.h74
-rwxr-xr-xlib/backupclient/BackupStoreFilename.cpp279
-rwxr-xr-xlib/backupclient/BackupStoreFilename.h85
-rwxr-xr-xlib/backupclient/BackupStoreFilenameClear.cpp335
-rwxr-xr-xlib/backupclient/BackupStoreFilenameClear.h60
-rw-r--r--lib/backupclient/BackupStoreObjectDump.cpp210
-rwxr-xr-xlib/backupclient/BackupStoreObjectMagic.h31
-rwxr-xr-xlib/backupclient/Makefile.extra16
34 files changed, 8622 insertions, 0 deletions
diff --git a/lib/backupclient/BackupClientCryptoKeys.cpp b/lib/backupclient/BackupClientCryptoKeys.cpp
new file mode 100755
index 00000000..015dadd2
--- /dev/null
+++ b/lib/backupclient/BackupClientCryptoKeys.cpp
@@ -0,0 +1,67 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientCryptoKeys.cpp
+// Purpose: function for setting up all the backup client keys
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "BackupClientCryptoKeys.h"
+#include "FileStream.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupStoreException.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreFile.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientCryptoKeys_Setup(const char *)
+// Purpose: Read in the key material file, and set keys to all the backup elements required.
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename)
+{
+ // Read in the key material
+ unsigned char KeyMaterial[BACKUPCRYPTOKEYS_FILE_SIZE];
+
+ // Open the file
+ FileStream file(KeyMaterialFilename);
+ // Read in data
+ if(!file.ReadFullBuffer(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntLoadClientKeyMaterial)
+ }
+
+ // Tell the filename how to encrypt
+ BackupStoreFilenameClear::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH,
+ KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH);
+ BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish);
+
+ // Tell the attributes how to encrypt
+ BackupClientFileAttributes::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH);
+ // and the secret for hashing
+ BackupClientFileAttributes::SetAttributeHashSecret(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH);
+
+ // Tell the files how to encrypt
+ BackupStoreFile::SetBlowfishKeys(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH,
+ KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH);
+#ifndef PLATFORM_OLD_OPENSSL
+ // Use AES where available
+ BackupStoreFile::SetAESKey(KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH);
+#endif
+
+ // Wipe the key material from memory
+ ::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE);
+}
+
+
+
diff --git a/lib/backupclient/BackupClientCryptoKeys.h b/lib/backupclient/BackupClientCryptoKeys.h
new file mode 100755
index 00000000..5e3a7df2
--- /dev/null
+++ b/lib/backupclient/BackupClientCryptoKeys.h
@@ -0,0 +1,55 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientCryptoKeys.h
+// Purpose: Format of crypto keys file, and function for setting everything up
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTCRYTOKEYS__H
+#define BACKUPCLIENTCRYTOKEYS__H
+
+
+// All keys are the maximum size that Blowfish supports. Since only the
+// setup time is affected by key length (encryption same speed whatever)
+// there is no disadvantage to using long keys as they are never
+// transmitted and are static over long periods of time.
+
+
+// All sizes in bytes. Some gaps deliberately left in the used material.
+
+// How long the key material file is expected to be
+#define BACKUPCRYPTOKEYS_FILE_SIZE 1024
+
+// key for encrypting filenames (448 bits)
+#define BACKUPCRYPTOKEYS_FILENAME_KEY_START 0
+#define BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH 56
+#define BACKUPCRYPTOKEYS_FILENAME_IV_START (0 + BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH)
+#define BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH 8
+
+// key for encrypting attributes (448 bits)
+#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START (BACKUPCRYPTOKEYS_FILENAME_KEY_START+64)
+#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH 56
+
+// Blowfish key for encrypting file data (448 bits (max blowfish key length))
+#define BACKUPCRYPTOKEYS_FILE_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START+64)
+#define BACKUPCRYPTOKEYS_FILE_KEY_LENGTH 56
+
+// key for encrypting file block index entries
+#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START (BACKUPCRYPTOKEYS_FILE_KEY_START+64)
+#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH 56
+
+// Secret for hashing attributes
+#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START (BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START+64)
+#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH 128
+
+// AES key for encrypting file data (256 bits (max AES key length))
+#define BACKUPCRYPTOKEYS_FILE_AES_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START+128)
+#define BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH 32
+
+
+void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename);
+
+#endif // BACKUPCLIENTCRYTOKEYS__H
+
diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp
new file mode 100755
index 00000000..4aa4edba
--- /dev/null
+++ b/lib/backupclient/BackupClientFileAttributes.cpp
@@ -0,0 +1,773 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientFileAttributes.cpp
+// Purpose: Storage of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <unistd.h>
+#include <limits.h>
+
+#include "BackupClientFileAttributes.h"
+#include "CommonException.h"
+#include "FileModificationTime.h"
+#include "BoxTimeToUnix.h"
+#include "BackupStoreException.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "MD5Digest.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+#define ATTRIBUTETYPE_GENERIC_UNIX 1
+
+#define ATTRIBUTE_ENCODING_BLOWFISH 2
+
+typedef struct
+{
+ int32_t AttributeType;
+ u_int32_t UID;
+ u_int32_t GID;
+ u_int64_t ModificationTime;
+ u_int64_t AttrModificationTime;
+ u_int32_t UserDefinedFlags;
+ u_int32_t FileGenerationNumber;
+ u_int16_t Mode;
+ // Symbolic link filename may follow
+} attr_StreamFormat;
+
+// This has wire packing so it's compatible across platforms
+// Use wider than necessary sizes, just to be careful.
+typedef struct
+{
+ int32_t uid, gid, mode;
+} attributeHashData;
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+
+#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256
+
+// Hide private static variables from the rest of the world
+// -- don't put them as static class variables to avoid openssl/evp.h being
+// included all over the project.
+namespace
+{
+ CipherContext sBlowfishEncrypt;
+ CipherContext sBlowfishDecrypt;
+ uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH];
+ int sAttributeHashSecretLength = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::BackupClientFileAttributes()
+// Purpose: Default constructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::BackupClientFileAttributes()
+ : mpClearAttributes(0)
+{
+ ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &)
+// Purpose: Copy constructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy)
+ : StreamableMemBlock(rToCopy), // base class does the hard work
+ mpClearAttributes(0)
+{
+}
+BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy)
+ : StreamableMemBlock(rToCopy), // base class does the hard work
+ mpClearAttributes(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::~BackupClientFileAttributes()
+// Purpose: Destructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::~BackupClientFileAttributes()
+{
+ if(mpClearAttributes)
+ {
+ delete mpClearAttributes;
+ mpClearAttributes = 0;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &)
+// Purpose: Assignment operator
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr)
+{
+ StreamableMemBlock::Set(rAttr);
+ RemoveClear(); // make sure no decrypted version held
+ return *this;
+}
+// Assume users play nice
+BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr)
+{
+ StreamableMemBlock::Set(rAttr);
+ RemoveClear(); // make sure no decrypted version held
+ return *this;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &)
+// Purpose: Comparison operator
+// Created: 2003/10/09
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const
+{
+ EnsureClearAvailable();
+ rAttr.EnsureClearAvailable();
+
+ return mpClearAttributes->operator==(*rAttr.mpClearAttributes);
+}
+// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data.
+/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const
+{
+ StreamableMemBlock *pDecoded = 0;
+
+ try
+ {
+ EnsureClearAvailable();
+ StreamableMemBlock *pDecoded = MakeClear(rAttr);
+
+ // Compare using clear version
+ bool compared = mpClearAttributes->operator==(rAttr);
+
+ // Delete temporary
+ delete pDecoded;
+
+ return compared;
+ }
+ catch(...)
+ {
+ delete pDecoded;
+ throw;
+ }
+}*/
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool)
+// Purpose: Compare, optionally ignoring the attribute modification time and/or modification time, and some data which is
+// irrelevant in practise (eg file generation number)
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime, bool IgnoreModTime) const
+{
+ EnsureClearAvailable();
+ rAttr.EnsureClearAvailable();
+
+ // Check sizes are the same, as a first check
+ if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize())
+ {
+ return false;
+ }
+
+ // Then check the elements of the two things
+ // Bytes are checked in network order, but this doesn't matter as we're only checking for equality.
+ attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+ attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer();
+
+ if(a1->AttributeType != a2->AttributeType
+ || a1->UID != a2->UID
+ || a1->GID != a2->GID
+ || a1->UserDefinedFlags != a2->UserDefinedFlags
+ || a1->Mode != a2->Mode)
+ {
+ return false;
+ }
+
+ if(!IgnoreModTime)
+ {
+ if(a1->ModificationTime != a2->ModificationTime)
+ {
+ return false;
+ }
+ }
+
+ if(!IgnoreAttrModTime)
+ {
+ if(a1->AttrModificationTime != a2->AttrModificationTime)
+ {
+ return false;
+ }
+ }
+
+ // Check symlink string?
+ unsigned int size = mpClearAttributes->GetSize();
+ if(size > sizeof(attr_StreamFormat))
+ {
+ // Symlink strings don't match
+ if(::memcmp(a1 + 1, a2 + 1, size - sizeof(attr_StreamFormat)) != 0)
+ {
+ return false;
+ }
+ }
+
+ // Passes all test, must be OK
+ return true;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadAttributes(const char *)
+// Purpose: Read the attributes of the file, and store them ready for streaming.
+// Optionally retrieve the modification time and attribute modification time.
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroModificationTimes, box_time_t *pModTime,
+ box_time_t *pAttrModTime, int64_t *pFileSize, ino_t *pInodeNumber, bool *pHasMultipleLinks)
+{
+ StreamableMemBlock *pnewAttr = 0;
+ try
+ {
+ struct stat st;
+ if(::lstat(Filename, &st) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ // Modification times etc
+ if(pModTime) {*pModTime = FileModificationTime(st);}
+ if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);}
+ if(pFileSize) {*pFileSize = st.st_size;}
+ if(pInodeNumber) {*pInodeNumber = st.st_ino;}
+ if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);}
+
+ // Is it a link?
+ if((st.st_mode & S_IFMT) == S_IFLNK)
+ {
+ ReadAttributesLink(Filename, &st, ZeroModificationTimes);
+ return;
+ }
+ ASSERT((st.st_mode & S_IFMT) != S_IFLNK);
+
+ // Now, can allocate the block
+ pnewAttr = new StreamableMemBlock(sizeof(attr_StreamFormat));
+
+ // Fill in the entries
+ attr_StreamFormat *pattr = (attr_StreamFormat*)pnewAttr->GetBuffer();
+
+#define FILL_IN_ATTRIBUTES \
+ ::memset(pattr, 0, sizeof(attr_StreamFormat)); \
+ ASSERT(pattr != 0); \
+ pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX); \
+ pattr->UID = htonl(st.st_uid); \
+ pattr->GID = htonl(st.st_gid); \
+ if(ZeroModificationTimes) \
+ { \
+ pattr->ModificationTime = 0; \
+ pattr->AttrModificationTime = 0; \
+ } \
+ else \
+ { \
+ pattr->ModificationTime = hton64(FileModificationTime(st)); \
+ pattr->AttrModificationTime = hton64(FileAttrModificationTime(st)); \
+ } \
+ pattr->Mode = htons(st.st_mode);
+#ifdef PLATFORM_stat_NO_st_flags
+ #define FILL_IN_ATTRIBUTES_2 \
+ pattr->UserDefinedFlags = 0; \
+ pattr->FileGenerationNumber = 0;
+#else
+ #define FILL_IN_ATTRIBUTES_2 \
+ pattr->UserDefinedFlags = htonl(st.st_flags); \
+ pattr->FileGenerationNumber = htonl(st.st_gen);
+#endif
+
+ FILL_IN_ATTRIBUTES
+ FILL_IN_ATTRIBUTES_2
+
+ // Attributes ready. Encrypt into this block
+ EncryptAttr(*pnewAttr);
+
+ // Store the new attributes
+ RemoveClear();
+ mpClearAttributes = pnewAttr;
+ pnewAttr = 0;
+ }
+ catch(...)
+ {
+ // clean up
+ delete pnewAttr;
+ pnewAttr = 0;
+ throw;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadAttributesLink()
+// Purpose: Private function, handles the case where a symbolic link is needed
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::ReadAttributesLink(const char *Filename, void *pst, bool ZeroModificationTimes)
+{
+ StreamableMemBlock *pnewAttr = 0;
+ try
+ {
+ // Avoid needing to have struct stat available when including header file
+ struct stat &st = *((struct stat *)pst);
+ // Make sure we're only called for symbolic links
+ ASSERT((st.st_mode & S_IFMT) == S_IFLNK);
+
+ // Get the filename the link is linked to
+ char linkedTo[PATH_MAX+4];
+ int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX);
+ if(linkedToSize == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ // Now, can allocate the block
+ pnewAttr = new StreamableMemBlock(sizeof(attr_StreamFormat) + linkedToSize + 1);
+
+ // Fill in the entries
+ attr_StreamFormat *pattr = (attr_StreamFormat*)pnewAttr->GetBuffer();
+
+ FILL_IN_ATTRIBUTES
+ FILL_IN_ATTRIBUTES_2
+
+ // Add the path name for the symbolic link
+ ::memcpy(pattr + 1, linkedTo, linkedToSize);
+ // Make sure it's neatly terminated
+ ((char*)(pattr + 1))[linkedToSize] = '\0';
+
+ // Attributes ready. Encrypt into this block
+ EncryptAttr(*pnewAttr);
+
+ // Store the new attributes
+ RemoveClear();
+ mpClearAttributes = pnewAttr;
+ pnewAttr = 0;
+ }
+ catch(...)
+ {
+ // clean up
+ delete pnewAttr;
+ pnewAttr = 0;
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::WriteAttributes(const char *)
+// Purpose: Apply the stored attributes to the file
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::WriteAttributes(const char *Filename) const
+{
+ // Got something loaded
+ if(GetSize() <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Make sure there are clear attributes to use
+ EnsureClearAvailable();
+ ASSERT(mpClearAttributes != 0);
+
+ // Check if the decrypted attributes are small enough, and the type of attributes stored
+ if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+ }
+ int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+ ASSERT(type != 0);
+ if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX)
+ {
+ // Don't know what to do with these
+ THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+ }
+
+ // Check there is enough space for an attributes block
+ if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat))
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Get pointer to structure
+ attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+
+ // is it a symlink?
+ int16_t mode = ntohs(pattr->Mode);
+ if((mode & S_IFMT) == S_IFLNK)
+ {
+ // Check things are sensible
+ if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1)
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Make a symlink, first deleting anything in the way
+ ::unlink(Filename);
+ if(::symlink((char*)(pattr + 1), Filename) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+
+ // If working as root, set user IDs
+ if(::geteuid() == 0)
+ {
+ #ifdef PLATFORM_LCHOWN_NOT_SUPPORTED
+ // only if not a link, can't set their owner on this platform
+ if((mode & S_IFMT) != S_IFLNK)
+ {
+ // Not a link, use normal chown
+ if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+ #else
+ if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) // use the version which sets things on symlinks
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ #endif
+ }
+
+ // Stop now if symlink, because otherwise it'll just be applied to the target
+ if((mode & S_IFMT) == S_IFLNK)
+ {
+ return;
+ }
+
+ // Set modification time?
+ box_time_t modtime = ntoh64(pattr->ModificationTime);
+ if(modtime != 0)
+ {
+ // Work out times as timevals
+ struct timeval times[2];
+ BoxTimeToTimeval(modtime, times[1]);
+ // Copy access time as well, why not, got to set it to something
+ times[0] = times[1];
+ // Attr modification time will be changed anyway, nothing that can be done about it
+
+ // Try to apply
+ if(::utimes(Filename, times) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+
+ // Apply everything else... (allowable mode flags only)
+ if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID | S_ISGID | S_ISVTX)) != 0) // mode must be done last (think setuid)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::IsSymLink()
+// Purpose: Do these attributes represent a symbolic link?
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::IsSymLink() const
+{
+ EnsureClearAvailable();
+
+ // Got the right kind of thing?
+ if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Get the type of attributes stored
+ int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+ ASSERT(type != 0);
+ if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat))
+ {
+ // Check link
+ attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+ return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK;
+ }
+
+ return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::RemoveClear()
+// Purpose: Private. Deletes any clear version of the attributes that may be held
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::RemoveClear() const
+{
+ if(mpClearAttributes)
+ {
+ delete mpClearAttributes;
+ }
+ mpClearAttributes = 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::EnsureClearAvailable()
+// Purpose: Private. Makes sure the clear version is available
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::EnsureClearAvailable() const
+{
+ if(mpClearAttributes == 0)
+ {
+ mpClearAttributes = MakeClear(*this);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &)
+// Purpose: Static. Decrypts stored attributes.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted)
+{
+ // New block
+ StreamableMemBlock *pdecrypted = 0;
+
+ try
+ {
+ // Check the block is big enough for IV and header
+ int ivSize = sBlowfishEncrypt.GetIVLength();
+ if(rEncrypted.GetSize() <= (ivSize + 1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes);
+ }
+
+ // How much space is needed for the output?
+ int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize);
+
+ // Allocate it
+ pdecrypted = new StreamableMemBlock(maxDecryptedSize);
+
+ // ptr to block
+ uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer();
+
+ // Check that the header has right type
+ if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH)
+ {
+ THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding);
+ }
+
+ // Set IV
+ sBlowfishDecrypt.SetIV(encBlock + 1);
+
+ // Decrypt
+ int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1));
+
+ // Resize block to fit
+ pdecrypted->ResizeBlock(decryptedSize);
+ }
+ catch(...)
+ {
+ delete pdecrypted;
+ pdecrypted = 0;
+ }
+
+ return pdecrypted;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int)
+// Purpose: Static. Sets the key to use for encryption and decryption.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength)
+{
+ // IVs set later
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &)
+// Purpose: Private. Encrypt the given attributes into this block.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt)
+{
+ // Free any existing block
+ FreeBlock();
+
+ // Work out the maximum amount of space we need
+ int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize());
+ // And the size of the IV
+ int ivSize = sBlowfishEncrypt.GetIVLength();
+
+ // Allocate this space
+ AllocateBlock(maxEncryptedSize + ivSize + 1);
+
+ // Store the encoding byte
+ uint8_t *block = (uint8_t*)GetBuffer();
+ block[0] = ATTRIBUTE_ENCODING_BLOWFISH;
+
+ // Generate and store an IV for this attribute block
+ int ivSize2 = 0;
+ const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2);
+ ASSERT(ivSize == ivSize2);
+
+ // Copy into the encrypted block
+ ::memcpy(block + 1, iv, ivSize);
+
+ // Do the transform
+ int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize());
+
+ // Resize this block
+ ResizeBlock(encrytedSize + ivSize + 1);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int)
+// Purpose: Set the secret for the filename attribute hash
+// Created: 25/4/04
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength)
+{
+ if(SecretLength > (int)sizeof(sAttributeHashSecret))
+ {
+ SecretLength = sizeof(sAttributeHashSecret);
+ }
+ if(SecretLength < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Copy
+ ::memcpy(sAttributeHashSecret, pSecret, SecretLength);
+ sAttributeHashSecretLength = SecretLength;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::GenerateAttributeHash(struct stat &, const std::string &)
+// Purpose: Generate a 64 bit hash from the attributes, used to detect changes.
+// Include filename in the hash, so that it changes from one file to another,
+// so don't reveal identical attributes.
+// Created: 25/4/04
+//
+// --------------------------------------------------------------------------
+uint64_t BackupClientFileAttributes::GenerateAttributeHash(struct stat &st, const std::string &rFilename)
+{
+ if(sAttributeHashSecretLength == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet)
+ }
+
+ // Assemble stuff we're interested in
+ attributeHashData hashData;
+ memset(&hashData, 0, sizeof(hashData));
+ // Use network byte order and large sizes to be cross platform
+ hashData.uid = htonl(st.st_uid);
+ hashData.gid = htonl(st.st_gid);
+ hashData.mode = htonl(st.st_mode);
+
+ // Create a MD5 hash of the data, filename, and secret
+ MD5Digest digest;
+ digest.Add(&hashData, sizeof(hashData));
+ digest.Add(rFilename.c_str(), rFilename.size());
+ digest.Add(sAttributeHashSecret, sAttributeHashSecretLength);
+ digest.Finish();
+
+ // Return the first 64 bits of the hash
+ uint64_t result = *((uint64_t *)(digest.DigestAsData()));
+ return result;
+}
+
+
+
diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h
new file mode 100755
index 00000000..dd930684
--- /dev/null
+++ b/lib/backupclient/BackupClientFileAttributes.h
@@ -0,0 +1,70 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientFileAttributes.h
+// Purpose: Storage of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTFILEATTRIBUTES__H
+#define BACKUPCLIENTFILEATTRIBUTES__H
+
+#include <string>
+
+#include "StreamableMemBlock.h"
+#include "BoxTime.h"
+
+struct stat;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupClientFileAttributes
+// Purpose: Storage, streaming and application of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+class BackupClientFileAttributes : public StreamableMemBlock
+{
+public:
+ BackupClientFileAttributes();
+ BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy);
+ BackupClientFileAttributes(const StreamableMemBlock &rToCopy);
+ ~BackupClientFileAttributes();
+ BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr);
+ BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr);
+ bool operator==(const BackupClientFileAttributes &rAttr) const;
+// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous?
+
+ bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const;
+
+ // Prevent access to base class members accidently
+ void Set();
+
+ void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false,
+ box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0,
+ ino_t *pInodeNumber = 0, bool *pHasMultipleLinks = 0);
+ void WriteAttributes(const char *Filename) const;
+
+ bool IsSymLink() const;
+
+ static void SetBlowfishKey(const void *pKey, int KeyLength);
+ static void SetAttributeHashSecret(const void *pSecret, int SecretLength);
+
+ static uint64_t GenerateAttributeHash(struct stat &st, const std::string &rFilename);
+
+private:
+ void ReadAttributesLink(const char *Filename, void *pst, bool ZeroModificationTimes);
+
+ void RemoveClear() const;
+ void EnsureClearAvailable() const;
+ static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted);
+ void EncryptAttr(const StreamableMemBlock &rToEncrypt);
+
+private:
+ mutable StreamableMemBlock *mpClearAttributes;
+};
+
+#endif // BACKUPCLIENTFILEATTRIBUTES__H
+
diff --git a/lib/backupclient/BackupClientMakeExcludeList.cpp b/lib/backupclient/BackupClientMakeExcludeList.cpp
new file mode 100755
index 00000000..0615faaa
--- /dev/null
+++ b/lib/backupclient/BackupClientMakeExcludeList.cpp
@@ -0,0 +1,75 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientMakeExcludeList.cpp
+// Purpose: Makes exclude lists from bbbackupd config location entries
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupClientMakeExcludeList.h"
+#include "Configuration.h"
+#include "ExcludeList.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientMakeExcludeList(const Configuration &, const char *, const char *)
+// Purpose: Given a Configuration object corresponding to a bbackupd Location, and the
+// two names of the keys for definite and regex entries, return a ExcludeList.
+// Or 0 if it isn't required.
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName,
+ const char *AlwaysIncludeDefiniteName, const char *AlwaysIncludeRegexName)
+{
+ // Check that at least one of the entries exists
+ if(!rConfig.KeyExists(DefiniteName) && !rConfig.KeyExists(RegexName))
+ {
+ // Neither exists -- return 0 as an Exclude list isn't required.
+ return 0;
+ }
+
+ // Create the exclude list
+ ExcludeList *pexclude = new ExcludeList;
+
+ try
+ {
+ // Definite names to add?
+ if(rConfig.KeyExists(DefiniteName))
+ {
+ pexclude->AddDefiniteEntries(rConfig.GetKeyValue(DefiniteName));
+ }
+ // Regular expressions to add?
+ if(rConfig.KeyExists(RegexName))
+ {
+ pexclude->AddRegexEntries(rConfig.GetKeyValue(RegexName));
+ }
+
+ // Add a "always include" list?
+ if(AlwaysIncludeDefiniteName != 0 && AlwaysIncludeRegexName != 0)
+ {
+ // This will accept NULL as a valid argument, so safe to do this.
+ pexclude->SetAlwaysIncludeList(
+ BackupClientMakeExcludeList(rConfig, AlwaysIncludeDefiniteName, AlwaysIncludeRegexName)
+ );
+ }
+ }
+ catch(...)
+ {
+ // Clean up
+ delete pexclude;
+ throw;
+ }
+
+ return pexclude;
+}
+
+
+
diff --git a/lib/backupclient/BackupClientMakeExcludeList.h b/lib/backupclient/BackupClientMakeExcludeList.h
new file mode 100755
index 00000000..d5dd8af4
--- /dev/null
+++ b/lib/backupclient/BackupClientMakeExcludeList.h
@@ -0,0 +1,48 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientMakeExcludeList.h
+// Purpose: Makes exclude lists from bbbackupd config location entries
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTMAKEEXCLUDELIST__H
+#define BACKUPCLIENTMAKEEXCLUDELIST__H
+
+class ExcludeList;
+class Configuration;
+
+ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName,
+ const char *AlwaysIncludeDefiniteName = 0, const char *AlwaysIncludeRegexName = 0);
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientMakeExcludeList_Files(const Configuration &)
+// Purpose: Create a exclude list from config file entries for files. May return 0.
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+inline ExcludeList *BackupClientMakeExcludeList_Files(const Configuration &rConfig)
+{
+ return BackupClientMakeExcludeList(rConfig, "ExcludeFile", "ExcludeFilesRegex", "AlwaysIncludeFile", "AlwaysIncludeFilesRegex");
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientMakeExcludeList_Dirs(const Configuration &)
+// Purpose: Create a exclude list from config file entries for directories. May return 0.
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+inline ExcludeList *BackupClientMakeExcludeList_Dirs(const Configuration &rConfig)
+{
+ return BackupClientMakeExcludeList(rConfig, "ExcludeDir", "ExcludeDirsRegex", "AlwaysIncludeDir", "AlwaysIncludeDirsRegex");
+}
+
+
+#endif // BACKUPCLIENTMAKEEXCLUDELIST__H
+
diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp
new file mode 100755
index 00000000..ebe1d365
--- /dev/null
+++ b/lib/backupclient/BackupClientRestore.cpp
@@ -0,0 +1,468 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientRestore.cpp
+// Purpose:
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string>
+#include <set>
+#include <unistd.h>
+#include <limits.h>
+#include <stdio.h>
+
+#include "BackupClientRestore.h"
+#include "autogen_BackupProtocolClient.h"
+#include "CommonException.h"
+#include "BackupClientFileAttributes.h"
+#include "IOStream.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "CollectInBufferStream.h"
+#include "FileStream.h"
+#include "Utils.h"
+
+#include "MemLeakFindOn.h"
+
+#define MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES (128*1024)
+
+class RestoreResumeInfo
+{
+public:
+ // constructor
+ RestoreResumeInfo()
+ : mNextLevelID(0),
+ mpNextLevel(0)
+ {
+ }
+
+ // destructor
+ ~RestoreResumeInfo()
+ {
+ delete mpNextLevel;
+ mpNextLevel = 0;
+ }
+
+ // Get a next level object
+ RestoreResumeInfo &AddLevel(int64_t ID, const std::string &rLocalName)
+ {
+ ASSERT(mpNextLevel == 0 && mNextLevelID == 0);
+ mpNextLevel = new RestoreResumeInfo;
+ mNextLevelID = ID;
+ mNextLevelLocalName = rLocalName;
+ return *mpNextLevel;
+ }
+
+ // Remove the next level info
+ void RemoveLevel()
+ {
+ ASSERT(mpNextLevel != 0 && mNextLevelID != 0);
+ delete mpNextLevel;
+ mpNextLevel = 0;
+ mNextLevelID = 0;
+ mNextLevelLocalName.erase();
+ }
+
+ void Save(const std::string &rFilename) const
+ {
+ // TODO: use proper buffered streams when they're done
+ // Build info in memory buffer
+ CollectInBufferStream write;
+
+ // Save this level
+ SaveLevel(write);
+
+ // Store in file
+ write.SetForReading();
+ FileStream file(rFilename.c_str(), O_WRONLY | O_CREAT);
+ write.CopyStreamTo(file, IOStream::TimeOutInfinite, 8*1024 /* large buffer */);
+ }
+
+ void SaveLevel(IOStream &rWrite) const
+ {
+ // Write the restored objects
+ int64_t numObjects = mRestoredObjects.size();
+ rWrite.Write(&numObjects, sizeof(numObjects));
+ for(std::set<int64_t>::const_iterator i(mRestoredObjects.begin()); i != mRestoredObjects.end(); ++i)
+ {
+ int64_t id = *i;
+ rWrite.Write(&id, sizeof(id));
+ }
+
+ // Next level?
+ if(mpNextLevel != 0)
+ {
+ // ID
+ rWrite.Write(&mNextLevelID, sizeof(mNextLevelID));
+ // Name string
+ int32_t nsize = mNextLevelLocalName.size();
+ rWrite.Write(&nsize, sizeof(nsize));
+ rWrite.Write(mNextLevelLocalName.c_str(), nsize);
+ // And then the level itself
+ mpNextLevel->SaveLevel(rWrite);
+ }
+ else
+ {
+ // Just write a zero
+ int64_t zero = 0;
+ rWrite.Write(&zero, sizeof(zero));
+ }
+ }
+
+ // Not written to be efficient -- shouldn't be called very often.
+ bool Load(const std::string &rFilename)
+ {
+ // Delete and reset if necessary
+ if(mpNextLevel != 0)
+ {
+ RemoveLevel();
+ }
+
+ // Open file
+ FileStream file(rFilename.c_str());
+
+ // Load this level
+ return LoadLevel(file);
+ }
+
+ #define CHECKED_READ(x, s) if(!rRead.ReadFullBuffer(x, s, 0)) {return false;}
+ bool LoadLevel(IOStream &rRead)
+ {
+ // Load the restored objects list
+ mRestoredObjects.clear();
+ int64_t numObjects = 0;
+ CHECKED_READ(&numObjects, sizeof(numObjects));
+ for(int64_t o = 0; o < numObjects; ++o)
+ {
+ int64_t id;
+ CHECKED_READ(&id, sizeof(id));
+ mRestoredObjects.insert(id);
+ }
+
+ // ID of next level?
+ int64_t nextID = 0;
+ CHECKED_READ(&nextID, sizeof(nextID));
+ if(nextID != 0)
+ {
+ // Load the next level!
+ std::string name;
+ int32_t nsize = 0;
+ CHECKED_READ(&nsize, sizeof(nsize));
+ char n[PATH_MAX];
+ if(nsize > PATH_MAX) return false;
+ CHECKED_READ(n, nsize);
+ name.assign(n, nsize);
+
+ // Create a new level
+ mpNextLevel = new RestoreResumeInfo;
+ mNextLevelID = nextID;
+ mNextLevelLocalName = name;
+
+ // And ask it to read itself in
+ if(!mpNextLevel->LoadLevel(rRead))
+ {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ // List of objects at this level which have been done already
+ std::set<int64_t> mRestoredObjects;
+ // Next level ID
+ int64_t mNextLevelID;
+ // Pointer to next level
+ RestoreResumeInfo *mpNextLevel;
+ // Local filename of next level
+ std::string mNextLevelLocalName;
+};
+
+// parameters structure
+typedef struct
+{
+ bool PrintDots;
+ bool RestoreDeleted;
+ std::string mRestoreResumeInfoFilename;
+ RestoreResumeInfo mResumeInfo;
+} RestoreParams;
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientRestoreDir(BackupProtocolClient &, int64_t, const char *, bool)
+// Purpose: Restore a directory
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t DirectoryID, std::string &rLocalDirectoryName,
+ RestoreParams &Params, RestoreResumeInfo &rLevel)
+{
+ // If we're resuming... check that we haven't got a next level to look at
+ if(rLevel.mpNextLevel != 0)
+ {
+ // Recurse immediately
+ std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + rLevel.mNextLevelLocalName);
+ BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, localDirname, Params, *rLevel.mpNextLevel);
+
+ // Add it to the list of done itmes
+ rLevel.mRestoredObjects.insert(rLevel.mNextLevelID);
+
+ // Remove the level for the recursed directory
+ rLevel.RemoveLevel();
+ }
+
+ // Save the resumption information
+ Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+
+ // Create the local directory (if not already done) -- path and owner set later, just use restrictive owner mode
+ switch(ObjectExists(rLocalDirectoryName.c_str()))
+ {
+ case ObjectExists_Dir:
+ // Do nothing
+ break;
+ case ObjectExists_File:
+ {
+ // File exists with this name, which is fun. Get rid of it.
+ ::printf("WARNING: File present with name '%s', removing out of the way of restored directory. Use specific restore with ID to restore this object.", rLocalDirectoryName.c_str());
+ if(::unlink(rLocalDirectoryName.c_str()) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+ TRACE1("In restore, directory name collision with file %s", rLocalDirectoryName.c_str());
+ }
+ // follow through to... (no break)
+ case ObjectExists_NoObject:
+ if(::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+ break;
+ default:
+ ASSERT(false);
+ break;
+ }
+
+ // Fetch the directory listing from the server -- getting a list of files which is approparite to the restore type
+ rConnection.QueryListDirectory(
+ DirectoryID,
+ Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING),
+ BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)),
+ true /* want attributes */);
+
+ // Retrieve the directory from the stream following
+ BackupStoreDirectory dir;
+ std::auto_ptr<IOStream> dirstream(rConnection.ReceiveStream());
+ dir.ReadFromStream(*dirstream, rConnection.GetTimeout());
+
+ // Apply attributes to the directory
+ const StreamableMemBlock &dirAttrBlock(dir.GetAttributes());
+ BackupClientFileAttributes dirAttr(dirAttrBlock);
+ dirAttr.WriteAttributes(rLocalDirectoryName.c_str());
+
+ int64_t bytesWrittenSinceLastRestoreInfoSave = 0;
+
+ // Process files
+ {
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+ {
+ // Check ID hasn't already been done
+ if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end())
+ {
+ // Local name
+ BackupStoreFilenameClear nm(en->GetName());
+ std::string localFilename(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename());
+
+ // Unlink anything which already exists -- for resuming restores, we can't overwrite files already there.
+ ::unlink(localFilename.c_str());
+
+ // Request it from the store
+ rConnection.QueryGetFile(DirectoryID, en->GetObjectID());
+
+ // Stream containing encoded file
+ std::auto_ptr<IOStream> objectStream(rConnection.ReceiveStream());
+
+ // Decode the file -- need to do different things depending on whether
+ // the directory entry has additional attributes
+ if(en->HasAttributes())
+ {
+ // Use these attributes
+ const StreamableMemBlock &storeAttr(en->GetAttributes());
+ BackupClientFileAttributes attr(storeAttr);
+ BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout(), &attr);
+ }
+ else
+ {
+ // Use attributes stored in file
+ BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout());
+ }
+
+ // Progress display?
+ if(Params.PrintDots)
+ {
+ printf(".");
+ fflush(stdout);
+ }
+
+ // Add it to the list of done itmes
+ rLevel.mRestoredObjects.insert(en->GetObjectID());
+
+ // Save restore info?
+ int64_t fileSize;
+ if(FileExists(localFilename.c_str(), &fileSize, true /* treat links as not existing */))
+ {
+ // File exists...
+ bytesWrittenSinceLastRestoreInfoSave += fileSize;
+
+ if(bytesWrittenSinceLastRestoreInfoSave > MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES)
+ {
+ // Save the restore info, in case it's needed later
+ Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+ bytesWrittenSinceLastRestoreInfoSave = 0;
+ }
+ }
+ }
+ }
+ }
+
+ // Make sure the restore info has been saved
+ if(bytesWrittenSinceLastRestoreInfoSave != 0)
+ {
+ // Save the restore info, in case it's needed later
+ Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+ bytesWrittenSinceLastRestoreInfoSave = 0;
+ }
+
+
+ // Recuse to directories
+ {
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
+ {
+ // Check ID hasn't already been done
+ if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end())
+ {
+ // Local name
+ BackupStoreFilenameClear nm(en->GetName());
+ std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename());
+
+ // Add the level for the next entry
+ RestoreResumeInfo &rnextLevel(rLevel.AddLevel(en->GetObjectID(), nm.GetClearFilename()));
+
+ // Recurse
+ BackupClientRestoreDir(rConnection, en->GetObjectID(), localDirname, Params, rnextLevel);
+
+ // Remove the level for the above call
+ rLevel.RemoveLevel();
+
+ // Add it to the list of done itmes
+ rLevel.mRestoredObjects.insert(en->GetObjectID());
+ }
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientRestore(BackupProtocolClient &, int64_t, const char *, bool, bool)
+// Purpose: Restore a directory on the server to a local directory on the disc.
+//
+// The local directory must not already exist.
+//
+// If a restore is aborted for any reason, then it may be resumed if
+// Resume == true. If Resume == false and resumption is possible, then
+// Restore_ResumePossible is returned.
+//
+// Set RestoreDeleted to restore a deleted directory. This may not give the
+// directory structure when it was deleted, because files may have been deleted
+// within it before it was deleted.
+//
+// Returns Restore_TargetExists if the target directory exists, but
+// there is no restore possible. (Won't attempt to overwrite things.)
+//
+// Returns Restore_Complete on success. (Exceptions on error.)
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName,
+ bool PrintDots, bool RestoreDeleted, bool UndeleteAfterRestoreDeleted, bool Resume)
+{
+ // Parameter block
+ RestoreParams params;
+ params.PrintDots = PrintDots;
+ params.RestoreDeleted = RestoreDeleted;
+ params.mRestoreResumeInfoFilename = LocalDirectoryName;
+ params.mRestoreResumeInfoFilename += ".boxbackupresume";
+
+ // Target exists?
+ int targetExistance = ObjectExists(LocalDirectoryName);
+
+ // Does any resumption information exist?
+ bool doingResume = false;
+ if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && targetExistance == ObjectExists_Dir)
+ {
+ if(!Resume)
+ {
+ // Caller didn't specify that resume should be done, so refuse to do it
+ // but say why.
+ return Restore_ResumePossible;
+ }
+
+ // Attempt to load the resume info file
+ if(!params.mResumeInfo.Load(params.mRestoreResumeInfoFilename))
+ {
+ // failed -- bad file, so things have gone a bit wrong
+ return Restore_TargetExists;
+ }
+
+ // Flag as doing resume so next check isn't actually performed
+ doingResume = true;
+ }
+
+ // Does the directory already exist?
+ if(targetExistance != ObjectExists_NoObject && !doingResume)
+ {
+ // Don't do anything in this case!
+ return Restore_TargetExists;
+ }
+
+ // Restore the directory
+ std::string localName(LocalDirectoryName);
+ BackupClientRestoreDir(rConnection, DirectoryID, localName, params, params.mResumeInfo);
+
+ // Undelete the directory on the server?
+ if(RestoreDeleted && UndeleteAfterRestoreDeleted)
+ {
+ // Send the command
+ rConnection.QueryUndeleteDirectory(DirectoryID);
+ }
+
+ // Finish progress display?
+ if(PrintDots)
+ {
+ printf("\n");
+ fflush(stdout);
+ }
+
+ // Delete the resume information file
+ ::unlink(params.mRestoreResumeInfoFilename.c_str());
+
+ return Restore_Complete;
+}
+
+
+
+
diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h
new file mode 100755
index 00000000..4b382771
--- /dev/null
+++ b/lib/backupclient/BackupClientRestore.h
@@ -0,0 +1,26 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientRestore.h
+// Purpose: Functions to restore files from a backup store
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSCLIENTRESTORE_H
+#define BACKUPSCLIENTRESTORE__H
+
+class BackupProtocolClient;
+
+enum
+{
+ Restore_Complete = 0,
+ Restore_ResumePossible = 1,
+ Restore_TargetExists = 2
+};
+
+int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName,
+ bool PrintDots = false, bool RestoreDeleted = false, bool UndeleteAfterRestoreDeleted = false, bool Resume = false);
+
+#endif // BACKUPSCLIENTRESTORE__H
+
diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp
new file mode 100755
index 00000000..eec2ceaf
--- /dev/null
+++ b/lib/backupclient/BackupDaemonConfigVerify.cpp
@@ -0,0 +1,105 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupDaemonConfigVerify.cpp
+// Purpose: Configuration file definition for bbackupd
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupDaemonConfigVerify.h"
+#include "Daemon.h"
+#include "BoxPortsAndFiles.h"
+
+#include "MemLeakFindOn.h"
+
+
+static const ConfigurationVerifyKey backuplocationkeys[] =
+{
+ {"ExcludeFile", 0, ConfigTest_MultiValueAllowed, 0},
+ {"ExcludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"ExcludeDir", 0, ConfigTest_MultiValueAllowed, 0},
+ {"ExcludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeFile", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeDir", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"Path", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+static const ConfigurationVerify backuplocations[] =
+{
+ {
+ "*",
+ 0,
+ backuplocationkeys,
+ ConfigTest_LastEntry,
+ 0
+ }
+};
+
+static const ConfigurationVerifyKey verifyserverkeys[] =
+{
+ DAEMON_VERIFY_SERVER_KEYS
+};
+
+static const ConfigurationVerify verifyserver[] =
+{
+ {
+ "Server",
+ 0,
+ verifyserverkeys,
+ ConfigTest_Exists,
+ 0
+ },
+ {
+ "BackupLocations",
+ backuplocations,
+ 0,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+ }
+};
+
+static const ConfigurationVerifyKey verifyrootkeys[] =
+{
+ {"AccountNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+
+ {"UpdateStoreInterval", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"MinimumFileAge", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"MaxUploadWait", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"MaxFileTimeInFuture", "172800", ConfigTest_IsInt, 0}, // file is uploaded if the file is this much in the future (2 days default)
+
+ {"AutomaticBackup", "yes", ConfigTest_IsBool, 0},
+
+ {"SyncAllowScript", 0, 0, 0}, // optional script to run to see if the sync should be started now
+ // return "now" if it's allowed, or a number of seconds if it's not
+
+ {"MaximumDiffingTime", 0, ConfigTest_IsInt, 0},
+
+ {"FileTrackingSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"DiffingUploadSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"StoreHostname", 0, ConfigTest_Exists, 0},
+ {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file
+
+ {"CommandSocket", 0, 0, 0}, // not compulsory to have this
+
+ {"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full
+
+ {"CertificateFile", 0, ConfigTest_Exists, 0},
+ {"PrivateKeyFile", 0, ConfigTest_Exists, 0},
+ {"TrustedCAsFile", 0, ConfigTest_Exists, 0},
+ {"KeysFile", 0, ConfigTest_Exists, 0},
+ {"DataDirectory", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+const ConfigurationVerify BackupDaemonConfigVerify =
+{
+ "root",
+ verifyserver,
+ verifyrootkeys,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+};
+
diff --git a/lib/backupclient/BackupDaemonConfigVerify.h b/lib/backupclient/BackupDaemonConfigVerify.h
new file mode 100755
index 00000000..fefa703c
--- /dev/null
+++ b/lib/backupclient/BackupDaemonConfigVerify.h
@@ -0,0 +1,18 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupDaemonConfigVerify.h
+// Purpose: Configuration file definition for bbackupd
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPDAEMONCONFIGVERIFY__H
+#define BACKUPDAEMONCONFIGVERIFY__H
+
+#include "Configuration.h"
+
+extern const ConfigurationVerify BackupDaemonConfigVerify;
+
+#endif // BACKUPDAEMONCONFIGVERIFY__H
+
diff --git a/lib/backupclient/BackupStoreConstants.h b/lib/backupclient/BackupStoreConstants.h
new file mode 100755
index 00000000..8254e918
--- /dev/null
+++ b/lib/backupclient/BackupStoreConstants.h
@@ -0,0 +1,53 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreContants.h
+// Purpose: constants for the backup system
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECONSTANTS__H
+#define BACKUPSTORECONSTANTS__H
+
+#define BACKUPSTORE_ROOT_DIRECTORY_ID 1
+
+#define BACKUP_STORE_SERVER_VERSION 1
+
+// Minimum size for a chunk to be compressed
+#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256
+
+// min and max sizes for blocks
+#define BACKUP_FILE_MIN_BLOCK_SIZE 4096
+#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024)
+
+// Increase the block size if there are more than this number of blocks
+#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096
+
+// Avoid creating blocks smaller than this
+#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128
+
+// Maximum number of sizes to do an rsync-like scan for
+#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 8
+
+// When doing rsync scans, do not scan for blocks smaller than
+#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 256
+
+// A limit to stop diffing running out of control: If more than this
+// times the number of blocks in the original index are found, stop
+// looking. This stops really bad cases of diffing files containing
+// all the same byte using huge amounts of memory and processor time.
+// This is a multiple of the number of blocks in the diff from file.
+#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096
+
+// How many seconds to wait before deleting unused root directory entries?
+#ifndef NDEBUG
+ // Debug: 30 seconds (easier to test)
+ #define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER 30
+#else
+ // Release: 2 days (plenty of time for sysadmins to notice, or change their mind)
+ #define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER 172800
+#endif
+
+#endif // BACKUPSTORECONSTANTS__H
+
diff --git a/lib/backupclient/BackupStoreDirectory.cpp b/lib/backupclient/BackupStoreDirectory.cpp
new file mode 100755
index 00000000..31fed78d
--- /dev/null
+++ b/lib/backupclient/BackupStoreDirectory.cpp
@@ -0,0 +1,563 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory.h
+// Purpose: Representation of a backup directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+
+#include "BackupStoreDirectory.h"
+#include "IOStream.h"
+#include "BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int32_t mNumEntries;
+ int64_t mObjectID; // this object ID
+ int64_t mContainerID; // ID of container
+ uint64_t mAttributesModTime;
+ int32_t mOptionsPresent; // bit mask of optional sections / features present
+ // Then a StreamableMemBlock for attributes
+} dir_StreamFormat;
+
+enum
+{
+ Option_DependencyInfoPresent = 1
+} dir_StreamFormatOptions;
+
+typedef struct
+{
+ uint64_t mModificationTime;
+ int64_t mObjectID;
+ int64_t mSizeInBlocks;
+ uint64_t mAttributesHash;
+ int16_t mFlags; // order smaller items after bigger ones (for alignment)
+ // Then a BackupStoreFilename
+ // Then a StreamableMemBlock for attributes
+} en_StreamFormat;
+
+typedef struct
+{
+ int64_t mDependsNewer;
+ int64_t mDependsOlder;
+} en_StreamFormatDepends;
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::BackupStoreDirectory()
+// Purpose: Constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::BackupStoreDirectory()
+ : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0)
+{
+ ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t)
+// Purpose: Constructor giving object and container IDs
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID)
+ : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::~BackupStoreDirectory()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::~BackupStoreDirectory()
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+ {
+ delete (*i);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int)
+// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Get the header
+ dir_StreamFormat hdr;
+ if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic value...
+ if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat)
+ }
+
+ // Get data
+ mObjectID = ntoh64(hdr.mObjectID);
+ mContainerID = ntoh64(hdr.mContainerID);
+ mAttributesModTime = ntoh64(hdr.mAttributesModTime);
+
+ // Options
+ int32_t options = ntohl(hdr.mOptionsPresent);
+
+ // Get attributes
+ mAttributes.ReadFromStream(rStream, Timeout);
+
+ // Decode count
+ int count = ntohl(hdr.mNumEntries);
+
+ // Clear existing list
+ mEntries.clear();
+
+ // Read them in!
+ for(int c = 0; c < count; ++c)
+ {
+ Entry *pen = new Entry;
+ try
+ {
+ // Read from stream
+ pen->ReadFromStream(rStream, Timeout);
+
+ // Add to list
+ mEntries.push_back(pen);
+ }
+ catch(...)
+ {
+ delete pen;
+ throw;
+ }
+ }
+
+ // Read in dependency info?
+ if(options & Option_DependencyInfoPresent)
+ {
+ // Read in extra dependency data
+ for(int c = 0; c < count; ++c)
+ {
+ mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout);
+ }
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool)
+// Purpose: Writes a selection of entries to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const
+{
+ // Get count of entries
+ int32_t count = mEntries.size();
+ if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Need to count the entries
+ count = 0;
+ Iterator i(*this);
+ while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0)
+ {
+ count++;
+ }
+ }
+
+ // Check that sensible IDs have been set
+ ASSERT(mObjectID != 0);
+ ASSERT(mContainerID != 0);
+
+ // Need dependency info?
+ bool dependencyInfoRequired = false;
+ if(StreamDependencyInfo)
+ {
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ if(pen->HasDependencies())
+ {
+ dependencyInfoRequired = true;
+ }
+ }
+ }
+
+ // Options
+ int32_t options = 0;
+ if(dependencyInfoRequired) options |= Option_DependencyInfoPresent;
+
+ // Build header
+ dir_StreamFormat hdr;
+ hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE);
+ hdr.mNumEntries = htonl(count);
+ hdr.mObjectID = hton64(mObjectID);
+ hdr.mContainerID = hton64(mContainerID);
+ hdr.mAttributesModTime = hton64(mAttributesModTime);
+ hdr.mOptionsPresent = htonl(options);
+
+ // Write header
+ rStream.Write(&hdr, sizeof(hdr));
+
+ // Write the attributes?
+ if(StreamAttributes)
+ {
+ mAttributes.WriteToStream(rStream);
+ }
+ else
+ {
+ // Write a blank header instead
+ StreamableMemBlock::WriteEmptyBlockToStream(rStream);
+ }
+
+ // Then write all the entries
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ pen->WriteToStream(rStream);
+ }
+
+ // Write dependency info?
+ if(dependencyInfoRequired)
+ {
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ pen->WriteToStreamDependencyInfo(rStream);
+ }
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddEntry(const Entry &)
+// Purpose: Adds entry to directory (no checking)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy)
+{
+ Entry *pnew = new Entry(rEntryToCopy);
+ try
+ {
+ mEntries.push_back(pnew);
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+
+ return pnew;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
+// Purpose: Adds entry to directory (no checking)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime)
+{
+ Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime);
+ try
+ {
+ mEntries.push_back(pnew);
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+
+ return pnew;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::DeleteEntry(int64_t)
+// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::DeleteEntry(int64_t ObjectID)
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin());
+ i != mEntries.end(); ++i)
+ {
+ if((*i)->mObjectID == ObjectID)
+ {
+ // Delete
+ delete (*i);
+ // Remove from list
+ mEntries.erase(i);
+ // Done
+ return;
+ }
+ }
+
+ // Not found
+ THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::FindEntryByID(int64_t)
+// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist.
+// Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const
+{
+ for(std::vector<Entry*>::const_iterator i(mEntries.begin());
+ i != mEntries.end(); ++i)
+ {
+ if((*i)->mObjectID == ObjectID)
+ {
+ // Found
+ return (*i);
+ }
+ }
+
+ // Not found
+ return 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry()
+// Purpose: Constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry()
+ : mModificationTime(0),
+ mObjectID(0),
+ mSizeInBlocks(0),
+ mFlags(0),
+ mAttributesHash(0),
+ mMinMarkNumber(0),
+ mMarkNumber(0),
+ mDependsNewer(0),
+ mDependsOlder(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::~Entry()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::~Entry()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry(const Entry &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry(const Entry &rToCopy)
+ : mName(rToCopy.mName),
+ mModificationTime(rToCopy.mModificationTime),
+ mObjectID(rToCopy.mObjectID),
+ mSizeInBlocks(rToCopy.mSizeInBlocks),
+ mFlags(rToCopy.mFlags),
+ mAttributesHash(rToCopy.mAttributesHash),
+ mAttributes(rToCopy.mAttributes),
+ mMinMarkNumber(rToCopy.mMinMarkNumber),
+ mMarkNumber(rToCopy.mMarkNumber),
+ mDependsNewer(rToCopy.mDependsNewer),
+ mDependsOlder(rToCopy.mDependsOlder)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
+// Purpose: Constructor from values
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash)
+ : mName(rName),
+ mModificationTime(ModificationTime),
+ mObjectID(ObjectID),
+ mSizeInBlocks(SizeInBlocks),
+ mFlags(Flags),
+ mAttributesHash(AttributesHash),
+ mMinMarkNumber(0),
+ mMarkNumber(0),
+ mDependsNewer(0),
+ mDependsOlder(0)
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int)
+// Purpose: Read an entry from a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Grab the raw bytes from the stream which compose the header
+ en_StreamFormat entry;
+ if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Do reading first before modifying the variables, to be more exception safe
+
+ // Get the filename
+ BackupStoreFilename name;
+ name.ReadFromStream(rStream, Timeout);
+
+ // Get the attributes
+ mAttributes.ReadFromStream(rStream, Timeout);
+
+ // Store the rest of the bits
+ mModificationTime = ntoh64(entry.mModificationTime);
+ mObjectID = ntoh64(entry.mObjectID);
+ mSizeInBlocks = ntoh64(entry.mSizeInBlocks);
+ mAttributesHash = ntoh64(entry.mAttributesHash);
+ mFlags = ntohs(entry.mFlags);
+ mName = name;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &)
+// Purpose: Writes the entry to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const
+{
+ // Build a structure
+ en_StreamFormat entry;
+ entry.mModificationTime = hton64(mModificationTime);
+ entry.mObjectID = hton64(mObjectID);
+ entry.mSizeInBlocks = hton64(mSizeInBlocks);
+ entry.mAttributesHash = hton64(mAttributesHash);
+ entry.mFlags = htons(mFlags);
+
+ // Write it
+ rStream.Write(&entry, sizeof(entry));
+
+ // Write the filename
+ mName.WriteToStream(rStream);
+
+ // Write any attributes
+ mAttributes.WriteToStream(rStream);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int)
+// Purpose: Read the optional dependency info from a stream
+// Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout)
+{
+ // Grab the raw bytes from the stream which compose the header
+ en_StreamFormatDepends depends;
+ if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Store the data
+ mDependsNewer = ntoh64(depends.mDependsNewer);
+ mDependsOlder = ntoh64(depends.mDependsOlder);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &)
+// Purpose: Write the optional dependency info to a stream
+// Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const
+{
+ // Build structure
+ en_StreamFormatDepends depends;
+ depends.mDependsNewer = hton64(mDependsNewer);
+ depends.mDependsOlder = hton64(mDependsOlder);
+ // Write
+ rStream.Write(&depends, sizeof(depends));
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreDirectory.h b/lib/backupclient/BackupStoreDirectory.h
new file mode 100755
index 00000000..958eee81
--- /dev/null
+++ b/lib/backupclient/BackupStoreDirectory.h
@@ -0,0 +1,268 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory.h
+// Purpose: Representation of a backup directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREDIRECTORY__H
+#define BACKUPSTOREDIRECTORY__H
+
+#include <string>
+#include <vector>
+
+#include "BackupStoreFilenameClear.h"
+#include "StreamableMemBlock.h"
+#include "BoxTime.h"
+
+class IOStream;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreDirectory
+// Purpose: In memory representation of a directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreDirectory
+{
+public:
+ BackupStoreDirectory();
+ BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID);
+private:
+ // Copying not allowed
+ BackupStoreDirectory(const BackupStoreDirectory &rToCopy);
+public:
+ ~BackupStoreDirectory();
+
+ class Entry
+ {
+ public:
+ friend class BackupStoreDirectory;
+
+ Entry();
+ ~Entry();
+ Entry(const Entry &rToCopy);
+ Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash);
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream) const;
+
+ const BackupStoreFilename &GetName() const {return mName;}
+ box_time_t GetModificationTime() const {return mModificationTime;}
+ int64_t GetObjectID() const {return mObjectID;}
+ int64_t GetSizeInBlocks() const {return mSizeInBlocks;}
+ int16_t GetFlags() const {return mFlags;}
+ void AddFlags(int16_t Flags) {mFlags |= Flags;}
+ void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;}
+
+ // Some things can be changed
+ void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;}
+ void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;}
+
+ // Attributes
+ bool HasAttributes() const {return !mAttributes.IsEmpty();}
+ void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;}
+ const StreamableMemBlock &GetAttributes() const {return mAttributes;}
+ uint64_t GetAttributesHash() const {return mAttributesHash;}
+
+ // Marks
+ // The lowest mark number a version of a file of this name has ever had
+ uint32_t GetMinMarkNumber() const {return mMinMarkNumber;}
+ // The mark number on this file
+ uint32_t GetMarkNumber() const {return mMarkNumber;}
+
+ // Make sure these flags are synced with those in backupprocotol.txt
+ // ListDirectory command
+ enum
+ {
+ Flags_INCLUDE_EVERYTHING = -1,
+ Flags_EXCLUDE_NOTHING = 0,
+ Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below!
+ Flags_File = 1,
+ Flags_Dir = 2,
+ Flags_Deleted = 4,
+ Flags_OldVersion = 8,
+ Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion
+ };
+ // characters for textual listing of files -- see bbackupquery/BackupQueries
+ #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR"
+
+ bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet)
+ {
+ return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet))
+ && ((mFlags & FlagsNotToBeSet) == 0);
+ };
+
+ // Get dependency info
+ // new version this depends on
+ int64_t GetDependsNewer() const {return mDependsNewer;}
+ void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;}
+ // older version which depends on this
+ int64_t GetDependsOlder() const {return mDependsOlder;}
+ void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;}
+
+ // Dependency info saving
+ bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;}
+ void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout);
+ void WriteToStreamDependencyInfo(IOStream &rStream) const;
+
+ private:
+ BackupStoreFilename mName;
+ box_time_t mModificationTime;
+ int64_t mObjectID;
+ int64_t mSizeInBlocks;
+ int16_t mFlags;
+ uint64_t mAttributesHash;
+ StreamableMemBlock mAttributes;
+ uint32_t mMinMarkNumber;
+ uint32_t mMarkNumber;
+
+ uint64_t mDependsNewer; // new version this depends on
+ uint64_t mDependsOlder; // older version which depends on this
+ };
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream,
+ int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING,
+ int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING,
+ bool StreamAttributes = true, bool StreamDependencyInfo = true) const;
+
+ Entry *AddEntry(const Entry &rEntryToCopy);
+ Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime);
+ void DeleteEntry(int64_t ObjectID);
+ Entry *FindEntryByID(int64_t ObjectID) const;
+
+ int64_t GetObjectID() const {return mObjectID;}
+ int64_t GetContainerID() const {return mContainerID;}
+
+ // Need to be able to update the container ID when moving objects
+ void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;}
+
+ // Purely for use of server -- not serialised into streams
+ int64_t GetRevisionID() const {return mRevisionID;}
+ void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;}
+
+ unsigned int GetNumberOfEntries() const {return mEntries.size();}
+
+ // User info -- not serialised into streams
+ int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;}
+ void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;}
+
+ // Attributes
+ bool HasAttributes() const {return !mAttributes.IsEmpty();}
+ void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;}
+ const StreamableMemBlock &GetAttributes() const {return mAttributes;}
+ box_time_t GetAttributesModTime() const {return mAttributesModTime;}
+
+ class Iterator
+ {
+ public:
+ Iterator(const BackupStoreDirectory &rDir)
+ : mrDir(rDir), i(rDir.mEntries.begin())
+ {
+ }
+
+ BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags
+ while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.end())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ // WARNING: This function is really very inefficient.
+ // Only use when you want to look up ONE filename, not in a loop looking up lots.
+ // In a looping situation, cache the decrypted filenames in another memory structure.
+ BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags or filename
+ while( (i != mrDir.mEntries.end())
+ && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) )
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.end())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ private:
+ const BackupStoreDirectory &mrDir;
+ std::vector<Entry*>::const_iterator i;
+ };
+
+ friend class Iterator;
+
+ class ReverseIterator
+ {
+ public:
+ ReverseIterator(const BackupStoreDirectory &rDir)
+ : mrDir(rDir), i(rDir.mEntries.rbegin())
+ {
+ }
+
+ BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags
+ while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.rend())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ private:
+ const BackupStoreDirectory &mrDir;
+ std::vector<Entry*>::const_reverse_iterator i;
+ };
+
+ friend class ReverseIterator;
+
+ // For recovery of the store
+ // Implemented in BackupStoreCheck2.cpp
+ bool CheckAndFix();
+ void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags);
+ bool NameInUse(const BackupStoreFilename &rName);
+ // Don't use these functions in normal code!
+
+ // For testing
+ void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;}
+
+ // Debug and diagonistics
+ void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere
+
+private:
+ int64_t mRevisionID;
+ int64_t mObjectID;
+ int64_t mContainerID;
+ std::vector<Entry*> mEntries;
+ box_time_t mAttributesModTime;
+ StreamableMemBlock mAttributes;
+ int64_t mUserInfo1;
+};
+
+#endif // BACKUPSTOREDIRECTORY__H
+
diff --git a/lib/backupclient/BackupStoreException.h b/lib/backupclient/BackupStoreException.h
new file mode 100755
index 00000000..981dfa60
--- /dev/null
+++ b/lib/backupclient/BackupStoreException.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreException.h
+// Purpose: Exception
+// Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREEXCEPTION__H
+#define BACKUPSTOREEXCEPTION__H
+
+// Compatibility
+#include "autogen_BackupStoreException.h"
+
+#endif // BACKUPSTOREEXCEPTION__H
+
diff --git a/lib/backupclient/BackupStoreException.txt b/lib/backupclient/BackupStoreException.txt
new file mode 100644
index 00000000..50c615b2
--- /dev/null
+++ b/lib/backupclient/BackupStoreException.txt
@@ -0,0 +1,70 @@
+EXCEPTION BackupStore 4
+
+Internal 0
+BadAccountDatabaseFile 1
+AccountDatabaseNoSuchEntry 2
+InvalidBackupStoreFilename 3
+UnknownFilenameEncoding 4
+CouldntReadEntireStructureFromStream 5
+BadDirectoryFormat 6
+CouldNotFindEntryInDirectory 7
+OutputFileAlreadyExists 8
+OSFileError 9
+StreamDoesntHaveRequiredFeatures 10
+BadBackupStoreFile 11
+CouldNotLoadStoreInfo 12
+BadStoreInfoOnLoad 13
+StoreInfoIsReadOnly 14
+StoreInfoDirNotInList 15
+StoreInfoBlockDeltaMakesValueNegative 16
+DirectoryHasBeenDeleted 17
+StoreInfoNotInitialised 18
+StoreInfoAlreadyLoaded 19
+StoreInfoNotLoaded 20
+ReadFileFromStreamTimedOut 21
+FileWrongSizeAfterBeingStored 22
+AddedFileDoesNotVerify 23
+StoreInfoForWrongAccount 24
+ContextIsReadOnly 25
+AttributesNotLoaded 26
+AttributesNotUnderstood 27
+WrongServerVersion 28 # client side
+ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account.
+NameAlreadyExistsInDirectory 30
+BerkelyDBFailure 31 # client side
+InodeMapIsReadOnly 32 # client side
+InodeMapNotOpen 33 # client side
+FilenameEncryptionKeyNotKnown 34
+FilenameEncryptionNoKeyForSpecifiedMethod 35
+FilenameEncryptionNotSetup 36
+CouldntLoadClientKeyMaterial 37
+BadEncryptedAttributes 38
+EncryptedAttributesHaveUnknownEncoding 39
+OutputSizeTooSmallForChunk 40
+BadEncodedChunk 41
+NotEnoughSpaceToDecodeChunk 42
+ChunkHasUnknownEncoding 43
+ChunkContainsBadCompressedData 44
+CantWriteToEncodedFileStream 45
+Temp_FileEncodeStreamDidntReadBuffer 46
+CantWriteToDecodedFileStream 47
+WhenDecodingExpectedToReadButCouldnt 48
+BackupStoreFileFailedIntegrityCheck 49
+ThereIsNoDataInASymLink 50
+IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51
+BlockEntryEncodingDidntGiveExpectedLength 52
+CouldNotFindUnusedIDDuringAllocation 53
+AddedFileExceedsStorageLimit 54
+CannotDiffAnIncompleteStoreFile 55
+CannotDecodeDiffedFilesWithoutCombining 56
+FailedToReadBlockOnCombine 57
+OnCombineFromFileIsIncomplete 58
+BadNotifySysadminEventCode 59
+InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60
+CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server?
+AttributeHashSecretNotSet 62
+AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES
+SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection.
+IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related
+DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory
+PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it.
diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp
new file mode 100755
index 00000000..2ea877b4
--- /dev/null
+++ b/lib/backupclient/BackupStoreFile.cpp
@@ -0,0 +1,1499 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFile.cpp
+// Purpose: Utils for manipulating files
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <unistd.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <new>
+#include <string.h>
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ #include <syslog.h>
+ #include <stdio.h>
+#endif
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFilename.h"
+#include "BackupStoreException.h"
+#include "IOStream.h"
+#include "Guards.h"
+#include "FileModificationTime.h"
+#include "FileStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreObjectMagic.h"
+#include "Compress.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "CipherAES.h"
+#include "BackupStoreConstants.h"
+#include "CollectInBufferStream.h"
+#include "RollingChecksum.h"
+#include "MD5Digest.h"
+#include "ReadGatherStream.h"
+#include "Random.h"
+#include "BackupStoreFileEncodeStream.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+
+// How big a buffer to use for copying files
+#define COPY_BUFFER_SIZE (8*1024)
+
+// Statistics
+BackupStoreFileStats BackupStoreFile::msStats = {0,0,0};
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool sWarnedAboutBackwardsCompatiblity = false;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &)
+// Purpose: Encode a file into something for storing on file server.
+// Requires a real filename so full info can be stored.
+//
+// Returns a stream. Most of the work is done by the stream
+// when data is actually requested -- the file will be held
+// open until the stream is deleted or the file finished.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime)
+{
+ // Create the stream
+ std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
+
+ // Do the initial setup
+ ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, 0 /* no recipe, just encode */,
+ ContainerID, rStoreFilename, pModificationTime);
+
+ // Return the stream for the caller
+ return stream;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &)
+// Purpose: Verify that an encoded file meets the format requirements.
+// Doesn't verify that the data is intact and can be decoded.
+// Optionally returns the ID of the file which it is diffed from,
+// and the (original) container ID.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut)
+{
+ // Get the size of the file
+ int64_t fileSize = rFile.BytesLeftToRead();
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Get the header...
+ file_StreamFormat hdr;
+ if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read header
+ return false;
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ return false;
+ }
+
+ // Get a filename, see if it loads OK
+ try
+ {
+ BackupStoreFilename fn;
+ fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+ }
+ catch(...)
+ {
+ // an error occured while reading it, so that's not good
+ return false;
+ }
+
+ // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not
+ try
+ {
+ int32_t size_s;
+ if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+ }
+ int size = ntohl(size_s);
+ // Skip forward the size
+ rFile.Seek(size, IOStream::SeekType_Relative);
+ }
+ catch(...)
+ {
+ // an error occured while reading it, so that's not good
+ return false;
+ }
+
+ // Get current position in file -- the end of the header
+ int64_t headerEnd = rFile.GetPosition();
+
+ // Get number of blocks
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+
+ // Calculate where the block index will be, check it's reasonable
+ int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+ if(blockIndexLoc < headerEnd)
+ {
+ // Not enough space left for the block index, let alone the blocks themselves
+ return false;
+ }
+
+ // Load the block index header
+ rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute);
+ file_BlockIndexHeader blkhdr;
+ if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read block index header -- assume bad file
+ return false;
+ }
+
+ // Check header
+ if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
+#endif
+ )
+ || (int64_t)ntoh64(blkhdr.mNumBlocks) != numBlocks)
+ {
+ // Bad header -- either magic value or number of blocks is wrong
+ return false;
+ }
+
+ // Flag for recording whether a block is referenced from another file
+ bool blockFromOtherFileReferenced = false;
+
+ // Read the index, checking that the length values all make sense
+ int64_t currentBlockStart = headerEnd;
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read block entry
+ file_BlockIndexEntry blk;
+ if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read block index entry -- assume bad file
+ return false;
+ }
+
+ // Check size and location
+ int64_t blkSize = ntoh64(blk.mEncodedSize);
+ if(blkSize <= 0)
+ {
+ // Mark that this file references another file
+ blockFromOtherFileReferenced = true;
+ }
+ else
+ {
+ // This block is actually in this file
+ if((currentBlockStart + blkSize) > blockIndexLoc)
+ {
+ // Encoded size makes the block run over the index
+ return false;
+ }
+
+ // Move the current block start ot the end of this block
+ currentBlockStart += blkSize;
+ }
+ }
+
+ // Check that there's no empty space
+ if(currentBlockStart != blockIndexLoc)
+ {
+ return false;
+ }
+
+ // Check that if another block is references, then the ID is there, and if one isn't there is no ID.
+ int64_t otherID = ntoh64(blkhdr.mOtherFileID);
+ if((otherID != 0 && blockFromOtherFileReferenced == false)
+ || (otherID == 0 && blockFromOtherFileReferenced == true))
+ {
+ // Doesn't look good!
+ return false;
+ }
+
+ // Does the caller want the other ID?
+ if(pDiffFromObjectIDOut)
+ {
+ *pDiffFromObjectIDOut = otherID;
+ }
+
+ // Does the caller want the container ID?
+ if(pContainerIDOut)
+ {
+ *pContainerIDOut = ntoh64(hdr.mContainerID);
+ }
+
+ // Passes all tests
+ return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeFile(IOStream &, const char *)
+// Purpose: Decode a file. Will set file attributes. File must not exist.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Does file exist?
+ struct stat st;
+ if(::stat(DecodedFilename, &st) == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists)
+ }
+
+ // Try, delete output file if error
+ try
+ {
+ // Make a stream for outputting this file
+ FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL);
+
+ // Get the decoding stream
+ std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr));
+
+ // Is it a symlink?
+ if(!stream->IsSymLink())
+ {
+ // Copy it out to the file
+ stream->CopyStreamTo(out);
+ }
+
+ // Write the attributes
+ stream->GetAttributes().WriteAttributes(DecodedFilename);
+ }
+ catch(...)
+ {
+ ::unlink(DecodedFilename);
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *)
+// Purpose: Return a stream which will decode the encrypted file data on the fly.
+// Accepts streams in block index first, or main header first, order. In the latter case,
+// the stream must be Seek()able.
+//
+// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow
+// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Create stream
+ std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout));
+
+ // Get it ready
+ stream->Setup(pAlterativeAttr);
+
+ // Return to caller
+ return stream;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int)
+// Purpose: Constructor
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout)
+ : mrEncodedFile(rEncodedFile),
+ mTimeout(Timeout),
+ mNumBlocks(0),
+ mpBlockIndex(0),
+ mpEncodedData(0),
+ mpClearData(0),
+ mClearDataSize(0),
+ mCurrentBlock(-1),
+ mCurrentBlockClearSize(0),
+ mPositionInCurrentBlock(0),
+ mEntryIVBase(42) // different to default value in the encoded stream!
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ , mIsOldVersion(false)
+#endif
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::~DecodedStream()
+// Purpose: Desctructor
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::DecodedStream::~DecodedStream()
+{
+ // Free any allocated memory
+ if(mpBlockIndex)
+ {
+ ::free(mpBlockIndex);
+ }
+ if(mpEncodedData)
+ {
+ BackupStoreFile::CodingChunkFree(mpEncodedData);
+ }
+ if(mpClearData)
+ {
+ ::free(mpClearData);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *)
+// Purpose: Get the stream ready to decode -- reads in headers
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Get the size of the file
+ int64_t fileSize = mrEncodedFile.BytesLeftToRead();
+
+ // Get the magic number to work out which order the stream is in
+ int32_t magic;
+ if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read magic value
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ bool inFileOrder = true;
+ switch(ntohl(magic))
+ {
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
+ mIsOldVersion = true;
+ // control flows on
+#endif
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
+ inFileOrder = true;
+ break;
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0:
+ mIsOldVersion = true;
+ // control flows on
+#endif
+ case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1:
+ inFileOrder = false;
+ break;
+
+ default:
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // If not in file order, then the index list must be read now
+ if(!inFileOrder)
+ {
+ ReadBlockIndex(true /* have already read and verified the magic number */);
+ }
+
+ // Get header
+ file_StreamFormat hdr;
+ if(inFileOrder)
+ {
+ // Read the header, without the magic number
+ if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic),
+ 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ // Put in magic number
+ hdr.mMagicValue = magic;
+ }
+ else
+ {
+ // Not in file order, so need to read the full header
+ if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Get the filename
+ mFilename.ReadFromStream(mrEncodedFile, mTimeout);
+
+ // Get the attributes (either from stream, or supplied attributes)
+ if(pAlterativeAttr != 0)
+ {
+ // Read dummy attributes
+ BackupClientFileAttributes attr;
+ attr.ReadFromStream(mrEncodedFile, mTimeout);
+
+ // Set to supplied attributes
+ mAttributes = *pAlterativeAttr;
+ }
+ else
+ {
+ // Read the attributes from the stream
+ mAttributes.ReadFromStream(mrEncodedFile, mTimeout);
+ }
+
+ // If it is in file order, go and read the file attributes
+ // Requires that the stream can seek
+ if(inFileOrder)
+ {
+ // Make sure the file size is known
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Store current location (beginning of encoded blocks)
+ int64_t endOfHeaderPos = mrEncodedFile.GetPosition();
+
+ // Work out where the index is
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+ int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+
+ // Seek to that position
+ mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute);
+
+ // Read the block index
+ ReadBlockIndex(false /* magic number still to be read */);
+
+ // Seek back to the end of header position, ready for reading the chunks
+ mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute);
+ }
+
+ // Check view of blocks from block header and file header match
+ if(mNumBlocks != (int64_t)ntoh64(hdr.mNumBlocks))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Need to allocate some memory for the two blocks for reading encoded data, and clear data
+ if(mNumBlocks > 0)
+ {
+ // Find the maximum encoded data size
+ int32_t maxEncodedDataSize = 0;
+ const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
+ ASSERT(entry != 0);
+ for(int64_t e = 0; e < mNumBlocks; e++)
+ {
+ // Get the clear and encoded size
+ int32_t encodedSize = ntoh64(entry[e].mEncodedSize);
+ ASSERT(encodedSize > 0);
+
+ // Larger?
+ if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize;
+ }
+
+ // Allocate those blocks!
+ mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32);
+
+ // Allocate the block for the clear data, using the hint from the header.
+ // If this is wrong, things will exception neatly later on, so it can't be used
+ // to do anything more than cause an error on downloading.
+ mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32;
+ mpClearData = (uint8_t*)::malloc(mClearDataSize);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool)
+// Purpose: Read the block index from the stream, and store in internal buffer (minus header)
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead)
+{
+ // Header
+ file_BlockIndexHeader blkhdr;
+
+ // Read it in -- way depends on how whether the magic number has already been read
+ if(MagicAlreadyRead)
+ {
+ // Read the header, without the magic number
+ if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue),
+ 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+ else
+ {
+ // Magic not already read, so need to read the full header
+ if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Check magic value
+ if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ }
+
+ // Get the number of blocks out of the header
+ mNumBlocks = ntoh64(blkhdr.mNumBlocks);
+
+ // Read the IV base
+ mEntryIVBase = ntoh64(blkhdr.mEntryIVBase);
+
+ // Load the block entries in?
+ if(mNumBlocks > 0)
+ {
+ // How big is the index?
+ int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks;
+
+ // Allocate some memory
+ mpBlockIndex = ::malloc(indexSize);
+ if(mpBlockIndex == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Read it in
+ if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Read(void *, int, int)
+// Purpose: As interface. Reads decrpyted data.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Symlinks don't have data. So can't read it. Not even zero bytes.
+ if(IsSymLink())
+ {
+ // Don't allow reading in this case
+ THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink);
+ }
+
+ // Already finished?
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // At end of stream, nothing to do
+ return 0;
+ }
+
+ int bytesToRead = NBytes;
+ uint8_t *output = (uint8_t*)pBuffer;
+
+ while(bytesToRead > 0 && mCurrentBlock < mNumBlocks)
+ {
+ // Anything left in the current block?
+ if(mPositionInCurrentBlock < mCurrentBlockClearSize)
+ {
+ // Copy data out of this buffer
+ int s = mCurrentBlockClearSize - mPositionInCurrentBlock;
+ if(s > bytesToRead) s = bytesToRead; // limit to requested data
+
+ // Copy
+ ::memcpy(output, mpClearData + mPositionInCurrentBlock, s);
+
+ // Update positions
+ output += s;
+ mPositionInCurrentBlock += s;
+ bytesToRead -= s;
+ }
+
+ // Need to get some more data?
+ if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize)
+ {
+ // Number of next block
+ ++mCurrentBlock;
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // Stop now!
+ break;
+ }
+
+ // Get the size from the block index
+ const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
+ int32_t encodedSize = ntoh64(entry[mCurrentBlock].mEncodedSize);
+ if(encodedSize <= 0)
+ {
+ // The caller is attempting to decode a file which is the direct result of a diff
+ // operation, and so does not contain all the data.
+ // It needs to be combined with the previous version first.
+ THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining)
+ }
+
+ // Load in next block
+ if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Decode the data
+ mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize);
+
+ // Calculate IV for this entry
+ uint64_t iv = mEntryIVBase;
+ iv += mCurrentBlock;
+ // Convert to network byte order before encrypting with it, so that restores work on
+ // platforms with different endiannesses.
+ iv = hton64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Make sure this is the right size
+ if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
+ {
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ if(!mIsOldVersion)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the
+ // IV for the encrypted section. Try again, with the thing the other way round
+ iv = box_swap64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+ if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ else
+ {
+ // Warn and log this issue
+ if(!sWarnedAboutBackwardsCompatiblity)
+ {
+ ::printf("WARNING: Decoded one or more files using backwards compatibility mode for block index.\n");
+ ::syslog(LOG_ERR, "WARNING: Decoded one or more files using backwards compatibility mode for block index.\n");
+ sWarnedAboutBackwardsCompatiblity = true;
+ }
+ }
+#else
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+#endif
+ }
+
+ // Check the digest
+ MD5Digest md5;
+ md5.Add(mpClearData, mCurrentBlockClearSize);
+ md5.Finish();
+ if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum))
+ {
+ THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck)
+ }
+
+ // Set vars to say what's happening
+ mPositionInCurrentBlock = 0;
+ }
+ }
+
+ ASSERT(bytesToRead >= 0);
+ ASSERT(bytesToRead <= NBytes);
+
+ return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::IsSymLink()
+// Purpose: Is the unencoded file actually a symlink?
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::IsSymLink()
+{
+ // First, check in with the attributes
+ if(!mAttributes.IsSymLink())
+ {
+ return false;
+ }
+
+ // So the attributes think it is a symlink.
+ // Consistency check...
+ if(mNumBlocks != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Write(const void *, int)
+// Purpose: As interface. Throws exception, as you can't write to this stream.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::StreamDataLeft()
+// Purpose: As interface. Any data left?
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::StreamDataLeft()
+{
+ return mCurrentBlock < mNumBlocks;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::StreamClosed()
+// Purpose: As interface. Always returns true, no writing allowed.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::StreamClosed()
+{
+ // Can't write to this stream!
+ return true;
+}
+
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::SetBlowfishKey(const void *, int)
+// Purpose: Static. Sets the key to use for encryption and decryption.
+// Created: 7/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength)
+{
+ // IVs set later
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+
+ sBlowfishEncryptBlockEntry.Reset();
+ sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
+ sBlowfishEncryptBlockEntry.UsePadding(false);
+ sBlowfishDecryptBlockEntry.Reset();
+ sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
+ sBlowfishDecryptBlockEntry.UsePadding(false);
+}
+
+
+#ifndef PLATFORM_OLD_OPENSSL
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::SetAESKey(const void *, int)
+// Purpose: Sets the AES key to use for file data encryption. Will select AES as
+// the cipher to use when encrypting.
+// Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength)
+{
+ // Setup context
+ sAESEncrypt.Reset();
+ sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sAESDecrypt.Reset();
+ sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
+
+ // Set encryption to use this key, instead of the "default" blowfish key
+ spEncrypt = &sAESEncrypt;
+ sEncryptCipherType = HEADER_AES_ENCODING;
+}
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int)
+// Purpose: The maximum output size of a block, given the chunk size
+// Created: 7/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize)
+{
+ // Calculate... the maximum size of output by first the largest it could be after compression,
+ // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck
+ // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing
+ // an problem where a rather non-compressable file didn't fit in a block buffer.)
+ return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1
+ + sBlowfishEncrypt.GetIVLength() + 128;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &)
+// Purpose: Encodes a chunk (encryption, possible compressed beforehand)
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput)
+{
+ ASSERT(spEncrypt != 0);
+
+ // Check there's some space in the output block
+ if(rOutput.mBufferSize < 256)
+ {
+ rOutput.Reallocate(256);
+ }
+
+ // Check alignment of the block
+ ASSERT((((uint32_t)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+
+ // Want to compress it?
+ bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE);
+
+ // Build header
+ uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT;
+ if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED;
+
+ // Store header
+ rOutput.mpBuffer[0] = header;
+ int outOffset = 1;
+
+ // Setup cipher, and store the IV
+ int ivLen = 0;
+ const void *iv = spEncrypt->SetRandomIV(ivLen);
+ ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen);
+ outOffset += ivLen;
+
+ // Start encryption process
+ spEncrypt->Begin();
+
+ #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \
+ { \
+ if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \
+ { \
+ rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \
+ } \
+ }
+
+ // Encode the chunk
+ if(compressChunk)
+ {
+ // buffer to compress into
+ uint8_t buffer[2048];
+
+ // Set compressor with all the chunk as an input
+ Compress<true> compress;
+ compress.Input(Chunk, ChunkSize);
+ compress.FinishInput();
+
+ // Get and encrypt output
+ while(!compress.OutputHasFinished())
+ {
+ int s = compress.Output(buffer, sizeof(buffer));
+ if(s > 0)
+ {
+ ENCODECHUNK_CHECK_SPACE(s)
+ outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s);
+ }
+ else
+ {
+ // Should never happen, as we put all the input in in one go.
+ // So if this happens, it means there's a logical problem somewhere
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+ }
+ ENCODECHUNK_CHECK_SPACE(16)
+ outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
+ }
+ else
+ {
+ // Straight encryption
+ ENCODECHUNK_CHECK_SPACE(ChunkSize)
+ outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize);
+ ENCODECHUNK_CHECK_SPACE(16)
+ outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
+ }
+
+ ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check
+
+ return outOffset;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int)
+// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find
+// the extra output buffer size needed before calling.
+// See notes in EncodeChunk() for notes re alignment of the
+// encoded data.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize)
+{
+ // Check alignment of the encoded block
+ ASSERT((((uint32_t)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+
+ // First check
+ if(EncodedSize < 1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
+ }
+
+ const uint8_t *input = (uint8_t*)Encoded;
+
+ // Get header, make checks, etc
+ uint8_t header = input[0];
+ bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED;
+ uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT);
+ if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING)
+ {
+ THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding)
+ }
+
+#ifndef PLATFORM_OLD_OPENSSL
+ // Choose cipher
+ CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt);
+#else
+ // AES not supported with this version of OpenSSL
+ if(encodingType == HEADER_AES_ENCODING)
+ {
+ THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL)
+ }
+ CipherContext &cipher(sBlowfishDecrypt);
+#endif
+
+ // Check enough space for header, an IV and one byte of input
+ int ivLen = cipher.GetIVLength();
+ if(EncodedSize < (1 + ivLen + 1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
+ }
+
+ // Set IV in decrypt context, and start
+ cipher.SetIV(input + 1);
+ cipher.Begin();
+
+ // Setup vars for code
+ int inOffset = 1 + ivLen;
+ uint8_t *output = (uint8_t*)Output;
+ int outOffset = 0;
+
+ // Do action
+ if(chunkCompressed)
+ {
+ // Do things in chunks
+ uint8_t buffer[2048];
+ int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer));
+
+ // Decompressor
+ Compress<false> decompress;
+
+ while(inOffset < EncodedSize)
+ {
+ // Decrypt a block
+ int bl = inputBlockLen;
+ if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long
+ int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl);
+ inOffset += bl;
+
+ // Decompress the decrypted data
+ if(s > 0)
+ {
+ decompress.Input(buffer, s);
+ int os = 0;
+ do
+ {
+ os = decompress.Output(output + outOffset, OutputSize - outOffset);
+ outOffset += os;
+ } while(os > 0);
+
+ // Check that there's space left in the output buffer -- there always should be
+ if(outOffset >= OutputSize)
+ {
+ THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
+ }
+ }
+ }
+
+ // Get any compressed data remaining in the cipher context and compression
+ int s = cipher.Final(buffer, sizeof(buffer));
+ decompress.Input(buffer, s);
+ decompress.FinishInput();
+ while(!decompress.OutputHasFinished())
+ {
+ int os = decompress.Output(output + outOffset, OutputSize - outOffset);
+ outOffset += os;
+
+ // Check that there's space left in the output buffer -- there always should be
+ if(outOffset >= OutputSize)
+ {
+ THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
+ }
+ }
+ }
+ else
+ {
+ // Easy decryption
+ outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset);
+ outOffset += cipher.Final(output + outOffset, OutputSize - outOffset);
+ }
+
+ return outOffset;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool)
+// Purpose: Returns a stream which gives a Stream order version of the encoded file.
+// If TakeOwnership == true, then the input stream will be deleted when the
+// returned stream is deleted.
+// The input stream must be seekable.
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership)
+{
+ ASSERT(pStream != 0);
+
+ // Get the size of the file
+ int64_t fileSize = pStream->BytesLeftToRead();
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Read the header
+ int bytesRead = 0;
+ file_StreamFormat hdr;
+ bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead);
+
+ // Seek backwards to put the file pointer back where it was before we started this
+ pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative);
+
+ // Check we got a block
+ if(!readBlock)
+ {
+ // Couldn't read header -- assume file bad
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Get number of blocks
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+
+ // Calculate where the block index will be, check it's reasonable
+ int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+ int64_t blockIndexLoc = fileSize - blockIndexSize;
+ if(blockIndexLoc < 0)
+ {
+ // Doesn't look good!
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Build a reordered stream
+ std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership));
+
+ // Set it up...
+ ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get()));
+ int component = rreordered.AddComponent(pStream);
+ // Send out the block index
+ rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc);
+ // And then the rest of the file
+ rreordered.AddBlock(component, blockIndexLoc, true, 0);
+
+ return reordered;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ResetStats()
+// Purpose: Reset the gathered statistics
+// Created: 20/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::ResetStats()
+{
+ msStats.mBytesInEncodedFiles = 0;
+ msStats.mBytesAlreadyOnServer = 0;
+ msStats.mTotalFileStreamSize = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &)
+// Purpose: Compares the contents of a file against the checksums contained in the
+// block index. Returns true if the checksums match, meaning the file is
+// extremely likely to match the original. Will always consume the entire index.
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout)
+{
+ // is it a symlink?
+ bool sourceIsSymlink = false;
+ {
+ struct stat st;
+ if(::lstat(Filename, &st) == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ if((st.st_mode & S_IFMT) == S_IFLNK)
+ {
+ sourceIsSymlink = true;
+ }
+ }
+
+ // Open file, if it's not a symlink
+ std::auto_ptr<FileStream> in;
+ if(!sourceIsSymlink)
+ {
+ in.reset(new FileStream(Filename));
+ }
+
+ // Read header
+ file_BlockIndexHeader hdr;
+ if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic
+ if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0);
+#endif
+
+ // Get basic information
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+ uint64_t entryIVBase = ntoh64(hdr.mEntryIVBase);
+
+ //TODO: Verify that these sizes look reasonable
+
+ // setup
+ void *data = 0;
+ int32_t dataSize = -1;
+ bool matches = true;
+ int64_t totalSizeInBlockIndex = 0;
+
+ try
+ {
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read an entry from the stream
+ file_BlockIndexEntry entry;
+ if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read entry
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Calculate IV for this entry
+ uint64_t iv = entryIVBase;
+ iv += b;
+ iv = hton64(iv);
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ if(isOldVersion)
+ {
+ // Reverse the IV for compatibility
+ iv = box_swap64(iv);
+ }
+#endif
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry.mEnEnc, sizeof(entry.mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Size of block
+ int32_t blockClearSize = ntohl(entryEnc.mSize);
+ if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ totalSizeInBlockIndex += blockClearSize;
+
+ // Make sure there's enough memory allocated to load the block in
+ if(dataSize < blockClearSize)
+ {
+ // Too small, free the block if it's already allocated
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+ // Allocate a block
+ data = ::malloc(blockClearSize + 128);
+ if(data == 0)
+ {
+ throw std::bad_alloc();
+ }
+ dataSize = blockClearSize + 128;
+ }
+
+ // Load in the block from the file, if it's not a symlink
+ if(!sourceIsSymlink)
+ {
+ if(in->Read(data, blockClearSize) != blockClearSize)
+ {
+ // Not enough data left in the file, can't possibly match
+ matches = false;
+ }
+ else
+ {
+ // Check the checksum
+ MD5Digest md5;
+ md5.Add(data, blockClearSize);
+ md5.Finish();
+ if(!md5.DigestMatches(entryEnc.mStrongChecksum))
+ {
+ // Checksum didn't match
+ matches = false;
+ }
+ }
+ }
+
+ // Keep on going regardless, to make sure the entire block index stream is read
+ // -- must always be consistent about what happens with the stream.
+ }
+ }
+ catch(...)
+ {
+ // clean up in case of errors
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+ throw;
+ }
+
+ // free block
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+
+ // Check for data left over if it's not a symlink
+ if(!sourceIsSymlink)
+ {
+ // Anything left to read in the file?
+ if(in->BytesLeftToRead() != 0)
+ {
+ // File has extra data at the end
+ matches = false;
+ }
+ }
+
+ // Symlinks must have zero size on server
+ if(sourceIsSymlink)
+ {
+ matches = (totalSizeInBlockIndex == 0);
+ }
+
+ return matches;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer()
+// Purpose: Constructor
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::EncodingBuffer::EncodingBuffer()
+ : mpBuffer(0),
+ mBufferSize(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer()
+// Purpose: Destructor
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::EncodingBuffer::~EncodingBuffer()
+{
+ if(mpBuffer != 0)
+ {
+ BackupStoreFile::CodingChunkFree(mpBuffer);
+ mpBuffer = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::Allocate(int)
+// Purpose: Do initial allocation of block
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::EncodingBuffer::Allocate(int Size)
+{
+ ASSERT(mpBuffer == 0);
+ uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ mpBuffer = buffer;
+ mBufferSize = Size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::Reallocate(int)
+// Purpose: Reallocate the block. Try not to call this, it has to copy
+// the entire contents as the block can't be reallocated straight.
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize)
+{
+ TRACE2("Reallocating EncodingBuffer from %d to %d\n", mBufferSize, NewSize);
+ ASSERT(mpBuffer != 0);
+ uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ // Copy data
+ ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize);
+
+ // Free old
+ BackupStoreFile::CodingChunkFree(mpBuffer);
+
+ // Store new buffer
+ mpBuffer = buffer;
+ mBufferSize = NewSize;
+}
+
+
diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h
new file mode 100755
index 00000000..0274107b
--- /dev/null
+++ b/lib/backupclient/BackupStoreFile.h
@@ -0,0 +1,194 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFile.h
+// Purpose: Utils for manipulating files
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILE__H
+#define BACKUPSTOREFILE__H
+
+#include "IOStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreFilename.h"
+
+#include <memory>
+
+typedef struct
+{
+ int64_t mBytesInEncodedFiles;
+ int64_t mBytesAlreadyOnServer;
+ int64_t mTotalFileStreamSize;
+} BackupStoreFileStats;
+
+// Uncomment to disable backwards compatibility
+//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+
+
+// Output buffer to EncodeChunk and input data to DecodeChunk must
+// have specific alignment, see function comments.
+#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16
+#define BACKUPSTOREFILE_CODING_OFFSET 15
+
+// Have some memory allocation commands, note closing "Off" at end of file.
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFile
+// Purpose: Class to hold together utils for maniplating files.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class BackupStoreFile
+{
+public:
+ class DecodedStream : public IOStream
+ {
+ friend class BackupStoreFile;
+ private:
+ DecodedStream(IOStream &rEncodedFile, int Timeout);
+ DecodedStream(const DecodedStream &); // not allowed
+ DecodedStream &operator=(const DecodedStream &); // not allowed
+ public:
+ ~DecodedStream();
+
+ // Stream functions
+ virtual int Read(void *pBuffer, int NBytes, int Timeout);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+ // Accessor functions
+ const BackupClientFileAttributes &GetAttributes() {return mAttributes;}
+ const BackupStoreFilename &GetFilename() {return mFilename;}
+ int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests
+
+ bool IsSymLink();
+
+ private:
+ void Setup(const BackupClientFileAttributes *pAlterativeAttr);
+ void ReadBlockIndex(bool MagicAlreadyRead);
+
+ private:
+ IOStream &mrEncodedFile;
+ int mTimeout;
+ BackupClientFileAttributes mAttributes;
+ BackupStoreFilename mFilename;
+ int64_t mNumBlocks;
+ void *mpBlockIndex;
+ uint8_t *mpEncodedData;
+ uint8_t *mpClearData;
+ int mClearDataSize;
+ int mCurrentBlock;
+ int mCurrentBlockClearSize;
+ int mPositionInCurrentBlock;
+ uint64_t mEntryIVBase;
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool mIsOldVersion;
+#endif
+ };
+
+
+ // Main interface
+ static std::auto_ptr<IOStream> EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0);
+ static std::auto_ptr<IOStream> EncodeFileDiff(const char *Filename, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
+ int Timeout, int64_t *pModificationTime = 0, bool *pIsCompletelyDifferent = 0);
+ static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0);
+ static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut);
+ static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut);
+ static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0);
+ static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
+ static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
+ static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout);
+ static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false);
+
+ // Stream manipulation
+ static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership);
+ static void MoveStreamPositionToBlockIndex(IOStream &rStream);
+
+ // Crypto setup
+ static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength);
+#ifndef PLATFORM_OLD_OPENSSL
+ static void SetAESKey(const void *pKey, int KeyLength);
+#endif
+
+ // Allocation of properly aligning chunks for decoding and encoding chunks
+ inline static void *CodingChunkAlloc(int Size)
+ {
+ uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3));
+ if(a == 0) return 0;
+ // Align to main block size
+ ASSERT(sizeof(uint32_t) == sizeof(void*)); // make sure casting the right pointer size, will need to fix on platforms with 64 bit pointers
+ uint32_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE - (((uint32_t)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE);
+ uint8_t *b = (a + adjustment);
+ // Store adjustment
+ *b = (uint8_t)adjustment;
+ // Return offset
+ return b + BACKUPSTOREFILE_CODING_OFFSET;
+ }
+ inline static void CodingChunkFree(void *Block)
+ {
+ // Check alignment is as expected
+ ASSERT(sizeof(uint32_t) == sizeof(void*)); // make sure casting the right pointer size, will need to fix on platforms with 64 bit pointers
+ ASSERT((((uint32_t)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+ uint8_t *a = (uint8_t*)Block;
+ a -= BACKUPSTOREFILE_CODING_OFFSET;
+ // Adjust downwards...
+ a -= *a;
+ free(a);
+ }
+
+ // Limits
+ static void SetMaximumDiffingTime(int Seconds);
+
+ // Building blocks
+ class EncodingBuffer
+ {
+ public:
+ EncodingBuffer();
+ ~EncodingBuffer();
+ private:
+ // No copying
+ EncodingBuffer(const EncodingBuffer &);
+ EncodingBuffer &operator=(const EncodingBuffer &);
+ public:
+ void Allocate(int Size);
+ void Reallocate(int NewSize);
+
+ uint8_t *mpBuffer;
+ int mBufferSize;
+ };
+ static int MaxBlockSizeForChunkSize(int ChunkSize);
+ static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput);
+
+ // Caller should know how big the output size is, but also allocate a bit more memory to cover various
+ // overheads allowed for in checks
+ static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize)
+ {
+ // Plenty big enough
+ return KnownChunkSize + 256;
+ }
+ static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize);
+
+ // Statisitics, not designed to be completely reliable
+ static void ResetStats();
+ static BackupStoreFileStats msStats;
+
+ // For debug
+#ifndef NDEBUG
+ static bool TraceDetailsOfDiffProcess;
+#endif
+
+ // For decoding encoded files
+ static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile);
+};
+
+#include "MemLeakFindOff.h"
+
+#endif // BACKUPSTOREFILE__H
+
diff --git a/lib/backupclient/BackupStoreFileCmbDiff.cpp b/lib/backupclient/BackupStoreFileCmbDiff.cpp
new file mode 100644
index 00000000..039b00eb
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCmbDiff.cpp
@@ -0,0 +1,326 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCmbDiff.cpp
+// Purpose: Combine two diffs together
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <stdlib.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CombineDiffs(IOStream &, IOStream &, IOStream &rOut)
+// Purpose: Given two diffs, combine them into a single diff, to produce a diff
+// which, combined with the original file, creates the result of applying
+// rDiff, then rDiff2. Two opens of rDiff2 are required
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut)
+{
+ // Skip header of first diff, record where the data starts, and skip to the index
+ int64_t diff1DataStarts = 0;
+ {
+ // Read the header for the From file
+ file_StreamFormat diff1Hdr;
+ if(!rDiff1.ReadFullBuffer(&diff1Hdr, sizeof(diff1Hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diff1Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Skip over the filename and attributes of the From file
+ // BLOCK
+ {
+ BackupStoreFilename filename2;
+ filename2.ReadFromStream(rDiff1, IOStream::TimeOutInfinite);
+ int32_t size_s;
+ if(!rDiff1.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+ }
+ int size = ntohl(size_s);
+ // Skip forward the size
+ rDiff1.Seek(size, IOStream::SeekType_Relative);
+ }
+ // Record position
+ diff1DataStarts = rDiff1.GetPosition();
+ // Skip to index
+ rDiff1.Seek(0 - (((ntoh64(diff1Hdr.mNumBlocks)) * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+ }
+
+ // Read the index of the first diff
+ // Header first
+ file_BlockIndexHeader diff1IdxHdr;
+ if(!rDiff1.ReadFullBuffer(&diff1IdxHdr, sizeof(diff1IdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(diff1IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ int64_t diff1NumBlocks = ntoh64(diff1IdxHdr.mNumBlocks);
+ // Allocate some memory
+ int64_t *diff1BlockStartPositions = (int64_t*)::malloc((diff1NumBlocks + 1) * sizeof(int64_t));
+ if(diff1BlockStartPositions == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Buffer data
+ void *buffer = 0;
+ int bufferSize = 0;
+
+ try
+ {
+ // Then the entries:
+ // For each entry, want to know if it's in the file, and if so, how big it is.
+ // We'll store this as an array of file positions in the file, with an additioal
+ // entry on the end so that we can work out the length of the last block.
+ // If an entry isn't in the file, then store 0 - (position in other file).
+ int64_t diff1Position = diff1DataStarts;
+ for(int64_t b = 0; b < diff1NumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff1.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Where's the block?
+ int64_t blockEn = ntoh64(e.mEncodedSize);
+ if(blockEn <= 0)
+ {
+ // Just store the negated block number
+ diff1BlockStartPositions[b] = blockEn;
+ }
+ else
+ {
+ // Block is present in this file
+ diff1BlockStartPositions[b] = diff1Position;
+ diff1Position += blockEn;
+ }
+ }
+
+ // Finish off the list, so the last entry can have it's size calcuated.
+ diff1BlockStartPositions[diff1NumBlocks] = diff1Position;
+
+ // Now read the second diff's header, copying it to the out file
+ file_StreamFormat diff2Hdr;
+ if(!rDiff2.ReadFullBuffer(&diff2Hdr, sizeof(diff2Hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diff2Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Copy
+ rOut.Write(&diff2Hdr, sizeof(diff2Hdr));
+ // Copy over filename and attributes
+ // BLOCK
+ {
+ BackupStoreFilename filename;
+ filename.ReadFromStream(rDiff2, IOStream::TimeOutInfinite);
+ filename.WriteToStream(rOut);
+ StreamableMemBlock attr;
+ attr.ReadFromStream(rDiff2, IOStream::TimeOutInfinite);
+ attr.WriteToStream(rOut);
+ }
+
+ // Get to the index of rDiff2b, and read the header
+ MoveStreamPositionToBlockIndex(rDiff2b);
+ file_BlockIndexHeader diff2IdxHdr;
+ if(!rDiff2b.ReadFullBuffer(&diff2IdxHdr, sizeof(diff2IdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(diff2IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ int64_t diff2NumBlocks = ntoh64(diff2IdxHdr.mNumBlocks);
+ int64_t diff2IndexEntriesStart = rDiff2b.GetPosition();
+
+ // Then read all the entries
+ int64_t diff2FilePosition = rDiff2.GetPosition();
+ for(int64_t b = 0; b < diff2NumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // What do to next about copying data
+ bool copyBlock = false;
+ int copySize = 0;
+ int64_t copyFrom = 0;
+ bool fromFileDiff1 = false;
+
+ // Where's the block?
+ int64_t blockEn = ntoh64(e.mEncodedSize);
+ if(blockEn > 0)
+ {
+ // Block is present in this file -- copy to out
+ copyBlock = true;
+ copyFrom = diff2FilePosition;
+ copySize = (int)blockEn;
+
+ // Move pointer onwards
+ diff2FilePosition += blockEn;
+ }
+ else
+ {
+ // Block isn't present here -- is it present in the old one?
+ int64_t blockIndex = 0 - blockEn;
+ if(blockIndex < 0 || blockIndex > diff1NumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ if(diff1BlockStartPositions[blockIndex] > 0)
+ {
+ // Block is in the old diff file, copy it across
+ copyBlock = true;
+ copyFrom = diff1BlockStartPositions[blockIndex];
+ int nb = blockIndex + 1;
+ while(diff1BlockStartPositions[nb] <= 0)
+ {
+ // This is safe, because the last entry will terminate it properly!
+ ++nb;
+ ASSERT(nb <= diff1NumBlocks);
+ }
+ copySize = diff1BlockStartPositions[nb] - copyFrom;
+ fromFileDiff1 = true;
+ }
+ }
+ //TRACE4("%d %d %lld %d\n", copyBlock, copySize, copyFrom, fromFileDiff1);
+
+ // Copy data to the output file?
+ if(copyBlock)
+ {
+ // Allocate enough space
+ if(bufferSize < copySize || buffer == 0)
+ {
+ // Free old block
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ bufferSize = 0;
+ }
+ // Allocate new block
+ buffer = ::malloc(copySize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ bufferSize = copySize;
+ }
+ ASSERT(bufferSize >= copySize);
+
+ // Load in the data
+ if(fromFileDiff1)
+ {
+ rDiff1.Seek(copyFrom, IOStream::SeekType_Absolute);
+ if(!rDiff1.ReadFullBuffer(buffer, copySize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ }
+ else
+ {
+ rDiff2.Seek(copyFrom, IOStream::SeekType_Absolute);
+ if(!rDiff2.ReadFullBuffer(buffer, copySize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ }
+ // Write out data
+ rOut.Write(buffer, copySize);
+ }
+ }
+
+ // Write the modified header
+ diff2IdxHdr.mOtherFileID = diff1IdxHdr.mOtherFileID;
+ rOut.Write(&diff2IdxHdr, sizeof(diff2IdxHdr));
+
+ // Then we'll write out the index, reading the data again
+ rDiff2b.Seek(diff2IndexEntriesStart, IOStream::SeekType_Absolute);
+ for(int64_t b = 0; b < diff2NumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Where's the block?
+ int64_t blockEn = ntoh64(e.mEncodedSize);
+
+ // If it's not in this file, it needs modification...
+ if(blockEn <= 0)
+ {
+ int64_t blockIndex = 0 - blockEn;
+ // In another file. Need to translate this against the other diff
+ if(diff1BlockStartPositions[blockIndex] > 0)
+ {
+ // Block is in the first diff file, stick in size
+ int nb = blockIndex + 1;
+ while(diff1BlockStartPositions[nb] <= 0)
+ {
+ // This is safe, because the last entry will terminate it properly!
+ ++nb;
+ ASSERT(nb <= diff1NumBlocks);
+ }
+ int64_t size = diff1BlockStartPositions[nb] - diff1BlockStartPositions[blockIndex];
+ e.mEncodedSize = hton64(size);
+ }
+ else
+ {
+ // Block in the original file, use translated value
+ e.mEncodedSize = hton64(diff1BlockStartPositions[blockIndex]);
+ }
+ }
+
+ // Write entry
+ rOut.Write(&e, sizeof(e));
+ }
+ }
+ catch(...)
+ {
+ // clean up
+ ::free(diff1BlockStartPositions);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+ throw;
+ }
+
+ // Clean up allocated memory
+ ::free(diff1BlockStartPositions);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+}
+
diff --git a/lib/backupclient/BackupStoreFileCmbIdx.cpp b/lib/backupclient/BackupStoreFileCmbIdx.cpp
new file mode 100644
index 00000000..253001c2
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCmbIdx.cpp
@@ -0,0 +1,324 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCmbIdx.cpp
+// Purpose: Combine indicies of a delta file and the file it's a diff from.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <string.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+// Hide from outside world
+namespace
+{
+
+class BSFCombinedIndexStream : public IOStream
+{
+public:
+ BSFCombinedIndexStream(IOStream *pDiff);
+ ~BSFCombinedIndexStream();
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+ virtual void Initialise(IOStream &rFrom);
+
+private:
+ IOStream *mpDiff;
+ bool mIsInitialised;
+ bool mHeaderWritten;
+ file_BlockIndexHeader mHeader;
+ int64_t mNumEntriesToGo;
+ int64_t mNumEntriesInFromFile;
+ int64_t *mFromBlockSizes; // NOTE: Entries in network byte order
+};
+
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CombineFileIndices(IOStream &, IOStream &, bool)
+// Purpose: Given a diff file and the file it's a diff from, return a stream from which
+// can be read the index of the combined file, without actually combining them.
+// The stream of the diff must have a lifetime greater than or equal to the
+// lifetime of the returned stream object. The full "from" file stream
+// only needs to exist during the actual function call.
+// If you pass in dodgy files which aren't related, then you will either
+// get an error or bad results. So don't do that.
+// If DiffIsIndexOnly is true, then rDiff is assumed to be a stream positioned
+// at the beginning of the block index. Similarly for FromIsIndexOnly.
+// WARNING: Reads of the returned streams with buffer sizes less than 64 bytes
+// will not return any data.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly, bool FromIsIndexOnly)
+{
+ // Reposition file pointers?
+ if(!DiffIsIndexOnly)
+ {
+ MoveStreamPositionToBlockIndex(rDiff);
+ }
+ if(!FromIsIndexOnly)
+ {
+ MoveStreamPositionToBlockIndex(rFrom);
+ }
+
+ // Create object
+ std::auto_ptr<IOStream> stream(new BSFCombinedIndexStream(&rDiff));
+
+ // Initialise it
+ ((BSFCombinedIndexStream *)stream.get())->Initialise(rFrom);
+
+ // And return the stream
+ return stream;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::BSFCombinedIndexStream()
+// Purpose: Private class. Constructor.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+BSFCombinedIndexStream::BSFCombinedIndexStream(IOStream *pDiff)
+ : mpDiff(pDiff),
+ mIsInitialised(false),
+ mHeaderWritten(false),
+ mNumEntriesToGo(0),
+ mNumEntriesInFromFile(0),
+ mFromBlockSizes(0)
+{
+ ASSERT(mpDiff != 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::~BSFCombinedIndexStream()
+// Purpose: Private class. Destructor.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+BSFCombinedIndexStream::~BSFCombinedIndexStream()
+{
+ if(mFromBlockSizes != 0)
+ {
+ ::free(mFromBlockSizes);
+ mFromBlockSizes = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::Initialise(IOStream &)
+// Purpose: Private class. Initalise from the streams (diff passed in constructor).
+// Both streams must have file pointer positioned at the block index.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+void BSFCombinedIndexStream::Initialise(IOStream &rFrom)
+{
+ // Paranoia is good.
+ if(mIsInitialised)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Look at the diff file: Read in the header
+ if(!mpDiff->ReadFullBuffer(&mHeader, sizeof(mHeader), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(mHeader.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Read relevant data.
+ mNumEntriesToGo = ntoh64(mHeader.mNumBlocks);
+
+ // Adjust a bit to reflect the fact it's no longer a diff
+ mHeader.mOtherFileID = hton64(0);
+
+ // Now look at the from file: Read header
+ file_BlockIndexHeader fromHdr;
+ if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Then... allocate memory for the list of sizes
+ mNumEntriesInFromFile = ntoh64(fromHdr.mNumBlocks);
+ mFromBlockSizes = (int64_t*)::malloc(mNumEntriesInFromFile * sizeof(int64_t));
+ if(mFromBlockSizes == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // And read them all in!
+ for(int64_t b = 0; b < mNumEntriesInFromFile; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check that the from file isn't a delta in itself
+ if(ntoh64(e.mEncodedSize) <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+ }
+
+ // Store size (in network byte order)
+ mFromBlockSizes[b] = e.mEncodedSize;
+ }
+
+ // Flag as initialised
+ mIsInitialised = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::Read(void *, int, int)
+// Purpose: Private class. As interface.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Paranoia is good.
+ if(!mIsInitialised || mFromBlockSizes == 0 || mpDiff == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ int written = 0;
+
+ // Header output yet?
+ if(!mHeaderWritten)
+ {
+ // Enough space?
+ if(NBytes < (int)sizeof(mHeader)) return 0;
+
+ // Copy in
+ ::memcpy(pBuffer, &mHeader, sizeof(mHeader));
+ NBytes -= sizeof(mHeader);
+ written += sizeof(mHeader);
+
+ // Flag it's done
+ mHeaderWritten = true;
+ }
+
+ // How many entries can be written?
+ int entriesToWrite = NBytes / sizeof(file_BlockIndexEntry);
+ if(entriesToWrite > mNumEntriesToGo)
+ {
+ entriesToWrite = mNumEntriesToGo;
+ }
+
+ // Setup ready to go
+ file_BlockIndexEntry *poutput = (file_BlockIndexEntry*)(((uint8_t*)pBuffer) + written);
+
+ // Write entries
+ for(int b = 0; b < entriesToWrite; ++b)
+ {
+ if(!mpDiff->ReadFullBuffer(&(poutput[b]), sizeof(file_BlockIndexEntry), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Does this need adjusting?
+ int s = ntoh64(poutput[b].mEncodedSize);
+ if(s <= 0)
+ {
+ // A reference to a block in the from file
+ int block = 0 - s;
+ ASSERT(block >= 0);
+ if(block >= mNumEntriesInFromFile)
+ {
+ // That's not good, the block doesn't exist
+ THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+ }
+
+ // Adjust the entry in the buffer
+ poutput[b].mEncodedSize = mFromBlockSizes[block]; // stored in network byte order, no translation necessary
+ }
+ }
+
+ // Update written count
+ written += entriesToWrite * sizeof(file_BlockIndexEntry);
+ mNumEntriesToGo -= entriesToWrite;
+
+ return written;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::Write(const void *, int)
+// Purpose: Private class. As interface.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::StreamDataLeft()
+// Purpose: Private class. As interface
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+bool BSFCombinedIndexStream::StreamDataLeft()
+{
+ return (!mHeaderWritten) || (mNumEntriesToGo > 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::StreamClosed()
+// Purpose: Private class. As interface.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+bool BSFCombinedIndexStream::StreamClosed()
+{
+ return true; // doesn't do writing
+}
+
diff --git a/lib/backupclient/BackupStoreFileCombine.cpp b/lib/backupclient/BackupStoreFileCombine.cpp
new file mode 100755
index 00000000..562a32d9
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCombine.cpp
@@ -0,0 +1,410 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCombine.cpp
+// Purpose: File combining for BackupStoreFile
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+#include "FileStream.h"
+
+#include "MemLeakFindOn.h"
+
+typedef struct
+{
+ int64_t mFilePosition;
+} FromIndexEntry;
+
+static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries);
+static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut);
+static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut);
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CombineFile(IOStream &, IOStream &, IOStream &)
+// Purpose: Where rDiff is a store file which is incomplete as a result of a
+// diffing operation, rFrom is the file it is diffed from, and
+// rOut is the stream in which to place the result, the old file
+// and new file are combined into a file containing all the data.
+// rDiff2 is the same file as rDiff, opened again to get two
+// independent streams to the same file.
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut)
+{
+ // Read and copy the header.
+ file_StreamFormat hdr;
+ if(!rDiff.ReadFullBuffer(&hdr, sizeof(hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Copy
+ rOut.Write(&hdr, sizeof(hdr));
+ // Copy over filename and attributes
+ // BLOCK
+ {
+ BackupStoreFilename filename;
+ filename.ReadFromStream(rDiff, IOStream::TimeOutInfinite);
+ filename.WriteToStream(rOut);
+ StreamableMemBlock attr;
+ attr.ReadFromStream(rDiff, IOStream::TimeOutInfinite);
+ attr.WriteToStream(rOut);
+ }
+
+ // Read the header for the From file
+ file_StreamFormat fromHdr;
+ if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Skip over the filename and attributes of the From file
+ // BLOCK
+ {
+ BackupStoreFilename filename2;
+ filename2.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ int32_t size_s;
+ if(!rFrom.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+ }
+ int size = ntohl(size_s);
+ // Skip forward the size
+ rFrom.Seek(size, IOStream::SeekType_Relative);
+ }
+
+ // Allocate memory for the block index of the From file
+ int64_t fromNumBlocks = ntoh64(fromHdr.mNumBlocks);
+ // NOTE: An extra entry is required so that the length of the last block can be calculated
+ FromIndexEntry *pFromIndex = (FromIndexEntry*)::malloc((fromNumBlocks+1) * sizeof(FromIndexEntry));
+ if(pFromIndex == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ try
+ {
+ // Load the index from the From file, calculating the offsets in the
+ // file as we go along, and enforce that everything should be present.
+ LoadFromIndex(rFrom, pFromIndex, fromNumBlocks);
+
+ // Read in the block index of the Diff file in small chunks, and output data
+ // for each block, either from this file, or the other file.
+ int64_t diffNumBlocks = ntoh64(hdr.mNumBlocks);
+ CopyData(rDiff /* positioned at start of data */, rDiff2, diffNumBlocks, rFrom, pFromIndex, fromNumBlocks, rOut);
+
+ // Read in the block index again, and output the new block index, simply
+ // filling in the sizes of blocks from the old file.
+ WriteNewIndex(rDiff, diffNumBlocks, pFromIndex, fromNumBlocks, rOut);
+
+ // Free buffers
+ ::free(pFromIndex);
+ pFromIndex = 0;
+ }
+ catch(...)
+ {
+ // Clean up
+ if(pFromIndex != 0)
+ {
+ ::free(pFromIndex);
+ pFromIndex = 0;
+ }
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static LoadFromIndex(IOStream &, FromIndexEntry *, int64_t)
+// Purpose: Static. Load the index from the From file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries)
+{
+ ASSERT(pIndex != 0);
+ ASSERT(NumEntries >= 0);
+
+ // Get the starting point in the file
+ int64_t filePos = rFrom.GetPosition();
+
+ // Jump to the end of the file to read the index
+ rFrom.Seek(0 - ((NumEntries * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+
+ // Read block index header
+ file_BlockIndexHeader blkhdr;
+ if(!rFrom.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || (int64_t)ntoh64(blkhdr.mNumBlocks) != NumEntries)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // And then the block entries
+ for(int64_t b = 0; b < NumEntries; ++b)
+ {
+ // Read
+ file_BlockIndexEntry en;
+ if(!rFrom.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // Add to list
+ pIndex[b].mFilePosition = filePos;
+
+ // Encoded size?
+ int64_t encodedSize = ntoh64(en.mEncodedSize);
+ // Check that the block is actually there
+ if(encodedSize <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+ }
+
+ // Move file pointer on
+ filePos += encodedSize;
+ }
+
+ // Store the position in the very last entry, so the size of the last entry can be calculated
+ pIndex[NumEntries].mFilePosition = filePos;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static CopyData(IOStream &, IOStream &, int64_t, IOStream &, FromIndexEntry *, int64_t, IOStream &)
+// Purpose: Static. Copy data from the Diff and From file to the out file.
+// rDiffData is at beginning of data.
+// rDiffIndex at any position.
+// rFrom is at any position.
+// rOut is after the header, ready for data
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks,
+ IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut)
+{
+ // Jump to the end of the diff file to read the index
+ rDiffIndex.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+
+ // Read block index header
+ file_BlockIndexHeader diffBlkhdr;
+ if(!rDiffIndex.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || (int64_t)ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Record where the From file is
+ int64_t fromPos = rFrom.GetPosition();
+
+ // Buffer data
+ void *buffer = 0;
+ int bufferSize = 0;
+
+ try
+ {
+ // Read the blocks in!
+ for(int64_t b = 0; b < DiffNumBlocks; ++b)
+ {
+ // Read
+ file_BlockIndexEntry en;
+ if(!rDiffIndex.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // What's the size value stored in the entry
+ int64_t encodedSize = ntoh64(en.mEncodedSize);
+
+ // How much data will be read?
+ int32_t blockSize = 0;
+ if(encodedSize > 0)
+ {
+ // The block is actually in the diff file
+ blockSize = encodedSize;
+ }
+ else
+ {
+ // It's in the from file. First, check to see if it's valid
+ int64_t blockIdx = (0 - encodedSize);
+ if(blockIdx > FromNumBlocks)
+ {
+ // References a block which doesn't actually exist
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Calculate size. This operation is safe because of the extra entry at the end
+ blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition;
+ }
+ ASSERT(blockSize > 0);
+
+ // Make sure there's memory available to copy this
+ if(bufferSize < blockSize || buffer == 0)
+ {
+ // Free old block
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ bufferSize = 0;
+ }
+ // Allocate new block
+ buffer = ::malloc(blockSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ bufferSize = blockSize;
+ }
+ ASSERT(bufferSize >= blockSize);
+
+ // Load in data from one of the files
+ if(encodedSize > 0)
+ {
+ // Load from diff file
+ if(!rDiffData.ReadFullBuffer(buffer, blockSize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ }
+ else
+ {
+ // Locate and read the data from the from file
+ int64_t blockIdx = (0 - encodedSize);
+ // Seek if necessary
+ if(fromPos != pFromIndex[blockIdx].mFilePosition)
+ {
+ rFrom.Seek(pFromIndex[blockIdx].mFilePosition, IOStream::SeekType_Absolute);
+ fromPos = pFromIndex[blockIdx].mFilePosition;
+ }
+ // Read
+ if(!rFrom.ReadFullBuffer(buffer, blockSize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // Update fromPos to current position
+ fromPos += blockSize;
+ }
+
+ // Write data to out file
+ rOut.Write(buffer, blockSize);
+ }
+
+ // Free buffer, if allocated
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ }
+ }
+ catch(...)
+ {
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ }
+ throw;
+ }
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static WriteNewIndex(IOStream &, int64_t, FromIndexEntry *, int64_t, IOStream &)
+// Purpose: Write the index to the out file, just copying from the diff file and
+// adjusting the entries.
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut)
+{
+ // Jump to the end of the diff file to read the index
+ rDiff.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+
+ // Read block index header
+ file_BlockIndexHeader diffBlkhdr;
+ if(!rDiff.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || (int64_t)ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Write it out with a blanked out other file ID
+ diffBlkhdr.mOtherFileID = hton64(0);
+ rOut.Write(&diffBlkhdr, sizeof(diffBlkhdr));
+
+ // Rewrite the index
+ for(int64_t b = 0; b < DiffNumBlocks; ++b)
+ {
+ file_BlockIndexEntry en;
+ if(!rDiff.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // What's the size value stored in the entry
+ int64_t encodedSize = ntoh64(en.mEncodedSize);
+
+ // Need to adjust it?
+ if(encodedSize <= 0)
+ {
+ // This actually refers to a block in the from file. So rewrite this.
+ int64_t blockIdx = (0 - encodedSize);
+ if(blockIdx > FromNumBlocks)
+ {
+ // References a block which doesn't actually exist
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Calculate size. This operation is safe because of the extra entry at the end
+ int32_t blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition;
+ // Then replace entry
+ en.mEncodedSize = hton64(((uint64_t)blockSize));
+ }
+
+ // Write entry
+ rOut.Write(&en, sizeof(en));
+ }
+}
+
+
+
+
+
diff --git a/lib/backupclient/BackupStoreFileCryptVar.cpp b/lib/backupclient/BackupStoreFileCryptVar.cpp
new file mode 100755
index 00000000..eeed64e4
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCryptVar.cpp
@@ -0,0 +1,31 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCryptVar.cpp
+// Purpose: Cryptographic keys for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFileWire.h"
+
+#include "MemLeakFindOn.h"
+
+CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt;
+CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt;
+
+#ifndef PLATFORM_OLD_OPENSSL
+ CipherContext BackupStoreFileCryptVar::sAESEncrypt;
+ CipherContext BackupStoreFileCryptVar::sAESDecrypt;
+#endif
+
+// Default to blowfish
+CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt;
+uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING;
+
+CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry;
+CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry;
+
diff --git a/lib/backupclient/BackupStoreFileCryptVar.h b/lib/backupclient/BackupStoreFileCryptVar.h
new file mode 100755
index 00000000..00a34f71
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCryptVar.h
@@ -0,0 +1,39 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCryptVar.h
+// Purpose: Cryptographic keys for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILECRYPTVAR__H
+#define BACKUPSTOREFILECRYPTVAR__H
+
+#include "CipherContext.h"
+
+// Hide private static variables from the rest of the world by putting them
+// as static variables in a namespace.
+// -- don't put them as static class variables to avoid openssl/evp.h being
+// included all over the project.
+namespace BackupStoreFileCryptVar
+{
+ // Keys for the main file data
+ extern CipherContext sBlowfishEncrypt;
+ extern CipherContext sBlowfishDecrypt;
+ // Use AES when available
+#ifndef PLATFORM_OLD_OPENSSL
+ extern CipherContext sAESEncrypt;
+ extern CipherContext sAESDecrypt;
+#endif
+ // How encoding will be done
+ extern CipherContext *spEncrypt;
+ extern uint8_t sEncryptCipherType;
+
+ // Keys for the block indicies
+ extern CipherContext sBlowfishEncryptBlockEntry;
+ extern CipherContext sBlowfishDecryptBlockEntry;
+}
+
+#endif // BACKUPSTOREFILECRYPTVAR__H
+
diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp
new file mode 100755
index 00000000..b72ce328
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileDiff.cpp
@@ -0,0 +1,973 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileDiff.cpp
+// Purpose: Functions relating to diffing BackupStoreFiles
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <map>
+#include <signal.h>
+#include <sys/time.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreFileEncodeStream.h"
+#include "BackupStoreConstants.h"
+#include "FileStream.h"
+#include "RollingChecksum.h"
+#include "MD5Digest.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+using namespace BackupStoreFileCreation;
+
+// By default, don't trace out details of the diff as we go along -- would fill up logs significantly.
+// But it's useful for the test.
+#ifndef NDEBUG
+ bool BackupStoreFile::TraceDetailsOfDiffProcess = false;
+#endif
+
+static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis);
+static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]);
+static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]);
+static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable);
+static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, uint16_t Hash, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks);
+static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile);
+
+// Avoid running on too long with diffs
+static int sMaximumDiffTime = 10; // maximum time to spend diffing
+static bool sDiffTimedOut = false;
+static bool sSetTimerSignelHandler = false;
+static void TimerSignalHandler(int signal);
+static void StartDiffTimer();
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::SetMaximumDiffingTime(int)
+// Purpose: Sets the maximum time to spend diffing, in seconds. Time is
+// process virutal time.
+// Created: 19/3/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetMaximumDiffingTime(int Seconds)
+{
+ sMaximumDiffTime = Seconds;
+ TRACE1("Set maximum diffing time to %d seconds\n", Seconds);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &)
+// Purpose: Move the file pointer in this stream to just before the block index.
+// Assumes that the stream is at the beginning, seekable, and
+// reading from the stream is OK.
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream)
+{
+ // Size of file
+ int64_t fileSize = rStream.BytesLeftToRead();
+
+ // Get header
+ file_StreamFormat hdr;
+
+ // Read the header
+ if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Work out where the index is
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+ int64_t blockHeaderPosFromEnd = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+
+ // Sanity check
+ if(blockHeaderPosFromEnd > static_cast<int64_t>(fileSize - sizeof(file_StreamFormat)))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Seek to that position
+ rStream.Seek(0 - blockHeaderPosFromEnd, IOStream::SeekType_End);
+
+ // Done. Stream now in right position (as long as the file is formatted correctly)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *)
+// Purpose: Similar to EncodeFile, but takes the object ID of the file it's
+// diffing from, and the index of the blocks in a stream. It'll then
+// calculate which blocks can be reused from that old file.
+// The timeout is the timeout value for reading the diff block index.
+// If pIsCompletelyDifferent != 0, it will be set to true if the
+// the two files are completely different (do not share any block), false otherwise.
+//
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff(const char *Filename, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
+ int Timeout, int64_t *pModificationTime, bool *pIsCompletelyDifferent)
+{
+ // Is it a symlink?
+ {
+ struct stat st;
+ if(::lstat(Filename, &st) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ if((st.st_mode & S_IFLNK) == S_IFLNK)
+ {
+ // Don't do diffs for symlinks
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = true;
+ }
+ return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime);
+ }
+ }
+
+ // Load in the blocks
+ BlocksAvailableEntry *pindex = 0;
+ int64_t blocksInIndex = 0;
+ bool canDiffFromThis = false;
+ LoadIndex(rDiffFromBlockIndex, DiffFromObjectID, &pindex, blocksInIndex, Timeout, canDiffFromThis);
+ //TRACE1("Diff: Blocks in index: %lld\n", blocksInIndex);
+
+ if(!canDiffFromThis)
+ {
+ // Don't do diffing...
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = true;
+ }
+ return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime);
+ }
+
+ // Pointer to recipe we're going to create
+ BackupStoreFileEncodeStream::Recipe *precipe = 0;
+
+ // Start the timeout timer, so that the operation doesn't continue for ever
+ StartDiffTimer();
+
+ try
+ {
+ // Find which sizes should be scanned
+ int32_t sizesToScan[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+ FindMostUsedSizes(pindex, blocksInIndex, sizesToScan);
+
+ // Flag for reporting to the user
+ bool completelyDifferent;
+
+ // BLOCK
+ {
+ // Search the file to find matching blocks
+ std::map<int64_t, int64_t> foundBlocks; // map of offset in file to index in block index
+ int64_t sizeOfInputFile = 0;
+ // BLOCK
+ {
+ FileStream file(Filename);
+ // Get size of file
+ sizeOfInputFile = file.BytesLeftToRead();
+ // Find all those lovely matching blocks
+ SearchForMatchingBlocks(file, foundBlocks, pindex, blocksInIndex, sizesToScan);
+
+ // Is it completely different?
+ completelyDifferent = (foundBlocks.size() == 0);
+ }
+
+ // Create a recipe -- if the two files are completely different, don't put the from file ID in the recipe.
+ precipe = new BackupStoreFileEncodeStream::Recipe(pindex, blocksInIndex, completelyDifferent?(0):(DiffFromObjectID));
+ BlocksAvailableEntry *pindexKeptRef = pindex; // we need this later, but must set pindex == 0 now, because of exceptions
+ pindex = 0; // Recipe now has ownership
+
+ // Fill it in
+ GenerateRecipe(*precipe, pindexKeptRef, blocksInIndex, foundBlocks, sizeOfInputFile);
+ }
+ // foundBlocks no longer required
+
+ // Create the stream
+ std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
+
+ // Do the initial setup
+ ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime);
+ precipe = 0; // Stream has taken ownership of this
+
+ // Tell user about completely different status?
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = completelyDifferent;
+ }
+
+ // Return the stream for the caller
+ return stream;
+ }
+ catch(...)
+ {
+ // cleanup
+ if(pindex != 0)
+ {
+ ::free(pindex);
+ pindex = 0;
+ }
+ if(precipe != 0)
+ {
+ delete precipe;
+ precipe = 0;
+ }
+ throw;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static LoadIndex(IOStream &, int64_t, BlocksAvailableEntry **, int64_t, bool &)
+// Purpose: Read in an index, and decrypt, and store in the in memory block format.
+// rCanDiffFromThis is set to false if the version of the from file is too old.
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis)
+{
+ // Reset
+ rNumBlocksOut = 0;
+ rCanDiffFromThis = false;
+
+ // Read header
+ file_BlockIndexHeader hdr;
+ if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ // Check against backwards comptaibility stuff
+ if(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0))
+ {
+ // Won't diff against old version
+
+ // Absorb rest of stream
+ char buffer[2048];
+ while(rBlockIndex.StreamDataLeft())
+ {
+ rBlockIndex.Read(buffer, sizeof(buffer), 1000 /* 1 sec timeout */);
+ }
+
+ // Tell caller
+ rCanDiffFromThis = false;
+ return;
+ }
+#endif
+
+ // Check magic
+ if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Check that we're not trying to diff against a file which references blocks from another file
+ if(((int64_t)ntoh64(hdr.mOtherFileID)) != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile)
+ }
+
+ // Mark as an acceptable diff.
+ rCanDiffFromThis = true;
+
+ // Get basic information
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+ uint64_t entryIVBase = ntoh64(hdr.mEntryIVBase);
+
+ //TODO: Verify that these sizes look reasonable
+
+ // Allocate space for the index
+ BlocksAvailableEntry *pindex = (BlocksAvailableEntry*)::malloc(sizeof(BlocksAvailableEntry) * numBlocks);
+ if(pindex == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ try
+ {
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read an entry from the stream
+ file_BlockIndexEntry entry;
+ if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read entry
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Calculate IV for this entry
+ uint64_t iv = entryIVBase;
+ iv += b;
+ // Network byte order
+ iv = hton64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry.mEnEnc, sizeof(entry.mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Check that we're not trying to diff against a file which references blocks from another file
+ if(((int64_t)ntoh64(entry.mEncodedSize)) <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile)
+ }
+
+ // Store all the required information
+ pindex[b].mpNextInHashList = 0; // hash list not set up yet
+ pindex[b].mSize = ntohl(entryEnc.mSize);
+ pindex[b].mWeakChecksum = ntohl(entryEnc.mWeakChecksum);
+ ::memcpy(pindex[b].mStrongChecksum, entryEnc.mStrongChecksum, sizeof(pindex[b].mStrongChecksum));
+ }
+
+ // Store index pointer for called
+ ASSERT(ppIndex != 0);
+ *ppIndex = pindex;
+
+ // Store number of blocks for caller
+ rNumBlocksOut = numBlocks;
+
+ }
+ catch(...)
+ {
+ // clean up and send the exception along its way
+ ::free(pindex);
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static FindMostUsedSizes(BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+// Purpose: Finds the most commonly used block sizes in the index
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+{
+ // Array for lengths
+ int64_t sizeCounts[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+
+ // Set arrays to lots of zeros (= unused entries)
+ for(int l = 0; l < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++l)
+ {
+ Sizes[l] = 0;
+ sizeCounts[l] = 0;
+ }
+
+ // Array for collecting sizes
+ std::map<int32_t, int64_t> foundSizes;
+
+ // Run through blocks and make a count of the entries
+ for(int64_t b = 0; b < NumBlocks; ++b)
+ {
+ // Only if the block size is bigger than the minimum size we'll scan for
+ if(pIndex[b].mSize > BACKUP_FILE_DIFF_MIN_BLOCK_SIZE)
+ {
+ // Find entry?
+ std::map<int32_t, int64_t>::const_iterator f(foundSizes.find(pIndex[b].mSize));
+ if(f != foundSizes.end())
+ {
+ // Increment existing entry
+ foundSizes[pIndex[b].mSize] = foundSizes[pIndex[b].mSize] + 1;
+ }
+ else
+ {
+ // New entry
+ foundSizes[pIndex[b].mSize] = 1;
+ }
+ }
+ }
+
+ // Make the block sizes
+ for(std::map<int32_t, int64_t>::const_iterator i(foundSizes.begin()); i != foundSizes.end(); ++i)
+ {
+ // Find the position of the size in the array
+ for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
+ {
+ if(i->second > sizeCounts[t])
+ {
+ // Then this size belong before this entry -- shuffle them up
+ for(int s = (BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1); s >= t; --s)
+ {
+ Sizes[s] = Sizes[s-1];
+ sizeCounts[s] = sizeCounts[s-1];
+ }
+
+ // Insert this size
+ Sizes[t] = i->first;
+ sizeCounts[t] = i->second;
+
+ // Shouldn't do any more searching
+ break;
+ }
+ }
+ }
+
+ // trace the size table in debug builds
+#ifndef NDEBUG
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
+ {
+ TRACE3("Diff block size %d: %d (count = %lld)\n", t, Sizes[t], sizeCounts[t]);
+ }
+ }
+#endif
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+// Purpose: Find the matching blocks within the file.
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks,
+ BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+{
+ // Allocate the hash lookup table
+ BlocksAvailableEntry **phashTable = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024));
+
+ // Choose a size for the buffer, just a little bit more than the maximum block size
+ int32_t bufSize = Sizes[0];
+ for(int z = 1; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z)
+ {
+ if(Sizes[z] > bufSize) bufSize = Sizes[z];
+ }
+ bufSize += 4;
+ ASSERT(bufSize > Sizes[0]);
+ ASSERT(bufSize > 0);
+ if(bufSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // TODO: Use buffered file class
+ // Because we read in the file a scanned block size at a time, it is likely to be
+ // inefficient. Probably will be much better to use a buffering IOStream class which
+ // reads data in at the size of the filesystem block size.
+
+ // Allocate the buffers.
+ uint8_t *pbuffer0 = (uint8_t *)::malloc(bufSize);
+ uint8_t *pbuffer1 = (uint8_t *)::malloc(bufSize);
+ try
+ {
+ // Check buffer allocation
+ if(pbuffer0 == 0 || pbuffer1 == 0 || phashTable == 0)
+ {
+ // If a buffer got alloocated, it will be cleaned up in the catch block
+ throw std::bad_alloc();
+ }
+
+ // Flag to abort the run, if too many blocks are found -- avoid using
+ // huge amounts of processor time when files contain many similar blocks.
+ bool abortSearch = false;
+
+ // Search for each block size in turn
+ // NOTE: Do the smallest size first, so that the scheme for adding
+ // entries in the found list works as expected and replaces smallers block
+ // with larger blocks when it finds matches at the same offset in the file.
+ for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s)
+ {
+ ASSERT(Sizes[s] <= bufSize);
+ //TRACE2("Diff pass %d, for block size %d\n", s, Sizes[s]);
+
+ // Check we haven't finished
+ if(Sizes[s] == 0)
+ {
+ // empty entry, try next size
+ continue;
+ }
+
+ // Set up the hash table entries
+ SetupHashTable(pIndex, NumBlocks, Sizes[s], phashTable);
+
+ // Shift file position to beginning
+ rFile.Seek(0, IOStream::SeekType_Absolute);
+
+ // Read first block
+ if(rFile.Read(pbuffer0, Sizes[s]) != Sizes[s])
+ {
+ // Size of file too short to match -- do next size
+ continue;
+ }
+
+ // Setup block pointers
+ uint8_t *beginnings = pbuffer0;
+ uint8_t *endings = pbuffer1;
+ int offset = 0;
+
+ // Calculate the first checksum, ready for rolling
+ RollingChecksum rolling(beginnings, Sizes[s]);
+
+ // Then roll, until the file is exhausted
+ int64_t fileBlockNumber = 0;
+ int rollOverInitialBytes = 0;
+ while(true)
+ {
+ // Load in another block of data, and record how big it is
+ int bytesInEndings = rFile.Read(endings, Sizes[s]);
+
+ // Skip any bytes from a previous matched block
+ while(rollOverInitialBytes > 0 && offset < bytesInEndings)
+ {
+ rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
+ ++offset;
+ --rollOverInitialBytes;
+ }
+
+ while(offset < bytesInEndings)
+ {
+ // Put is current checksum in hash list?
+ uint16_t hash = rolling.GetComponentForHashing();
+ if(phashTable[hash] != 0)
+ {
+ if(SecondStageMatch(phashTable[hash], hash, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks))
+ {
+ // Block matched, roll the checksum forward to the next block without doing
+ // any more comparisons, because these are pointless (as any more matches will be ignored when
+ // the receipe is generated) and just take up valuable processor time. Edge cases are
+ // especially nasty, using huge amounts of time and memory.
+ int skip = Sizes[s];
+ while(offset < bytesInEndings && skip > 0)
+ {
+ rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
+ ++offset;
+ --skip;
+ }
+ // Not all the bytes necessary will have been skipped, so get them
+ // skipped after the next block is loaded.
+ rollOverInitialBytes = skip;
+
+ // End this loop, so the final byte isn't used again
+ break;
+ }
+
+ if(static_cast<int64_t>(rFoundBlocks.size()) > (NumBlocks * BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE)
+ || sDiffTimedOut)
+ {
+ abortSearch = true;
+ break;
+ }
+ }
+
+ // Roll checksum forward
+ rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
+
+ // Increment offset
+ ++offset;
+ }
+
+ if(abortSearch) break;
+
+ // Finished?
+ if(bytesInEndings != Sizes[s])
+ {
+ // No more data in file -- check the final block
+ // (Do a copy and paste of 5 lines of code instead of introducing a comparison for
+ // each byte of the file)
+ uint16_t hash = rolling.GetComponentForHashing();
+ if(phashTable[hash] != 0)
+ {
+ SecondStageMatch(phashTable[hash], hash, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks);
+ }
+
+ // finish
+ break;
+ }
+
+ // Switch buffers, reset offset
+ beginnings = endings;
+ endings = (beginnings == pbuffer0)?(pbuffer1):(pbuffer0); // ie the other buffer
+ offset = 0;
+
+ // And count the blocks which have been done
+ ++fileBlockNumber;
+ }
+
+ if(abortSearch) break;
+ }
+
+ // Free buffers and hash table
+ ::free(pbuffer1);
+ pbuffer1 = 0;
+ ::free(pbuffer0);
+ pbuffer0 = 0;
+ ::free(phashTable);
+ phashTable = 0;
+ }
+ catch(...)
+ {
+ // Cleanup and throw
+ if(pbuffer1 != 0) ::free(pbuffer1);
+ if(pbuffer0 != 0) ::free(pbuffer0);
+ if(phashTable != 0) ::free(phashTable);
+ throw;
+ }
+
+#ifndef NDEBUG
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ // Trace out the found blocks in debug mode
+ TRACE0("Diff: list of found blocks\n======== ======== ======== ========\n Offset BlkIdx Size Movement\n");
+ for(std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); i != rFoundBlocks.end(); ++i)
+ {
+ int64_t orgLoc = 0;
+ for(int64_t b = 0; b < i->second; ++b)
+ {
+ orgLoc += pIndex[b].mSize;
+ }
+ TRACE4("%8lld %8lld %8lld %8lld\n", i->first, i->second,
+ (int64_t)(pIndex[i->second].mSize), i->first - orgLoc);
+ }
+ TRACE0("======== ======== ======== ========\n");
+ }
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static SetupHashTable(BlocksAvailableEntry *, int64_t, in32_t, BlocksAvailableEntry **)
+// Purpose: Set up the hash table ready for a scan
+// Created: 14/1/04
+//
+// --------------------------------------------------------------------------
+static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable)
+{
+ // Set all entries in the hash table to zero
+ ::memset(pHashTable, 0, (sizeof(BlocksAvailableEntry *) * (64*1024)));
+
+ // Scan through the blocks, building the hash table
+ for(int64_t b = 0; b < NumBlocks; ++b)
+ {
+ // Only look at the required block size
+ if(pIndex[b].mSize == BlockSize)
+ {
+ // Get the value under which to hash this entry
+ uint16_t hash = RollingChecksum::ExtractHashingComponent(pIndex[b].mWeakChecksum);
+
+ // Already present in table?
+ if(pHashTable[hash] != 0)
+ {
+ //TRACE1("Another hash entry for %d found\n", hash);
+ // Yes -- need to set the pointer in this entry to the current entry to build the linked list
+ pIndex[b].mpNextInHashList = pHashTable[hash];
+ }
+
+ // Put a pointer to this entry in the hash table
+ pHashTable[hash] = pIndex + b;
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static bool SecondStageMatch(xxx)
+// Purpose: When a match in the hash table is found, scan for second stage match using strong checksum.
+// Created: 14/1/04
+//
+// --------------------------------------------------------------------------
+static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, uint16_t Hash, uint8_t *pBeginnings, uint8_t *pEndings,
+ int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks)
+{
+ // Check parameters
+ ASSERT(pBeginnings != 0);
+ ASSERT(pEndings != 0);
+ ASSERT(Offset >= 0);
+ ASSERT(BlockSize > 0);
+ ASSERT(pFirstInHashList != 0);
+ ASSERT(pIndex != 0);
+
+ // Calculate the strong MD5 digest for this block
+ MD5Digest strong;
+ // Add the data from the beginnings
+ strong.Add(pBeginnings + Offset, BlockSize - Offset);
+ // Add any data from the endings
+ if(Offset > 0)
+ {
+ strong.Add(pEndings, Offset);
+ }
+ strong.Finish();
+
+ // Then go through the entries in the hash list, comparing with the strong digest calculated
+ BlocksAvailableEntry *scan = pFirstInHashList;
+ //TRACE0("second stage match\n");
+ while(scan != 0)
+ {
+ //TRACE3("scan size %d, block size %d, hash %d\n", scan->mSize, BlockSize, Hash);
+ ASSERT(scan->mSize == BlockSize);
+ ASSERT(RollingChecksum::ExtractHashingComponent(scan->mWeakChecksum) == Hash);
+
+ // Compare?
+ if(strong.DigestMatches(scan->mStrongChecksum))
+ {
+ //TRACE0("Match!\n");
+ // Found! Add to list of found blocks...
+ int64_t fileOffset = (FileBlockNumber * BlockSize) + Offset;
+ int64_t blockIndex = (scan - pIndex); // pointer arthmitic is frowed apon. But most efficient way of doing it here -- alternative is to use more memory
+
+ // Because we search for smallest blocks first, if there's an entry there
+ // we'll just be replacing it with a larger one, which is great news.
+ rFoundBlocks[fileOffset] = blockIndex;
+
+ // No point in searching further, report success
+ return true;
+ }
+
+ // Next
+ scan = scan->mpNextInHashList;
+ }
+
+ // Not matched
+ return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static GenerateRecipe(BackupStoreFileEncodeStream::Recipe &, BlocksAvailableEntry *, int64_t, std::map<int64_t, int64_t> &)
+// Purpose: Fills in the recipe from the found block list
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex,
+ int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile)
+{
+ // NOTE: This function could be a lot more sophisiticated. For example, if
+ // a small block overlaps a big block like this
+ // ****
+ // *******************************
+ // then the small block will be used, not the big one. But it'd be better to
+ // just ignore the small block and keep the big one. However, some stats should
+ // be gathered about real world files before writing complex code which might
+ // go wrong.
+
+ // Initialise a blank instruction
+ BackupStoreFileEncodeStream::RecipeInstruction instruction;
+ #define RESET_INSTRUCTION \
+ instruction.mSpaceBefore = 0; \
+ instruction.mBlocks = 0; \
+ instruction.mpStartBlock = 0;
+ RESET_INSTRUCTION
+
+ // First, a special case for when there are no found blocks
+ if(rFoundBlocks.size() == 0)
+ {
+ // No blocks, just a load of space
+ instruction.mSpaceBefore = SizeOfInputFile;
+ rRecipe.push_back(instruction);
+
+ #ifndef NDEBUG
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ TRACE1("Diff: Default recipe generated, %lld bytes of file\n", SizeOfInputFile);
+ }
+ #endif
+
+ // Don't do anything
+ return;
+ }
+
+ // Current location
+ int64_t loc = 0;
+
+ // Then iterate through the list, generating the recipe
+ std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin());
+ ASSERT(i != rFoundBlocks.end()); // check logic
+
+ // Counting for debug tracing
+#ifndef NDEBUG
+ int64_t debug_NewBytesFound = 0;
+ int64_t debug_OldBlocksUsed = 0;
+#endif
+
+ for(; i != rFoundBlocks.end(); ++i)
+ {
+ // Remember... map is (position in file) -> (index of block in pIndex)
+
+ if(i->first < loc)
+ {
+ // This block overlaps the last one
+ continue;
+ }
+ else if(i->first > loc)
+ {
+ // There's a gap between the end of the last thing and this block.
+ // If there's an instruction waiting, push it onto the list
+ if(instruction.mSpaceBefore != 0 || instruction.mpStartBlock != 0)
+ {
+ rRecipe.push_back(instruction);
+ }
+ // Start a new instruction, with the gap ready
+ RESET_INSTRUCTION
+ instruction.mSpaceBefore = i->first - loc;
+ // Move location forward to match
+ loc += instruction.mSpaceBefore;
+#ifndef NDEBUG
+ debug_NewBytesFound += instruction.mSpaceBefore;
+#endif
+ }
+
+ // First, does the current instruction need pushing back, because this block is not
+ // sequential to the last one?
+ if(instruction.mpStartBlock != 0 && (pIndex + i->second) != (instruction.mpStartBlock + instruction.mBlocks))
+ {
+ rRecipe.push_back(instruction);
+ RESET_INSTRUCTION
+ }
+
+ // Add in this block
+ if(instruction.mpStartBlock == 0)
+ {
+ // This block starts a new instruction
+ instruction.mpStartBlock = pIndex + i->second;
+ instruction.mBlocks = 1;
+ }
+ else
+ {
+ // It continues the previous section of blocks
+ instruction.mBlocks += 1;
+ }
+
+#ifndef NDEBUG
+ debug_OldBlocksUsed++;
+#endif
+
+ // Move location forward
+ loc += pIndex[i->second].mSize;
+ }
+
+ // Push the last instruction generated
+ rRecipe.push_back(instruction);
+
+ // Is there any space left at the end which needs sending?
+ if(loc != SizeOfInputFile)
+ {
+ RESET_INSTRUCTION
+ instruction.mSpaceBefore = SizeOfInputFile - loc;
+#ifndef NDEBUG
+ debug_NewBytesFound += instruction.mSpaceBefore;
+#endif
+ rRecipe.push_back(instruction);
+ }
+
+ // dump out the recipe
+#ifndef NDEBUG
+ TRACE2("Diff: %lld new bytes found, %lld old blocks used\n", debug_NewBytesFound, debug_OldBlocksUsed);
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ TRACE1("Diff: Recipe generated (size %d)\n======== ========= ========\nSpace b4 FirstBlk NumBlks\n", rRecipe.size());
+ {
+ for(unsigned int e = 0; e < rRecipe.size(); ++e)
+ {
+ char b[64];
+ sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex));
+ TRACE3("%8lld %s %8lld\n", rRecipe[e].mSpaceBefore, (rRecipe[e].mpStartBlock == 0)?" -":b, (int64_t)rRecipe[e].mBlocks);
+ }
+ }
+ TRACE0("======== ========= ========\n");
+ }
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static TimerSignalHandler(int)
+// Purpose: Signal handler
+// Created: 19/3/04
+//
+// --------------------------------------------------------------------------
+void TimerSignalHandler(int signal)
+{
+ sDiffTimedOut = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static StartDiffTimer()
+// Purpose: Starts the diff timeout timer
+// Created: 19/3/04
+//
+// --------------------------------------------------------------------------
+void StartDiffTimer()
+{
+ // Set timer signal handler
+ if(!sSetTimerSignelHandler)
+ {
+ ::signal(SIGVTALRM, TimerSignalHandler);
+ sSetTimerSignelHandler = true;
+ }
+
+ struct itimerval timeout;
+ // Don't want this to repeat
+ timeout.it_interval.tv_sec = 0;
+ timeout.it_interval.tv_usec = 0;
+ // Single timeout after the specified number of seconds
+ timeout.it_value.tv_sec = sMaximumDiffTime;
+ timeout.it_value.tv_usec = 0;
+ // Set timer
+ if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0)
+ {
+ TRACE0("WARNING: couldn't set diff timeout\n");
+ }
+
+ // Unset flag (last thing)
+ sDiffTimedOut = false;
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp
new file mode 100755
index 00000000..20e1fd80
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileEncodeStream.cpp
@@ -0,0 +1,675 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileEncodeStream.cpp
+// Purpose: Implement stream-based file encoding for the backup store
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupStoreFileEncodeStream.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BoxTime.h"
+#include "BackupClientFileAttributes.h"
+#include "FileStream.h"
+#include "RollingChecksum.h"
+#include "Random.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream
+// Purpose: Constructor (opens file)
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::BackupStoreFileEncodeStream()
+ : mpRecipe(0),
+ mpFile(0),
+ mStatus(Status_Header),
+ mSendData(true),
+ mTotalBlocks(0),
+ mAbsoluteBlockNumber(-1),
+ mInstructionNumber(-1),
+ mNumBlocks(0),
+ mCurrentBlock(-1),
+ mCurrentBlockEncodedSize(0),
+ mPositionInCurrentBlock(0),
+ mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE),
+ mLastBlockSize(0),
+ mpRawBuffer(0),
+ mAllocatedBufferSize(0),
+ mEntryIVBase(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
+// Purpose: Destructor
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
+{
+ // Free buffers
+ if(mpRawBuffer)
+ {
+ ::free(mpRawBuffer);
+ mpRawBuffer = 0;
+ }
+
+ // Close the file, which we might have open
+ if(mpFile)
+ {
+ delete mpFile;
+ mpFile = 0;
+ }
+
+ // Free the recipe
+ if(mpRecipe != 0)
+ {
+ delete mpRecipe;
+ mpRecipe = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *)
+// Purpose: Reads file information, and builds file header reading for sending.
+// Takes ownership of the Recipe.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEncodeStream::Recipe *pRecipe,
+ int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime)
+{
+ // Pointer to a blank recipe which we might create
+ BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0;
+
+ try
+ {
+ // Get file attributes
+ box_time_t modTime = 0;
+ int64_t fileSize = 0;
+ BackupClientFileAttributes attr;
+ attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime,
+ 0 /* not interested in attr mod time */, &fileSize);
+
+ // Might need to create a blank recipe...
+ if(pRecipe == 0)
+ {
+ pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0);
+
+ BackupStoreFileEncodeStream::RecipeInstruction instruction;
+ instruction.mSpaceBefore = fileSize; // whole file
+ instruction.mBlocks = 0; // no blocks
+ instruction.mpStartBlock = 0; // no block
+ pblankRecipe->push_back(instruction);
+
+ pRecipe = pblankRecipe;
+ }
+
+ // Tell caller?
+ if(pModificationTime != 0)
+ {
+ *pModificationTime = modTime;
+ }
+
+ // Go through each instruction in the recipe and work out how many blocks
+ // it will add, and the max clear size of these blocks
+ int maxBlockClearSize = 0;
+ for(uint64_t inst = 0; inst < pRecipe->size(); ++inst)
+ {
+ if((*pRecipe)[inst].mSpaceBefore > 0)
+ {
+ // Calculate the number of blocks the space before requires
+ int64_t numBlocks;
+ int32_t blockSize, lastBlockSize;
+ CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize);
+ // Add to accumlated total
+ mTotalBlocks += numBlocks;
+ // Update maximum clear size
+ if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize;
+ if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize;
+ }
+
+ // Add number of blocks copied from the previous file
+ mTotalBlocks += (*pRecipe)[inst].mBlocks;
+
+ // Check for bad things
+ if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Run through blocks to get the max clear size
+ for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b)
+ {
+ if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize;
+ }
+ }
+
+ // Send data? (symlinks don't have any data in them)
+ mSendData = !attr.IsSymLink();
+
+ // If not data is being sent, then the max clear block size is zero
+ if(!mSendData)
+ {
+ maxBlockClearSize = 0;
+ }
+
+ // Header
+ file_StreamFormat hdr;
+ hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1);
+ hdr.mNumBlocks = (mSendData)?(hton64(mTotalBlocks)):(0);
+ hdr.mContainerID = hton64(ContainerID);
+ hdr.mModificationTime = hton64(modTime);
+ // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size
+ hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128);
+ hdr.mOptions = 0; // no options defined yet
+
+ // Write header to stream
+ mData.Write(&hdr, sizeof(hdr));
+
+ // Write filename to stream
+ rStoreFilename.WriteToStream(mData);
+
+ // Write attributes to stream
+ attr.WriteToStream(mData);
+
+ // Allocate some buffers for writing data
+ if(mSendData)
+ {
+ // Open the file
+ mpFile = new FileStream(Filename);
+
+ // Work out the largest possible block required for the encoded data
+ mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize);
+
+ // Then allocate two blocks of this size
+ mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize);
+ if(mpRawBuffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+#ifndef NDEBUG
+ // In debug builds, make sure that the reallocation code is exercised.
+ mEncodedBuffer.Allocate(mAllocatedBufferSize / 4);
+#else
+ mEncodedBuffer.Allocate(mAllocatedBufferSize);
+#endif
+ }
+ else
+ {
+ // Write an empty block index for the symlink
+ file_BlockIndexHeader blkhdr;
+ blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
+ blkhdr.mOtherFileID = hton64(0); // not other file ID
+ blkhdr.mEntryIVBase = hton64(0);
+ blkhdr.mNumBlocks = hton64(0);
+ mData.Write(&blkhdr, sizeof(blkhdr));
+ }
+
+ // Ready for reading
+ mData.SetForReading();
+
+ // Update stats
+ BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize;
+
+ // Finally, store the pointer to the recipe, when we know exceptions won't occur
+ mpRecipe = pRecipe;
+ }
+ catch(...)
+ {
+ // Clean up any blank recipe
+ if(pblankRecipe != 0)
+ {
+ delete pblankRecipe;
+ pblankRecipe = 0;
+ }
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &)
+// Purpose: Calculates the sizes of blocks in a section of the file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut)
+{
+ // How many blocks, and how big?
+ rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2;
+ do
+ {
+ rBlockSizeOut *= 2;
+
+ rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut;
+
+ } while(rBlockSizeOut <= BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER);
+
+ // Last block size
+ rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut);
+
+ // Avoid small blocks?
+ if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN
+ && rNumBlocksOut > 1)
+ {
+ // Add the small bit of data to the last block
+ --rNumBlocksOut;
+ rLastBlockSizeOut += rBlockSizeOut;
+ }
+
+ // checks!
+ ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize);
+ //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut);
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Read(void *, int, int)
+// Purpose: As interface -- generates encoded file data on the fly from the raw file
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Check there's something to do.
+ if(mStatus == Status_Finished)
+ {
+ return 0;
+ }
+
+ int bytesToRead = NBytes;
+ uint8_t *buffer = (uint8_t*)pBuffer;
+
+ while(bytesToRead > 0 && mStatus != Status_Finished)
+ {
+ if(mStatus == Status_Header || mStatus == Status_BlockListing)
+ {
+ // Header or block listing phase -- send from the buffered stream
+
+ // Send bytes from the data buffer
+ int b = mData.Read(buffer, bytesToRead, Timeout);
+ bytesToRead -= b;
+ buffer += b;
+
+ // Check to see if all the data has been used from this stream
+ if(!mData.StreamDataLeft())
+ {
+ // Yes, move on to next phase (or finish, if there's no file data)
+ if(!mSendData)
+ {
+ mStatus = Status_Finished;
+ }
+ else
+ {
+ // Reset the buffer so it can be used for the next phase
+ mData.Reset();
+
+ // Get buffer ready for index?
+ if(mStatus == Status_Header)
+ {
+ // Just finished doing the stream header, create the block index header
+ file_BlockIndexHeader blkhdr;
+ blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
+ ASSERT(mpRecipe != 0);
+ blkhdr.mOtherFileID = hton64(mpRecipe->GetOtherFileID());
+ blkhdr.mNumBlocks = hton64(mTotalBlocks);
+
+ // Generate the IV base
+ Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase));
+ blkhdr.mEntryIVBase = hton64(mEntryIVBase);
+
+ mData.Write(&blkhdr, sizeof(blkhdr));
+ }
+
+ ++mStatus;
+ }
+ }
+ }
+ else if(mStatus == Status_Blocks)
+ {
+ // Block sending phase
+
+ if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize)
+ {
+ // Next block!
+ ++mCurrentBlock;
+ ++mAbsoluteBlockNumber;
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // Output extra blocks for this instruction and move forward in file
+ if(mInstructionNumber >= 0)
+ {
+ SkipPreviousBlocksInInstruction();
+ }
+
+ // Is there another instruction to go?
+ ++mInstructionNumber;
+
+ // Skip instructions which don't contain any data
+ while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size())
+ && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0)
+ {
+ SkipPreviousBlocksInInstruction();
+ ++mInstructionNumber;
+ }
+
+ if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size()))
+ {
+ // End of blocks, go to next phase
+ ++mStatus;
+
+ // Set the data to reading so the index can be written
+ mData.SetForReading();
+ }
+ else
+ {
+ // Get ready for this instruction
+ SetForInstruction();
+ }
+ }
+
+ // Can't use 'else' here as SetForInstruction() will change this
+ if(mCurrentBlock < mNumBlocks)
+ {
+ EncodeCurrentBlock();
+ }
+ }
+
+ // Send data from the current block (if there's data to send)
+ if(mPositionInCurrentBlock < mCurrentBlockEncodedSize)
+ {
+ // How much data to put in the buffer?
+ int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock;
+ if(s > bytesToRead) s = bytesToRead;
+
+ // Copy it in
+ ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s);
+
+ // Update variables
+ bytesToRead -= s;
+ buffer += s;
+ mPositionInCurrentBlock += s;
+ }
+ }
+ else
+ {
+ // Should never get here, as it'd be an invalid status
+ ASSERT(false);
+ }
+ }
+
+ // Add encoded size to stats
+ BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead);
+
+ // Return size of data to caller
+ return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction()
+// Purpose: Private. Stores the blocks of the old file referenced in the current
+// instruction into the index and skips over the data in the file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction()
+{
+ // Check something is necessary
+ if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0)
+ {
+ return;
+ }
+
+ // Index of the first block in old file (being diffed from)
+ int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock);
+
+ int64_t sizeToSkip = 0;
+
+ for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b)
+ {
+ // Update stats
+ BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
+
+ // Store the entry
+ StoreBlockIndexEntry(0 - (firstIndex + b),
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize,
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum,
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum);
+
+ // Increment the absolute block number -- kept encryption IV in sync
+ ++mAbsoluteBlockNumber;
+
+ // Add the size of this block to the size to skip
+ sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
+ }
+
+ // Move forward in the stream
+ mpFile->Seek(sizeToSkip, IOStream::SeekType_Relative);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::SetForInstruction()
+// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::SetForInstruction()
+{
+ // Calculate block sizes
+ CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize);
+
+ // Set variables
+ mCurrentBlock = 0;
+ mCurrentBlockEncodedSize = 0;
+ mPositionInCurrentBlock = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock()
+// Purpose: Private. Encodes the current block, and writes the block data to the index
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::EncodeCurrentBlock()
+{
+ // How big is the block, raw?
+ int blockRawSize = mBlockSize;
+ if(mCurrentBlock == (mNumBlocks - 1))
+ {
+ blockRawSize = mLastBlockSize;
+ }
+ ASSERT(blockRawSize < mAllocatedBufferSize);
+
+ // Check file open
+ if(mpFile == 0)
+ {
+ // File should be open, but isn't. So logical error.
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Read the data in
+ if(!mpFile->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */))
+ {
+ // TODO: Do something more intelligent, and abort this upload because the file
+ // has changed
+ THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer)
+ }
+
+ // Encode it
+ mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer);
+
+ //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize);
+
+ // Create block listing data -- generate checksums
+ RollingChecksum weakChecksum(mpRawBuffer, blockRawSize);
+ MD5Digest strongChecksum;
+ strongChecksum.Add(mpRawBuffer, blockRawSize);
+ strongChecksum.Finish();
+
+ // Add entry to the index
+ StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData());
+
+ // Set vars to reading this block
+ mPositionInCurrentBlock = 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *)
+// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream.
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum)
+{
+ // First, the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ entryEnc.mSize = htonl(ClearSize);
+ entryEnc.mWeakChecksum = htonl(WeakChecksum);
+ ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum));
+
+ // Then the clear section
+ file_BlockIndexEntry entry;
+ entry.mEncodedSize = hton64(((uint64_t)EncSizeOrBlkIndex));
+
+ // Then encrypt the encryted section
+ // Generate the IV from the block number
+ if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase))
+ {
+ THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements)
+ }
+ uint64_t iv = mEntryIVBase;
+ iv += mAbsoluteBlockNumber;
+ // Convert to network byte order before encrypting with it, so that restores work on
+ // platforms with different endiannesses.
+ iv = hton64(iv);
+ sBlowfishEncryptBlockEntry.SetIV(&iv);
+
+ // Encode the data
+ int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc));
+ if(encodedSize != sizeof(entry.mEnEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Save to data block for sending at the end of the stream
+ mData.Write(&entry, sizeof(entry));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Write(const void *, int)
+// Purpose: As interface. Exceptions.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StreamDataLeft()
+// Purpose: As interface -- end of stream reached?
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFileEncodeStream::StreamDataLeft()
+{
+ return (mStatus != Status_Finished);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StreamClosed()
+// Purpose: As interface
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFileEncodeStream::StreamClosed()
+{
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t)
+// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex,
+ int64_t NumBlocksInIndex, int64_t OtherFileID)
+ : mpBlockIndex(pBlockIndex),
+ mNumBlocksInIndex(NumBlocksInIndex),
+ mOtherFileID(OtherFileID)
+{
+ ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0))
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Recipe::~Recipe()
+// Purpose: Destructor
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::Recipe::~Recipe()
+{
+ // Free the block index, if there is one
+ if(mpBlockIndex != 0)
+ {
+ ::free(mpBlockIndex);
+ }
+}
+
+
+
+
diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h
new file mode 100755
index 00000000..1c748798
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileEncodeStream.h
@@ -0,0 +1,127 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileEncodeStream.h
+// Purpose: Implement stream-based file encoding for the backup store
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILEENCODESTREAM__H
+#define BACKUPSTOREFILEENCODESTREAM__H
+
+#include <vector>
+
+#include "IOStream.h"
+#include "BackupStoreFilename.h"
+#include "CollectInBufferStream.h"
+#include "MD5Digest.h"
+#include "BackupStoreFile.h"
+
+namespace BackupStoreFileCreation
+{
+ // Diffing and creation of files share some implementation details.
+ typedef struct _BlocksAvailableEntry
+ {
+ struct _BlocksAvailableEntry *mpNextInHashList;
+ int32_t mSize; // size in clear
+ uint32_t mWeakChecksum; // weak, rolling checksum
+ uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum
+ } BlocksAvailableEntry;
+
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFileEncodeStream
+// Purpose: Encode a file into a stream
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+class BackupStoreFileEncodeStream : public IOStream
+{
+public:
+ BackupStoreFileEncodeStream();
+ ~BackupStoreFileEncodeStream();
+
+ typedef struct
+ {
+ int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go
+ int32_t mBlocks; // number of block to reuse, starting at this one
+ BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null
+ } RecipeInstruction;
+
+ class Recipe : public std::vector<RecipeInstruction>
+ {
+ // NOTE: This class is rather tied in with the implementation of diffing.
+ public:
+ Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex,
+ int64_t OtherFileID = 0);
+ ~Recipe();
+
+ int64_t GetOtherFileID() {return mOtherFileID;}
+ int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock)
+ {
+ return pBlock - mpBlockIndex;
+ }
+
+ private:
+ BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex;
+ int64_t mNumBlocksInIndex;
+ int64_t mOtherFileID;
+ };
+
+ void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime);
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+private:
+ enum
+ {
+ Status_Header = 0,
+ Status_Blocks = 1,
+ Status_BlockListing = 2,
+ Status_Finished = 3
+ };
+
+private:
+ void EncodeCurrentBlock();
+ void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut);
+ void SkipPreviousBlocksInInstruction();
+ void SetForInstruction();
+ void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum);
+
+private:
+ Recipe *mpRecipe;
+ IOStream *mpFile; // source file
+ CollectInBufferStream mData; // buffer for header and index entries
+ int mStatus;
+ bool mSendData; // true if there's file data to send (ie not a symlink)
+ int64_t mTotalBlocks; // Total number of blocks in the file
+ int64_t mAbsoluteBlockNumber; // The absolute block number currently being output
+ // Instruction number
+ int64_t mInstructionNumber;
+ // All the below are within the current instruction
+ int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases
+ int64_t mCurrentBlock;
+ int32_t mCurrentBlockEncodedSize;
+ int32_t mPositionInCurrentBlock; // for reading out
+ int32_t mBlockSize; // Basic block size of most of the blocks in the file
+ int32_t mLastBlockSize; // the size (unencoded) of the last block in the file
+ // Buffers
+ uint8_t *mpRawBuffer; // buffer for raw data
+ BackupStoreFile::EncodingBuffer mEncodedBuffer;
+ // buffer for encoded data
+ int32_t mAllocatedBufferSize; // size of above two allocated blocks
+ uint64_t mEntryIVBase; // base for block entry IV
+};
+
+
+
+#endif // BACKUPSTOREFILEENCODESTREAM__H
+
diff --git a/lib/backupclient/BackupStoreFileRevDiff.cpp b/lib/backupclient/BackupStoreFileRevDiff.cpp
new file mode 100644
index 00000000..f1dc52d8
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileRevDiff.cpp
@@ -0,0 +1,258 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileRevDiff.cpp
+// Purpose: Reverse a patch, to build a new patch from new to old files
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <stdlib.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t)
+// Purpose: Reverse a patch, to build a new patch from new to old files. Takes
+// two independent copies to the From file, for efficiency.
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent)
+{
+ // Read and copy the header from the from file to the out file -- beginnings of the patch
+ file_StreamFormat hdr;
+ if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Copy
+ rOut.Write(&hdr, sizeof(hdr));
+ // Copy over filename and attributes
+ // BLOCK
+ {
+ BackupStoreFilename filename;
+ filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ filename.WriteToStream(rOut);
+ StreamableMemBlock attr;
+ attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ attr.WriteToStream(rOut);
+ }
+
+ // Build an index of common blocks.
+ // For each block in the from file, we want to know it's index in the
+ // diff file. Allocate memory for this information.
+ int64_t fromNumBlocks = ntoh64(hdr.mNumBlocks);
+ int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t));
+ if(pfromIndexInfo == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Buffer data
+ void *buffer = 0;
+ int bufferSize = 0;
+
+ // flag
+ bool isCompletelyDifferent = true;
+
+ try
+ {
+ // Initialise the index to be all 0, ie not filled in yet
+ for(int64_t i = 0; i < fromNumBlocks; ++i)
+ {
+ pfromIndexInfo[i] = 0;
+ }
+
+ // Within the from file, skip to the index
+ MoveStreamPositionToBlockIndex(rDiff);
+
+ // Read in header of index
+ file_BlockIndexHeader diffIdxHdr;
+ if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // And then read in each entry
+ int64_t diffNumBlocks = ntoh64(diffIdxHdr.mNumBlocks);
+ for(int64_t b = 0; b < diffNumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Where's the block?
+ int64_t blockEn = ntoh64(e.mEncodedSize);
+ if(blockEn > 0)
+ {
+ // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file
+ }
+ else
+ {
+ // Block is in the original file, store which block it is in this file
+ int64_t fromIndex = 0 - blockEn;
+ if(fromIndex < 0 || fromIndex >= fromNumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles)
+ }
+
+ // Store information about where it is in the new file
+ // NOTE: This is slight different to how it'll be stored in the final index.
+ pfromIndexInfo[fromIndex] = -1 - b;
+ }
+ }
+
+ // Open the index for the second copy of the from file
+ MoveStreamPositionToBlockIndex(rFrom2);
+
+ // Read in header of index
+ file_BlockIndexHeader fromIdxHdr;
+ if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || ntoh64(fromIdxHdr.mOtherFileID) != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // So, we can now start building the data in the file
+ int64_t filePosition = rFrom.GetPosition();
+ for(int64_t b = 0; b < fromNumBlocks; ++b)
+ {
+ // Read entry from from index
+ file_BlockIndexEntry e;
+ if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Get size
+ int64_t blockSize = hton64(e.mEncodedSize);
+ if(blockSize < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Copy this block?
+ if(pfromIndexInfo[b] == 0)
+ {
+ // Copy it, first move to file location
+ rFrom.Seek(filePosition, IOStream::SeekType_Absolute);
+
+ // Make sure there's memory available to copy this
+ if(bufferSize < blockSize || buffer == 0)
+ {
+ // Free old block
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ bufferSize = 0;
+ }
+ // Allocate new block
+ buffer = ::malloc(blockSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ bufferSize = blockSize;
+ }
+ ASSERT(bufferSize >= blockSize);
+
+ // Copy the block
+ if(!rFrom.ReadFullBuffer(buffer, blockSize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ rOut.Write(buffer, blockSize);
+
+ // Store the size
+ pfromIndexInfo[b] = blockSize;
+ }
+ else
+ {
+ // Block isn't needed, so it's not completely different
+ isCompletelyDifferent = false;
+ }
+ filePosition += blockSize;
+ }
+
+ // Then write the index, modified header first
+ fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(hton64(ObjectIDOfFrom));
+ rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr));
+
+ // Move to start of index entries
+ rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute);
+
+ // Then copy modified entries
+ for(int64_t b = 0; b < fromNumBlocks; ++b)
+ {
+ // Read entry from from index
+ file_BlockIndexEntry e;
+ if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Modify...
+ int64_t s = pfromIndexInfo[b];
+ // Adjust to reflect real block index (remember 0 has a different meaning here)
+ if(s < 0) ++s;
+ // Insert
+ e.mEncodedSize = hton64(s);
+ // Write
+ rOut.Write(&e, sizeof(e));
+ }
+ }
+ catch(...)
+ {
+ ::free(pfromIndexInfo);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+ throw;
+ }
+
+ // Free memory used (oh for finally {} blocks)
+ ::free(pfromIndexInfo);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+
+ // return completely different flag
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = isCompletelyDifferent;
+ }
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFileWire.h b/lib/backupclient/BackupStoreFileWire.h
new file mode 100755
index 00000000..5b1bc819
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileWire.h
@@ -0,0 +1,74 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileWire.h
+// Purpose: On the wire / disc formats for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILEWIRE__H
+#define BACKUPSTOREFILEWIRE__H
+
+#include "MD5Digest.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int64_t mNumBlocks; // number of blocks contained in the file
+ int64_t mContainerID;
+ int64_t mModificationTime;
+ int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block
+ int32_t mOptions; // bitmask of options used
+ // Then a BackupStoreFilename
+ // Then a BackupClientFileAttributes
+} file_StreamFormat;
+
+typedef struct
+{
+ int32_t mMagicValue; // different magic value
+ int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index
+ uint64_t mEntryIVBase; // base value for block IV
+ int64_t mNumBlocks; // repeat of value in file header
+} file_BlockIndexHeader;
+
+typedef struct
+{
+ int32_t mSize; // size in clear
+ uint32_t mWeakChecksum; // weak, rolling checksum
+ uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum
+} file_BlockIndexEntryEnc;
+
+typedef struct
+{
+ union
+ {
+ int64_t mEncodedSize; // size encoded, if > 0
+ int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0
+ };
+ uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section
+} file_BlockIndexEntry;
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// header for blocks of compressed data in files
+#define HEADER_CHUNK_IS_COMPRESSED 1 // bit
+#define HEADER_ENCODING_SHIFT 1 // shift value
+#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7
+#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7
+
+
+#endif // BACKUPSTOREFILEWIRE__H
+
diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp
new file mode 100755
index 00000000..fbfe3313
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilename.cpp
@@ -0,0 +1,279 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilename.cpp
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreFilename.h"
+#include "Protocol.h"
+#include "BackupStoreException.h"
+#include "IOStream.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::BackupStoreFilename()
+// Purpose: Default constructor -- creates an invalid filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::BackupStoreFilename()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy)
+ : BackupStoreFilename_base(rToCopy)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::~BackupStoreFilename()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::~BackupStoreFilename()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::CheckValid(bool)
+// Purpose: Checks the encoded filename for validity
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const
+{
+ bool ok = true;
+
+ if(size() < 2)
+ {
+ // Isn't long enough to have a header
+ ok = false;
+ }
+ else
+ {
+ // Check size is consistent
+ unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(*this);
+ if(dsize != size())
+ {
+ ok = false;
+ }
+
+ // And encoding is an accepted value
+ unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this);
+ if(encoding < Encoding_Min || encoding > Encoding_Max)
+ {
+ ok = false;
+ }
+ }
+
+ // Exception?
+ if(!ok && ExceptionIfInvalid)
+ {
+ THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename)
+ }
+
+ return ok;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::ReadFromProtocol(Protocol &)
+// Purpose: Reads the filename from the protocol object
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol)
+{
+ // Read the header
+ char hdr[2];
+ rProtocol.Read(hdr, 2);
+
+ // How big is it?
+ int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
+
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ std::string data;
+ rProtocol.Read(data, dsize - 2);
+
+ // assign to this string, storing the header and the extra data
+ assign(hdr, 2);
+ append(data.c_str(), data.size());
+
+ // Check it
+ CheckValid();
+
+ // Alert derived classes
+ EncodedFilenameChanged();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::WriteToProtocol(Protocol &)
+// Purpose: Writes the filename to the protocol object
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const
+{
+ CheckValid();
+
+ rProtocol.Write(c_str(), size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::ReadFromStream(IOStream &)
+// Purpose: Reads the filename from a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Read the header
+ char hdr[2];
+ if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // How big is it?
+ unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
+
+ // Assume most filenames are small
+ char buf[256];
+ if(dsize < sizeof(buf))
+ {
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ // Copy in header
+ buf[0] = hdr[0]; buf[1] = hdr[1];
+
+ // assign to this string, storing the header and the extra data
+ assign(buf, dsize);
+ }
+ else
+ {
+ // Block of memory to hold it
+ MemoryBlockGuard<char*> dataB(dsize+2);
+ char *data = dataB;
+
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ // Copy in header
+ data[0] = hdr[0]; data[1] = hdr[1];
+
+ // assign to this string, storing the header and the extra data
+ assign(data, dsize);
+ }
+
+ // Check it
+ CheckValid();
+
+ // Alert derived classes
+ EncodedFilenameChanged();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::WriteToStream(IOStream &)
+// Purpose: Writes the filename to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::WriteToStream(IOStream &rStream) const
+{
+ CheckValid();
+
+ rStream.Write(c_str(), size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::EncodedFilenameChanged()
+// Purpose: The encoded filename stored has changed
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::EncodedFilenameChanged()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::IsEncrypted()
+// Purpose: Returns true if the filename is stored using an encrypting encoding
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFilename::IsEncrypted() const
+{
+ return BACKUPSTOREFILENAME_GET_ENCODING(*this) != Encoding_Clear;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::SetAsClearFilename(const char *)
+// Purpose: Sets this object to be a valid filename, but with a filename in the clear.
+// Used on the server to create filenames when there's no way of encrypting it.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::SetAsClearFilename(const char *Clear)
+{
+ // Make std::string from the clear name
+ std::string toEncode(Clear);
+
+ // Make an encoded string
+ char hdr[2];
+ BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear);
+ std::string encoded(hdr, 2);
+ encoded += toEncode;
+ ASSERT(encoded.size() == toEncode.size() + 2);
+
+ // Store the encoded string
+ assign(encoded);
+
+ // Stuff which must be done
+ EncodedFilenameChanged();
+ CheckValid(false);
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h
new file mode 100755
index 00000000..4c951b6f
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilename.h
@@ -0,0 +1,85 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilename.h
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILENAME__H
+#define BACKUPSTOREFILENAME__H
+
+#include <string>
+
+class Protocol;
+class IOStream;
+
+// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+// don't define this -- the problem of memory usage still appears without this.
+// It's just that this class really showed up the problem. Instead, malloc allocation
+// is globally defined in BoxPlatform.h, for troublesome libraries.
+
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+ // Use a malloc_allocated string, because the STL default allocators really screw up with
+ // memory allocation, particularly with this class.
+ // Makes a few things a bit messy and inefficient with conversions.
+ // Given up using this, and use global malloc allocation instead, but thought it
+ // worth leaving this code in just in case it's useful for the future.
+ typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base;
+ // If this is changed, change GetClearFilename() back to returning a reference.
+#else
+ typedef std::string BackupStoreFilename_base;
+#endif // PLATFORM_HAVE_STL_MALLOC_ALLOC
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFilename
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreFilename : public BackupStoreFilename_base
+{
+public:
+ BackupStoreFilename();
+ BackupStoreFilename(const BackupStoreFilename &rToCopy);
+ virtual ~BackupStoreFilename();
+
+ bool CheckValid(bool ExceptionIfInvalid = true) const;
+
+ void ReadFromProtocol(Protocol &rProtocol);
+ void WriteToProtocol(Protocol &rProtocol) const;
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream) const;
+
+ void SetAsClearFilename(const char *Clear);
+
+ // Check that it's encrypted
+ bool IsEncrypted() const;
+
+ // These enumerated types belong in the base class so
+ // the CheckValid() function can make sure that the encoding
+ // is a valid encoding
+ enum
+ {
+ Encoding_Min = 1,
+ Encoding_Clear = 1,
+ Encoding_Blowfish = 2,
+ Encoding_Max = 2
+ };
+
+protected:
+ virtual void EncodedFilenameChanged();
+};
+
+// On the wire utilities for class and derived class
+#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2)
+#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3)
+
+#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;}
+
+#endif // BACKUPSTOREFILENAME__H
+
diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp
new file mode 100755
index 00000000..3069f612
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilenameClear.cpp
@@ -0,0 +1,335 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilenameClear.cpp
+// Purpose: BackupStoreFilenames in the clear
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupStoreException.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// Hide private variables from the rest of the world
+namespace
+{
+ int sEncodeMethod = BackupStoreFilename::Encoding_Clear;
+ CipherContext sBlowfishEncrypt;
+ CipherContext sBlowfishDecrypt;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear()
+// Purpose: Default constructor, creates an invalid filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &)
+// Purpose: Creates a filename, encoding from the given string
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode)
+{
+ SetClearFilename(rToEncode);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy)
+ : BackupStoreFilename(rToCopy),
+ mClearFilename(rToCopy.mClearFilename)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
+// Purpose: Copy from base class
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
+ : BackupStoreFilename(rToCopy)
+{
+ // Will get a clear filename when it's required
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::~BackupStoreFilenameClear()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::GetClearFilename()
+// Purpose: Get the unencoded filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+const std::string BackupStoreFilenameClear::GetClearFilename() const
+{
+ MakeClearAvailable();
+ // When modifying, remember to change back to reference return if at all possible
+ // -- returns an object rather than a reference to allow easy use with other code.
+ return std::string(mClearFilename.c_str(), mClearFilename.size());
+}
+#else
+const std::string &BackupStoreFilenameClear::GetClearFilename() const
+{
+ MakeClearAvailable();
+ return mClearFilename;
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &)
+// Purpose: Encode and make available the clear filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode)
+{
+ // Only allow Blowfish encodings
+ if(sEncodeMethod != Encoding_Blowfish)
+ {
+ THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup)
+ }
+
+ // Make an encoded string with blowfish encryption
+ EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish);
+
+ // Store the clear filename
+ mClearFilename.assign(rToEncode.c_str(), rToEncode.size());
+
+ // Make sure we did the right thing
+ if(!CheckValid(false))
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::MakeClearAvailable()
+// Purpose: Private. Make sure the clear filename is available
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::MakeClearAvailable() const
+{
+ if(!mClearFilename.empty())
+ return; // nothing to do
+
+ // Check valid
+ CheckValid();
+
+ // Decode the header
+ int size = BACKUPSTOREFILENAME_GET_SIZE(*this);
+ int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this);
+
+ // Decode based on encoding given in the header
+ switch(encoding)
+ {
+ case Encoding_Clear:
+ TRACE0("**** BackupStoreFilename encoded with Clear encoding ****\n");
+ mClearFilename.assign(c_str() + 2, size - 2);
+ break;
+
+ case Encoding_Blowfish:
+ DecryptEncoded(sBlowfishDecrypt);
+ break;
+
+ default:
+ THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding)
+ break;
+ }
+}
+
+
+// Buffer for encoding and decoding -- do this all in one single buffer to
+// avoid lots of string allocation, which stuffs up memory usage.
+// These static memory vars are, of course, not thread safe, but we don't use threads.
+#ifndef NDEBUG
+static int sEncDecBufferSize = 2; // small size for debug builds
+#else
+static int sEncDecBufferSize = 256;
+#endif
+static MemoryBlockGuard<uint8_t *> spEncDecBuffer(sEncDecBufferSize);
+
+// fudge to stop leak reporting
+#ifdef BOX_MEMORY_LEAK_TESTING
+namespace
+{
+ class leak_off
+ {
+ public:
+ leak_off()
+ {
+ MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer);
+ }
+ };
+ leak_off dont_report_as_leak;
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int)
+// Purpose: Private. Assigns the encoded filename string, encrypting.
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding)
+{
+ // Work out max size
+ int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4;
+
+ // Make sure encode/decode buffer has enough space
+ if(sEncDecBufferSize < maxOutSize)
+ {
+ TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, maxOutSize);
+ spEncDecBuffer.Resize(maxOutSize);
+ sEncDecBufferSize = maxOutSize;
+ }
+
+ // Pointer to buffer
+ uint8_t *buffer = spEncDecBuffer;
+ MEMLEAKFINDER_NOT_A_LEAK(buffer);
+
+ // Encode -- do entire block in one go
+ int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size());
+ // and add in header size
+ encSize += 2;
+
+ // Adjust header
+ BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding);
+
+ // Store the encoded string
+ assign((char*)buffer, encSize);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &)
+// Purpose: Decrypt the encoded filename using the cipher context
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const
+{
+ // Work out max size
+ int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(size()) + 4;
+
+ // Make sure encode/decode buffer has enough space
+ if(sEncDecBufferSize < maxOutSize)
+ {
+ TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, maxOutSize);
+ spEncDecBuffer.Resize(maxOutSize);
+ sEncDecBufferSize = maxOutSize;
+ }
+
+ // Pointer to buffer
+ uint8_t *buffer = spEncDecBuffer;
+ MEMLEAKFINDER_NOT_A_LEAK(buffer);
+
+ // Decrypt
+ const char *str = c_str() + 2;
+ int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, size() - 2);
+
+ // Assign to this
+ mClearFilename.assign((char*)buffer, sizeOut);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::EncodedFilenameChanged()
+// Purpose: The encoded filename stored has changed
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::EncodedFilenameChanged()
+{
+ BackupStoreFilename::EncodedFilenameChanged();
+
+ // Delete stored filename in clear
+ mClearFilename.erase();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int)
+// Purpose: Set the key used for Blowfish encryption of filenames
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength)
+{
+ // Initialisation vector not used. Can't use a different vector for each filename as
+ // that would stop comparisions on the server working.
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength);
+ sBlowfishEncrypt.SetIV(pIV);
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength);
+ sBlowfishDecrypt.SetIV(pIV);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetEncodingMethod(int)
+// Purpose: Set the encoding method used for filenames
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetEncodingMethod(int Method)
+{
+ sEncodeMethod = Method;
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFilenameClear.h b/lib/backupclient/BackupStoreFilenameClear.h
new file mode 100755
index 00000000..d4c45701
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilenameClear.h
@@ -0,0 +1,60 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilenameClear.h
+// Purpose: BackupStoreFilenames in the clear
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILENAMECLEAR__H
+#define BACKUPSTOREFILENAMECLEAR__H
+
+#include "BackupStoreFilename.h"
+
+class CipherContext;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFilenameClear
+// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreFilenameClear : public BackupStoreFilename
+{
+public:
+ BackupStoreFilenameClear();
+ BackupStoreFilenameClear(const std::string &rToEncode);
+ BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy);
+ BackupStoreFilenameClear(const BackupStoreFilename &rToCopy);
+ virtual ~BackupStoreFilenameClear();
+
+ // Because we need to use a different allocator for this class to avoid
+ // nasty things happening, can't return this as a reference. Which is a
+ // pity. But probably not too bad.
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+ const std::string GetClearFilename() const;
+#else
+ const std::string &GetClearFilename() const;
+#endif
+ void SetClearFilename(const std::string &rToEncode);
+
+ // Setup for encryption of filenames
+ static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength);
+ static void SetEncodingMethod(int Method);
+
+protected:
+ void MakeClearAvailable() const;
+ virtual void EncodedFilenameChanged();
+ void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding);
+ void DecryptEncoded(CipherContext &rCipherContext) const;
+
+private:
+ mutable BackupStoreFilename_base mClearFilename;
+};
+
+#endif // BACKUPSTOREFILENAMECLEAR__H
+
+
diff --git a/lib/backupclient/BackupStoreObjectDump.cpp b/lib/backupclient/BackupStoreObjectDump.cpp
new file mode 100644
index 00000000..caa0e82f
--- /dev/null
+++ b/lib/backupclient/BackupStoreObjectDump.cpp
@@ -0,0 +1,210 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreObjectDump.cpp
+// Purpose: Implementations of dumping objects to stdout/TRACE
+// Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <map>
+
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreFilename.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreObjectMagic.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static void OutputLine(FILE *, bool, const char *, ...)
+// Purpose: Output a line for the object dumping, to file and/or trace...
+// Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+static void OutputLine(FILE *file, bool ToTrace, const char *format, ...)
+{
+ char text[512];
+ int r = 0;
+ va_list ap;
+ va_start(ap, format);
+ r = vsnprintf(text, sizeof(text), format, ap);
+ va_end(ap);
+
+ if(file != 0)
+ {
+ ::fprintf(file, "%s", text);
+ }
+ if(ToTrace)
+ {
+ TRACE1("%s", text);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace)
+// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere)
+// Dump the contents to a file, or trace.
+// Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace)
+{
+ FILE *file = (FILE*)clibFileHandle;
+
+ OutputLine(file, ToTrace, "Directory object.\nObject ID: %llx\nContainer ID: %llx\nNumber entries: %d\n"\
+ "Attributes mod time: %llx\nAttributes size: %d\n", mObjectID, mContainerID, mEntries.size(),
+ mAttributesModTime, mAttributes.GetSize());
+
+ // So repeated filenames can be illustrated, even though they can't be decoded
+ std::map<BackupStoreFilename, int> nameNum;
+ int nameNumI = 0;
+
+ // Dump items
+ OutputLine(file, ToTrace, "Items:\nID Size AttrHash AtSz NSz NIdx Flags\n");
+ for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+ {
+ // Choose file name index number for this file
+ std::map<BackupStoreFilename, int>::iterator nn(nameNum.find((*i)->GetName()));
+ int ni = nameNumI;
+ if(nn != nameNum.end())
+ {
+ ni = nn->second;
+ }
+ else
+ {
+ nameNum[(*i)->GetName()] = nameNumI;
+ ++nameNumI;
+ }
+
+ // Do dependencies
+ char depends[128];
+ depends[0] = '\0';
+ int depends_l = 0;
+ if((*i)->GetDependsNewer() != 0)
+ {
+ depends_l += ::sprintf(depends + depends_l, " depNew(%llx)", (*i)->GetDependsNewer());
+ }
+ if((*i)->GetDependsOlder() != 0)
+ {
+ depends_l += ::sprintf(depends + depends_l, " depOld(%llx)", (*i)->GetDependsOlder());
+ }
+
+ // Output item
+ int16_t f = (*i)->GetFlags();
+ OutputLine(file, ToTrace, "%06llx %4lld %016llx %4d %3d %4d%s%s%s%s%s%s\n",
+ (*i)->GetObjectID(),
+ (*i)->GetSizeInBlocks(),
+ (*i)->GetAttributesHash(),
+ (*i)->GetAttributes().GetSize(),
+ (*i)->GetName().size(),
+ ni,
+ ((f & BackupStoreDirectory::Entry::Flags_File)?" file":""),
+ ((f & BackupStoreDirectory::Entry::Flags_Dir)?" dir":""),
+ ((f & BackupStoreDirectory::Entry::Flags_Deleted)?" del":""),
+ ((f & BackupStoreDirectory::Entry::Flags_OldVersion)?" old":""),
+ ((f & BackupStoreDirectory::Entry::Flags_RemoveASAP)?" removeASAP":""),
+ depends);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DumpFile(void *, bool, IOStream &)
+// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere)
+// Dump the contents to a file, or trace.
+// Created: 4/5/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile)
+{
+ FILE *file = (FILE*)clibFileHandle;
+
+ // Read header
+ file_StreamFormat hdr;
+ if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr),
+ 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Check and output header info
+ if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V0))
+ {
+ OutputLine(file, ToTrace, "File header doesn't have the correct magic, aborting dump\n");
+ return;
+ }
+
+ OutputLine(file, ToTrace, "File object.\nContainer ID: %llx\nModification time: %llx\n"\
+ "Max block clear size: %d\nOptions: %08x\nNum blocks: %d\n", ntoh64(hdr.mContainerID),
+ ntoh64(hdr.mModificationTime), ntohl(hdr.mMaxBlockClearSize), ntohl(hdr.mOptions),
+ ntoh64(hdr.mNumBlocks));
+
+ // Read the next two objects
+ BackupStoreFilename fn;
+ fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+ OutputLine(file, ToTrace, "Filename size: %d\n", fn.size());
+
+ BackupClientFileAttributes attr;
+ attr.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+ OutputLine(file, ToTrace, "Attributes size: %d\n", attr.GetSize());
+
+ // Dump the blocks
+ rFile.Seek(0, IOStream::SeekType_Absolute);
+ BackupStoreFile::MoveStreamPositionToBlockIndex(rFile);
+
+ // Read in header
+ file_BlockIndexHeader bhdr;
+ rFile.ReadFullBuffer(&bhdr, sizeof(bhdr), 0);
+ if(bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ && bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0))
+ {
+ OutputLine(file, ToTrace, "WARNING: Block header doesn't have the correct magic\n");
+ }
+ // number of blocks
+ int64_t nblocks = ntoh64(bhdr.mNumBlocks);
+ OutputLine(file, ToTrace, "Other file ID (for block refs): %llx\nNum blocks (in blk hdr): %lld\n",
+ ntoh64(bhdr.mOtherFileID), nblocks);
+
+ // Dump info about each block
+ OutputLine(file, ToTrace, "======== ===== ==========\n Index Where EncSz/Idx\n");
+ int64_t nnew = 0, nold = 0;
+ for(int64_t b = 0; b < nblocks; ++b)
+ {
+ file_BlockIndexEntry en;
+ if(!rFile.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ OutputLine(file, ToTrace, "Didn't manage to read block %lld from file\n", b);
+ continue;
+ }
+ int64_t s = ntoh64(en.mEncodedSize);
+ if(s > 0)
+ {
+ nnew++;
+ TRACE2("%8lld this s=%8lld\n", b, s);
+ }
+ else
+ {
+ nold++;
+ TRACE2("%8lld other i=%8lld\n", b, 0 - s);
+ }
+ }
+ TRACE0("======== ===== ==========\n");
+}
+
diff --git a/lib/backupclient/BackupStoreObjectMagic.h b/lib/backupclient/BackupStoreObjectMagic.h
new file mode 100755
index 00000000..7ee600a2
--- /dev/null
+++ b/lib/backupclient/BackupStoreObjectMagic.h
@@ -0,0 +1,31 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreObjectMagic.h
+// Purpose: Magic values for the start of objects in the backup store
+// Created: 19/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREOBJECTMAGIC__H
+#define BACKUPSTOREOBJECTMAGIC__H
+
+// Each of these values is the first 4 bytes of the object file.
+// Remember to swap from network to host byte order.
+
+// Magic value for file streams
+#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65
+// Do not use v0 in any new code!
+#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45
+
+// Magic for the block index at the file stream -- used to
+// ensure streams are reordered as expected
+#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478
+// Do not use v0 in any new code!
+#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B
+
+// Magic value for directory streams
+#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F
+
+#endif // BACKUPSTOREOBJECTMAGIC__H
+
diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra
new file mode 100755
index 00000000..66203e3c
--- /dev/null
+++ b/lib/backupclient/Makefile.extra
@@ -0,0 +1,16 @@
+
+MAKEPROTOCOL = ../../lib/server/makeprotocol.pl
+
+GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt
+
+# AUTOGEN SEEDING
+autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt
+ perl $(GEN_CMD_SRV)
+
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt
+ perl $(MAKEEXCEPTION) BackupStoreException.txt
+