summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorBen Summers <ben@fluffy.co.uk>2005-10-14 08:50:54 +0000
committerBen Summers <ben@fluffy.co.uk>2005-10-14 08:50:54 +0000
commit99f8ce096bc5569adbfea1911dbcda24c28d8d8b (patch)
tree049c302161fea1f2f6223e1e8f3c40d9e8aadc8b /lib
Box Backup 0.09 with a few tweeks
Diffstat (limited to 'lib')
-rwxr-xr-xlib/backupclient/BackupClientCryptoKeys.cpp67
-rwxr-xr-xlib/backupclient/BackupClientCryptoKeys.h55
-rwxr-xr-xlib/backupclient/BackupClientFileAttributes.cpp773
-rwxr-xr-xlib/backupclient/BackupClientFileAttributes.h70
-rwxr-xr-xlib/backupclient/BackupClientMakeExcludeList.cpp75
-rwxr-xr-xlib/backupclient/BackupClientMakeExcludeList.h48
-rwxr-xr-xlib/backupclient/BackupClientRestore.cpp468
-rwxr-xr-xlib/backupclient/BackupClientRestore.h26
-rwxr-xr-xlib/backupclient/BackupDaemonConfigVerify.cpp105
-rwxr-xr-xlib/backupclient/BackupDaemonConfigVerify.h18
-rwxr-xr-xlib/backupclient/BackupStoreConstants.h53
-rwxr-xr-xlib/backupclient/BackupStoreDirectory.cpp563
-rwxr-xr-xlib/backupclient/BackupStoreDirectory.h268
-rwxr-xr-xlib/backupclient/BackupStoreException.h17
-rw-r--r--lib/backupclient/BackupStoreException.txt70
-rwxr-xr-xlib/backupclient/BackupStoreFile.cpp1499
-rwxr-xr-xlib/backupclient/BackupStoreFile.h194
-rw-r--r--lib/backupclient/BackupStoreFileCmbDiff.cpp326
-rw-r--r--lib/backupclient/BackupStoreFileCmbIdx.cpp324
-rwxr-xr-xlib/backupclient/BackupStoreFileCombine.cpp410
-rwxr-xr-xlib/backupclient/BackupStoreFileCryptVar.cpp31
-rwxr-xr-xlib/backupclient/BackupStoreFileCryptVar.h39
-rwxr-xr-xlib/backupclient/BackupStoreFileDiff.cpp973
-rwxr-xr-xlib/backupclient/BackupStoreFileEncodeStream.cpp675
-rwxr-xr-xlib/backupclient/BackupStoreFileEncodeStream.h127
-rw-r--r--lib/backupclient/BackupStoreFileRevDiff.cpp258
-rwxr-xr-xlib/backupclient/BackupStoreFileWire.h74
-rwxr-xr-xlib/backupclient/BackupStoreFilename.cpp279
-rwxr-xr-xlib/backupclient/BackupStoreFilename.h85
-rwxr-xr-xlib/backupclient/BackupStoreFilenameClear.cpp335
-rwxr-xr-xlib/backupclient/BackupStoreFilenameClear.h60
-rw-r--r--lib/backupclient/BackupStoreObjectDump.cpp210
-rwxr-xr-xlib/backupclient/BackupStoreObjectMagic.h31
-rwxr-xr-xlib/backupclient/Makefile.extra16
-rwxr-xr-xlib/backupstore/BackupStoreAccountDatabase.cpp370
-rwxr-xr-xlib/backupstore/BackupStoreAccountDatabase.h75
-rwxr-xr-xlib/backupstore/BackupStoreAccounts.cpp162
-rwxr-xr-xlib/backupstore/BackupStoreAccounts.h47
-rw-r--r--lib/backupstore/BackupStoreCheck.cpp745
-rw-r--r--lib/backupstore/BackupStoreCheck.h199
-rw-r--r--lib/backupstore/BackupStoreCheck2.cpp841
-rw-r--r--lib/backupstore/BackupStoreCheckData.cpp205
-rwxr-xr-xlib/backupstore/BackupStoreConfigVerify.cpp48
-rwxr-xr-xlib/backupstore/BackupStoreConfigVerify.h18
-rwxr-xr-xlib/backupstore/BackupStoreInfo.cpp592
-rwxr-xr-xlib/backupstore/BackupStoreInfo.h111
-rwxr-xr-xlib/backupstore/StoreStructure.cpp95
-rwxr-xr-xlib/backupstore/StoreStructure.h32
-rwxr-xr-xlib/common/BannerText.h17
-rw-r--r--lib/common/BeginStructPackForWire.h23
-rwxr-xr-xlib/common/Box.h180
-rwxr-xr-xlib/common/BoxException.cpp21
-rwxr-xr-xlib/common/BoxException.h37
-rwxr-xr-xlib/common/BoxPlatform.h236
-rwxr-xr-xlib/common/BoxPortsAndFiles.h31
-rwxr-xr-xlib/common/BoxTime.cpp32
-rwxr-xr-xlib/common/BoxTime.h44
-rwxr-xr-xlib/common/BoxTimeToText.cpp41
-rwxr-xr-xlib/common/BoxTimeToText.h19
-rwxr-xr-xlib/common/BoxTimeToUnix.h30
-rwxr-xr-xlib/common/CollectInBufferStream.cpp274
-rwxr-xr-xlib/common/CollectInBufferStream.h60
-rwxr-xr-xlib/common/CommonException.h17
-rw-r--r--lib/common/CommonException.txt44
-rwxr-xr-xlib/common/Configuration.cpp741
-rwxr-xr-xlib/common/Configuration.h98
-rw-r--r--lib/common/Conversion.h98
-rw-r--r--lib/common/ConversionException.txt8
-rw-r--r--lib/common/ConversionString.cpp123
-rwxr-xr-xlib/common/DebugAssertFailed.cpp32
-rwxr-xr-xlib/common/DebugMemLeakFinder.cpp377
-rwxr-xr-xlib/common/DebugPrintf.cpp72
-rw-r--r--lib/common/EndStructPackForWire.h23
-rw-r--r--lib/common/EventWatchFilesystemObject.cpp99
-rw-r--r--lib/common/EventWatchFilesystemObject.h48
-rwxr-xr-xlib/common/ExcludeList.cpp219
-rwxr-xr-xlib/common/ExcludeList.h65
-rwxr-xr-xlib/common/FdGetLine.cpp211
-rwxr-xr-xlib/common/FdGetLine.h61
-rwxr-xr-xlib/common/FileModificationTime.h57
-rwxr-xr-xlib/common/FileStream.cpp245
-rwxr-xr-xlib/common/FileStream.h46
-rwxr-xr-xlib/common/Guards.h112
-rwxr-xr-xlib/common/IOStream.cpp229
-rwxr-xr-xlib/common/IOStream.h67
-rwxr-xr-xlib/common/IOStreamGetLine.cpp227
-rwxr-xr-xlib/common/IOStreamGetLine.h71
-rwxr-xr-xlib/common/LinuxWorkaround.cpp73
-rwxr-xr-xlib/common/LinuxWorkaround.h20
-rwxr-xr-xlib/common/MainHelper.h42
-rwxr-xr-xlib/common/Makefile.extra11
-rwxr-xr-xlib/common/MemBlockStream.cpp235
-rwxr-xr-xlib/common/MemBlockStream.h52
-rwxr-xr-xlib/common/MemLeakFindOff.h27
-rwxr-xr-xlib/common/MemLeakFindOn.h25
-rwxr-xr-xlib/common/MemLeakFinder.h60
-rwxr-xr-xlib/common/NamedLock.cpp144
-rwxr-xr-xlib/common/NamedLock.h41
-rwxr-xr-xlib/common/PartialReadStream.cpp135
-rwxr-xr-xlib/common/PartialReadStream.h46
-rwxr-xr-xlib/common/ReadGatherStream.cpp262
-rwxr-xr-xlib/common/ReadGatherStream.h67
-rwxr-xr-xlib/common/StreamableMemBlock.cpp365
-rwxr-xr-xlib/common/StreamableMemBlock.h71
-rwxr-xr-xlib/common/TemporaryDirectory.h26
-rwxr-xr-xlib/common/Test.h165
-rwxr-xr-xlib/common/UnixUser.cpp121
-rwxr-xr-xlib/common/UnixUser.h37
-rwxr-xr-xlib/common/Utils.cpp159
-rwxr-xr-xlib/common/Utils.h36
-rw-r--r--lib/common/WaitForEvent.cpp194
-rw-r--r--lib/common/WaitForEvent.h146
-rwxr-xr-xlib/common/makeexception.pl277
-rwxr-xr-xlib/compress/Compress.h195
-rwxr-xr-xlib/compress/CompressException.h17
-rw-r--r--lib/compress/CompressException.txt12
-rw-r--r--lib/compress/CompressStream.cpp425
-rw-r--r--lib/compress/CompressStream.h62
-rwxr-xr-xlib/compress/Makefile.extra7
-rw-r--r--lib/crypto/CipherAES.cpp163
-rw-r--r--lib/crypto/CipherAES.h50
-rwxr-xr-xlib/crypto/CipherBlowfish.cpp220
-rwxr-xr-xlib/crypto/CipherBlowfish.h60
-rwxr-xr-xlib/crypto/CipherContext.cpp615
-rwxr-xr-xlib/crypto/CipherContext.h83
-rwxr-xr-xlib/crypto/CipherDescription.cpp73
-rwxr-xr-xlib/crypto/CipherDescription.h59
-rwxr-xr-xlib/crypto/CipherException.h17
-rw-r--r--lib/crypto/CipherException.txt18
-rwxr-xr-xlib/crypto/MD5Digest.cpp82
-rwxr-xr-xlib/crypto/MD5Digest.h57
-rwxr-xr-xlib/crypto/Makefile.extra7
-rwxr-xr-xlib/crypto/Random.cpp127
-rwxr-xr-xlib/crypto/Random.h25
-rwxr-xr-xlib/crypto/RollingChecksum.cpp38
-rwxr-xr-xlib/crypto/RollingChecksum.h96
-rwxr-xr-xlib/raidfile/Makefile.extra7
-rwxr-xr-xlib/raidfile/RaidFileController.cpp217
-rwxr-xr-xlib/raidfile/RaidFileController.h107
-rwxr-xr-xlib/raidfile/RaidFileException.h17
-rw-r--r--lib/raidfile/RaidFileException.txt25
-rwxr-xr-xlib/raidfile/RaidFileRead.cpp1701
-rwxr-xr-xlib/raidfile/RaidFileRead.h72
-rwxr-xr-xlib/raidfile/RaidFileUtil.cpp188
-rwxr-xr-xlib/raidfile/RaidFileUtil.h97
-rwxr-xr-xlib/raidfile/RaidFileWrite.cpp817
-rwxr-xr-xlib/raidfile/RaidFileWrite.h66
-rwxr-xr-xlib/raidfile/raidfile-config97
-rw-r--r--lib/server/ConnectionException.txt27
-rwxr-xr-xlib/server/Daemon.cpp530
-rwxr-xr-xlib/server/Daemon.h75
-rw-r--r--lib/server/LocalProcessStream.cpp101
-rw-r--r--lib/server/LocalProcessStream.h19
-rwxr-xr-xlib/server/Makefile.extra11
-rwxr-xr-xlib/server/Protocol.cpp1120
-rwxr-xr-xlib/server/Protocol.h201
-rwxr-xr-xlib/server/ProtocolObject.cpp125
-rwxr-xr-xlib/server/ProtocolObject.h41
-rwxr-xr-xlib/server/ProtocolUncertainStream.cpp189
-rwxr-xr-xlib/server/ProtocolUncertainStream.h47
-rwxr-xr-xlib/server/ProtocolWire.h43
-rwxr-xr-xlib/server/SSLLib.cpp83
-rwxr-xr-xlib/server/SSLLib.h36
-rwxr-xr-xlib/server/ServerException.h46
-rw-r--r--lib/server/ServerException.txt39
-rwxr-xr-xlib/server/ServerStream.h340
-rwxr-xr-xlib/server/ServerTLS.h80
-rwxr-xr-xlib/server/Socket.cpp171
-rwxr-xr-xlib/server/Socket.h47
-rwxr-xr-xlib/server/SocketListen.h265
-rwxr-xr-xlib/server/SocketStream.cpp405
-rwxr-xr-xlib/server/SocketStream.h56
-rwxr-xr-xlib/server/SocketStreamTLS.cpp457
-rwxr-xr-xlib/server/SocketStreamTLS.h60
-rwxr-xr-xlib/server/TLSContext.cpp120
-rwxr-xr-xlib/server/TLSContext.h41
-rwxr-xr-xlib/server/makeprotocol.pl993
177 files changed, 31121 insertions, 0 deletions
diff --git a/lib/backupclient/BackupClientCryptoKeys.cpp b/lib/backupclient/BackupClientCryptoKeys.cpp
new file mode 100755
index 00000000..015dadd2
--- /dev/null
+++ b/lib/backupclient/BackupClientCryptoKeys.cpp
@@ -0,0 +1,67 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientCryptoKeys.cpp
+// Purpose: function for setting up all the backup client keys
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "BackupClientCryptoKeys.h"
+#include "FileStream.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupStoreException.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreFile.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientCryptoKeys_Setup(const char *)
+// Purpose: Read in the key material file, and set keys to all the backup elements required.
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename)
+{
+ // Read in the key material
+ unsigned char KeyMaterial[BACKUPCRYPTOKEYS_FILE_SIZE];
+
+ // Open the file
+ FileStream file(KeyMaterialFilename);
+ // Read in data
+ if(!file.ReadFullBuffer(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntLoadClientKeyMaterial)
+ }
+
+ // Tell the filename how to encrypt
+ BackupStoreFilenameClear::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH,
+ KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH);
+ BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish);
+
+ // Tell the attributes how to encrypt
+ BackupClientFileAttributes::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH);
+ // and the secret for hashing
+ BackupClientFileAttributes::SetAttributeHashSecret(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH);
+
+ // Tell the files how to encrypt
+ BackupStoreFile::SetBlowfishKeys(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH,
+ KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH);
+#ifndef PLATFORM_OLD_OPENSSL
+ // Use AES where available
+ BackupStoreFile::SetAESKey(KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH);
+#endif
+
+ // Wipe the key material from memory
+ ::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE);
+}
+
+
+
diff --git a/lib/backupclient/BackupClientCryptoKeys.h b/lib/backupclient/BackupClientCryptoKeys.h
new file mode 100755
index 00000000..5e3a7df2
--- /dev/null
+++ b/lib/backupclient/BackupClientCryptoKeys.h
@@ -0,0 +1,55 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientCryptoKeys.h
+// Purpose: Format of crypto keys file, and function for setting everything up
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTCRYTOKEYS__H
+#define BACKUPCLIENTCRYTOKEYS__H
+
+
+// All keys are the maximum size that Blowfish supports. Since only the
+// setup time is affected by key length (encryption same speed whatever)
+// there is no disadvantage to using long keys as they are never
+// transmitted and are static over long periods of time.
+
+
+// All sizes in bytes. Some gaps deliberately left in the used material.
+
+// How long the key material file is expected to be
+#define BACKUPCRYPTOKEYS_FILE_SIZE 1024
+
+// key for encrypting filenames (448 bits)
+#define BACKUPCRYPTOKEYS_FILENAME_KEY_START 0
+#define BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH 56
+#define BACKUPCRYPTOKEYS_FILENAME_IV_START (0 + BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH)
+#define BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH 8
+
+// key for encrypting attributes (448 bits)
+#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START (BACKUPCRYPTOKEYS_FILENAME_KEY_START+64)
+#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH 56
+
+// Blowfish key for encrypting file data (448 bits (max blowfish key length))
+#define BACKUPCRYPTOKEYS_FILE_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START+64)
+#define BACKUPCRYPTOKEYS_FILE_KEY_LENGTH 56
+
+// key for encrypting file block index entries
+#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START (BACKUPCRYPTOKEYS_FILE_KEY_START+64)
+#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH 56
+
+// Secret for hashing attributes
+#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START (BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START+64)
+#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH 128
+
+// AES key for encrypting file data (256 bits (max AES key length))
+#define BACKUPCRYPTOKEYS_FILE_AES_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START+128)
+#define BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH 32
+
+
+void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename);
+
+#endif // BACKUPCLIENTCRYTOKEYS__H
+
diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp
new file mode 100755
index 00000000..4aa4edba
--- /dev/null
+++ b/lib/backupclient/BackupClientFileAttributes.cpp
@@ -0,0 +1,773 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientFileAttributes.cpp
+// Purpose: Storage of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <unistd.h>
+#include <limits.h>
+
+#include "BackupClientFileAttributes.h"
+#include "CommonException.h"
+#include "FileModificationTime.h"
+#include "BoxTimeToUnix.h"
+#include "BackupStoreException.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "MD5Digest.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+#define ATTRIBUTETYPE_GENERIC_UNIX 1
+
+#define ATTRIBUTE_ENCODING_BLOWFISH 2
+
+typedef struct
+{
+ int32_t AttributeType;
+ u_int32_t UID;
+ u_int32_t GID;
+ u_int64_t ModificationTime;
+ u_int64_t AttrModificationTime;
+ u_int32_t UserDefinedFlags;
+ u_int32_t FileGenerationNumber;
+ u_int16_t Mode;
+ // Symbolic link filename may follow
+} attr_StreamFormat;
+
+// This has wire packing so it's compatible across platforms
+// Use wider than necessary sizes, just to be careful.
+typedef struct
+{
+ int32_t uid, gid, mode;
+} attributeHashData;
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+
+#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256
+
+// Hide private static variables from the rest of the world
+// -- don't put them as static class variables to avoid openssl/evp.h being
+// included all over the project.
+namespace
+{
+ CipherContext sBlowfishEncrypt;
+ CipherContext sBlowfishDecrypt;
+ uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH];
+ int sAttributeHashSecretLength = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::BackupClientFileAttributes()
+// Purpose: Default constructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::BackupClientFileAttributes()
+ : mpClearAttributes(0)
+{
+ ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &)
+// Purpose: Copy constructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy)
+ : StreamableMemBlock(rToCopy), // base class does the hard work
+ mpClearAttributes(0)
+{
+}
+BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy)
+ : StreamableMemBlock(rToCopy), // base class does the hard work
+ mpClearAttributes(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::~BackupClientFileAttributes()
+// Purpose: Destructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::~BackupClientFileAttributes()
+{
+ if(mpClearAttributes)
+ {
+ delete mpClearAttributes;
+ mpClearAttributes = 0;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &)
+// Purpose: Assignment operator
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr)
+{
+ StreamableMemBlock::Set(rAttr);
+ RemoveClear(); // make sure no decrypted version held
+ return *this;
+}
+// Assume users play nice
+BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr)
+{
+ StreamableMemBlock::Set(rAttr);
+ RemoveClear(); // make sure no decrypted version held
+ return *this;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &)
+// Purpose: Comparison operator
+// Created: 2003/10/09
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const
+{
+ EnsureClearAvailable();
+ rAttr.EnsureClearAvailable();
+
+ return mpClearAttributes->operator==(*rAttr.mpClearAttributes);
+}
+// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data.
+/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const
+{
+ StreamableMemBlock *pDecoded = 0;
+
+ try
+ {
+ EnsureClearAvailable();
+ StreamableMemBlock *pDecoded = MakeClear(rAttr);
+
+ // Compare using clear version
+ bool compared = mpClearAttributes->operator==(rAttr);
+
+ // Delete temporary
+ delete pDecoded;
+
+ return compared;
+ }
+ catch(...)
+ {
+ delete pDecoded;
+ throw;
+ }
+}*/
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool)
+// Purpose: Compare, optionally ignoring the attribute modification time and/or modification time, and some data which is
+// irrelevant in practise (eg file generation number)
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime, bool IgnoreModTime) const
+{
+ EnsureClearAvailable();
+ rAttr.EnsureClearAvailable();
+
+ // Check sizes are the same, as a first check
+ if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize())
+ {
+ return false;
+ }
+
+ // Then check the elements of the two things
+ // Bytes are checked in network order, but this doesn't matter as we're only checking for equality.
+ attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+ attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer();
+
+ if(a1->AttributeType != a2->AttributeType
+ || a1->UID != a2->UID
+ || a1->GID != a2->GID
+ || a1->UserDefinedFlags != a2->UserDefinedFlags
+ || a1->Mode != a2->Mode)
+ {
+ return false;
+ }
+
+ if(!IgnoreModTime)
+ {
+ if(a1->ModificationTime != a2->ModificationTime)
+ {
+ return false;
+ }
+ }
+
+ if(!IgnoreAttrModTime)
+ {
+ if(a1->AttrModificationTime != a2->AttrModificationTime)
+ {
+ return false;
+ }
+ }
+
+ // Check symlink string?
+ unsigned int size = mpClearAttributes->GetSize();
+ if(size > sizeof(attr_StreamFormat))
+ {
+ // Symlink strings don't match
+ if(::memcmp(a1 + 1, a2 + 1, size - sizeof(attr_StreamFormat)) != 0)
+ {
+ return false;
+ }
+ }
+
+ // Passes all test, must be OK
+ return true;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadAttributes(const char *)
+// Purpose: Read the attributes of the file, and store them ready for streaming.
+// Optionally retrieve the modification time and attribute modification time.
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroModificationTimes, box_time_t *pModTime,
+ box_time_t *pAttrModTime, int64_t *pFileSize, ino_t *pInodeNumber, bool *pHasMultipleLinks)
+{
+ StreamableMemBlock *pnewAttr = 0;
+ try
+ {
+ struct stat st;
+ if(::lstat(Filename, &st) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ // Modification times etc
+ if(pModTime) {*pModTime = FileModificationTime(st);}
+ if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);}
+ if(pFileSize) {*pFileSize = st.st_size;}
+ if(pInodeNumber) {*pInodeNumber = st.st_ino;}
+ if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);}
+
+ // Is it a link?
+ if((st.st_mode & S_IFMT) == S_IFLNK)
+ {
+ ReadAttributesLink(Filename, &st, ZeroModificationTimes);
+ return;
+ }
+ ASSERT((st.st_mode & S_IFMT) != S_IFLNK);
+
+ // Now, can allocate the block
+ pnewAttr = new StreamableMemBlock(sizeof(attr_StreamFormat));
+
+ // Fill in the entries
+ attr_StreamFormat *pattr = (attr_StreamFormat*)pnewAttr->GetBuffer();
+
+#define FILL_IN_ATTRIBUTES \
+ ::memset(pattr, 0, sizeof(attr_StreamFormat)); \
+ ASSERT(pattr != 0); \
+ pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX); \
+ pattr->UID = htonl(st.st_uid); \
+ pattr->GID = htonl(st.st_gid); \
+ if(ZeroModificationTimes) \
+ { \
+ pattr->ModificationTime = 0; \
+ pattr->AttrModificationTime = 0; \
+ } \
+ else \
+ { \
+ pattr->ModificationTime = hton64(FileModificationTime(st)); \
+ pattr->AttrModificationTime = hton64(FileAttrModificationTime(st)); \
+ } \
+ pattr->Mode = htons(st.st_mode);
+#ifdef PLATFORM_stat_NO_st_flags
+ #define FILL_IN_ATTRIBUTES_2 \
+ pattr->UserDefinedFlags = 0; \
+ pattr->FileGenerationNumber = 0;
+#else
+ #define FILL_IN_ATTRIBUTES_2 \
+ pattr->UserDefinedFlags = htonl(st.st_flags); \
+ pattr->FileGenerationNumber = htonl(st.st_gen);
+#endif
+
+ FILL_IN_ATTRIBUTES
+ FILL_IN_ATTRIBUTES_2
+
+ // Attributes ready. Encrypt into this block
+ EncryptAttr(*pnewAttr);
+
+ // Store the new attributes
+ RemoveClear();
+ mpClearAttributes = pnewAttr;
+ pnewAttr = 0;
+ }
+ catch(...)
+ {
+ // clean up
+ delete pnewAttr;
+ pnewAttr = 0;
+ throw;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadAttributesLink()
+// Purpose: Private function, handles the case where a symbolic link is needed
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::ReadAttributesLink(const char *Filename, void *pst, bool ZeroModificationTimes)
+{
+ StreamableMemBlock *pnewAttr = 0;
+ try
+ {
+ // Avoid needing to have struct stat available when including header file
+ struct stat &st = *((struct stat *)pst);
+ // Make sure we're only called for symbolic links
+ ASSERT((st.st_mode & S_IFMT) == S_IFLNK);
+
+ // Get the filename the link is linked to
+ char linkedTo[PATH_MAX+4];
+ int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX);
+ if(linkedToSize == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ // Now, can allocate the block
+ pnewAttr = new StreamableMemBlock(sizeof(attr_StreamFormat) + linkedToSize + 1);
+
+ // Fill in the entries
+ attr_StreamFormat *pattr = (attr_StreamFormat*)pnewAttr->GetBuffer();
+
+ FILL_IN_ATTRIBUTES
+ FILL_IN_ATTRIBUTES_2
+
+ // Add the path name for the symbolic link
+ ::memcpy(pattr + 1, linkedTo, linkedToSize);
+ // Make sure it's neatly terminated
+ ((char*)(pattr + 1))[linkedToSize] = '\0';
+
+ // Attributes ready. Encrypt into this block
+ EncryptAttr(*pnewAttr);
+
+ // Store the new attributes
+ RemoveClear();
+ mpClearAttributes = pnewAttr;
+ pnewAttr = 0;
+ }
+ catch(...)
+ {
+ // clean up
+ delete pnewAttr;
+ pnewAttr = 0;
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::WriteAttributes(const char *)
+// Purpose: Apply the stored attributes to the file
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::WriteAttributes(const char *Filename) const
+{
+ // Got something loaded
+ if(GetSize() <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Make sure there are clear attributes to use
+ EnsureClearAvailable();
+ ASSERT(mpClearAttributes != 0);
+
+ // Check if the decrypted attributes are small enough, and the type of attributes stored
+ if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+ }
+ int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+ ASSERT(type != 0);
+ if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX)
+ {
+ // Don't know what to do with these
+ THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+ }
+
+ // Check there is enough space for an attributes block
+ if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat))
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Get pointer to structure
+ attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+
+ // is it a symlink?
+ int16_t mode = ntohs(pattr->Mode);
+ if((mode & S_IFMT) == S_IFLNK)
+ {
+ // Check things are sensible
+ if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1)
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Make a symlink, first deleting anything in the way
+ ::unlink(Filename);
+ if(::symlink((char*)(pattr + 1), Filename) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+
+ // If working as root, set user IDs
+ if(::geteuid() == 0)
+ {
+ #ifdef PLATFORM_LCHOWN_NOT_SUPPORTED
+ // only if not a link, can't set their owner on this platform
+ if((mode & S_IFMT) != S_IFLNK)
+ {
+ // Not a link, use normal chown
+ if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+ #else
+ if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) // use the version which sets things on symlinks
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ #endif
+ }
+
+ // Stop now if symlink, because otherwise it'll just be applied to the target
+ if((mode & S_IFMT) == S_IFLNK)
+ {
+ return;
+ }
+
+ // Set modification time?
+ box_time_t modtime = ntoh64(pattr->ModificationTime);
+ if(modtime != 0)
+ {
+ // Work out times as timevals
+ struct timeval times[2];
+ BoxTimeToTimeval(modtime, times[1]);
+ // Copy access time as well, why not, got to set it to something
+ times[0] = times[1];
+ // Attr modification time will be changed anyway, nothing that can be done about it
+
+ // Try to apply
+ if(::utimes(Filename, times) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+
+ // Apply everything else... (allowable mode flags only)
+ if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID | S_ISGID | S_ISVTX)) != 0) // mode must be done last (think setuid)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::IsSymLink()
+// Purpose: Do these attributes represent a symbolic link?
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::IsSymLink() const
+{
+ EnsureClearAvailable();
+
+ // Got the right kind of thing?
+ if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Get the type of attributes stored
+ int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+ ASSERT(type != 0);
+ if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat))
+ {
+ // Check link
+ attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+ return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK;
+ }
+
+ return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::RemoveClear()
+// Purpose: Private. Deletes any clear version of the attributes that may be held
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::RemoveClear() const
+{
+ if(mpClearAttributes)
+ {
+ delete mpClearAttributes;
+ }
+ mpClearAttributes = 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::EnsureClearAvailable()
+// Purpose: Private. Makes sure the clear version is available
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::EnsureClearAvailable() const
+{
+ if(mpClearAttributes == 0)
+ {
+ mpClearAttributes = MakeClear(*this);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &)
+// Purpose: Static. Decrypts stored attributes.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted)
+{
+ // New block
+ StreamableMemBlock *pdecrypted = 0;
+
+ try
+ {
+ // Check the block is big enough for IV and header
+ int ivSize = sBlowfishEncrypt.GetIVLength();
+ if(rEncrypted.GetSize() <= (ivSize + 1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes);
+ }
+
+ // How much space is needed for the output?
+ int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize);
+
+ // Allocate it
+ pdecrypted = new StreamableMemBlock(maxDecryptedSize);
+
+ // ptr to block
+ uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer();
+
+ // Check that the header has right type
+ if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH)
+ {
+ THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding);
+ }
+
+ // Set IV
+ sBlowfishDecrypt.SetIV(encBlock + 1);
+
+ // Decrypt
+ int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1));
+
+ // Resize block to fit
+ pdecrypted->ResizeBlock(decryptedSize);
+ }
+ catch(...)
+ {
+ delete pdecrypted;
+ pdecrypted = 0;
+ }
+
+ return pdecrypted;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int)
+// Purpose: Static. Sets the key to use for encryption and decryption.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength)
+{
+ // IVs set later
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &)
+// Purpose: Private. Encrypt the given attributes into this block.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt)
+{
+ // Free any existing block
+ FreeBlock();
+
+ // Work out the maximum amount of space we need
+ int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize());
+ // And the size of the IV
+ int ivSize = sBlowfishEncrypt.GetIVLength();
+
+ // Allocate this space
+ AllocateBlock(maxEncryptedSize + ivSize + 1);
+
+ // Store the encoding byte
+ uint8_t *block = (uint8_t*)GetBuffer();
+ block[0] = ATTRIBUTE_ENCODING_BLOWFISH;
+
+ // Generate and store an IV for this attribute block
+ int ivSize2 = 0;
+ const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2);
+ ASSERT(ivSize == ivSize2);
+
+ // Copy into the encrypted block
+ ::memcpy(block + 1, iv, ivSize);
+
+ // Do the transform
+ int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize());
+
+ // Resize this block
+ ResizeBlock(encrytedSize + ivSize + 1);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int)
+// Purpose: Set the secret for the filename attribute hash
+// Created: 25/4/04
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength)
+{
+ if(SecretLength > (int)sizeof(sAttributeHashSecret))
+ {
+ SecretLength = sizeof(sAttributeHashSecret);
+ }
+ if(SecretLength < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Copy
+ ::memcpy(sAttributeHashSecret, pSecret, SecretLength);
+ sAttributeHashSecretLength = SecretLength;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::GenerateAttributeHash(struct stat &, const std::string &)
+// Purpose: Generate a 64 bit hash from the attributes, used to detect changes.
+// Include filename in the hash, so that it changes from one file to another,
+// so don't reveal identical attributes.
+// Created: 25/4/04
+//
+// --------------------------------------------------------------------------
+uint64_t BackupClientFileAttributes::GenerateAttributeHash(struct stat &st, const std::string &rFilename)
+{
+ if(sAttributeHashSecretLength == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet)
+ }
+
+ // Assemble stuff we're interested in
+ attributeHashData hashData;
+ memset(&hashData, 0, sizeof(hashData));
+ // Use network byte order and large sizes to be cross platform
+ hashData.uid = htonl(st.st_uid);
+ hashData.gid = htonl(st.st_gid);
+ hashData.mode = htonl(st.st_mode);
+
+ // Create a MD5 hash of the data, filename, and secret
+ MD5Digest digest;
+ digest.Add(&hashData, sizeof(hashData));
+ digest.Add(rFilename.c_str(), rFilename.size());
+ digest.Add(sAttributeHashSecret, sAttributeHashSecretLength);
+ digest.Finish();
+
+ // Return the first 64 bits of the hash
+ uint64_t result = *((uint64_t *)(digest.DigestAsData()));
+ return result;
+}
+
+
+
diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h
new file mode 100755
index 00000000..dd930684
--- /dev/null
+++ b/lib/backupclient/BackupClientFileAttributes.h
@@ -0,0 +1,70 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientFileAttributes.h
+// Purpose: Storage of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTFILEATTRIBUTES__H
+#define BACKUPCLIENTFILEATTRIBUTES__H
+
+#include <string>
+
+#include "StreamableMemBlock.h"
+#include "BoxTime.h"
+
+struct stat;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupClientFileAttributes
+// Purpose: Storage, streaming and application of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+class BackupClientFileAttributes : public StreamableMemBlock
+{
+public:
+ BackupClientFileAttributes();
+ BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy);
+ BackupClientFileAttributes(const StreamableMemBlock &rToCopy);
+ ~BackupClientFileAttributes();
+ BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr);
+ BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr);
+ bool operator==(const BackupClientFileAttributes &rAttr) const;
+// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous?
+
+ bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const;
+
+ // Prevent access to base class members accidently
+ void Set();
+
+ void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false,
+ box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0,
+ ino_t *pInodeNumber = 0, bool *pHasMultipleLinks = 0);
+ void WriteAttributes(const char *Filename) const;
+
+ bool IsSymLink() const;
+
+ static void SetBlowfishKey(const void *pKey, int KeyLength);
+ static void SetAttributeHashSecret(const void *pSecret, int SecretLength);
+
+ static uint64_t GenerateAttributeHash(struct stat &st, const std::string &rFilename);
+
+private:
+ void ReadAttributesLink(const char *Filename, void *pst, bool ZeroModificationTimes);
+
+ void RemoveClear() const;
+ void EnsureClearAvailable() const;
+ static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted);
+ void EncryptAttr(const StreamableMemBlock &rToEncrypt);
+
+private:
+ mutable StreamableMemBlock *mpClearAttributes;
+};
+
+#endif // BACKUPCLIENTFILEATTRIBUTES__H
+
diff --git a/lib/backupclient/BackupClientMakeExcludeList.cpp b/lib/backupclient/BackupClientMakeExcludeList.cpp
new file mode 100755
index 00000000..0615faaa
--- /dev/null
+++ b/lib/backupclient/BackupClientMakeExcludeList.cpp
@@ -0,0 +1,75 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientMakeExcludeList.cpp
+// Purpose: Makes exclude lists from bbbackupd config location entries
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupClientMakeExcludeList.h"
+#include "Configuration.h"
+#include "ExcludeList.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientMakeExcludeList(const Configuration &, const char *, const char *)
+// Purpose: Given a Configuration object corresponding to a bbackupd Location, and the
+// two names of the keys for definite and regex entries, return a ExcludeList.
+// Or 0 if it isn't required.
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName,
+ const char *AlwaysIncludeDefiniteName, const char *AlwaysIncludeRegexName)
+{
+ // Check that at least one of the entries exists
+ if(!rConfig.KeyExists(DefiniteName) && !rConfig.KeyExists(RegexName))
+ {
+ // Neither exists -- return 0 as an Exclude list isn't required.
+ return 0;
+ }
+
+ // Create the exclude list
+ ExcludeList *pexclude = new ExcludeList;
+
+ try
+ {
+ // Definite names to add?
+ if(rConfig.KeyExists(DefiniteName))
+ {
+ pexclude->AddDefiniteEntries(rConfig.GetKeyValue(DefiniteName));
+ }
+ // Regular expressions to add?
+ if(rConfig.KeyExists(RegexName))
+ {
+ pexclude->AddRegexEntries(rConfig.GetKeyValue(RegexName));
+ }
+
+ // Add a "always include" list?
+ if(AlwaysIncludeDefiniteName != 0 && AlwaysIncludeRegexName != 0)
+ {
+ // This will accept NULL as a valid argument, so safe to do this.
+ pexclude->SetAlwaysIncludeList(
+ BackupClientMakeExcludeList(rConfig, AlwaysIncludeDefiniteName, AlwaysIncludeRegexName)
+ );
+ }
+ }
+ catch(...)
+ {
+ // Clean up
+ delete pexclude;
+ throw;
+ }
+
+ return pexclude;
+}
+
+
+
diff --git a/lib/backupclient/BackupClientMakeExcludeList.h b/lib/backupclient/BackupClientMakeExcludeList.h
new file mode 100755
index 00000000..d5dd8af4
--- /dev/null
+++ b/lib/backupclient/BackupClientMakeExcludeList.h
@@ -0,0 +1,48 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientMakeExcludeList.h
+// Purpose: Makes exclude lists from bbbackupd config location entries
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTMAKEEXCLUDELIST__H
+#define BACKUPCLIENTMAKEEXCLUDELIST__H
+
+class ExcludeList;
+class Configuration;
+
+ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName,
+ const char *AlwaysIncludeDefiniteName = 0, const char *AlwaysIncludeRegexName = 0);
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientMakeExcludeList_Files(const Configuration &)
+// Purpose: Create a exclude list from config file entries for files. May return 0.
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+inline ExcludeList *BackupClientMakeExcludeList_Files(const Configuration &rConfig)
+{
+ return BackupClientMakeExcludeList(rConfig, "ExcludeFile", "ExcludeFilesRegex", "AlwaysIncludeFile", "AlwaysIncludeFilesRegex");
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientMakeExcludeList_Dirs(const Configuration &)
+// Purpose: Create a exclude list from config file entries for directories. May return 0.
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+inline ExcludeList *BackupClientMakeExcludeList_Dirs(const Configuration &rConfig)
+{
+ return BackupClientMakeExcludeList(rConfig, "ExcludeDir", "ExcludeDirsRegex", "AlwaysIncludeDir", "AlwaysIncludeDirsRegex");
+}
+
+
+#endif // BACKUPCLIENTMAKEEXCLUDELIST__H
+
diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp
new file mode 100755
index 00000000..ebe1d365
--- /dev/null
+++ b/lib/backupclient/BackupClientRestore.cpp
@@ -0,0 +1,468 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientRestore.cpp
+// Purpose:
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string>
+#include <set>
+#include <unistd.h>
+#include <limits.h>
+#include <stdio.h>
+
+#include "BackupClientRestore.h"
+#include "autogen_BackupProtocolClient.h"
+#include "CommonException.h"
+#include "BackupClientFileAttributes.h"
+#include "IOStream.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "CollectInBufferStream.h"
+#include "FileStream.h"
+#include "Utils.h"
+
+#include "MemLeakFindOn.h"
+
+#define MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES (128*1024)
+
+class RestoreResumeInfo
+{
+public:
+ // constructor
+ RestoreResumeInfo()
+ : mNextLevelID(0),
+ mpNextLevel(0)
+ {
+ }
+
+ // destructor
+ ~RestoreResumeInfo()
+ {
+ delete mpNextLevel;
+ mpNextLevel = 0;
+ }
+
+ // Get a next level object
+ RestoreResumeInfo &AddLevel(int64_t ID, const std::string &rLocalName)
+ {
+ ASSERT(mpNextLevel == 0 && mNextLevelID == 0);
+ mpNextLevel = new RestoreResumeInfo;
+ mNextLevelID = ID;
+ mNextLevelLocalName = rLocalName;
+ return *mpNextLevel;
+ }
+
+ // Remove the next level info
+ void RemoveLevel()
+ {
+ ASSERT(mpNextLevel != 0 && mNextLevelID != 0);
+ delete mpNextLevel;
+ mpNextLevel = 0;
+ mNextLevelID = 0;
+ mNextLevelLocalName.erase();
+ }
+
+ void Save(const std::string &rFilename) const
+ {
+ // TODO: use proper buffered streams when they're done
+ // Build info in memory buffer
+ CollectInBufferStream write;
+
+ // Save this level
+ SaveLevel(write);
+
+ // Store in file
+ write.SetForReading();
+ FileStream file(rFilename.c_str(), O_WRONLY | O_CREAT);
+ write.CopyStreamTo(file, IOStream::TimeOutInfinite, 8*1024 /* large buffer */);
+ }
+
+ void SaveLevel(IOStream &rWrite) const
+ {
+ // Write the restored objects
+ int64_t numObjects = mRestoredObjects.size();
+ rWrite.Write(&numObjects, sizeof(numObjects));
+ for(std::set<int64_t>::const_iterator i(mRestoredObjects.begin()); i != mRestoredObjects.end(); ++i)
+ {
+ int64_t id = *i;
+ rWrite.Write(&id, sizeof(id));
+ }
+
+ // Next level?
+ if(mpNextLevel != 0)
+ {
+ // ID
+ rWrite.Write(&mNextLevelID, sizeof(mNextLevelID));
+ // Name string
+ int32_t nsize = mNextLevelLocalName.size();
+ rWrite.Write(&nsize, sizeof(nsize));
+ rWrite.Write(mNextLevelLocalName.c_str(), nsize);
+ // And then the level itself
+ mpNextLevel->SaveLevel(rWrite);
+ }
+ else
+ {
+ // Just write a zero
+ int64_t zero = 0;
+ rWrite.Write(&zero, sizeof(zero));
+ }
+ }
+
+ // Not written to be efficient -- shouldn't be called very often.
+ bool Load(const std::string &rFilename)
+ {
+ // Delete and reset if necessary
+ if(mpNextLevel != 0)
+ {
+ RemoveLevel();
+ }
+
+ // Open file
+ FileStream file(rFilename.c_str());
+
+ // Load this level
+ return LoadLevel(file);
+ }
+
+ #define CHECKED_READ(x, s) if(!rRead.ReadFullBuffer(x, s, 0)) {return false;}
+ bool LoadLevel(IOStream &rRead)
+ {
+ // Load the restored objects list
+ mRestoredObjects.clear();
+ int64_t numObjects = 0;
+ CHECKED_READ(&numObjects, sizeof(numObjects));
+ for(int64_t o = 0; o < numObjects; ++o)
+ {
+ int64_t id;
+ CHECKED_READ(&id, sizeof(id));
+ mRestoredObjects.insert(id);
+ }
+
+ // ID of next level?
+ int64_t nextID = 0;
+ CHECKED_READ(&nextID, sizeof(nextID));
+ if(nextID != 0)
+ {
+ // Load the next level!
+ std::string name;
+ int32_t nsize = 0;
+ CHECKED_READ(&nsize, sizeof(nsize));
+ char n[PATH_MAX];
+ if(nsize > PATH_MAX) return false;
+ CHECKED_READ(n, nsize);
+ name.assign(n, nsize);
+
+ // Create a new level
+ mpNextLevel = new RestoreResumeInfo;
+ mNextLevelID = nextID;
+ mNextLevelLocalName = name;
+
+ // And ask it to read itself in
+ if(!mpNextLevel->LoadLevel(rRead))
+ {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ // List of objects at this level which have been done already
+ std::set<int64_t> mRestoredObjects;
+ // Next level ID
+ int64_t mNextLevelID;
+ // Pointer to next level
+ RestoreResumeInfo *mpNextLevel;
+ // Local filename of next level
+ std::string mNextLevelLocalName;
+};
+
+// parameters structure
+typedef struct
+{
+ bool PrintDots;
+ bool RestoreDeleted;
+ std::string mRestoreResumeInfoFilename;
+ RestoreResumeInfo mResumeInfo;
+} RestoreParams;
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientRestoreDir(BackupProtocolClient &, int64_t, const char *, bool)
+// Purpose: Restore a directory
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t DirectoryID, std::string &rLocalDirectoryName,
+ RestoreParams &Params, RestoreResumeInfo &rLevel)
+{
+ // If we're resuming... check that we haven't got a next level to look at
+ if(rLevel.mpNextLevel != 0)
+ {
+ // Recurse immediately
+ std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + rLevel.mNextLevelLocalName);
+ BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, localDirname, Params, *rLevel.mpNextLevel);
+
+ // Add it to the list of done itmes
+ rLevel.mRestoredObjects.insert(rLevel.mNextLevelID);
+
+ // Remove the level for the recursed directory
+ rLevel.RemoveLevel();
+ }
+
+ // Save the resumption information
+ Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+
+ // Create the local directory (if not already done) -- path and owner set later, just use restrictive owner mode
+ switch(ObjectExists(rLocalDirectoryName.c_str()))
+ {
+ case ObjectExists_Dir:
+ // Do nothing
+ break;
+ case ObjectExists_File:
+ {
+ // File exists with this name, which is fun. Get rid of it.
+ ::printf("WARNING: File present with name '%s', removing out of the way of restored directory. Use specific restore with ID to restore this object.", rLocalDirectoryName.c_str());
+ if(::unlink(rLocalDirectoryName.c_str()) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+ TRACE1("In restore, directory name collision with file %s", rLocalDirectoryName.c_str());
+ }
+ // follow through to... (no break)
+ case ObjectExists_NoObject:
+ if(::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+ break;
+ default:
+ ASSERT(false);
+ break;
+ }
+
+ // Fetch the directory listing from the server -- getting a list of files which is approparite to the restore type
+ rConnection.QueryListDirectory(
+ DirectoryID,
+ Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING),
+ BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)),
+ true /* want attributes */);
+
+ // Retrieve the directory from the stream following
+ BackupStoreDirectory dir;
+ std::auto_ptr<IOStream> dirstream(rConnection.ReceiveStream());
+ dir.ReadFromStream(*dirstream, rConnection.GetTimeout());
+
+ // Apply attributes to the directory
+ const StreamableMemBlock &dirAttrBlock(dir.GetAttributes());
+ BackupClientFileAttributes dirAttr(dirAttrBlock);
+ dirAttr.WriteAttributes(rLocalDirectoryName.c_str());
+
+ int64_t bytesWrittenSinceLastRestoreInfoSave = 0;
+
+ // Process files
+ {
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+ {
+ // Check ID hasn't already been done
+ if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end())
+ {
+ // Local name
+ BackupStoreFilenameClear nm(en->GetName());
+ std::string localFilename(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename());
+
+ // Unlink anything which already exists -- for resuming restores, we can't overwrite files already there.
+ ::unlink(localFilename.c_str());
+
+ // Request it from the store
+ rConnection.QueryGetFile(DirectoryID, en->GetObjectID());
+
+ // Stream containing encoded file
+ std::auto_ptr<IOStream> objectStream(rConnection.ReceiveStream());
+
+ // Decode the file -- need to do different things depending on whether
+ // the directory entry has additional attributes
+ if(en->HasAttributes())
+ {
+ // Use these attributes
+ const StreamableMemBlock &storeAttr(en->GetAttributes());
+ BackupClientFileAttributes attr(storeAttr);
+ BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout(), &attr);
+ }
+ else
+ {
+ // Use attributes stored in file
+ BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout());
+ }
+
+ // Progress display?
+ if(Params.PrintDots)
+ {
+ printf(".");
+ fflush(stdout);
+ }
+
+ // Add it to the list of done itmes
+ rLevel.mRestoredObjects.insert(en->GetObjectID());
+
+ // Save restore info?
+ int64_t fileSize;
+ if(FileExists(localFilename.c_str(), &fileSize, true /* treat links as not existing */))
+ {
+ // File exists...
+ bytesWrittenSinceLastRestoreInfoSave += fileSize;
+
+ if(bytesWrittenSinceLastRestoreInfoSave > MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES)
+ {
+ // Save the restore info, in case it's needed later
+ Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+ bytesWrittenSinceLastRestoreInfoSave = 0;
+ }
+ }
+ }
+ }
+ }
+
+ // Make sure the restore info has been saved
+ if(bytesWrittenSinceLastRestoreInfoSave != 0)
+ {
+ // Save the restore info, in case it's needed later
+ Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+ bytesWrittenSinceLastRestoreInfoSave = 0;
+ }
+
+
+ // Recuse to directories
+ {
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
+ {
+ // Check ID hasn't already been done
+ if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end())
+ {
+ // Local name
+ BackupStoreFilenameClear nm(en->GetName());
+ std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename());
+
+ // Add the level for the next entry
+ RestoreResumeInfo &rnextLevel(rLevel.AddLevel(en->GetObjectID(), nm.GetClearFilename()));
+
+ // Recurse
+ BackupClientRestoreDir(rConnection, en->GetObjectID(), localDirname, Params, rnextLevel);
+
+ // Remove the level for the above call
+ rLevel.RemoveLevel();
+
+ // Add it to the list of done itmes
+ rLevel.mRestoredObjects.insert(en->GetObjectID());
+ }
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientRestore(BackupProtocolClient &, int64_t, const char *, bool, bool)
+// Purpose: Restore a directory on the server to a local directory on the disc.
+//
+// The local directory must not already exist.
+//
+// If a restore is aborted for any reason, then it may be resumed if
+// Resume == true. If Resume == false and resumption is possible, then
+// Restore_ResumePossible is returned.
+//
+// Set RestoreDeleted to restore a deleted directory. This may not give the
+// directory structure when it was deleted, because files may have been deleted
+// within it before it was deleted.
+//
+// Returns Restore_TargetExists if the target directory exists, but
+// there is no restore possible. (Won't attempt to overwrite things.)
+//
+// Returns Restore_Complete on success. (Exceptions on error.)
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName,
+ bool PrintDots, bool RestoreDeleted, bool UndeleteAfterRestoreDeleted, bool Resume)
+{
+ // Parameter block
+ RestoreParams params;
+ params.PrintDots = PrintDots;
+ params.RestoreDeleted = RestoreDeleted;
+ params.mRestoreResumeInfoFilename = LocalDirectoryName;
+ params.mRestoreResumeInfoFilename += ".boxbackupresume";
+
+ // Target exists?
+ int targetExistance = ObjectExists(LocalDirectoryName);
+
+ // Does any resumption information exist?
+ bool doingResume = false;
+ if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && targetExistance == ObjectExists_Dir)
+ {
+ if(!Resume)
+ {
+ // Caller didn't specify that resume should be done, so refuse to do it
+ // but say why.
+ return Restore_ResumePossible;
+ }
+
+ // Attempt to load the resume info file
+ if(!params.mResumeInfo.Load(params.mRestoreResumeInfoFilename))
+ {
+ // failed -- bad file, so things have gone a bit wrong
+ return Restore_TargetExists;
+ }
+
+ // Flag as doing resume so next check isn't actually performed
+ doingResume = true;
+ }
+
+ // Does the directory already exist?
+ if(targetExistance != ObjectExists_NoObject && !doingResume)
+ {
+ // Don't do anything in this case!
+ return Restore_TargetExists;
+ }
+
+ // Restore the directory
+ std::string localName(LocalDirectoryName);
+ BackupClientRestoreDir(rConnection, DirectoryID, localName, params, params.mResumeInfo);
+
+ // Undelete the directory on the server?
+ if(RestoreDeleted && UndeleteAfterRestoreDeleted)
+ {
+ // Send the command
+ rConnection.QueryUndeleteDirectory(DirectoryID);
+ }
+
+ // Finish progress display?
+ if(PrintDots)
+ {
+ printf("\n");
+ fflush(stdout);
+ }
+
+ // Delete the resume information file
+ ::unlink(params.mRestoreResumeInfoFilename.c_str());
+
+ return Restore_Complete;
+}
+
+
+
+
diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h
new file mode 100755
index 00000000..4b382771
--- /dev/null
+++ b/lib/backupclient/BackupClientRestore.h
@@ -0,0 +1,26 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientRestore.h
+// Purpose: Functions to restore files from a backup store
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSCLIENTRESTORE_H
+#define BACKUPSCLIENTRESTORE__H
+
+class BackupProtocolClient;
+
+enum
+{
+ Restore_Complete = 0,
+ Restore_ResumePossible = 1,
+ Restore_TargetExists = 2
+};
+
+int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName,
+ bool PrintDots = false, bool RestoreDeleted = false, bool UndeleteAfterRestoreDeleted = false, bool Resume = false);
+
+#endif // BACKUPSCLIENTRESTORE__H
+
diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp
new file mode 100755
index 00000000..eec2ceaf
--- /dev/null
+++ b/lib/backupclient/BackupDaemonConfigVerify.cpp
@@ -0,0 +1,105 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupDaemonConfigVerify.cpp
+// Purpose: Configuration file definition for bbackupd
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupDaemonConfigVerify.h"
+#include "Daemon.h"
+#include "BoxPortsAndFiles.h"
+
+#include "MemLeakFindOn.h"
+
+
+static const ConfigurationVerifyKey backuplocationkeys[] =
+{
+ {"ExcludeFile", 0, ConfigTest_MultiValueAllowed, 0},
+ {"ExcludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"ExcludeDir", 0, ConfigTest_MultiValueAllowed, 0},
+ {"ExcludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeFile", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeDir", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"Path", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+static const ConfigurationVerify backuplocations[] =
+{
+ {
+ "*",
+ 0,
+ backuplocationkeys,
+ ConfigTest_LastEntry,
+ 0
+ }
+};
+
+static const ConfigurationVerifyKey verifyserverkeys[] =
+{
+ DAEMON_VERIFY_SERVER_KEYS
+};
+
+static const ConfigurationVerify verifyserver[] =
+{
+ {
+ "Server",
+ 0,
+ verifyserverkeys,
+ ConfigTest_Exists,
+ 0
+ },
+ {
+ "BackupLocations",
+ backuplocations,
+ 0,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+ }
+};
+
+static const ConfigurationVerifyKey verifyrootkeys[] =
+{
+ {"AccountNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+
+ {"UpdateStoreInterval", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"MinimumFileAge", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"MaxUploadWait", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"MaxFileTimeInFuture", "172800", ConfigTest_IsInt, 0}, // file is uploaded if the file is this much in the future (2 days default)
+
+ {"AutomaticBackup", "yes", ConfigTest_IsBool, 0},
+
+ {"SyncAllowScript", 0, 0, 0}, // optional script to run to see if the sync should be started now
+ // return "now" if it's allowed, or a number of seconds if it's not
+
+ {"MaximumDiffingTime", 0, ConfigTest_IsInt, 0},
+
+ {"FileTrackingSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"DiffingUploadSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"StoreHostname", 0, ConfigTest_Exists, 0},
+ {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file
+
+ {"CommandSocket", 0, 0, 0}, // not compulsory to have this
+
+ {"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full
+
+ {"CertificateFile", 0, ConfigTest_Exists, 0},
+ {"PrivateKeyFile", 0, ConfigTest_Exists, 0},
+ {"TrustedCAsFile", 0, ConfigTest_Exists, 0},
+ {"KeysFile", 0, ConfigTest_Exists, 0},
+ {"DataDirectory", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+const ConfigurationVerify BackupDaemonConfigVerify =
+{
+ "root",
+ verifyserver,
+ verifyrootkeys,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+};
+
diff --git a/lib/backupclient/BackupDaemonConfigVerify.h b/lib/backupclient/BackupDaemonConfigVerify.h
new file mode 100755
index 00000000..fefa703c
--- /dev/null
+++ b/lib/backupclient/BackupDaemonConfigVerify.h
@@ -0,0 +1,18 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupDaemonConfigVerify.h
+// Purpose: Configuration file definition for bbackupd
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPDAEMONCONFIGVERIFY__H
+#define BACKUPDAEMONCONFIGVERIFY__H
+
+#include "Configuration.h"
+
+extern const ConfigurationVerify BackupDaemonConfigVerify;
+
+#endif // BACKUPDAEMONCONFIGVERIFY__H
+
diff --git a/lib/backupclient/BackupStoreConstants.h b/lib/backupclient/BackupStoreConstants.h
new file mode 100755
index 00000000..8254e918
--- /dev/null
+++ b/lib/backupclient/BackupStoreConstants.h
@@ -0,0 +1,53 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreContants.h
+// Purpose: constants for the backup system
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECONSTANTS__H
+#define BACKUPSTORECONSTANTS__H
+
+#define BACKUPSTORE_ROOT_DIRECTORY_ID 1
+
+#define BACKUP_STORE_SERVER_VERSION 1
+
+// Minimum size for a chunk to be compressed
+#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256
+
+// min and max sizes for blocks
+#define BACKUP_FILE_MIN_BLOCK_SIZE 4096
+#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024)
+
+// Increase the block size if there are more than this number of blocks
+#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096
+
+// Avoid creating blocks smaller than this
+#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128
+
+// Maximum number of sizes to do an rsync-like scan for
+#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 8
+
+// When doing rsync scans, do not scan for blocks smaller than
+#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 256
+
+// A limit to stop diffing running out of control: If more than this
+// times the number of blocks in the original index are found, stop
+// looking. This stops really bad cases of diffing files containing
+// all the same byte using huge amounts of memory and processor time.
+// This is a multiple of the number of blocks in the diff from file.
+#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096
+
+// How many seconds to wait before deleting unused root directory entries?
+#ifndef NDEBUG
+ // Debug: 30 seconds (easier to test)
+ #define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER 30
+#else
+ // Release: 2 days (plenty of time for sysadmins to notice, or change their mind)
+ #define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER 172800
+#endif
+
+#endif // BACKUPSTORECONSTANTS__H
+
diff --git a/lib/backupclient/BackupStoreDirectory.cpp b/lib/backupclient/BackupStoreDirectory.cpp
new file mode 100755
index 00000000..31fed78d
--- /dev/null
+++ b/lib/backupclient/BackupStoreDirectory.cpp
@@ -0,0 +1,563 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory.h
+// Purpose: Representation of a backup directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+
+#include "BackupStoreDirectory.h"
+#include "IOStream.h"
+#include "BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int32_t mNumEntries;
+ int64_t mObjectID; // this object ID
+ int64_t mContainerID; // ID of container
+ uint64_t mAttributesModTime;
+ int32_t mOptionsPresent; // bit mask of optional sections / features present
+ // Then a StreamableMemBlock for attributes
+} dir_StreamFormat;
+
+enum
+{
+ Option_DependencyInfoPresent = 1
+} dir_StreamFormatOptions;
+
+typedef struct
+{
+ uint64_t mModificationTime;
+ int64_t mObjectID;
+ int64_t mSizeInBlocks;
+ uint64_t mAttributesHash;
+ int16_t mFlags; // order smaller items after bigger ones (for alignment)
+ // Then a BackupStoreFilename
+ // Then a StreamableMemBlock for attributes
+} en_StreamFormat;
+
+typedef struct
+{
+ int64_t mDependsNewer;
+ int64_t mDependsOlder;
+} en_StreamFormatDepends;
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::BackupStoreDirectory()
+// Purpose: Constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::BackupStoreDirectory()
+ : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0)
+{
+ ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t)
+// Purpose: Constructor giving object and container IDs
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID)
+ : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::~BackupStoreDirectory()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::~BackupStoreDirectory()
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+ {
+ delete (*i);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int)
+// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Get the header
+ dir_StreamFormat hdr;
+ if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic value...
+ if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat)
+ }
+
+ // Get data
+ mObjectID = ntoh64(hdr.mObjectID);
+ mContainerID = ntoh64(hdr.mContainerID);
+ mAttributesModTime = ntoh64(hdr.mAttributesModTime);
+
+ // Options
+ int32_t options = ntohl(hdr.mOptionsPresent);
+
+ // Get attributes
+ mAttributes.ReadFromStream(rStream, Timeout);
+
+ // Decode count
+ int count = ntohl(hdr.mNumEntries);
+
+ // Clear existing list
+ mEntries.clear();
+
+ // Read them in!
+ for(int c = 0; c < count; ++c)
+ {
+ Entry *pen = new Entry;
+ try
+ {
+ // Read from stream
+ pen->ReadFromStream(rStream, Timeout);
+
+ // Add to list
+ mEntries.push_back(pen);
+ }
+ catch(...)
+ {
+ delete pen;
+ throw;
+ }
+ }
+
+ // Read in dependency info?
+ if(options & Option_DependencyInfoPresent)
+ {
+ // Read in extra dependency data
+ for(int c = 0; c < count; ++c)
+ {
+ mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout);
+ }
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool)
+// Purpose: Writes a selection of entries to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const
+{
+ // Get count of entries
+ int32_t count = mEntries.size();
+ if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Need to count the entries
+ count = 0;
+ Iterator i(*this);
+ while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0)
+ {
+ count++;
+ }
+ }
+
+ // Check that sensible IDs have been set
+ ASSERT(mObjectID != 0);
+ ASSERT(mContainerID != 0);
+
+ // Need dependency info?
+ bool dependencyInfoRequired = false;
+ if(StreamDependencyInfo)
+ {
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ if(pen->HasDependencies())
+ {
+ dependencyInfoRequired = true;
+ }
+ }
+ }
+
+ // Options
+ int32_t options = 0;
+ if(dependencyInfoRequired) options |= Option_DependencyInfoPresent;
+
+ // Build header
+ dir_StreamFormat hdr;
+ hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE);
+ hdr.mNumEntries = htonl(count);
+ hdr.mObjectID = hton64(mObjectID);
+ hdr.mContainerID = hton64(mContainerID);
+ hdr.mAttributesModTime = hton64(mAttributesModTime);
+ hdr.mOptionsPresent = htonl(options);
+
+ // Write header
+ rStream.Write(&hdr, sizeof(hdr));
+
+ // Write the attributes?
+ if(StreamAttributes)
+ {
+ mAttributes.WriteToStream(rStream);
+ }
+ else
+ {
+ // Write a blank header instead
+ StreamableMemBlock::WriteEmptyBlockToStream(rStream);
+ }
+
+ // Then write all the entries
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ pen->WriteToStream(rStream);
+ }
+
+ // Write dependency info?
+ if(dependencyInfoRequired)
+ {
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ pen->WriteToStreamDependencyInfo(rStream);
+ }
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddEntry(const Entry &)
+// Purpose: Adds entry to directory (no checking)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy)
+{
+ Entry *pnew = new Entry(rEntryToCopy);
+ try
+ {
+ mEntries.push_back(pnew);
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+
+ return pnew;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
+// Purpose: Adds entry to directory (no checking)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime)
+{
+ Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime);
+ try
+ {
+ mEntries.push_back(pnew);
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+
+ return pnew;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::DeleteEntry(int64_t)
+// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::DeleteEntry(int64_t ObjectID)
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin());
+ i != mEntries.end(); ++i)
+ {
+ if((*i)->mObjectID == ObjectID)
+ {
+ // Delete
+ delete (*i);
+ // Remove from list
+ mEntries.erase(i);
+ // Done
+ return;
+ }
+ }
+
+ // Not found
+ THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::FindEntryByID(int64_t)
+// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist.
+// Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const
+{
+ for(std::vector<Entry*>::const_iterator i(mEntries.begin());
+ i != mEntries.end(); ++i)
+ {
+ if((*i)->mObjectID == ObjectID)
+ {
+ // Found
+ return (*i);
+ }
+ }
+
+ // Not found
+ return 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry()
+// Purpose: Constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry()
+ : mModificationTime(0),
+ mObjectID(0),
+ mSizeInBlocks(0),
+ mFlags(0),
+ mAttributesHash(0),
+ mMinMarkNumber(0),
+ mMarkNumber(0),
+ mDependsNewer(0),
+ mDependsOlder(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::~Entry()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::~Entry()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry(const Entry &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry(const Entry &rToCopy)
+ : mName(rToCopy.mName),
+ mModificationTime(rToCopy.mModificationTime),
+ mObjectID(rToCopy.mObjectID),
+ mSizeInBlocks(rToCopy.mSizeInBlocks),
+ mFlags(rToCopy.mFlags),
+ mAttributesHash(rToCopy.mAttributesHash),
+ mAttributes(rToCopy.mAttributes),
+ mMinMarkNumber(rToCopy.mMinMarkNumber),
+ mMarkNumber(rToCopy.mMarkNumber),
+ mDependsNewer(rToCopy.mDependsNewer),
+ mDependsOlder(rToCopy.mDependsOlder)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
+// Purpose: Constructor from values
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash)
+ : mName(rName),
+ mModificationTime(ModificationTime),
+ mObjectID(ObjectID),
+ mSizeInBlocks(SizeInBlocks),
+ mFlags(Flags),
+ mAttributesHash(AttributesHash),
+ mMinMarkNumber(0),
+ mMarkNumber(0),
+ mDependsNewer(0),
+ mDependsOlder(0)
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int)
+// Purpose: Read an entry from a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Grab the raw bytes from the stream which compose the header
+ en_StreamFormat entry;
+ if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Do reading first before modifying the variables, to be more exception safe
+
+ // Get the filename
+ BackupStoreFilename name;
+ name.ReadFromStream(rStream, Timeout);
+
+ // Get the attributes
+ mAttributes.ReadFromStream(rStream, Timeout);
+
+ // Store the rest of the bits
+ mModificationTime = ntoh64(entry.mModificationTime);
+ mObjectID = ntoh64(entry.mObjectID);
+ mSizeInBlocks = ntoh64(entry.mSizeInBlocks);
+ mAttributesHash = ntoh64(entry.mAttributesHash);
+ mFlags = ntohs(entry.mFlags);
+ mName = name;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &)
+// Purpose: Writes the entry to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const
+{
+ // Build a structure
+ en_StreamFormat entry;
+ entry.mModificationTime = hton64(mModificationTime);
+ entry.mObjectID = hton64(mObjectID);
+ entry.mSizeInBlocks = hton64(mSizeInBlocks);
+ entry.mAttributesHash = hton64(mAttributesHash);
+ entry.mFlags = htons(mFlags);
+
+ // Write it
+ rStream.Write(&entry, sizeof(entry));
+
+ // Write the filename
+ mName.WriteToStream(rStream);
+
+ // Write any attributes
+ mAttributes.WriteToStream(rStream);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int)
+// Purpose: Read the optional dependency info from a stream
+// Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout)
+{
+ // Grab the raw bytes from the stream which compose the header
+ en_StreamFormatDepends depends;
+ if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Store the data
+ mDependsNewer = ntoh64(depends.mDependsNewer);
+ mDependsOlder = ntoh64(depends.mDependsOlder);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &)
+// Purpose: Write the optional dependency info to a stream
+// Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const
+{
+ // Build structure
+ en_StreamFormatDepends depends;
+ depends.mDependsNewer = hton64(mDependsNewer);
+ depends.mDependsOlder = hton64(mDependsOlder);
+ // Write
+ rStream.Write(&depends, sizeof(depends));
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreDirectory.h b/lib/backupclient/BackupStoreDirectory.h
new file mode 100755
index 00000000..958eee81
--- /dev/null
+++ b/lib/backupclient/BackupStoreDirectory.h
@@ -0,0 +1,268 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory.h
+// Purpose: Representation of a backup directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREDIRECTORY__H
+#define BACKUPSTOREDIRECTORY__H
+
+#include <string>
+#include <vector>
+
+#include "BackupStoreFilenameClear.h"
+#include "StreamableMemBlock.h"
+#include "BoxTime.h"
+
+class IOStream;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreDirectory
+// Purpose: In memory representation of a directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreDirectory
+{
+public:
+ BackupStoreDirectory();
+ BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID);
+private:
+ // Copying not allowed
+ BackupStoreDirectory(const BackupStoreDirectory &rToCopy);
+public:
+ ~BackupStoreDirectory();
+
+ class Entry
+ {
+ public:
+ friend class BackupStoreDirectory;
+
+ Entry();
+ ~Entry();
+ Entry(const Entry &rToCopy);
+ Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash);
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream) const;
+
+ const BackupStoreFilename &GetName() const {return mName;}
+ box_time_t GetModificationTime() const {return mModificationTime;}
+ int64_t GetObjectID() const {return mObjectID;}
+ int64_t GetSizeInBlocks() const {return mSizeInBlocks;}
+ int16_t GetFlags() const {return mFlags;}
+ void AddFlags(int16_t Flags) {mFlags |= Flags;}
+ void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;}
+
+ // Some things can be changed
+ void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;}
+ void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;}
+
+ // Attributes
+ bool HasAttributes() const {return !mAttributes.IsEmpty();}
+ void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;}
+ const StreamableMemBlock &GetAttributes() const {return mAttributes;}
+ uint64_t GetAttributesHash() const {return mAttributesHash;}
+
+ // Marks
+ // The lowest mark number a version of a file of this name has ever had
+ uint32_t GetMinMarkNumber() const {return mMinMarkNumber;}
+ // The mark number on this file
+ uint32_t GetMarkNumber() const {return mMarkNumber;}
+
+ // Make sure these flags are synced with those in backupprocotol.txt
+ // ListDirectory command
+ enum
+ {
+ Flags_INCLUDE_EVERYTHING = -1,
+ Flags_EXCLUDE_NOTHING = 0,
+ Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below!
+ Flags_File = 1,
+ Flags_Dir = 2,
+ Flags_Deleted = 4,
+ Flags_OldVersion = 8,
+ Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion
+ };
+ // characters for textual listing of files -- see bbackupquery/BackupQueries
+ #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR"
+
+ bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet)
+ {
+ return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet))
+ && ((mFlags & FlagsNotToBeSet) == 0);
+ };
+
+ // Get dependency info
+ // new version this depends on
+ int64_t GetDependsNewer() const {return mDependsNewer;}
+ void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;}
+ // older version which depends on this
+ int64_t GetDependsOlder() const {return mDependsOlder;}
+ void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;}
+
+ // Dependency info saving
+ bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;}
+ void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout);
+ void WriteToStreamDependencyInfo(IOStream &rStream) const;
+
+ private:
+ BackupStoreFilename mName;
+ box_time_t mModificationTime;
+ int64_t mObjectID;
+ int64_t mSizeInBlocks;
+ int16_t mFlags;
+ uint64_t mAttributesHash;
+ StreamableMemBlock mAttributes;
+ uint32_t mMinMarkNumber;
+ uint32_t mMarkNumber;
+
+ uint64_t mDependsNewer; // new version this depends on
+ uint64_t mDependsOlder; // older version which depends on this
+ };
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream,
+ int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING,
+ int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING,
+ bool StreamAttributes = true, bool StreamDependencyInfo = true) const;
+
+ Entry *AddEntry(const Entry &rEntryToCopy);
+ Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime);
+ void DeleteEntry(int64_t ObjectID);
+ Entry *FindEntryByID(int64_t ObjectID) const;
+
+ int64_t GetObjectID() const {return mObjectID;}
+ int64_t GetContainerID() const {return mContainerID;}
+
+ // Need to be able to update the container ID when moving objects
+ void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;}
+
+ // Purely for use of server -- not serialised into streams
+ int64_t GetRevisionID() const {return mRevisionID;}
+ void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;}
+
+ unsigned int GetNumberOfEntries() const {return mEntries.size();}
+
+ // User info -- not serialised into streams
+ int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;}
+ void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;}
+
+ // Attributes
+ bool HasAttributes() const {return !mAttributes.IsEmpty();}
+ void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;}
+ const StreamableMemBlock &GetAttributes() const {return mAttributes;}
+ box_time_t GetAttributesModTime() const {return mAttributesModTime;}
+
+ class Iterator
+ {
+ public:
+ Iterator(const BackupStoreDirectory &rDir)
+ : mrDir(rDir), i(rDir.mEntries.begin())
+ {
+ }
+
+ BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags
+ while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.end())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ // WARNING: This function is really very inefficient.
+ // Only use when you want to look up ONE filename, not in a loop looking up lots.
+ // In a looping situation, cache the decrypted filenames in another memory structure.
+ BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags or filename
+ while( (i != mrDir.mEntries.end())
+ && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) )
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.end())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ private:
+ const BackupStoreDirectory &mrDir;
+ std::vector<Entry*>::const_iterator i;
+ };
+
+ friend class Iterator;
+
+ class ReverseIterator
+ {
+ public:
+ ReverseIterator(const BackupStoreDirectory &rDir)
+ : mrDir(rDir), i(rDir.mEntries.rbegin())
+ {
+ }
+
+ BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags
+ while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.rend())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ private:
+ const BackupStoreDirectory &mrDir;
+ std::vector<Entry*>::const_reverse_iterator i;
+ };
+
+ friend class ReverseIterator;
+
+ // For recovery of the store
+ // Implemented in BackupStoreCheck2.cpp
+ bool CheckAndFix();
+ void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags);
+ bool NameInUse(const BackupStoreFilename &rName);
+ // Don't use these functions in normal code!
+
+ // For testing
+ void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;}
+
+ // Debug and diagonistics
+ void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere
+
+private:
+ int64_t mRevisionID;
+ int64_t mObjectID;
+ int64_t mContainerID;
+ std::vector<Entry*> mEntries;
+ box_time_t mAttributesModTime;
+ StreamableMemBlock mAttributes;
+ int64_t mUserInfo1;
+};
+
+#endif // BACKUPSTOREDIRECTORY__H
+
diff --git a/lib/backupclient/BackupStoreException.h b/lib/backupclient/BackupStoreException.h
new file mode 100755
index 00000000..981dfa60
--- /dev/null
+++ b/lib/backupclient/BackupStoreException.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreException.h
+// Purpose: Exception
+// Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREEXCEPTION__H
+#define BACKUPSTOREEXCEPTION__H
+
+// Compatibility
+#include "autogen_BackupStoreException.h"
+
+#endif // BACKUPSTOREEXCEPTION__H
+
diff --git a/lib/backupclient/BackupStoreException.txt b/lib/backupclient/BackupStoreException.txt
new file mode 100644
index 00000000..50c615b2
--- /dev/null
+++ b/lib/backupclient/BackupStoreException.txt
@@ -0,0 +1,70 @@
+EXCEPTION BackupStore 4
+
+Internal 0
+BadAccountDatabaseFile 1
+AccountDatabaseNoSuchEntry 2
+InvalidBackupStoreFilename 3
+UnknownFilenameEncoding 4
+CouldntReadEntireStructureFromStream 5
+BadDirectoryFormat 6
+CouldNotFindEntryInDirectory 7
+OutputFileAlreadyExists 8
+OSFileError 9
+StreamDoesntHaveRequiredFeatures 10
+BadBackupStoreFile 11
+CouldNotLoadStoreInfo 12
+BadStoreInfoOnLoad 13
+StoreInfoIsReadOnly 14
+StoreInfoDirNotInList 15
+StoreInfoBlockDeltaMakesValueNegative 16
+DirectoryHasBeenDeleted 17
+StoreInfoNotInitialised 18
+StoreInfoAlreadyLoaded 19
+StoreInfoNotLoaded 20
+ReadFileFromStreamTimedOut 21
+FileWrongSizeAfterBeingStored 22
+AddedFileDoesNotVerify 23
+StoreInfoForWrongAccount 24
+ContextIsReadOnly 25
+AttributesNotLoaded 26
+AttributesNotUnderstood 27
+WrongServerVersion 28 # client side
+ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account.
+NameAlreadyExistsInDirectory 30
+BerkelyDBFailure 31 # client side
+InodeMapIsReadOnly 32 # client side
+InodeMapNotOpen 33 # client side
+FilenameEncryptionKeyNotKnown 34
+FilenameEncryptionNoKeyForSpecifiedMethod 35
+FilenameEncryptionNotSetup 36
+CouldntLoadClientKeyMaterial 37
+BadEncryptedAttributes 38
+EncryptedAttributesHaveUnknownEncoding 39
+OutputSizeTooSmallForChunk 40
+BadEncodedChunk 41
+NotEnoughSpaceToDecodeChunk 42
+ChunkHasUnknownEncoding 43
+ChunkContainsBadCompressedData 44
+CantWriteToEncodedFileStream 45
+Temp_FileEncodeStreamDidntReadBuffer 46
+CantWriteToDecodedFileStream 47
+WhenDecodingExpectedToReadButCouldnt 48
+BackupStoreFileFailedIntegrityCheck 49
+ThereIsNoDataInASymLink 50
+IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51
+BlockEntryEncodingDidntGiveExpectedLength 52
+CouldNotFindUnusedIDDuringAllocation 53
+AddedFileExceedsStorageLimit 54
+CannotDiffAnIncompleteStoreFile 55
+CannotDecodeDiffedFilesWithoutCombining 56
+FailedToReadBlockOnCombine 57
+OnCombineFromFileIsIncomplete 58
+BadNotifySysadminEventCode 59
+InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60
+CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server?
+AttributeHashSecretNotSet 62
+AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES
+SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection.
+IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related
+DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory
+PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it.
diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp
new file mode 100755
index 00000000..2ea877b4
--- /dev/null
+++ b/lib/backupclient/BackupStoreFile.cpp
@@ -0,0 +1,1499 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFile.cpp
+// Purpose: Utils for manipulating files
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <unistd.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <new>
+#include <string.h>
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ #include <syslog.h>
+ #include <stdio.h>
+#endif
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFilename.h"
+#include "BackupStoreException.h"
+#include "IOStream.h"
+#include "Guards.h"
+#include "FileModificationTime.h"
+#include "FileStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreObjectMagic.h"
+#include "Compress.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "CipherAES.h"
+#include "BackupStoreConstants.h"
+#include "CollectInBufferStream.h"
+#include "RollingChecksum.h"
+#include "MD5Digest.h"
+#include "ReadGatherStream.h"
+#include "Random.h"
+#include "BackupStoreFileEncodeStream.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+
+// How big a buffer to use for copying files
+#define COPY_BUFFER_SIZE (8*1024)
+
+// Statistics
+BackupStoreFileStats BackupStoreFile::msStats = {0,0,0};
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool sWarnedAboutBackwardsCompatiblity = false;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &)
+// Purpose: Encode a file into something for storing on file server.
+// Requires a real filename so full info can be stored.
+//
+// Returns a stream. Most of the work is done by the stream
+// when data is actually requested -- the file will be held
+// open until the stream is deleted or the file finished.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime)
+{
+ // Create the stream
+ std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
+
+ // Do the initial setup
+ ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, 0 /* no recipe, just encode */,
+ ContainerID, rStoreFilename, pModificationTime);
+
+ // Return the stream for the caller
+ return stream;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &)
+// Purpose: Verify that an encoded file meets the format requirements.
+// Doesn't verify that the data is intact and can be decoded.
+// Optionally returns the ID of the file which it is diffed from,
+// and the (original) container ID.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut)
+{
+ // Get the size of the file
+ int64_t fileSize = rFile.BytesLeftToRead();
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Get the header...
+ file_StreamFormat hdr;
+ if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read header
+ return false;
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ return false;
+ }
+
+ // Get a filename, see if it loads OK
+ try
+ {
+ BackupStoreFilename fn;
+ fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+ }
+ catch(...)
+ {
+ // an error occured while reading it, so that's not good
+ return false;
+ }
+
+ // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not
+ try
+ {
+ int32_t size_s;
+ if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+ }
+ int size = ntohl(size_s);
+ // Skip forward the size
+ rFile.Seek(size, IOStream::SeekType_Relative);
+ }
+ catch(...)
+ {
+ // an error occured while reading it, so that's not good
+ return false;
+ }
+
+ // Get current position in file -- the end of the header
+ int64_t headerEnd = rFile.GetPosition();
+
+ // Get number of blocks
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+
+ // Calculate where the block index will be, check it's reasonable
+ int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+ if(blockIndexLoc < headerEnd)
+ {
+ // Not enough space left for the block index, let alone the blocks themselves
+ return false;
+ }
+
+ // Load the block index header
+ rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute);
+ file_BlockIndexHeader blkhdr;
+ if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read block index header -- assume bad file
+ return false;
+ }
+
+ // Check header
+ if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
+#endif
+ )
+ || (int64_t)ntoh64(blkhdr.mNumBlocks) != numBlocks)
+ {
+ // Bad header -- either magic value or number of blocks is wrong
+ return false;
+ }
+
+ // Flag for recording whether a block is referenced from another file
+ bool blockFromOtherFileReferenced = false;
+
+ // Read the index, checking that the length values all make sense
+ int64_t currentBlockStart = headerEnd;
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read block entry
+ file_BlockIndexEntry blk;
+ if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read block index entry -- assume bad file
+ return false;
+ }
+
+ // Check size and location
+ int64_t blkSize = ntoh64(blk.mEncodedSize);
+ if(blkSize <= 0)
+ {
+ // Mark that this file references another file
+ blockFromOtherFileReferenced = true;
+ }
+ else
+ {
+ // This block is actually in this file
+ if((currentBlockStart + blkSize) > blockIndexLoc)
+ {
+ // Encoded size makes the block run over the index
+ return false;
+ }
+
+ // Move the current block start ot the end of this block
+ currentBlockStart += blkSize;
+ }
+ }
+
+ // Check that there's no empty space
+ if(currentBlockStart != blockIndexLoc)
+ {
+ return false;
+ }
+
+ // Check that if another block is references, then the ID is there, and if one isn't there is no ID.
+ int64_t otherID = ntoh64(blkhdr.mOtherFileID);
+ if((otherID != 0 && blockFromOtherFileReferenced == false)
+ || (otherID == 0 && blockFromOtherFileReferenced == true))
+ {
+ // Doesn't look good!
+ return false;
+ }
+
+ // Does the caller want the other ID?
+ if(pDiffFromObjectIDOut)
+ {
+ *pDiffFromObjectIDOut = otherID;
+ }
+
+ // Does the caller want the container ID?
+ if(pContainerIDOut)
+ {
+ *pContainerIDOut = ntoh64(hdr.mContainerID);
+ }
+
+ // Passes all tests
+ return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeFile(IOStream &, const char *)
+// Purpose: Decode a file. Will set file attributes. File must not exist.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Does file exist?
+ struct stat st;
+ if(::stat(DecodedFilename, &st) == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists)
+ }
+
+ // Try, delete output file if error
+ try
+ {
+ // Make a stream for outputting this file
+ FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL);
+
+ // Get the decoding stream
+ std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr));
+
+ // Is it a symlink?
+ if(!stream->IsSymLink())
+ {
+ // Copy it out to the file
+ stream->CopyStreamTo(out);
+ }
+
+ // Write the attributes
+ stream->GetAttributes().WriteAttributes(DecodedFilename);
+ }
+ catch(...)
+ {
+ ::unlink(DecodedFilename);
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *)
+// Purpose: Return a stream which will decode the encrypted file data on the fly.
+// Accepts streams in block index first, or main header first, order. In the latter case,
+// the stream must be Seek()able.
+//
+// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow
+// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Create stream
+ std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout));
+
+ // Get it ready
+ stream->Setup(pAlterativeAttr);
+
+ // Return to caller
+ return stream;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int)
+// Purpose: Constructor
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout)
+ : mrEncodedFile(rEncodedFile),
+ mTimeout(Timeout),
+ mNumBlocks(0),
+ mpBlockIndex(0),
+ mpEncodedData(0),
+ mpClearData(0),
+ mClearDataSize(0),
+ mCurrentBlock(-1),
+ mCurrentBlockClearSize(0),
+ mPositionInCurrentBlock(0),
+ mEntryIVBase(42) // different to default value in the encoded stream!
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ , mIsOldVersion(false)
+#endif
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::~DecodedStream()
+// Purpose: Desctructor
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::DecodedStream::~DecodedStream()
+{
+ // Free any allocated memory
+ if(mpBlockIndex)
+ {
+ ::free(mpBlockIndex);
+ }
+ if(mpEncodedData)
+ {
+ BackupStoreFile::CodingChunkFree(mpEncodedData);
+ }
+ if(mpClearData)
+ {
+ ::free(mpClearData);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *)
+// Purpose: Get the stream ready to decode -- reads in headers
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Get the size of the file
+ int64_t fileSize = mrEncodedFile.BytesLeftToRead();
+
+ // Get the magic number to work out which order the stream is in
+ int32_t magic;
+ if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read magic value
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ bool inFileOrder = true;
+ switch(ntohl(magic))
+ {
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
+ mIsOldVersion = true;
+ // control flows on
+#endif
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
+ inFileOrder = true;
+ break;
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0:
+ mIsOldVersion = true;
+ // control flows on
+#endif
+ case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1:
+ inFileOrder = false;
+ break;
+
+ default:
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // If not in file order, then the index list must be read now
+ if(!inFileOrder)
+ {
+ ReadBlockIndex(true /* have already read and verified the magic number */);
+ }
+
+ // Get header
+ file_StreamFormat hdr;
+ if(inFileOrder)
+ {
+ // Read the header, without the magic number
+ if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic),
+ 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ // Put in magic number
+ hdr.mMagicValue = magic;
+ }
+ else
+ {
+ // Not in file order, so need to read the full header
+ if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Get the filename
+ mFilename.ReadFromStream(mrEncodedFile, mTimeout);
+
+ // Get the attributes (either from stream, or supplied attributes)
+ if(pAlterativeAttr != 0)
+ {
+ // Read dummy attributes
+ BackupClientFileAttributes attr;
+ attr.ReadFromStream(mrEncodedFile, mTimeout);
+
+ // Set to supplied attributes
+ mAttributes = *pAlterativeAttr;
+ }
+ else
+ {
+ // Read the attributes from the stream
+ mAttributes.ReadFromStream(mrEncodedFile, mTimeout);
+ }
+
+ // If it is in file order, go and read the file attributes
+ // Requires that the stream can seek
+ if(inFileOrder)
+ {
+ // Make sure the file size is known
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Store current location (beginning of encoded blocks)
+ int64_t endOfHeaderPos = mrEncodedFile.GetPosition();
+
+ // Work out where the index is
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+ int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+
+ // Seek to that position
+ mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute);
+
+ // Read the block index
+ ReadBlockIndex(false /* magic number still to be read */);
+
+ // Seek back to the end of header position, ready for reading the chunks
+ mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute);
+ }
+
+ // Check view of blocks from block header and file header match
+ if(mNumBlocks != (int64_t)ntoh64(hdr.mNumBlocks))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Need to allocate some memory for the two blocks for reading encoded data, and clear data
+ if(mNumBlocks > 0)
+ {
+ // Find the maximum encoded data size
+ int32_t maxEncodedDataSize = 0;
+ const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
+ ASSERT(entry != 0);
+ for(int64_t e = 0; e < mNumBlocks; e++)
+ {
+ // Get the clear and encoded size
+ int32_t encodedSize = ntoh64(entry[e].mEncodedSize);
+ ASSERT(encodedSize > 0);
+
+ // Larger?
+ if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize;
+ }
+
+ // Allocate those blocks!
+ mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32);
+
+ // Allocate the block for the clear data, using the hint from the header.
+ // If this is wrong, things will exception neatly later on, so it can't be used
+ // to do anything more than cause an error on downloading.
+ mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32;
+ mpClearData = (uint8_t*)::malloc(mClearDataSize);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool)
+// Purpose: Read the block index from the stream, and store in internal buffer (minus header)
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead)
+{
+ // Header
+ file_BlockIndexHeader blkhdr;
+
+ // Read it in -- way depends on how whether the magic number has already been read
+ if(MagicAlreadyRead)
+ {
+ // Read the header, without the magic number
+ if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue),
+ 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+ else
+ {
+ // Magic not already read, so need to read the full header
+ if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Check magic value
+ if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ }
+
+ // Get the number of blocks out of the header
+ mNumBlocks = ntoh64(blkhdr.mNumBlocks);
+
+ // Read the IV base
+ mEntryIVBase = ntoh64(blkhdr.mEntryIVBase);
+
+ // Load the block entries in?
+ if(mNumBlocks > 0)
+ {
+ // How big is the index?
+ int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks;
+
+ // Allocate some memory
+ mpBlockIndex = ::malloc(indexSize);
+ if(mpBlockIndex == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Read it in
+ if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Read(void *, int, int)
+// Purpose: As interface. Reads decrpyted data.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Symlinks don't have data. So can't read it. Not even zero bytes.
+ if(IsSymLink())
+ {
+ // Don't allow reading in this case
+ THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink);
+ }
+
+ // Already finished?
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // At end of stream, nothing to do
+ return 0;
+ }
+
+ int bytesToRead = NBytes;
+ uint8_t *output = (uint8_t*)pBuffer;
+
+ while(bytesToRead > 0 && mCurrentBlock < mNumBlocks)
+ {
+ // Anything left in the current block?
+ if(mPositionInCurrentBlock < mCurrentBlockClearSize)
+ {
+ // Copy data out of this buffer
+ int s = mCurrentBlockClearSize - mPositionInCurrentBlock;
+ if(s > bytesToRead) s = bytesToRead; // limit to requested data
+
+ // Copy
+ ::memcpy(output, mpClearData + mPositionInCurrentBlock, s);
+
+ // Update positions
+ output += s;
+ mPositionInCurrentBlock += s;
+ bytesToRead -= s;
+ }
+
+ // Need to get some more data?
+ if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize)
+ {
+ // Number of next block
+ ++mCurrentBlock;
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // Stop now!
+ break;
+ }
+
+ // Get the size from the block index
+ const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
+ int32_t encodedSize = ntoh64(entry[mCurrentBlock].mEncodedSize);
+ if(encodedSize <= 0)
+ {
+ // The caller is attempting to decode a file which is the direct result of a diff
+ // operation, and so does not contain all the data.
+ // It needs to be combined with the previous version first.
+ THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining)
+ }
+
+ // Load in next block
+ if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Decode the data
+ mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize);
+
+ // Calculate IV for this entry
+ uint64_t iv = mEntryIVBase;
+ iv += mCurrentBlock;
+ // Convert to network byte order before encrypting with it, so that restores work on
+ // platforms with different endiannesses.
+ iv = hton64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Make sure this is the right size
+ if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
+ {
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ if(!mIsOldVersion)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the
+ // IV for the encrypted section. Try again, with the thing the other way round
+ iv = box_swap64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+ if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ else
+ {
+ // Warn and log this issue
+ if(!sWarnedAboutBackwardsCompatiblity)
+ {
+ ::printf("WARNING: Decoded one or more files using backwards compatibility mode for block index.\n");
+ ::syslog(LOG_ERR, "WARNING: Decoded one or more files using backwards compatibility mode for block index.\n");
+ sWarnedAboutBackwardsCompatiblity = true;
+ }
+ }
+#else
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+#endif
+ }
+
+ // Check the digest
+ MD5Digest md5;
+ md5.Add(mpClearData, mCurrentBlockClearSize);
+ md5.Finish();
+ if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum))
+ {
+ THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck)
+ }
+
+ // Set vars to say what's happening
+ mPositionInCurrentBlock = 0;
+ }
+ }
+
+ ASSERT(bytesToRead >= 0);
+ ASSERT(bytesToRead <= NBytes);
+
+ return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::IsSymLink()
+// Purpose: Is the unencoded file actually a symlink?
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::IsSymLink()
+{
+ // First, check in with the attributes
+ if(!mAttributes.IsSymLink())
+ {
+ return false;
+ }
+
+ // So the attributes think it is a symlink.
+ // Consistency check...
+ if(mNumBlocks != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Write(const void *, int)
+// Purpose: As interface. Throws exception, as you can't write to this stream.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::StreamDataLeft()
+// Purpose: As interface. Any data left?
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::StreamDataLeft()
+{
+ return mCurrentBlock < mNumBlocks;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::StreamClosed()
+// Purpose: As interface. Always returns true, no writing allowed.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::StreamClosed()
+{
+ // Can't write to this stream!
+ return true;
+}
+
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::SetBlowfishKey(const void *, int)
+// Purpose: Static. Sets the key to use for encryption and decryption.
+// Created: 7/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength)
+{
+ // IVs set later
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+
+ sBlowfishEncryptBlockEntry.Reset();
+ sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
+ sBlowfishEncryptBlockEntry.UsePadding(false);
+ sBlowfishDecryptBlockEntry.Reset();
+ sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
+ sBlowfishDecryptBlockEntry.UsePadding(false);
+}
+
+
+#ifndef PLATFORM_OLD_OPENSSL
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::SetAESKey(const void *, int)
+// Purpose: Sets the AES key to use for file data encryption. Will select AES as
+// the cipher to use when encrypting.
+// Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength)
+{
+ // Setup context
+ sAESEncrypt.Reset();
+ sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sAESDecrypt.Reset();
+ sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
+
+ // Set encryption to use this key, instead of the "default" blowfish key
+ spEncrypt = &sAESEncrypt;
+ sEncryptCipherType = HEADER_AES_ENCODING;
+}
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int)
+// Purpose: The maximum output size of a block, given the chunk size
+// Created: 7/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize)
+{
+ // Calculate... the maximum size of output by first the largest it could be after compression,
+ // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck
+ // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing
+ // an problem where a rather non-compressable file didn't fit in a block buffer.)
+ return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1
+ + sBlowfishEncrypt.GetIVLength() + 128;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &)
+// Purpose: Encodes a chunk (encryption, possible compressed beforehand)
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput)
+{
+ ASSERT(spEncrypt != 0);
+
+ // Check there's some space in the output block
+ if(rOutput.mBufferSize < 256)
+ {
+ rOutput.Reallocate(256);
+ }
+
+ // Check alignment of the block
+ ASSERT((((uint32_t)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+
+ // Want to compress it?
+ bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE);
+
+ // Build header
+ uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT;
+ if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED;
+
+ // Store header
+ rOutput.mpBuffer[0] = header;
+ int outOffset = 1;
+
+ // Setup cipher, and store the IV
+ int ivLen = 0;
+ const void *iv = spEncrypt->SetRandomIV(ivLen);
+ ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen);
+ outOffset += ivLen;
+
+ // Start encryption process
+ spEncrypt->Begin();
+
+ #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \
+ { \
+ if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \
+ { \
+ rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \
+ } \
+ }
+
+ // Encode the chunk
+ if(compressChunk)
+ {
+ // buffer to compress into
+ uint8_t buffer[2048];
+
+ // Set compressor with all the chunk as an input
+ Compress<true> compress;
+ compress.Input(Chunk, ChunkSize);
+ compress.FinishInput();
+
+ // Get and encrypt output
+ while(!compress.OutputHasFinished())
+ {
+ int s = compress.Output(buffer, sizeof(buffer));
+ if(s > 0)
+ {
+ ENCODECHUNK_CHECK_SPACE(s)
+ outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s);
+ }
+ else
+ {
+ // Should never happen, as we put all the input in in one go.
+ // So if this happens, it means there's a logical problem somewhere
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+ }
+ ENCODECHUNK_CHECK_SPACE(16)
+ outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
+ }
+ else
+ {
+ // Straight encryption
+ ENCODECHUNK_CHECK_SPACE(ChunkSize)
+ outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize);
+ ENCODECHUNK_CHECK_SPACE(16)
+ outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
+ }
+
+ ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check
+
+ return outOffset;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int)
+// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find
+// the extra output buffer size needed before calling.
+// See notes in EncodeChunk() for notes re alignment of the
+// encoded data.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize)
+{
+ // Check alignment of the encoded block
+ ASSERT((((uint32_t)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+
+ // First check
+ if(EncodedSize < 1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
+ }
+
+ const uint8_t *input = (uint8_t*)Encoded;
+
+ // Get header, make checks, etc
+ uint8_t header = input[0];
+ bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED;
+ uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT);
+ if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING)
+ {
+ THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding)
+ }
+
+#ifndef PLATFORM_OLD_OPENSSL
+ // Choose cipher
+ CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt);
+#else
+ // AES not supported with this version of OpenSSL
+ if(encodingType == HEADER_AES_ENCODING)
+ {
+ THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL)
+ }
+ CipherContext &cipher(sBlowfishDecrypt);
+#endif
+
+ // Check enough space for header, an IV and one byte of input
+ int ivLen = cipher.GetIVLength();
+ if(EncodedSize < (1 + ivLen + 1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
+ }
+
+ // Set IV in decrypt context, and start
+ cipher.SetIV(input + 1);
+ cipher.Begin();
+
+ // Setup vars for code
+ int inOffset = 1 + ivLen;
+ uint8_t *output = (uint8_t*)Output;
+ int outOffset = 0;
+
+ // Do action
+ if(chunkCompressed)
+ {
+ // Do things in chunks
+ uint8_t buffer[2048];
+ int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer));
+
+ // Decompressor
+ Compress<false> decompress;
+
+ while(inOffset < EncodedSize)
+ {
+ // Decrypt a block
+ int bl = inputBlockLen;
+ if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long
+ int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl);
+ inOffset += bl;
+
+ // Decompress the decrypted data
+ if(s > 0)
+ {
+ decompress.Input(buffer, s);
+ int os = 0;
+ do
+ {
+ os = decompress.Output(output + outOffset, OutputSize - outOffset);
+ outOffset += os;
+ } while(os > 0);
+
+ // Check that there's space left in the output buffer -- there always should be
+ if(outOffset >= OutputSize)
+ {
+ THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
+ }
+ }
+ }
+
+ // Get any compressed data remaining in the cipher context and compression
+ int s = cipher.Final(buffer, sizeof(buffer));
+ decompress.Input(buffer, s);
+ decompress.FinishInput();
+ while(!decompress.OutputHasFinished())
+ {
+ int os = decompress.Output(output + outOffset, OutputSize - outOffset);
+ outOffset += os;
+
+ // Check that there's space left in the output buffer -- there always should be
+ if(outOffset >= OutputSize)
+ {
+ THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
+ }
+ }
+ }
+ else
+ {
+ // Easy decryption
+ outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset);
+ outOffset += cipher.Final(output + outOffset, OutputSize - outOffset);
+ }
+
+ return outOffset;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool)
+// Purpose: Returns a stream which gives a Stream order version of the encoded file.
+// If TakeOwnership == true, then the input stream will be deleted when the
+// returned stream is deleted.
+// The input stream must be seekable.
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership)
+{
+ ASSERT(pStream != 0);
+
+ // Get the size of the file
+ int64_t fileSize = pStream->BytesLeftToRead();
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Read the header
+ int bytesRead = 0;
+ file_StreamFormat hdr;
+ bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead);
+
+ // Seek backwards to put the file pointer back where it was before we started this
+ pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative);
+
+ // Check we got a block
+ if(!readBlock)
+ {
+ // Couldn't read header -- assume file bad
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Get number of blocks
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+
+ // Calculate where the block index will be, check it's reasonable
+ int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+ int64_t blockIndexLoc = fileSize - blockIndexSize;
+ if(blockIndexLoc < 0)
+ {
+ // Doesn't look good!
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Build a reordered stream
+ std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership));
+
+ // Set it up...
+ ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get()));
+ int component = rreordered.AddComponent(pStream);
+ // Send out the block index
+ rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc);
+ // And then the rest of the file
+ rreordered.AddBlock(component, blockIndexLoc, true, 0);
+
+ return reordered;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ResetStats()
+// Purpose: Reset the gathered statistics
+// Created: 20/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::ResetStats()
+{
+ msStats.mBytesInEncodedFiles = 0;
+ msStats.mBytesAlreadyOnServer = 0;
+ msStats.mTotalFileStreamSize = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &)
+// Purpose: Compares the contents of a file against the checksums contained in the
+// block index. Returns true if the checksums match, meaning the file is
+// extremely likely to match the original. Will always consume the entire index.
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout)
+{
+ // is it a symlink?
+ bool sourceIsSymlink = false;
+ {
+ struct stat st;
+ if(::lstat(Filename, &st) == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ if((st.st_mode & S_IFMT) == S_IFLNK)
+ {
+ sourceIsSymlink = true;
+ }
+ }
+
+ // Open file, if it's not a symlink
+ std::auto_ptr<FileStream> in;
+ if(!sourceIsSymlink)
+ {
+ in.reset(new FileStream(Filename));
+ }
+
+ // Read header
+ file_BlockIndexHeader hdr;
+ if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic
+ if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0);
+#endif
+
+ // Get basic information
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+ uint64_t entryIVBase = ntoh64(hdr.mEntryIVBase);
+
+ //TODO: Verify that these sizes look reasonable
+
+ // setup
+ void *data = 0;
+ int32_t dataSize = -1;
+ bool matches = true;
+ int64_t totalSizeInBlockIndex = 0;
+
+ try
+ {
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read an entry from the stream
+ file_BlockIndexEntry entry;
+ if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read entry
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Calculate IV for this entry
+ uint64_t iv = entryIVBase;
+ iv += b;
+ iv = hton64(iv);
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ if(isOldVersion)
+ {
+ // Reverse the IV for compatibility
+ iv = box_swap64(iv);
+ }
+#endif
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry.mEnEnc, sizeof(entry.mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Size of block
+ int32_t blockClearSize = ntohl(entryEnc.mSize);
+ if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ totalSizeInBlockIndex += blockClearSize;
+
+ // Make sure there's enough memory allocated to load the block in
+ if(dataSize < blockClearSize)
+ {
+ // Too small, free the block if it's already allocated
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+ // Allocate a block
+ data = ::malloc(blockClearSize + 128);
+ if(data == 0)
+ {
+ throw std::bad_alloc();
+ }
+ dataSize = blockClearSize + 128;
+ }
+
+ // Load in the block from the file, if it's not a symlink
+ if(!sourceIsSymlink)
+ {
+ if(in->Read(data, blockClearSize) != blockClearSize)
+ {
+ // Not enough data left in the file, can't possibly match
+ matches = false;
+ }
+ else
+ {
+ // Check the checksum
+ MD5Digest md5;
+ md5.Add(data, blockClearSize);
+ md5.Finish();
+ if(!md5.DigestMatches(entryEnc.mStrongChecksum))
+ {
+ // Checksum didn't match
+ matches = false;
+ }
+ }
+ }
+
+ // Keep on going regardless, to make sure the entire block index stream is read
+ // -- must always be consistent about what happens with the stream.
+ }
+ }
+ catch(...)
+ {
+ // clean up in case of errors
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+ throw;
+ }
+
+ // free block
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+
+ // Check for data left over if it's not a symlink
+ if(!sourceIsSymlink)
+ {
+ // Anything left to read in the file?
+ if(in->BytesLeftToRead() != 0)
+ {
+ // File has extra data at the end
+ matches = false;
+ }
+ }
+
+ // Symlinks must have zero size on server
+ if(sourceIsSymlink)
+ {
+ matches = (totalSizeInBlockIndex == 0);
+ }
+
+ return matches;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer()
+// Purpose: Constructor
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::EncodingBuffer::EncodingBuffer()
+ : mpBuffer(0),
+ mBufferSize(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer()
+// Purpose: Destructor
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::EncodingBuffer::~EncodingBuffer()
+{
+ if(mpBuffer != 0)
+ {
+ BackupStoreFile::CodingChunkFree(mpBuffer);
+ mpBuffer = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::Allocate(int)
+// Purpose: Do initial allocation of block
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::EncodingBuffer::Allocate(int Size)
+{
+ ASSERT(mpBuffer == 0);
+ uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ mpBuffer = buffer;
+ mBufferSize = Size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::Reallocate(int)
+// Purpose: Reallocate the block. Try not to call this, it has to copy
+// the entire contents as the block can't be reallocated straight.
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize)
+{
+ TRACE2("Reallocating EncodingBuffer from %d to %d\n", mBufferSize, NewSize);
+ ASSERT(mpBuffer != 0);
+ uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ // Copy data
+ ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize);
+
+ // Free old
+ BackupStoreFile::CodingChunkFree(mpBuffer);
+
+ // Store new buffer
+ mpBuffer = buffer;
+ mBufferSize = NewSize;
+}
+
+
diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h
new file mode 100755
index 00000000..0274107b
--- /dev/null
+++ b/lib/backupclient/BackupStoreFile.h
@@ -0,0 +1,194 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFile.h
+// Purpose: Utils for manipulating files
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILE__H
+#define BACKUPSTOREFILE__H
+
+#include "IOStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreFilename.h"
+
+#include <memory>
+
+typedef struct
+{
+ int64_t mBytesInEncodedFiles;
+ int64_t mBytesAlreadyOnServer;
+ int64_t mTotalFileStreamSize;
+} BackupStoreFileStats;
+
+// Uncomment to disable backwards compatibility
+//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+
+
+// Output buffer to EncodeChunk and input data to DecodeChunk must
+// have specific alignment, see function comments.
+#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16
+#define BACKUPSTOREFILE_CODING_OFFSET 15
+
+// Have some memory allocation commands, note closing "Off" at end of file.
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFile
+// Purpose: Class to hold together utils for maniplating files.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class BackupStoreFile
+{
+public:
+ class DecodedStream : public IOStream
+ {
+ friend class BackupStoreFile;
+ private:
+ DecodedStream(IOStream &rEncodedFile, int Timeout);
+ DecodedStream(const DecodedStream &); // not allowed
+ DecodedStream &operator=(const DecodedStream &); // not allowed
+ public:
+ ~DecodedStream();
+
+ // Stream functions
+ virtual int Read(void *pBuffer, int NBytes, int Timeout);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+ // Accessor functions
+ const BackupClientFileAttributes &GetAttributes() {return mAttributes;}
+ const BackupStoreFilename &GetFilename() {return mFilename;}
+ int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests
+
+ bool IsSymLink();
+
+ private:
+ void Setup(const BackupClientFileAttributes *pAlterativeAttr);
+ void ReadBlockIndex(bool MagicAlreadyRead);
+
+ private:
+ IOStream &mrEncodedFile;
+ int mTimeout;
+ BackupClientFileAttributes mAttributes;
+ BackupStoreFilename mFilename;
+ int64_t mNumBlocks;
+ void *mpBlockIndex;
+ uint8_t *mpEncodedData;
+ uint8_t *mpClearData;
+ int mClearDataSize;
+ int mCurrentBlock;
+ int mCurrentBlockClearSize;
+ int mPositionInCurrentBlock;
+ uint64_t mEntryIVBase;
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool mIsOldVersion;
+#endif
+ };
+
+
+ // Main interface
+ static std::auto_ptr<IOStream> EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0);
+ static std::auto_ptr<IOStream> EncodeFileDiff(const char *Filename, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
+ int Timeout, int64_t *pModificationTime = 0, bool *pIsCompletelyDifferent = 0);
+ static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0);
+ static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut);
+ static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut);
+ static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0);
+ static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
+ static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
+ static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout);
+ static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false);
+
+ // Stream manipulation
+ static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership);
+ static void MoveStreamPositionToBlockIndex(IOStream &rStream);
+
+ // Crypto setup
+ static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength);
+#ifndef PLATFORM_OLD_OPENSSL
+ static void SetAESKey(const void *pKey, int KeyLength);
+#endif
+
+ // Allocation of properly aligning chunks for decoding and encoding chunks
+ inline static void *CodingChunkAlloc(int Size)
+ {
+ uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3));
+ if(a == 0) return 0;
+ // Align to main block size
+ ASSERT(sizeof(uint32_t) == sizeof(void*)); // make sure casting the right pointer size, will need to fix on platforms with 64 bit pointers
+ uint32_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE - (((uint32_t)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE);
+ uint8_t *b = (a + adjustment);
+ // Store adjustment
+ *b = (uint8_t)adjustment;
+ // Return offset
+ return b + BACKUPSTOREFILE_CODING_OFFSET;
+ }
+ inline static void CodingChunkFree(void *Block)
+ {
+ // Check alignment is as expected
+ ASSERT(sizeof(uint32_t) == sizeof(void*)); // make sure casting the right pointer size, will need to fix on platforms with 64 bit pointers
+ ASSERT((((uint32_t)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+ uint8_t *a = (uint8_t*)Block;
+ a -= BACKUPSTOREFILE_CODING_OFFSET;
+ // Adjust downwards...
+ a -= *a;
+ free(a);
+ }
+
+ // Limits
+ static void SetMaximumDiffingTime(int Seconds);
+
+ // Building blocks
+ class EncodingBuffer
+ {
+ public:
+ EncodingBuffer();
+ ~EncodingBuffer();
+ private:
+ // No copying
+ EncodingBuffer(const EncodingBuffer &);
+ EncodingBuffer &operator=(const EncodingBuffer &);
+ public:
+ void Allocate(int Size);
+ void Reallocate(int NewSize);
+
+ uint8_t *mpBuffer;
+ int mBufferSize;
+ };
+ static int MaxBlockSizeForChunkSize(int ChunkSize);
+ static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput);
+
+ // Caller should know how big the output size is, but also allocate a bit more memory to cover various
+ // overheads allowed for in checks
+ static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize)
+ {
+ // Plenty big enough
+ return KnownChunkSize + 256;
+ }
+ static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize);
+
+ // Statisitics, not designed to be completely reliable
+ static void ResetStats();
+ static BackupStoreFileStats msStats;
+
+ // For debug
+#ifndef NDEBUG
+ static bool TraceDetailsOfDiffProcess;
+#endif
+
+ // For decoding encoded files
+ static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile);
+};
+
+#include "MemLeakFindOff.h"
+
+#endif // BACKUPSTOREFILE__H
+
diff --git a/lib/backupclient/BackupStoreFileCmbDiff.cpp b/lib/backupclient/BackupStoreFileCmbDiff.cpp
new file mode 100644
index 00000000..039b00eb
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCmbDiff.cpp
@@ -0,0 +1,326 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCmbDiff.cpp
+// Purpose: Combine two diffs together
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <stdlib.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CombineDiffs(IOStream &, IOStream &, IOStream &rOut)
+// Purpose: Given two diffs, combine them into a single diff, to produce a diff
+// which, combined with the original file, creates the result of applying
+// rDiff, then rDiff2. Two opens of rDiff2 are required
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut)
+{
+ // Skip header of first diff, record where the data starts, and skip to the index
+ int64_t diff1DataStarts = 0;
+ {
+ // Read the header for the From file
+ file_StreamFormat diff1Hdr;
+ if(!rDiff1.ReadFullBuffer(&diff1Hdr, sizeof(diff1Hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diff1Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Skip over the filename and attributes of the From file
+ // BLOCK
+ {
+ BackupStoreFilename filename2;
+ filename2.ReadFromStream(rDiff1, IOStream::TimeOutInfinite);
+ int32_t size_s;
+ if(!rDiff1.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+ }
+ int size = ntohl(size_s);
+ // Skip forward the size
+ rDiff1.Seek(size, IOStream::SeekType_Relative);
+ }
+ // Record position
+ diff1DataStarts = rDiff1.GetPosition();
+ // Skip to index
+ rDiff1.Seek(0 - (((ntoh64(diff1Hdr.mNumBlocks)) * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+ }
+
+ // Read the index of the first diff
+ // Header first
+ file_BlockIndexHeader diff1IdxHdr;
+ if(!rDiff1.ReadFullBuffer(&diff1IdxHdr, sizeof(diff1IdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(diff1IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ int64_t diff1NumBlocks = ntoh64(diff1IdxHdr.mNumBlocks);
+ // Allocate some memory
+ int64_t *diff1BlockStartPositions = (int64_t*)::malloc((diff1NumBlocks + 1) * sizeof(int64_t));
+ if(diff1BlockStartPositions == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Buffer data
+ void *buffer = 0;
+ int bufferSize = 0;
+
+ try
+ {
+ // Then the entries:
+ // For each entry, want to know if it's in the file, and if so, how big it is.
+ // We'll store this as an array of file positions in the file, with an additioal
+ // entry on the end so that we can work out the length of the last block.
+ // If an entry isn't in the file, then store 0 - (position in other file).
+ int64_t diff1Position = diff1DataStarts;
+ for(int64_t b = 0; b < diff1NumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff1.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Where's the block?
+ int64_t blockEn = ntoh64(e.mEncodedSize);
+ if(blockEn <= 0)
+ {
+ // Just store the negated block number
+ diff1BlockStartPositions[b] = blockEn;
+ }
+ else
+ {
+ // Block is present in this file
+ diff1BlockStartPositions[b] = diff1Position;
+ diff1Position += blockEn;
+ }
+ }
+
+ // Finish off the list, so the last entry can have it's size calcuated.
+ diff1BlockStartPositions[diff1NumBlocks] = diff1Position;
+
+ // Now read the second diff's header, copying it to the out file
+ file_StreamFormat diff2Hdr;
+ if(!rDiff2.ReadFullBuffer(&diff2Hdr, sizeof(diff2Hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diff2Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Copy
+ rOut.Write(&diff2Hdr, sizeof(diff2Hdr));
+ // Copy over filename and attributes
+ // BLOCK
+ {
+ BackupStoreFilename filename;
+ filename.ReadFromStream(rDiff2, IOStream::TimeOutInfinite);
+ filename.WriteToStream(rOut);
+ StreamableMemBlock attr;
+ attr.ReadFromStream(rDiff2, IOStream::TimeOutInfinite);
+ attr.WriteToStream(rOut);
+ }
+
+ // Get to the index of rDiff2b, and read the header
+ MoveStreamPositionToBlockIndex(rDiff2b);
+ file_BlockIndexHeader diff2IdxHdr;
+ if(!rDiff2b.ReadFullBuffer(&diff2IdxHdr, sizeof(diff2IdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(diff2IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ int64_t diff2NumBlocks = ntoh64(diff2IdxHdr.mNumBlocks);
+ int64_t diff2IndexEntriesStart = rDiff2b.GetPosition();
+
+ // Then read all the entries
+ int64_t diff2FilePosition = rDiff2.GetPosition();
+ for(int64_t b = 0; b < diff2NumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // What do to next about copying data
+ bool copyBlock = false;
+ int copySize = 0;
+ int64_t copyFrom = 0;
+ bool fromFileDiff1 = false;
+
+ // Where's the block?
+ int64_t blockEn = ntoh64(e.mEncodedSize);
+ if(blockEn > 0)
+ {
+ // Block is present in this file -- copy to out
+ copyBlock = true;
+ copyFrom = diff2FilePosition;
+ copySize = (int)blockEn;
+
+ // Move pointer onwards
+ diff2FilePosition += blockEn;
+ }
+ else
+ {
+ // Block isn't present here -- is it present in the old one?
+ int64_t blockIndex = 0 - blockEn;
+ if(blockIndex < 0 || blockIndex > diff1NumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ if(diff1BlockStartPositions[blockIndex] > 0)
+ {
+ // Block is in the old diff file, copy it across
+ copyBlock = true;
+ copyFrom = diff1BlockStartPositions[blockIndex];
+ int nb = blockIndex + 1;
+ while(diff1BlockStartPositions[nb] <= 0)
+ {
+ // This is safe, because the last entry will terminate it properly!
+ ++nb;
+ ASSERT(nb <= diff1NumBlocks);
+ }
+ copySize = diff1BlockStartPositions[nb] - copyFrom;
+ fromFileDiff1 = true;
+ }
+ }
+ //TRACE4("%d %d %lld %d\n", copyBlock, copySize, copyFrom, fromFileDiff1);
+
+ // Copy data to the output file?
+ if(copyBlock)
+ {
+ // Allocate enough space
+ if(bufferSize < copySize || buffer == 0)
+ {
+ // Free old block
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ bufferSize = 0;
+ }
+ // Allocate new block
+ buffer = ::malloc(copySize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ bufferSize = copySize;
+ }
+ ASSERT(bufferSize >= copySize);
+
+ // Load in the data
+ if(fromFileDiff1)
+ {
+ rDiff1.Seek(copyFrom, IOStream::SeekType_Absolute);
+ if(!rDiff1.ReadFullBuffer(buffer, copySize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ }
+ else
+ {
+ rDiff2.Seek(copyFrom, IOStream::SeekType_Absolute);
+ if(!rDiff2.ReadFullBuffer(buffer, copySize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ }
+ // Write out data
+ rOut.Write(buffer, copySize);
+ }
+ }
+
+ // Write the modified header
+ diff2IdxHdr.mOtherFileID = diff1IdxHdr.mOtherFileID;
+ rOut.Write(&diff2IdxHdr, sizeof(diff2IdxHdr));
+
+ // Then we'll write out the index, reading the data again
+ rDiff2b.Seek(diff2IndexEntriesStart, IOStream::SeekType_Absolute);
+ for(int64_t b = 0; b < diff2NumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Where's the block?
+ int64_t blockEn = ntoh64(e.mEncodedSize);
+
+ // If it's not in this file, it needs modification...
+ if(blockEn <= 0)
+ {
+ int64_t blockIndex = 0 - blockEn;
+ // In another file. Need to translate this against the other diff
+ if(diff1BlockStartPositions[blockIndex] > 0)
+ {
+ // Block is in the first diff file, stick in size
+ int nb = blockIndex + 1;
+ while(diff1BlockStartPositions[nb] <= 0)
+ {
+ // This is safe, because the last entry will terminate it properly!
+ ++nb;
+ ASSERT(nb <= diff1NumBlocks);
+ }
+ int64_t size = diff1BlockStartPositions[nb] - diff1BlockStartPositions[blockIndex];
+ e.mEncodedSize = hton64(size);
+ }
+ else
+ {
+ // Block in the original file, use translated value
+ e.mEncodedSize = hton64(diff1BlockStartPositions[blockIndex]);
+ }
+ }
+
+ // Write entry
+ rOut.Write(&e, sizeof(e));
+ }
+ }
+ catch(...)
+ {
+ // clean up
+ ::free(diff1BlockStartPositions);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+ throw;
+ }
+
+ // Clean up allocated memory
+ ::free(diff1BlockStartPositions);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+}
+
diff --git a/lib/backupclient/BackupStoreFileCmbIdx.cpp b/lib/backupclient/BackupStoreFileCmbIdx.cpp
new file mode 100644
index 00000000..253001c2
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCmbIdx.cpp
@@ -0,0 +1,324 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCmbIdx.cpp
+// Purpose: Combine indicies of a delta file and the file it's a diff from.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <string.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+// Hide from outside world
+namespace
+{
+
+class BSFCombinedIndexStream : public IOStream
+{
+public:
+ BSFCombinedIndexStream(IOStream *pDiff);
+ ~BSFCombinedIndexStream();
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+ virtual void Initialise(IOStream &rFrom);
+
+private:
+ IOStream *mpDiff;
+ bool mIsInitialised;
+ bool mHeaderWritten;
+ file_BlockIndexHeader mHeader;
+ int64_t mNumEntriesToGo;
+ int64_t mNumEntriesInFromFile;
+ int64_t *mFromBlockSizes; // NOTE: Entries in network byte order
+};
+
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CombineFileIndices(IOStream &, IOStream &, bool)
+// Purpose: Given a diff file and the file it's a diff from, return a stream from which
+// can be read the index of the combined file, without actually combining them.
+// The stream of the diff must have a lifetime greater than or equal to the
+// lifetime of the returned stream object. The full "from" file stream
+// only needs to exist during the actual function call.
+// If you pass in dodgy files which aren't related, then you will either
+// get an error or bad results. So don't do that.
+// If DiffIsIndexOnly is true, then rDiff is assumed to be a stream positioned
+// at the beginning of the block index. Similarly for FromIsIndexOnly.
+// WARNING: Reads of the returned streams with buffer sizes less than 64 bytes
+// will not return any data.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly, bool FromIsIndexOnly)
+{
+ // Reposition file pointers?
+ if(!DiffIsIndexOnly)
+ {
+ MoveStreamPositionToBlockIndex(rDiff);
+ }
+ if(!FromIsIndexOnly)
+ {
+ MoveStreamPositionToBlockIndex(rFrom);
+ }
+
+ // Create object
+ std::auto_ptr<IOStream> stream(new BSFCombinedIndexStream(&rDiff));
+
+ // Initialise it
+ ((BSFCombinedIndexStream *)stream.get())->Initialise(rFrom);
+
+ // And return the stream
+ return stream;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::BSFCombinedIndexStream()
+// Purpose: Private class. Constructor.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+BSFCombinedIndexStream::BSFCombinedIndexStream(IOStream *pDiff)
+ : mpDiff(pDiff),
+ mIsInitialised(false),
+ mHeaderWritten(false),
+ mNumEntriesToGo(0),
+ mNumEntriesInFromFile(0),
+ mFromBlockSizes(0)
+{
+ ASSERT(mpDiff != 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::~BSFCombinedIndexStream()
+// Purpose: Private class. Destructor.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+BSFCombinedIndexStream::~BSFCombinedIndexStream()
+{
+ if(mFromBlockSizes != 0)
+ {
+ ::free(mFromBlockSizes);
+ mFromBlockSizes = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::Initialise(IOStream &)
+// Purpose: Private class. Initalise from the streams (diff passed in constructor).
+// Both streams must have file pointer positioned at the block index.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+void BSFCombinedIndexStream::Initialise(IOStream &rFrom)
+{
+ // Paranoia is good.
+ if(mIsInitialised)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Look at the diff file: Read in the header
+ if(!mpDiff->ReadFullBuffer(&mHeader, sizeof(mHeader), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(mHeader.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Read relevant data.
+ mNumEntriesToGo = ntoh64(mHeader.mNumBlocks);
+
+ // Adjust a bit to reflect the fact it's no longer a diff
+ mHeader.mOtherFileID = hton64(0);
+
+ // Now look at the from file: Read header
+ file_BlockIndexHeader fromHdr;
+ if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Then... allocate memory for the list of sizes
+ mNumEntriesInFromFile = ntoh64(fromHdr.mNumBlocks);
+ mFromBlockSizes = (int64_t*)::malloc(mNumEntriesInFromFile * sizeof(int64_t));
+ if(mFromBlockSizes == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // And read them all in!
+ for(int64_t b = 0; b < mNumEntriesInFromFile; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check that the from file isn't a delta in itself
+ if(ntoh64(e.mEncodedSize) <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+ }
+
+ // Store size (in network byte order)
+ mFromBlockSizes[b] = e.mEncodedSize;
+ }
+
+ // Flag as initialised
+ mIsInitialised = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::Read(void *, int, int)
+// Purpose: Private class. As interface.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Paranoia is good.
+ if(!mIsInitialised || mFromBlockSizes == 0 || mpDiff == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ int written = 0;
+
+ // Header output yet?
+ if(!mHeaderWritten)
+ {
+ // Enough space?
+ if(NBytes < (int)sizeof(mHeader)) return 0;
+
+ // Copy in
+ ::memcpy(pBuffer, &mHeader, sizeof(mHeader));
+ NBytes -= sizeof(mHeader);
+ written += sizeof(mHeader);
+
+ // Flag it's done
+ mHeaderWritten = true;
+ }
+
+ // How many entries can be written?
+ int entriesToWrite = NBytes / sizeof(file_BlockIndexEntry);
+ if(entriesToWrite > mNumEntriesToGo)
+ {
+ entriesToWrite = mNumEntriesToGo;
+ }
+
+ // Setup ready to go
+ file_BlockIndexEntry *poutput = (file_BlockIndexEntry*)(((uint8_t*)pBuffer) + written);
+
+ // Write entries
+ for(int b = 0; b < entriesToWrite; ++b)
+ {
+ if(!mpDiff->ReadFullBuffer(&(poutput[b]), sizeof(file_BlockIndexEntry), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Does this need adjusting?
+ int s = ntoh64(poutput[b].mEncodedSize);
+ if(s <= 0)
+ {
+ // A reference to a block in the from file
+ int block = 0 - s;
+ ASSERT(block >= 0);
+ if(block >= mNumEntriesInFromFile)
+ {
+ // That's not good, the block doesn't exist
+ THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+ }
+
+ // Adjust the entry in the buffer
+ poutput[b].mEncodedSize = mFromBlockSizes[block]; // stored in network byte order, no translation necessary
+ }
+ }
+
+ // Update written count
+ written += entriesToWrite * sizeof(file_BlockIndexEntry);
+ mNumEntriesToGo -= entriesToWrite;
+
+ return written;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::Write(const void *, int)
+// Purpose: Private class. As interface.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::StreamDataLeft()
+// Purpose: Private class. As interface
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+bool BSFCombinedIndexStream::StreamDataLeft()
+{
+ return (!mHeaderWritten) || (mNumEntriesToGo > 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::StreamClosed()
+// Purpose: Private class. As interface.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+bool BSFCombinedIndexStream::StreamClosed()
+{
+ return true; // doesn't do writing
+}
+
diff --git a/lib/backupclient/BackupStoreFileCombine.cpp b/lib/backupclient/BackupStoreFileCombine.cpp
new file mode 100755
index 00000000..562a32d9
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCombine.cpp
@@ -0,0 +1,410 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCombine.cpp
+// Purpose: File combining for BackupStoreFile
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+#include "FileStream.h"
+
+#include "MemLeakFindOn.h"
+
+typedef struct
+{
+ int64_t mFilePosition;
+} FromIndexEntry;
+
+static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries);
+static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut);
+static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut);
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CombineFile(IOStream &, IOStream &, IOStream &)
+// Purpose: Where rDiff is a store file which is incomplete as a result of a
+// diffing operation, rFrom is the file it is diffed from, and
+// rOut is the stream in which to place the result, the old file
+// and new file are combined into a file containing all the data.
+// rDiff2 is the same file as rDiff, opened again to get two
+// independent streams to the same file.
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut)
+{
+ // Read and copy the header.
+ file_StreamFormat hdr;
+ if(!rDiff.ReadFullBuffer(&hdr, sizeof(hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Copy
+ rOut.Write(&hdr, sizeof(hdr));
+ // Copy over filename and attributes
+ // BLOCK
+ {
+ BackupStoreFilename filename;
+ filename.ReadFromStream(rDiff, IOStream::TimeOutInfinite);
+ filename.WriteToStream(rOut);
+ StreamableMemBlock attr;
+ attr.ReadFromStream(rDiff, IOStream::TimeOutInfinite);
+ attr.WriteToStream(rOut);
+ }
+
+ // Read the header for the From file
+ file_StreamFormat fromHdr;
+ if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Skip over the filename and attributes of the From file
+ // BLOCK
+ {
+ BackupStoreFilename filename2;
+ filename2.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ int32_t size_s;
+ if(!rFrom.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+ }
+ int size = ntohl(size_s);
+ // Skip forward the size
+ rFrom.Seek(size, IOStream::SeekType_Relative);
+ }
+
+ // Allocate memory for the block index of the From file
+ int64_t fromNumBlocks = ntoh64(fromHdr.mNumBlocks);
+ // NOTE: An extra entry is required so that the length of the last block can be calculated
+ FromIndexEntry *pFromIndex = (FromIndexEntry*)::malloc((fromNumBlocks+1) * sizeof(FromIndexEntry));
+ if(pFromIndex == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ try
+ {
+ // Load the index from the From file, calculating the offsets in the
+ // file as we go along, and enforce that everything should be present.
+ LoadFromIndex(rFrom, pFromIndex, fromNumBlocks);
+
+ // Read in the block index of the Diff file in small chunks, and output data
+ // for each block, either from this file, or the other file.
+ int64_t diffNumBlocks = ntoh64(hdr.mNumBlocks);
+ CopyData(rDiff /* positioned at start of data */, rDiff2, diffNumBlocks, rFrom, pFromIndex, fromNumBlocks, rOut);
+
+ // Read in the block index again, and output the new block index, simply
+ // filling in the sizes of blocks from the old file.
+ WriteNewIndex(rDiff, diffNumBlocks, pFromIndex, fromNumBlocks, rOut);
+
+ // Free buffers
+ ::free(pFromIndex);
+ pFromIndex = 0;
+ }
+ catch(...)
+ {
+ // Clean up
+ if(pFromIndex != 0)
+ {
+ ::free(pFromIndex);
+ pFromIndex = 0;
+ }
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static LoadFromIndex(IOStream &, FromIndexEntry *, int64_t)
+// Purpose: Static. Load the index from the From file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries)
+{
+ ASSERT(pIndex != 0);
+ ASSERT(NumEntries >= 0);
+
+ // Get the starting point in the file
+ int64_t filePos = rFrom.GetPosition();
+
+ // Jump to the end of the file to read the index
+ rFrom.Seek(0 - ((NumEntries * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+
+ // Read block index header
+ file_BlockIndexHeader blkhdr;
+ if(!rFrom.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || (int64_t)ntoh64(blkhdr.mNumBlocks) != NumEntries)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // And then the block entries
+ for(int64_t b = 0; b < NumEntries; ++b)
+ {
+ // Read
+ file_BlockIndexEntry en;
+ if(!rFrom.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // Add to list
+ pIndex[b].mFilePosition = filePos;
+
+ // Encoded size?
+ int64_t encodedSize = ntoh64(en.mEncodedSize);
+ // Check that the block is actually there
+ if(encodedSize <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+ }
+
+ // Move file pointer on
+ filePos += encodedSize;
+ }
+
+ // Store the position in the very last entry, so the size of the last entry can be calculated
+ pIndex[NumEntries].mFilePosition = filePos;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static CopyData(IOStream &, IOStream &, int64_t, IOStream &, FromIndexEntry *, int64_t, IOStream &)
+// Purpose: Static. Copy data from the Diff and From file to the out file.
+// rDiffData is at beginning of data.
+// rDiffIndex at any position.
+// rFrom is at any position.
+// rOut is after the header, ready for data
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks,
+ IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut)
+{
+ // Jump to the end of the diff file to read the index
+ rDiffIndex.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+
+ // Read block index header
+ file_BlockIndexHeader diffBlkhdr;
+ if(!rDiffIndex.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || (int64_t)ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Record where the From file is
+ int64_t fromPos = rFrom.GetPosition();
+
+ // Buffer data
+ void *buffer = 0;
+ int bufferSize = 0;
+
+ try
+ {
+ // Read the blocks in!
+ for(int64_t b = 0; b < DiffNumBlocks; ++b)
+ {
+ // Read
+ file_BlockIndexEntry en;
+ if(!rDiffIndex.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // What's the size value stored in the entry
+ int64_t encodedSize = ntoh64(en.mEncodedSize);
+
+ // How much data will be read?
+ int32_t blockSize = 0;
+ if(encodedSize > 0)
+ {
+ // The block is actually in the diff file
+ blockSize = encodedSize;
+ }
+ else
+ {
+ // It's in the from file. First, check to see if it's valid
+ int64_t blockIdx = (0 - encodedSize);
+ if(blockIdx > FromNumBlocks)
+ {
+ // References a block which doesn't actually exist
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Calculate size. This operation is safe because of the extra entry at the end
+ blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition;
+ }
+ ASSERT(blockSize > 0);
+
+ // Make sure there's memory available to copy this
+ if(bufferSize < blockSize || buffer == 0)
+ {
+ // Free old block
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ bufferSize = 0;
+ }
+ // Allocate new block
+ buffer = ::malloc(blockSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ bufferSize = blockSize;
+ }
+ ASSERT(bufferSize >= blockSize);
+
+ // Load in data from one of the files
+ if(encodedSize > 0)
+ {
+ // Load from diff file
+ if(!rDiffData.ReadFullBuffer(buffer, blockSize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ }
+ else
+ {
+ // Locate and read the data from the from file
+ int64_t blockIdx = (0 - encodedSize);
+ // Seek if necessary
+ if(fromPos != pFromIndex[blockIdx].mFilePosition)
+ {
+ rFrom.Seek(pFromIndex[blockIdx].mFilePosition, IOStream::SeekType_Absolute);
+ fromPos = pFromIndex[blockIdx].mFilePosition;
+ }
+ // Read
+ if(!rFrom.ReadFullBuffer(buffer, blockSize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // Update fromPos to current position
+ fromPos += blockSize;
+ }
+
+ // Write data to out file
+ rOut.Write(buffer, blockSize);
+ }
+
+ // Free buffer, if allocated
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ }
+ }
+ catch(...)
+ {
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ }
+ throw;
+ }
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static WriteNewIndex(IOStream &, int64_t, FromIndexEntry *, int64_t, IOStream &)
+// Purpose: Write the index to the out file, just copying from the diff file and
+// adjusting the entries.
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut)
+{
+ // Jump to the end of the diff file to read the index
+ rDiff.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+
+ // Read block index header
+ file_BlockIndexHeader diffBlkhdr;
+ if(!rDiff.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || (int64_t)ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Write it out with a blanked out other file ID
+ diffBlkhdr.mOtherFileID = hton64(0);
+ rOut.Write(&diffBlkhdr, sizeof(diffBlkhdr));
+
+ // Rewrite the index
+ for(int64_t b = 0; b < DiffNumBlocks; ++b)
+ {
+ file_BlockIndexEntry en;
+ if(!rDiff.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // What's the size value stored in the entry
+ int64_t encodedSize = ntoh64(en.mEncodedSize);
+
+ // Need to adjust it?
+ if(encodedSize <= 0)
+ {
+ // This actually refers to a block in the from file. So rewrite this.
+ int64_t blockIdx = (0 - encodedSize);
+ if(blockIdx > FromNumBlocks)
+ {
+ // References a block which doesn't actually exist
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Calculate size. This operation is safe because of the extra entry at the end
+ int32_t blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition;
+ // Then replace entry
+ en.mEncodedSize = hton64(((uint64_t)blockSize));
+ }
+
+ // Write entry
+ rOut.Write(&en, sizeof(en));
+ }
+}
+
+
+
+
+
diff --git a/lib/backupclient/BackupStoreFileCryptVar.cpp b/lib/backupclient/BackupStoreFileCryptVar.cpp
new file mode 100755
index 00000000..eeed64e4
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCryptVar.cpp
@@ -0,0 +1,31 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCryptVar.cpp
+// Purpose: Cryptographic keys for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFileWire.h"
+
+#include "MemLeakFindOn.h"
+
+CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt;
+CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt;
+
+#ifndef PLATFORM_OLD_OPENSSL
+ CipherContext BackupStoreFileCryptVar::sAESEncrypt;
+ CipherContext BackupStoreFileCryptVar::sAESDecrypt;
+#endif
+
+// Default to blowfish
+CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt;
+uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING;
+
+CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry;
+CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry;
+
diff --git a/lib/backupclient/BackupStoreFileCryptVar.h b/lib/backupclient/BackupStoreFileCryptVar.h
new file mode 100755
index 00000000..00a34f71
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCryptVar.h
@@ -0,0 +1,39 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCryptVar.h
+// Purpose: Cryptographic keys for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILECRYPTVAR__H
+#define BACKUPSTOREFILECRYPTVAR__H
+
+#include "CipherContext.h"
+
+// Hide private static variables from the rest of the world by putting them
+// as static variables in a namespace.
+// -- don't put them as static class variables to avoid openssl/evp.h being
+// included all over the project.
+namespace BackupStoreFileCryptVar
+{
+ // Keys for the main file data
+ extern CipherContext sBlowfishEncrypt;
+ extern CipherContext sBlowfishDecrypt;
+ // Use AES when available
+#ifndef PLATFORM_OLD_OPENSSL
+ extern CipherContext sAESEncrypt;
+ extern CipherContext sAESDecrypt;
+#endif
+ // How encoding will be done
+ extern CipherContext *spEncrypt;
+ extern uint8_t sEncryptCipherType;
+
+ // Keys for the block indicies
+ extern CipherContext sBlowfishEncryptBlockEntry;
+ extern CipherContext sBlowfishDecryptBlockEntry;
+}
+
+#endif // BACKUPSTOREFILECRYPTVAR__H
+
diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp
new file mode 100755
index 00000000..b72ce328
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileDiff.cpp
@@ -0,0 +1,973 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileDiff.cpp
+// Purpose: Functions relating to diffing BackupStoreFiles
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <map>
+#include <signal.h>
+#include <sys/time.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreFileEncodeStream.h"
+#include "BackupStoreConstants.h"
+#include "FileStream.h"
+#include "RollingChecksum.h"
+#include "MD5Digest.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+using namespace BackupStoreFileCreation;
+
+// By default, don't trace out details of the diff as we go along -- would fill up logs significantly.
+// But it's useful for the test.
+#ifndef NDEBUG
+ bool BackupStoreFile::TraceDetailsOfDiffProcess = false;
+#endif
+
+static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis);
+static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]);
+static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]);
+static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable);
+static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, uint16_t Hash, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks);
+static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile);
+
+// Avoid running on too long with diffs
+static int sMaximumDiffTime = 10; // maximum time to spend diffing
+static bool sDiffTimedOut = false;
+static bool sSetTimerSignelHandler = false;
+static void TimerSignalHandler(int signal);
+static void StartDiffTimer();
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::SetMaximumDiffingTime(int)
+// Purpose: Sets the maximum time to spend diffing, in seconds. Time is
+// process virutal time.
+// Created: 19/3/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetMaximumDiffingTime(int Seconds)
+{
+ sMaximumDiffTime = Seconds;
+ TRACE1("Set maximum diffing time to %d seconds\n", Seconds);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &)
+// Purpose: Move the file pointer in this stream to just before the block index.
+// Assumes that the stream is at the beginning, seekable, and
+// reading from the stream is OK.
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream)
+{
+ // Size of file
+ int64_t fileSize = rStream.BytesLeftToRead();
+
+ // Get header
+ file_StreamFormat hdr;
+
+ // Read the header
+ if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Work out where the index is
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+ int64_t blockHeaderPosFromEnd = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+
+ // Sanity check
+ if(blockHeaderPosFromEnd > static_cast<int64_t>(fileSize - sizeof(file_StreamFormat)))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Seek to that position
+ rStream.Seek(0 - blockHeaderPosFromEnd, IOStream::SeekType_End);
+
+ // Done. Stream now in right position (as long as the file is formatted correctly)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *)
+// Purpose: Similar to EncodeFile, but takes the object ID of the file it's
+// diffing from, and the index of the blocks in a stream. It'll then
+// calculate which blocks can be reused from that old file.
+// The timeout is the timeout value for reading the diff block index.
+// If pIsCompletelyDifferent != 0, it will be set to true if the
+// the two files are completely different (do not share any block), false otherwise.
+//
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff(const char *Filename, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
+ int Timeout, int64_t *pModificationTime, bool *pIsCompletelyDifferent)
+{
+ // Is it a symlink?
+ {
+ struct stat st;
+ if(::lstat(Filename, &st) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ if((st.st_mode & S_IFLNK) == S_IFLNK)
+ {
+ // Don't do diffs for symlinks
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = true;
+ }
+ return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime);
+ }
+ }
+
+ // Load in the blocks
+ BlocksAvailableEntry *pindex = 0;
+ int64_t blocksInIndex = 0;
+ bool canDiffFromThis = false;
+ LoadIndex(rDiffFromBlockIndex, DiffFromObjectID, &pindex, blocksInIndex, Timeout, canDiffFromThis);
+ //TRACE1("Diff: Blocks in index: %lld\n", blocksInIndex);
+
+ if(!canDiffFromThis)
+ {
+ // Don't do diffing...
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = true;
+ }
+ return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime);
+ }
+
+ // Pointer to recipe we're going to create
+ BackupStoreFileEncodeStream::Recipe *precipe = 0;
+
+ // Start the timeout timer, so that the operation doesn't continue for ever
+ StartDiffTimer();
+
+ try
+ {
+ // Find which sizes should be scanned
+ int32_t sizesToScan[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+ FindMostUsedSizes(pindex, blocksInIndex, sizesToScan);
+
+ // Flag for reporting to the user
+ bool completelyDifferent;
+
+ // BLOCK
+ {
+ // Search the file to find matching blocks
+ std::map<int64_t, int64_t> foundBlocks; // map of offset in file to index in block index
+ int64_t sizeOfInputFile = 0;
+ // BLOCK
+ {
+ FileStream file(Filename);
+ // Get size of file
+ sizeOfInputFile = file.BytesLeftToRead();
+ // Find all those lovely matching blocks
+ SearchForMatchingBlocks(file, foundBlocks, pindex, blocksInIndex, sizesToScan);
+
+ // Is it completely different?
+ completelyDifferent = (foundBlocks.size() == 0);
+ }
+
+ // Create a recipe -- if the two files are completely different, don't put the from file ID in the recipe.
+ precipe = new BackupStoreFileEncodeStream::Recipe(pindex, blocksInIndex, completelyDifferent?(0):(DiffFromObjectID));
+ BlocksAvailableEntry *pindexKeptRef = pindex; // we need this later, but must set pindex == 0 now, because of exceptions
+ pindex = 0; // Recipe now has ownership
+
+ // Fill it in
+ GenerateRecipe(*precipe, pindexKeptRef, blocksInIndex, foundBlocks, sizeOfInputFile);
+ }
+ // foundBlocks no longer required
+
+ // Create the stream
+ std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
+
+ // Do the initial setup
+ ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime);
+ precipe = 0; // Stream has taken ownership of this
+
+ // Tell user about completely different status?
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = completelyDifferent;
+ }
+
+ // Return the stream for the caller
+ return stream;
+ }
+ catch(...)
+ {
+ // cleanup
+ if(pindex != 0)
+ {
+ ::free(pindex);
+ pindex = 0;
+ }
+ if(precipe != 0)
+ {
+ delete precipe;
+ precipe = 0;
+ }
+ throw;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static LoadIndex(IOStream &, int64_t, BlocksAvailableEntry **, int64_t, bool &)
+// Purpose: Read in an index, and decrypt, and store in the in memory block format.
+// rCanDiffFromThis is set to false if the version of the from file is too old.
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis)
+{
+ // Reset
+ rNumBlocksOut = 0;
+ rCanDiffFromThis = false;
+
+ // Read header
+ file_BlockIndexHeader hdr;
+ if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ // Check against backwards comptaibility stuff
+ if(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0))
+ {
+ // Won't diff against old version
+
+ // Absorb rest of stream
+ char buffer[2048];
+ while(rBlockIndex.StreamDataLeft())
+ {
+ rBlockIndex.Read(buffer, sizeof(buffer), 1000 /* 1 sec timeout */);
+ }
+
+ // Tell caller
+ rCanDiffFromThis = false;
+ return;
+ }
+#endif
+
+ // Check magic
+ if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Check that we're not trying to diff against a file which references blocks from another file
+ if(((int64_t)ntoh64(hdr.mOtherFileID)) != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile)
+ }
+
+ // Mark as an acceptable diff.
+ rCanDiffFromThis = true;
+
+ // Get basic information
+ int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+ uint64_t entryIVBase = ntoh64(hdr.mEntryIVBase);
+
+ //TODO: Verify that these sizes look reasonable
+
+ // Allocate space for the index
+ BlocksAvailableEntry *pindex = (BlocksAvailableEntry*)::malloc(sizeof(BlocksAvailableEntry) * numBlocks);
+ if(pindex == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ try
+ {
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read an entry from the stream
+ file_BlockIndexEntry entry;
+ if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read entry
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Calculate IV for this entry
+ uint64_t iv = entryIVBase;
+ iv += b;
+ // Network byte order
+ iv = hton64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry.mEnEnc, sizeof(entry.mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Check that we're not trying to diff against a file which references blocks from another file
+ if(((int64_t)ntoh64(entry.mEncodedSize)) <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile)
+ }
+
+ // Store all the required information
+ pindex[b].mpNextInHashList = 0; // hash list not set up yet
+ pindex[b].mSize = ntohl(entryEnc.mSize);
+ pindex[b].mWeakChecksum = ntohl(entryEnc.mWeakChecksum);
+ ::memcpy(pindex[b].mStrongChecksum, entryEnc.mStrongChecksum, sizeof(pindex[b].mStrongChecksum));
+ }
+
+ // Store index pointer for called
+ ASSERT(ppIndex != 0);
+ *ppIndex = pindex;
+
+ // Store number of blocks for caller
+ rNumBlocksOut = numBlocks;
+
+ }
+ catch(...)
+ {
+ // clean up and send the exception along its way
+ ::free(pindex);
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static FindMostUsedSizes(BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+// Purpose: Finds the most commonly used block sizes in the index
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+{
+ // Array for lengths
+ int64_t sizeCounts[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+
+ // Set arrays to lots of zeros (= unused entries)
+ for(int l = 0; l < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++l)
+ {
+ Sizes[l] = 0;
+ sizeCounts[l] = 0;
+ }
+
+ // Array for collecting sizes
+ std::map<int32_t, int64_t> foundSizes;
+
+ // Run through blocks and make a count of the entries
+ for(int64_t b = 0; b < NumBlocks; ++b)
+ {
+ // Only if the block size is bigger than the minimum size we'll scan for
+ if(pIndex[b].mSize > BACKUP_FILE_DIFF_MIN_BLOCK_SIZE)
+ {
+ // Find entry?
+ std::map<int32_t, int64_t>::const_iterator f(foundSizes.find(pIndex[b].mSize));
+ if(f != foundSizes.end())
+ {
+ // Increment existing entry
+ foundSizes[pIndex[b].mSize] = foundSizes[pIndex[b].mSize] + 1;
+ }
+ else
+ {
+ // New entry
+ foundSizes[pIndex[b].mSize] = 1;
+ }
+ }
+ }
+
+ // Make the block sizes
+ for(std::map<int32_t, int64_t>::const_iterator i(foundSizes.begin()); i != foundSizes.end(); ++i)
+ {
+ // Find the position of the size in the array
+ for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
+ {
+ if(i->second > sizeCounts[t])
+ {
+ // Then this size belong before this entry -- shuffle them up
+ for(int s = (BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1); s >= t; --s)
+ {
+ Sizes[s] = Sizes[s-1];
+ sizeCounts[s] = sizeCounts[s-1];
+ }
+
+ // Insert this size
+ Sizes[t] = i->first;
+ sizeCounts[t] = i->second;
+
+ // Shouldn't do any more searching
+ break;
+ }
+ }
+ }
+
+ // trace the size table in debug builds
+#ifndef NDEBUG
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
+ {
+ TRACE3("Diff block size %d: %d (count = %lld)\n", t, Sizes[t], sizeCounts[t]);
+ }
+ }
+#endif
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+// Purpose: Find the matching blocks within the file.
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks,
+ BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+{
+ // Allocate the hash lookup table
+ BlocksAvailableEntry **phashTable = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024));
+
+ // Choose a size for the buffer, just a little bit more than the maximum block size
+ int32_t bufSize = Sizes[0];
+ for(int z = 1; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z)
+ {
+ if(Sizes[z] > bufSize) bufSize = Sizes[z];
+ }
+ bufSize += 4;
+ ASSERT(bufSize > Sizes[0]);
+ ASSERT(bufSize > 0);
+ if(bufSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // TODO: Use buffered file class
+ // Because we read in the file a scanned block size at a time, it is likely to be
+ // inefficient. Probably will be much better to use a buffering IOStream class which
+ // reads data in at the size of the filesystem block size.
+
+ // Allocate the buffers.
+ uint8_t *pbuffer0 = (uint8_t *)::malloc(bufSize);
+ uint8_t *pbuffer1 = (uint8_t *)::malloc(bufSize);
+ try
+ {
+ // Check buffer allocation
+ if(pbuffer0 == 0 || pbuffer1 == 0 || phashTable == 0)
+ {
+ // If a buffer got alloocated, it will be cleaned up in the catch block
+ throw std::bad_alloc();
+ }
+
+ // Flag to abort the run, if too many blocks are found -- avoid using
+ // huge amounts of processor time when files contain many similar blocks.
+ bool abortSearch = false;
+
+ // Search for each block size in turn
+ // NOTE: Do the smallest size first, so that the scheme for adding
+ // entries in the found list works as expected and replaces smallers block
+ // with larger blocks when it finds matches at the same offset in the file.
+ for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s)
+ {
+ ASSERT(Sizes[s] <= bufSize);
+ //TRACE2("Diff pass %d, for block size %d\n", s, Sizes[s]);
+
+ // Check we haven't finished
+ if(Sizes[s] == 0)
+ {
+ // empty entry, try next size
+ continue;
+ }
+
+ // Set up the hash table entries
+ SetupHashTable(pIndex, NumBlocks, Sizes[s], phashTable);
+
+ // Shift file position to beginning
+ rFile.Seek(0, IOStream::SeekType_Absolute);
+
+ // Read first block
+ if(rFile.Read(pbuffer0, Sizes[s]) != Sizes[s])
+ {
+ // Size of file too short to match -- do next size
+ continue;
+ }
+
+ // Setup block pointers
+ uint8_t *beginnings = pbuffer0;
+ uint8_t *endings = pbuffer1;
+ int offset = 0;
+
+ // Calculate the first checksum, ready for rolling
+ RollingChecksum rolling(beginnings, Sizes[s]);
+
+ // Then roll, until the file is exhausted
+ int64_t fileBlockNumber = 0;
+ int rollOverInitialBytes = 0;
+ while(true)
+ {
+ // Load in another block of data, and record how big it is
+ int bytesInEndings = rFile.Read(endings, Sizes[s]);
+
+ // Skip any bytes from a previous matched block
+ while(rollOverInitialBytes > 0 && offset < bytesInEndings)
+ {
+ rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
+ ++offset;
+ --rollOverInitialBytes;
+ }
+
+ while(offset < bytesInEndings)
+ {
+ // Put is current checksum in hash list?
+ uint16_t hash = rolling.GetComponentForHashing();
+ if(phashTable[hash] != 0)
+ {
+ if(SecondStageMatch(phashTable[hash], hash, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks))
+ {
+ // Block matched, roll the checksum forward to the next block without doing
+ // any more comparisons, because these are pointless (as any more matches will be ignored when
+ // the receipe is generated) and just take up valuable processor time. Edge cases are
+ // especially nasty, using huge amounts of time and memory.
+ int skip = Sizes[s];
+ while(offset < bytesInEndings && skip > 0)
+ {
+ rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
+ ++offset;
+ --skip;
+ }
+ // Not all the bytes necessary will have been skipped, so get them
+ // skipped after the next block is loaded.
+ rollOverInitialBytes = skip;
+
+ // End this loop, so the final byte isn't used again
+ break;
+ }
+
+ if(static_cast<int64_t>(rFoundBlocks.size()) > (NumBlocks * BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE)
+ || sDiffTimedOut)
+ {
+ abortSearch = true;
+ break;
+ }
+ }
+
+ // Roll checksum forward
+ rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
+
+ // Increment offset
+ ++offset;
+ }
+
+ if(abortSearch) break;
+
+ // Finished?
+ if(bytesInEndings != Sizes[s])
+ {
+ // No more data in file -- check the final block
+ // (Do a copy and paste of 5 lines of code instead of introducing a comparison for
+ // each byte of the file)
+ uint16_t hash = rolling.GetComponentForHashing();
+ if(phashTable[hash] != 0)
+ {
+ SecondStageMatch(phashTable[hash], hash, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks);
+ }
+
+ // finish
+ break;
+ }
+
+ // Switch buffers, reset offset
+ beginnings = endings;
+ endings = (beginnings == pbuffer0)?(pbuffer1):(pbuffer0); // ie the other buffer
+ offset = 0;
+
+ // And count the blocks which have been done
+ ++fileBlockNumber;
+ }
+
+ if(abortSearch) break;
+ }
+
+ // Free buffers and hash table
+ ::free(pbuffer1);
+ pbuffer1 = 0;
+ ::free(pbuffer0);
+ pbuffer0 = 0;
+ ::free(phashTable);
+ phashTable = 0;
+ }
+ catch(...)
+ {
+ // Cleanup and throw
+ if(pbuffer1 != 0) ::free(pbuffer1);
+ if(pbuffer0 != 0) ::free(pbuffer0);
+ if(phashTable != 0) ::free(phashTable);
+ throw;
+ }
+
+#ifndef NDEBUG
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ // Trace out the found blocks in debug mode
+ TRACE0("Diff: list of found blocks\n======== ======== ======== ========\n Offset BlkIdx Size Movement\n");
+ for(std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); i != rFoundBlocks.end(); ++i)
+ {
+ int64_t orgLoc = 0;
+ for(int64_t b = 0; b < i->second; ++b)
+ {
+ orgLoc += pIndex[b].mSize;
+ }
+ TRACE4("%8lld %8lld %8lld %8lld\n", i->first, i->second,
+ (int64_t)(pIndex[i->second].mSize), i->first - orgLoc);
+ }
+ TRACE0("======== ======== ======== ========\n");
+ }
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static SetupHashTable(BlocksAvailableEntry *, int64_t, in32_t, BlocksAvailableEntry **)
+// Purpose: Set up the hash table ready for a scan
+// Created: 14/1/04
+//
+// --------------------------------------------------------------------------
+static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable)
+{
+ // Set all entries in the hash table to zero
+ ::memset(pHashTable, 0, (sizeof(BlocksAvailableEntry *) * (64*1024)));
+
+ // Scan through the blocks, building the hash table
+ for(int64_t b = 0; b < NumBlocks; ++b)
+ {
+ // Only look at the required block size
+ if(pIndex[b].mSize == BlockSize)
+ {
+ // Get the value under which to hash this entry
+ uint16_t hash = RollingChecksum::ExtractHashingComponent(pIndex[b].mWeakChecksum);
+
+ // Already present in table?
+ if(pHashTable[hash] != 0)
+ {
+ //TRACE1("Another hash entry for %d found\n", hash);
+ // Yes -- need to set the pointer in this entry to the current entry to build the linked list
+ pIndex[b].mpNextInHashList = pHashTable[hash];
+ }
+
+ // Put a pointer to this entry in the hash table
+ pHashTable[hash] = pIndex + b;
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static bool SecondStageMatch(xxx)
+// Purpose: When a match in the hash table is found, scan for second stage match using strong checksum.
+// Created: 14/1/04
+//
+// --------------------------------------------------------------------------
+static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, uint16_t Hash, uint8_t *pBeginnings, uint8_t *pEndings,
+ int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks)
+{
+ // Check parameters
+ ASSERT(pBeginnings != 0);
+ ASSERT(pEndings != 0);
+ ASSERT(Offset >= 0);
+ ASSERT(BlockSize > 0);
+ ASSERT(pFirstInHashList != 0);
+ ASSERT(pIndex != 0);
+
+ // Calculate the strong MD5 digest for this block
+ MD5Digest strong;
+ // Add the data from the beginnings
+ strong.Add(pBeginnings + Offset, BlockSize - Offset);
+ // Add any data from the endings
+ if(Offset > 0)
+ {
+ strong.Add(pEndings, Offset);
+ }
+ strong.Finish();
+
+ // Then go through the entries in the hash list, comparing with the strong digest calculated
+ BlocksAvailableEntry *scan = pFirstInHashList;
+ //TRACE0("second stage match\n");
+ while(scan != 0)
+ {
+ //TRACE3("scan size %d, block size %d, hash %d\n", scan->mSize, BlockSize, Hash);
+ ASSERT(scan->mSize == BlockSize);
+ ASSERT(RollingChecksum::ExtractHashingComponent(scan->mWeakChecksum) == Hash);
+
+ // Compare?
+ if(strong.DigestMatches(scan->mStrongChecksum))
+ {
+ //TRACE0("Match!\n");
+ // Found! Add to list of found blocks...
+ int64_t fileOffset = (FileBlockNumber * BlockSize) + Offset;
+ int64_t blockIndex = (scan - pIndex); // pointer arthmitic is frowed apon. But most efficient way of doing it here -- alternative is to use more memory
+
+ // Because we search for smallest blocks first, if there's an entry there
+ // we'll just be replacing it with a larger one, which is great news.
+ rFoundBlocks[fileOffset] = blockIndex;
+
+ // No point in searching further, report success
+ return true;
+ }
+
+ // Next
+ scan = scan->mpNextInHashList;
+ }
+
+ // Not matched
+ return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static GenerateRecipe(BackupStoreFileEncodeStream::Recipe &, BlocksAvailableEntry *, int64_t, std::map<int64_t, int64_t> &)
+// Purpose: Fills in the recipe from the found block list
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex,
+ int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile)
+{
+ // NOTE: This function could be a lot more sophisiticated. For example, if
+ // a small block overlaps a big block like this
+ // ****
+ // *******************************
+ // then the small block will be used, not the big one. But it'd be better to
+ // just ignore the small block and keep the big one. However, some stats should
+ // be gathered about real world files before writing complex code which might
+ // go wrong.
+
+ // Initialise a blank instruction
+ BackupStoreFileEncodeStream::RecipeInstruction instruction;
+ #define RESET_INSTRUCTION \
+ instruction.mSpaceBefore = 0; \
+ instruction.mBlocks = 0; \
+ instruction.mpStartBlock = 0;
+ RESET_INSTRUCTION
+
+ // First, a special case for when there are no found blocks
+ if(rFoundBlocks.size() == 0)
+ {
+ // No blocks, just a load of space
+ instruction.mSpaceBefore = SizeOfInputFile;
+ rRecipe.push_back(instruction);
+
+ #ifndef NDEBUG
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ TRACE1("Diff: Default recipe generated, %lld bytes of file\n", SizeOfInputFile);
+ }
+ #endif
+
+ // Don't do anything
+ return;
+ }
+
+ // Current location
+ int64_t loc = 0;
+
+ // Then iterate through the list, generating the recipe
+ std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin());
+ ASSERT(i != rFoundBlocks.end()); // check logic
+
+ // Counting for debug tracing
+#ifndef NDEBUG
+ int64_t debug_NewBytesFound = 0;
+ int64_t debug_OldBlocksUsed = 0;
+#endif
+
+ for(; i != rFoundBlocks.end(); ++i)
+ {
+ // Remember... map is (position in file) -> (index of block in pIndex)
+
+ if(i->first < loc)
+ {
+ // This block overlaps the last one
+ continue;
+ }
+ else if(i->first > loc)
+ {
+ // There's a gap between the end of the last thing and this block.
+ // If there's an instruction waiting, push it onto the list
+ if(instruction.mSpaceBefore != 0 || instruction.mpStartBlock != 0)
+ {
+ rRecipe.push_back(instruction);
+ }
+ // Start a new instruction, with the gap ready
+ RESET_INSTRUCTION
+ instruction.mSpaceBefore = i->first - loc;
+ // Move location forward to match
+ loc += instruction.mSpaceBefore;
+#ifndef NDEBUG
+ debug_NewBytesFound += instruction.mSpaceBefore;
+#endif
+ }
+
+ // First, does the current instruction need pushing back, because this block is not
+ // sequential to the last one?
+ if(instruction.mpStartBlock != 0 && (pIndex + i->second) != (instruction.mpStartBlock + instruction.mBlocks))
+ {
+ rRecipe.push_back(instruction);
+ RESET_INSTRUCTION
+ }
+
+ // Add in this block
+ if(instruction.mpStartBlock == 0)
+ {
+ // This block starts a new instruction
+ instruction.mpStartBlock = pIndex + i->second;
+ instruction.mBlocks = 1;
+ }
+ else
+ {
+ // It continues the previous section of blocks
+ instruction.mBlocks += 1;
+ }
+
+#ifndef NDEBUG
+ debug_OldBlocksUsed++;
+#endif
+
+ // Move location forward
+ loc += pIndex[i->second].mSize;
+ }
+
+ // Push the last instruction generated
+ rRecipe.push_back(instruction);
+
+ // Is there any space left at the end which needs sending?
+ if(loc != SizeOfInputFile)
+ {
+ RESET_INSTRUCTION
+ instruction.mSpaceBefore = SizeOfInputFile - loc;
+#ifndef NDEBUG
+ debug_NewBytesFound += instruction.mSpaceBefore;
+#endif
+ rRecipe.push_back(instruction);
+ }
+
+ // dump out the recipe
+#ifndef NDEBUG
+ TRACE2("Diff: %lld new bytes found, %lld old blocks used\n", debug_NewBytesFound, debug_OldBlocksUsed);
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ TRACE1("Diff: Recipe generated (size %d)\n======== ========= ========\nSpace b4 FirstBlk NumBlks\n", rRecipe.size());
+ {
+ for(unsigned int e = 0; e < rRecipe.size(); ++e)
+ {
+ char b[64];
+ sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex));
+ TRACE3("%8lld %s %8lld\n", rRecipe[e].mSpaceBefore, (rRecipe[e].mpStartBlock == 0)?" -":b, (int64_t)rRecipe[e].mBlocks);
+ }
+ }
+ TRACE0("======== ========= ========\n");
+ }
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static TimerSignalHandler(int)
+// Purpose: Signal handler
+// Created: 19/3/04
+//
+// --------------------------------------------------------------------------
+void TimerSignalHandler(int signal)
+{
+ sDiffTimedOut = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static StartDiffTimer()
+// Purpose: Starts the diff timeout timer
+// Created: 19/3/04
+//
+// --------------------------------------------------------------------------
+void StartDiffTimer()
+{
+ // Set timer signal handler
+ if(!sSetTimerSignelHandler)
+ {
+ ::signal(SIGVTALRM, TimerSignalHandler);
+ sSetTimerSignelHandler = true;
+ }
+
+ struct itimerval timeout;
+ // Don't want this to repeat
+ timeout.it_interval.tv_sec = 0;
+ timeout.it_interval.tv_usec = 0;
+ // Single timeout after the specified number of seconds
+ timeout.it_value.tv_sec = sMaximumDiffTime;
+ timeout.it_value.tv_usec = 0;
+ // Set timer
+ if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0)
+ {
+ TRACE0("WARNING: couldn't set diff timeout\n");
+ }
+
+ // Unset flag (last thing)
+ sDiffTimedOut = false;
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp
new file mode 100755
index 00000000..20e1fd80
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileEncodeStream.cpp
@@ -0,0 +1,675 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileEncodeStream.cpp
+// Purpose: Implement stream-based file encoding for the backup store
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupStoreFileEncodeStream.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BoxTime.h"
+#include "BackupClientFileAttributes.h"
+#include "FileStream.h"
+#include "RollingChecksum.h"
+#include "Random.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream
+// Purpose: Constructor (opens file)
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::BackupStoreFileEncodeStream()
+ : mpRecipe(0),
+ mpFile(0),
+ mStatus(Status_Header),
+ mSendData(true),
+ mTotalBlocks(0),
+ mAbsoluteBlockNumber(-1),
+ mInstructionNumber(-1),
+ mNumBlocks(0),
+ mCurrentBlock(-1),
+ mCurrentBlockEncodedSize(0),
+ mPositionInCurrentBlock(0),
+ mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE),
+ mLastBlockSize(0),
+ mpRawBuffer(0),
+ mAllocatedBufferSize(0),
+ mEntryIVBase(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
+// Purpose: Destructor
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
+{
+ // Free buffers
+ if(mpRawBuffer)
+ {
+ ::free(mpRawBuffer);
+ mpRawBuffer = 0;
+ }
+
+ // Close the file, which we might have open
+ if(mpFile)
+ {
+ delete mpFile;
+ mpFile = 0;
+ }
+
+ // Free the recipe
+ if(mpRecipe != 0)
+ {
+ delete mpRecipe;
+ mpRecipe = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *)
+// Purpose: Reads file information, and builds file header reading for sending.
+// Takes ownership of the Recipe.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEncodeStream::Recipe *pRecipe,
+ int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime)
+{
+ // Pointer to a blank recipe which we might create
+ BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0;
+
+ try
+ {
+ // Get file attributes
+ box_time_t modTime = 0;
+ int64_t fileSize = 0;
+ BackupClientFileAttributes attr;
+ attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime,
+ 0 /* not interested in attr mod time */, &fileSize);
+
+ // Might need to create a blank recipe...
+ if(pRecipe == 0)
+ {
+ pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0);
+
+ BackupStoreFileEncodeStream::RecipeInstruction instruction;
+ instruction.mSpaceBefore = fileSize; // whole file
+ instruction.mBlocks = 0; // no blocks
+ instruction.mpStartBlock = 0; // no block
+ pblankRecipe->push_back(instruction);
+
+ pRecipe = pblankRecipe;
+ }
+
+ // Tell caller?
+ if(pModificationTime != 0)
+ {
+ *pModificationTime = modTime;
+ }
+
+ // Go through each instruction in the recipe and work out how many blocks
+ // it will add, and the max clear size of these blocks
+ int maxBlockClearSize = 0;
+ for(uint64_t inst = 0; inst < pRecipe->size(); ++inst)
+ {
+ if((*pRecipe)[inst].mSpaceBefore > 0)
+ {
+ // Calculate the number of blocks the space before requires
+ int64_t numBlocks;
+ int32_t blockSize, lastBlockSize;
+ CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize);
+ // Add to accumlated total
+ mTotalBlocks += numBlocks;
+ // Update maximum clear size
+ if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize;
+ if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize;
+ }
+
+ // Add number of blocks copied from the previous file
+ mTotalBlocks += (*pRecipe)[inst].mBlocks;
+
+ // Check for bad things
+ if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Run through blocks to get the max clear size
+ for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b)
+ {
+ if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize;
+ }
+ }
+
+ // Send data? (symlinks don't have any data in them)
+ mSendData = !attr.IsSymLink();
+
+ // If not data is being sent, then the max clear block size is zero
+ if(!mSendData)
+ {
+ maxBlockClearSize = 0;
+ }
+
+ // Header
+ file_StreamFormat hdr;
+ hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1);
+ hdr.mNumBlocks = (mSendData)?(hton64(mTotalBlocks)):(0);
+ hdr.mContainerID = hton64(ContainerID);
+ hdr.mModificationTime = hton64(modTime);
+ // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size
+ hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128);
+ hdr.mOptions = 0; // no options defined yet
+
+ // Write header to stream
+ mData.Write(&hdr, sizeof(hdr));
+
+ // Write filename to stream
+ rStoreFilename.WriteToStream(mData);
+
+ // Write attributes to stream
+ attr.WriteToStream(mData);
+
+ // Allocate some buffers for writing data
+ if(mSendData)
+ {
+ // Open the file
+ mpFile = new FileStream(Filename);
+
+ // Work out the largest possible block required for the encoded data
+ mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize);
+
+ // Then allocate two blocks of this size
+ mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize);
+ if(mpRawBuffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+#ifndef NDEBUG
+ // In debug builds, make sure that the reallocation code is exercised.
+ mEncodedBuffer.Allocate(mAllocatedBufferSize / 4);
+#else
+ mEncodedBuffer.Allocate(mAllocatedBufferSize);
+#endif
+ }
+ else
+ {
+ // Write an empty block index for the symlink
+ file_BlockIndexHeader blkhdr;
+ blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
+ blkhdr.mOtherFileID = hton64(0); // not other file ID
+ blkhdr.mEntryIVBase = hton64(0);
+ blkhdr.mNumBlocks = hton64(0);
+ mData.Write(&blkhdr, sizeof(blkhdr));
+ }
+
+ // Ready for reading
+ mData.SetForReading();
+
+ // Update stats
+ BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize;
+
+ // Finally, store the pointer to the recipe, when we know exceptions won't occur
+ mpRecipe = pRecipe;
+ }
+ catch(...)
+ {
+ // Clean up any blank recipe
+ if(pblankRecipe != 0)
+ {
+ delete pblankRecipe;
+ pblankRecipe = 0;
+ }
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &)
+// Purpose: Calculates the sizes of blocks in a section of the file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut)
+{
+ // How many blocks, and how big?
+ rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2;
+ do
+ {
+ rBlockSizeOut *= 2;
+
+ rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut;
+
+ } while(rBlockSizeOut <= BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER);
+
+ // Last block size
+ rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut);
+
+ // Avoid small blocks?
+ if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN
+ && rNumBlocksOut > 1)
+ {
+ // Add the small bit of data to the last block
+ --rNumBlocksOut;
+ rLastBlockSizeOut += rBlockSizeOut;
+ }
+
+ // checks!
+ ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize);
+ //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut);
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Read(void *, int, int)
+// Purpose: As interface -- generates encoded file data on the fly from the raw file
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Check there's something to do.
+ if(mStatus == Status_Finished)
+ {
+ return 0;
+ }
+
+ int bytesToRead = NBytes;
+ uint8_t *buffer = (uint8_t*)pBuffer;
+
+ while(bytesToRead > 0 && mStatus != Status_Finished)
+ {
+ if(mStatus == Status_Header || mStatus == Status_BlockListing)
+ {
+ // Header or block listing phase -- send from the buffered stream
+
+ // Send bytes from the data buffer
+ int b = mData.Read(buffer, bytesToRead, Timeout);
+ bytesToRead -= b;
+ buffer += b;
+
+ // Check to see if all the data has been used from this stream
+ if(!mData.StreamDataLeft())
+ {
+ // Yes, move on to next phase (or finish, if there's no file data)
+ if(!mSendData)
+ {
+ mStatus = Status_Finished;
+ }
+ else
+ {
+ // Reset the buffer so it can be used for the next phase
+ mData.Reset();
+
+ // Get buffer ready for index?
+ if(mStatus == Status_Header)
+ {
+ // Just finished doing the stream header, create the block index header
+ file_BlockIndexHeader blkhdr;
+ blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
+ ASSERT(mpRecipe != 0);
+ blkhdr.mOtherFileID = hton64(mpRecipe->GetOtherFileID());
+ blkhdr.mNumBlocks = hton64(mTotalBlocks);
+
+ // Generate the IV base
+ Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase));
+ blkhdr.mEntryIVBase = hton64(mEntryIVBase);
+
+ mData.Write(&blkhdr, sizeof(blkhdr));
+ }
+
+ ++mStatus;
+ }
+ }
+ }
+ else if(mStatus == Status_Blocks)
+ {
+ // Block sending phase
+
+ if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize)
+ {
+ // Next block!
+ ++mCurrentBlock;
+ ++mAbsoluteBlockNumber;
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // Output extra blocks for this instruction and move forward in file
+ if(mInstructionNumber >= 0)
+ {
+ SkipPreviousBlocksInInstruction();
+ }
+
+ // Is there another instruction to go?
+ ++mInstructionNumber;
+
+ // Skip instructions which don't contain any data
+ while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size())
+ && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0)
+ {
+ SkipPreviousBlocksInInstruction();
+ ++mInstructionNumber;
+ }
+
+ if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size()))
+ {
+ // End of blocks, go to next phase
+ ++mStatus;
+
+ // Set the data to reading so the index can be written
+ mData.SetForReading();
+ }
+ else
+ {
+ // Get ready for this instruction
+ SetForInstruction();
+ }
+ }
+
+ // Can't use 'else' here as SetForInstruction() will change this
+ if(mCurrentBlock < mNumBlocks)
+ {
+ EncodeCurrentBlock();
+ }
+ }
+
+ // Send data from the current block (if there's data to send)
+ if(mPositionInCurrentBlock < mCurrentBlockEncodedSize)
+ {
+ // How much data to put in the buffer?
+ int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock;
+ if(s > bytesToRead) s = bytesToRead;
+
+ // Copy it in
+ ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s);
+
+ // Update variables
+ bytesToRead -= s;
+ buffer += s;
+ mPositionInCurrentBlock += s;
+ }
+ }
+ else
+ {
+ // Should never get here, as it'd be an invalid status
+ ASSERT(false);
+ }
+ }
+
+ // Add encoded size to stats
+ BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead);
+
+ // Return size of data to caller
+ return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction()
+// Purpose: Private. Stores the blocks of the old file referenced in the current
+// instruction into the index and skips over the data in the file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction()
+{
+ // Check something is necessary
+ if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0)
+ {
+ return;
+ }
+
+ // Index of the first block in old file (being diffed from)
+ int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock);
+
+ int64_t sizeToSkip = 0;
+
+ for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b)
+ {
+ // Update stats
+ BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
+
+ // Store the entry
+ StoreBlockIndexEntry(0 - (firstIndex + b),
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize,
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum,
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum);
+
+ // Increment the absolute block number -- kept encryption IV in sync
+ ++mAbsoluteBlockNumber;
+
+ // Add the size of this block to the size to skip
+ sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
+ }
+
+ // Move forward in the stream
+ mpFile->Seek(sizeToSkip, IOStream::SeekType_Relative);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::SetForInstruction()
+// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::SetForInstruction()
+{
+ // Calculate block sizes
+ CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize);
+
+ // Set variables
+ mCurrentBlock = 0;
+ mCurrentBlockEncodedSize = 0;
+ mPositionInCurrentBlock = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock()
+// Purpose: Private. Encodes the current block, and writes the block data to the index
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::EncodeCurrentBlock()
+{
+ // How big is the block, raw?
+ int blockRawSize = mBlockSize;
+ if(mCurrentBlock == (mNumBlocks - 1))
+ {
+ blockRawSize = mLastBlockSize;
+ }
+ ASSERT(blockRawSize < mAllocatedBufferSize);
+
+ // Check file open
+ if(mpFile == 0)
+ {
+ // File should be open, but isn't. So logical error.
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Read the data in
+ if(!mpFile->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */))
+ {
+ // TODO: Do something more intelligent, and abort this upload because the file
+ // has changed
+ THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer)
+ }
+
+ // Encode it
+ mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer);
+
+ //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize);
+
+ // Create block listing data -- generate checksums
+ RollingChecksum weakChecksum(mpRawBuffer, blockRawSize);
+ MD5Digest strongChecksum;
+ strongChecksum.Add(mpRawBuffer, blockRawSize);
+ strongChecksum.Finish();
+
+ // Add entry to the index
+ StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData());
+
+ // Set vars to reading this block
+ mPositionInCurrentBlock = 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *)
+// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream.
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum)
+{
+ // First, the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ entryEnc.mSize = htonl(ClearSize);
+ entryEnc.mWeakChecksum = htonl(WeakChecksum);
+ ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum));
+
+ // Then the clear section
+ file_BlockIndexEntry entry;
+ entry.mEncodedSize = hton64(((uint64_t)EncSizeOrBlkIndex));
+
+ // Then encrypt the encryted section
+ // Generate the IV from the block number
+ if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase))
+ {
+ THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements)
+ }
+ uint64_t iv = mEntryIVBase;
+ iv += mAbsoluteBlockNumber;
+ // Convert to network byte order before encrypting with it, so that restores work on
+ // platforms with different endiannesses.
+ iv = hton64(iv);
+ sBlowfishEncryptBlockEntry.SetIV(&iv);
+
+ // Encode the data
+ int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc));
+ if(encodedSize != sizeof(entry.mEnEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Save to data block for sending at the end of the stream
+ mData.Write(&entry, sizeof(entry));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Write(const void *, int)
+// Purpose: As interface. Exceptions.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StreamDataLeft()
+// Purpose: As interface -- end of stream reached?
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFileEncodeStream::StreamDataLeft()
+{
+ return (mStatus != Status_Finished);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StreamClosed()
+// Purpose: As interface
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFileEncodeStream::StreamClosed()
+{
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t)
+// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex,
+ int64_t NumBlocksInIndex, int64_t OtherFileID)
+ : mpBlockIndex(pBlockIndex),
+ mNumBlocksInIndex(NumBlocksInIndex),
+ mOtherFileID(OtherFileID)
+{
+ ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0))
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Recipe::~Recipe()
+// Purpose: Destructor
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::Recipe::~Recipe()
+{
+ // Free the block index, if there is one
+ if(mpBlockIndex != 0)
+ {
+ ::free(mpBlockIndex);
+ }
+}
+
+
+
+
diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h
new file mode 100755
index 00000000..1c748798
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileEncodeStream.h
@@ -0,0 +1,127 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileEncodeStream.h
+// Purpose: Implement stream-based file encoding for the backup store
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILEENCODESTREAM__H
+#define BACKUPSTOREFILEENCODESTREAM__H
+
+#include <vector>
+
+#include "IOStream.h"
+#include "BackupStoreFilename.h"
+#include "CollectInBufferStream.h"
+#include "MD5Digest.h"
+#include "BackupStoreFile.h"
+
+namespace BackupStoreFileCreation
+{
+ // Diffing and creation of files share some implementation details.
+ typedef struct _BlocksAvailableEntry
+ {
+ struct _BlocksAvailableEntry *mpNextInHashList;
+ int32_t mSize; // size in clear
+ uint32_t mWeakChecksum; // weak, rolling checksum
+ uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum
+ } BlocksAvailableEntry;
+
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFileEncodeStream
+// Purpose: Encode a file into a stream
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+class BackupStoreFileEncodeStream : public IOStream
+{
+public:
+ BackupStoreFileEncodeStream();
+ ~BackupStoreFileEncodeStream();
+
+ typedef struct
+ {
+ int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go
+ int32_t mBlocks; // number of block to reuse, starting at this one
+ BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null
+ } RecipeInstruction;
+
+ class Recipe : public std::vector<RecipeInstruction>
+ {
+ // NOTE: This class is rather tied in with the implementation of diffing.
+ public:
+ Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex,
+ int64_t OtherFileID = 0);
+ ~Recipe();
+
+ int64_t GetOtherFileID() {return mOtherFileID;}
+ int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock)
+ {
+ return pBlock - mpBlockIndex;
+ }
+
+ private:
+ BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex;
+ int64_t mNumBlocksInIndex;
+ int64_t mOtherFileID;
+ };
+
+ void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime);
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+private:
+ enum
+ {
+ Status_Header = 0,
+ Status_Blocks = 1,
+ Status_BlockListing = 2,
+ Status_Finished = 3
+ };
+
+private:
+ void EncodeCurrentBlock();
+ void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut);
+ void SkipPreviousBlocksInInstruction();
+ void SetForInstruction();
+ void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum);
+
+private:
+ Recipe *mpRecipe;
+ IOStream *mpFile; // source file
+ CollectInBufferStream mData; // buffer for header and index entries
+ int mStatus;
+ bool mSendData; // true if there's file data to send (ie not a symlink)
+ int64_t mTotalBlocks; // Total number of blocks in the file
+ int64_t mAbsoluteBlockNumber; // The absolute block number currently being output
+ // Instruction number
+ int64_t mInstructionNumber;
+ // All the below are within the current instruction
+ int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases
+ int64_t mCurrentBlock;
+ int32_t mCurrentBlockEncodedSize;
+ int32_t mPositionInCurrentBlock; // for reading out
+ int32_t mBlockSize; // Basic block size of most of the blocks in the file
+ int32_t mLastBlockSize; // the size (unencoded) of the last block in the file
+ // Buffers
+ uint8_t *mpRawBuffer; // buffer for raw data
+ BackupStoreFile::EncodingBuffer mEncodedBuffer;
+ // buffer for encoded data
+ int32_t mAllocatedBufferSize; // size of above two allocated blocks
+ uint64_t mEntryIVBase; // base for block entry IV
+};
+
+
+
+#endif // BACKUPSTOREFILEENCODESTREAM__H
+
diff --git a/lib/backupclient/BackupStoreFileRevDiff.cpp b/lib/backupclient/BackupStoreFileRevDiff.cpp
new file mode 100644
index 00000000..f1dc52d8
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileRevDiff.cpp
@@ -0,0 +1,258 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileRevDiff.cpp
+// Purpose: Reverse a patch, to build a new patch from new to old files
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <stdlib.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t)
+// Purpose: Reverse a patch, to build a new patch from new to old files. Takes
+// two independent copies to the From file, for efficiency.
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent)
+{
+ // Read and copy the header from the from file to the out file -- beginnings of the patch
+ file_StreamFormat hdr;
+ if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Copy
+ rOut.Write(&hdr, sizeof(hdr));
+ // Copy over filename and attributes
+ // BLOCK
+ {
+ BackupStoreFilename filename;
+ filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ filename.WriteToStream(rOut);
+ StreamableMemBlock attr;
+ attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ attr.WriteToStream(rOut);
+ }
+
+ // Build an index of common blocks.
+ // For each block in the from file, we want to know it's index in the
+ // diff file. Allocate memory for this information.
+ int64_t fromNumBlocks = ntoh64(hdr.mNumBlocks);
+ int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t));
+ if(pfromIndexInfo == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Buffer data
+ void *buffer = 0;
+ int bufferSize = 0;
+
+ // flag
+ bool isCompletelyDifferent = true;
+
+ try
+ {
+ // Initialise the index to be all 0, ie not filled in yet
+ for(int64_t i = 0; i < fromNumBlocks; ++i)
+ {
+ pfromIndexInfo[i] = 0;
+ }
+
+ // Within the from file, skip to the index
+ MoveStreamPositionToBlockIndex(rDiff);
+
+ // Read in header of index
+ file_BlockIndexHeader diffIdxHdr;
+ if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // And then read in each entry
+ int64_t diffNumBlocks = ntoh64(diffIdxHdr.mNumBlocks);
+ for(int64_t b = 0; b < diffNumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Where's the block?
+ int64_t blockEn = ntoh64(e.mEncodedSize);
+ if(blockEn > 0)
+ {
+ // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file
+ }
+ else
+ {
+ // Block is in the original file, store which block it is in this file
+ int64_t fromIndex = 0 - blockEn;
+ if(fromIndex < 0 || fromIndex >= fromNumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles)
+ }
+
+ // Store information about where it is in the new file
+ // NOTE: This is slight different to how it'll be stored in the final index.
+ pfromIndexInfo[fromIndex] = -1 - b;
+ }
+ }
+
+ // Open the index for the second copy of the from file
+ MoveStreamPositionToBlockIndex(rFrom2);
+
+ // Read in header of index
+ file_BlockIndexHeader fromIdxHdr;
+ if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || ntoh64(fromIdxHdr.mOtherFileID) != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // So, we can now start building the data in the file
+ int64_t filePosition = rFrom.GetPosition();
+ for(int64_t b = 0; b < fromNumBlocks; ++b)
+ {
+ // Read entry from from index
+ file_BlockIndexEntry e;
+ if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Get size
+ int64_t blockSize = hton64(e.mEncodedSize);
+ if(blockSize < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Copy this block?
+ if(pfromIndexInfo[b] == 0)
+ {
+ // Copy it, first move to file location
+ rFrom.Seek(filePosition, IOStream::SeekType_Absolute);
+
+ // Make sure there's memory available to copy this
+ if(bufferSize < blockSize || buffer == 0)
+ {
+ // Free old block
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ bufferSize = 0;
+ }
+ // Allocate new block
+ buffer = ::malloc(blockSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ bufferSize = blockSize;
+ }
+ ASSERT(bufferSize >= blockSize);
+
+ // Copy the block
+ if(!rFrom.ReadFullBuffer(buffer, blockSize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ rOut.Write(buffer, blockSize);
+
+ // Store the size
+ pfromIndexInfo[b] = blockSize;
+ }
+ else
+ {
+ // Block isn't needed, so it's not completely different
+ isCompletelyDifferent = false;
+ }
+ filePosition += blockSize;
+ }
+
+ // Then write the index, modified header first
+ fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(hton64(ObjectIDOfFrom));
+ rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr));
+
+ // Move to start of index entries
+ rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute);
+
+ // Then copy modified entries
+ for(int64_t b = 0; b < fromNumBlocks; ++b)
+ {
+ // Read entry from from index
+ file_BlockIndexEntry e;
+ if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Modify...
+ int64_t s = pfromIndexInfo[b];
+ // Adjust to reflect real block index (remember 0 has a different meaning here)
+ if(s < 0) ++s;
+ // Insert
+ e.mEncodedSize = hton64(s);
+ // Write
+ rOut.Write(&e, sizeof(e));
+ }
+ }
+ catch(...)
+ {
+ ::free(pfromIndexInfo);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+ throw;
+ }
+
+ // Free memory used (oh for finally {} blocks)
+ ::free(pfromIndexInfo);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+
+ // return completely different flag
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = isCompletelyDifferent;
+ }
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFileWire.h b/lib/backupclient/BackupStoreFileWire.h
new file mode 100755
index 00000000..5b1bc819
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileWire.h
@@ -0,0 +1,74 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileWire.h
+// Purpose: On the wire / disc formats for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILEWIRE__H
+#define BACKUPSTOREFILEWIRE__H
+
+#include "MD5Digest.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int64_t mNumBlocks; // number of blocks contained in the file
+ int64_t mContainerID;
+ int64_t mModificationTime;
+ int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block
+ int32_t mOptions; // bitmask of options used
+ // Then a BackupStoreFilename
+ // Then a BackupClientFileAttributes
+} file_StreamFormat;
+
+typedef struct
+{
+ int32_t mMagicValue; // different magic value
+ int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index
+ uint64_t mEntryIVBase; // base value for block IV
+ int64_t mNumBlocks; // repeat of value in file header
+} file_BlockIndexHeader;
+
+typedef struct
+{
+ int32_t mSize; // size in clear
+ uint32_t mWeakChecksum; // weak, rolling checksum
+ uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum
+} file_BlockIndexEntryEnc;
+
+typedef struct
+{
+ union
+ {
+ int64_t mEncodedSize; // size encoded, if > 0
+ int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0
+ };
+ uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section
+} file_BlockIndexEntry;
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// header for blocks of compressed data in files
+#define HEADER_CHUNK_IS_COMPRESSED 1 // bit
+#define HEADER_ENCODING_SHIFT 1 // shift value
+#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7
+#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7
+
+
+#endif // BACKUPSTOREFILEWIRE__H
+
diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp
new file mode 100755
index 00000000..fbfe3313
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilename.cpp
@@ -0,0 +1,279 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilename.cpp
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreFilename.h"
+#include "Protocol.h"
+#include "BackupStoreException.h"
+#include "IOStream.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::BackupStoreFilename()
+// Purpose: Default constructor -- creates an invalid filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::BackupStoreFilename()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy)
+ : BackupStoreFilename_base(rToCopy)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::~BackupStoreFilename()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::~BackupStoreFilename()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::CheckValid(bool)
+// Purpose: Checks the encoded filename for validity
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const
+{
+ bool ok = true;
+
+ if(size() < 2)
+ {
+ // Isn't long enough to have a header
+ ok = false;
+ }
+ else
+ {
+ // Check size is consistent
+ unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(*this);
+ if(dsize != size())
+ {
+ ok = false;
+ }
+
+ // And encoding is an accepted value
+ unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this);
+ if(encoding < Encoding_Min || encoding > Encoding_Max)
+ {
+ ok = false;
+ }
+ }
+
+ // Exception?
+ if(!ok && ExceptionIfInvalid)
+ {
+ THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename)
+ }
+
+ return ok;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::ReadFromProtocol(Protocol &)
+// Purpose: Reads the filename from the protocol object
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol)
+{
+ // Read the header
+ char hdr[2];
+ rProtocol.Read(hdr, 2);
+
+ // How big is it?
+ int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
+
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ std::string data;
+ rProtocol.Read(data, dsize - 2);
+
+ // assign to this string, storing the header and the extra data
+ assign(hdr, 2);
+ append(data.c_str(), data.size());
+
+ // Check it
+ CheckValid();
+
+ // Alert derived classes
+ EncodedFilenameChanged();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::WriteToProtocol(Protocol &)
+// Purpose: Writes the filename to the protocol object
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const
+{
+ CheckValid();
+
+ rProtocol.Write(c_str(), size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::ReadFromStream(IOStream &)
+// Purpose: Reads the filename from a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Read the header
+ char hdr[2];
+ if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // How big is it?
+ unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
+
+ // Assume most filenames are small
+ char buf[256];
+ if(dsize < sizeof(buf))
+ {
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ // Copy in header
+ buf[0] = hdr[0]; buf[1] = hdr[1];
+
+ // assign to this string, storing the header and the extra data
+ assign(buf, dsize);
+ }
+ else
+ {
+ // Block of memory to hold it
+ MemoryBlockGuard<char*> dataB(dsize+2);
+ char *data = dataB;
+
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ // Copy in header
+ data[0] = hdr[0]; data[1] = hdr[1];
+
+ // assign to this string, storing the header and the extra data
+ assign(data, dsize);
+ }
+
+ // Check it
+ CheckValid();
+
+ // Alert derived classes
+ EncodedFilenameChanged();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::WriteToStream(IOStream &)
+// Purpose: Writes the filename to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::WriteToStream(IOStream &rStream) const
+{
+ CheckValid();
+
+ rStream.Write(c_str(), size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::EncodedFilenameChanged()
+// Purpose: The encoded filename stored has changed
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::EncodedFilenameChanged()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::IsEncrypted()
+// Purpose: Returns true if the filename is stored using an encrypting encoding
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFilename::IsEncrypted() const
+{
+ return BACKUPSTOREFILENAME_GET_ENCODING(*this) != Encoding_Clear;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::SetAsClearFilename(const char *)
+// Purpose: Sets this object to be a valid filename, but with a filename in the clear.
+// Used on the server to create filenames when there's no way of encrypting it.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::SetAsClearFilename(const char *Clear)
+{
+ // Make std::string from the clear name
+ std::string toEncode(Clear);
+
+ // Make an encoded string
+ char hdr[2];
+ BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear);
+ std::string encoded(hdr, 2);
+ encoded += toEncode;
+ ASSERT(encoded.size() == toEncode.size() + 2);
+
+ // Store the encoded string
+ assign(encoded);
+
+ // Stuff which must be done
+ EncodedFilenameChanged();
+ CheckValid(false);
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h
new file mode 100755
index 00000000..4c951b6f
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilename.h
@@ -0,0 +1,85 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilename.h
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILENAME__H
+#define BACKUPSTOREFILENAME__H
+
+#include <string>
+
+class Protocol;
+class IOStream;
+
+// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+// don't define this -- the problem of memory usage still appears without this.
+// It's just that this class really showed up the problem. Instead, malloc allocation
+// is globally defined in BoxPlatform.h, for troublesome libraries.
+
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+ // Use a malloc_allocated string, because the STL default allocators really screw up with
+ // memory allocation, particularly with this class.
+ // Makes a few things a bit messy and inefficient with conversions.
+ // Given up using this, and use global malloc allocation instead, but thought it
+ // worth leaving this code in just in case it's useful for the future.
+ typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base;
+ // If this is changed, change GetClearFilename() back to returning a reference.
+#else
+ typedef std::string BackupStoreFilename_base;
+#endif // PLATFORM_HAVE_STL_MALLOC_ALLOC
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFilename
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreFilename : public BackupStoreFilename_base
+{
+public:
+ BackupStoreFilename();
+ BackupStoreFilename(const BackupStoreFilename &rToCopy);
+ virtual ~BackupStoreFilename();
+
+ bool CheckValid(bool ExceptionIfInvalid = true) const;
+
+ void ReadFromProtocol(Protocol &rProtocol);
+ void WriteToProtocol(Protocol &rProtocol) const;
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream) const;
+
+ void SetAsClearFilename(const char *Clear);
+
+ // Check that it's encrypted
+ bool IsEncrypted() const;
+
+ // These enumerated types belong in the base class so
+ // the CheckValid() function can make sure that the encoding
+ // is a valid encoding
+ enum
+ {
+ Encoding_Min = 1,
+ Encoding_Clear = 1,
+ Encoding_Blowfish = 2,
+ Encoding_Max = 2
+ };
+
+protected:
+ virtual void EncodedFilenameChanged();
+};
+
+// On the wire utilities for class and derived class
+#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2)
+#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3)
+
+#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;}
+
+#endif // BACKUPSTOREFILENAME__H
+
diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp
new file mode 100755
index 00000000..3069f612
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilenameClear.cpp
@@ -0,0 +1,335 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilenameClear.cpp
+// Purpose: BackupStoreFilenames in the clear
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupStoreException.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// Hide private variables from the rest of the world
+namespace
+{
+ int sEncodeMethod = BackupStoreFilename::Encoding_Clear;
+ CipherContext sBlowfishEncrypt;
+ CipherContext sBlowfishDecrypt;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear()
+// Purpose: Default constructor, creates an invalid filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &)
+// Purpose: Creates a filename, encoding from the given string
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode)
+{
+ SetClearFilename(rToEncode);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy)
+ : BackupStoreFilename(rToCopy),
+ mClearFilename(rToCopy.mClearFilename)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
+// Purpose: Copy from base class
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
+ : BackupStoreFilename(rToCopy)
+{
+ // Will get a clear filename when it's required
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::~BackupStoreFilenameClear()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::GetClearFilename()
+// Purpose: Get the unencoded filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+const std::string BackupStoreFilenameClear::GetClearFilename() const
+{
+ MakeClearAvailable();
+ // When modifying, remember to change back to reference return if at all possible
+ // -- returns an object rather than a reference to allow easy use with other code.
+ return std::string(mClearFilename.c_str(), mClearFilename.size());
+}
+#else
+const std::string &BackupStoreFilenameClear::GetClearFilename() const
+{
+ MakeClearAvailable();
+ return mClearFilename;
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &)
+// Purpose: Encode and make available the clear filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode)
+{
+ // Only allow Blowfish encodings
+ if(sEncodeMethod != Encoding_Blowfish)
+ {
+ THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup)
+ }
+
+ // Make an encoded string with blowfish encryption
+ EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish);
+
+ // Store the clear filename
+ mClearFilename.assign(rToEncode.c_str(), rToEncode.size());
+
+ // Make sure we did the right thing
+ if(!CheckValid(false))
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::MakeClearAvailable()
+// Purpose: Private. Make sure the clear filename is available
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::MakeClearAvailable() const
+{
+ if(!mClearFilename.empty())
+ return; // nothing to do
+
+ // Check valid
+ CheckValid();
+
+ // Decode the header
+ int size = BACKUPSTOREFILENAME_GET_SIZE(*this);
+ int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this);
+
+ // Decode based on encoding given in the header
+ switch(encoding)
+ {
+ case Encoding_Clear:
+ TRACE0("**** BackupStoreFilename encoded with Clear encoding ****\n");
+ mClearFilename.assign(c_str() + 2, size - 2);
+ break;
+
+ case Encoding_Blowfish:
+ DecryptEncoded(sBlowfishDecrypt);
+ break;
+
+ default:
+ THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding)
+ break;
+ }
+}
+
+
+// Buffer for encoding and decoding -- do this all in one single buffer to
+// avoid lots of string allocation, which stuffs up memory usage.
+// These static memory vars are, of course, not thread safe, but we don't use threads.
+#ifndef NDEBUG
+static int sEncDecBufferSize = 2; // small size for debug builds
+#else
+static int sEncDecBufferSize = 256;
+#endif
+static MemoryBlockGuard<uint8_t *> spEncDecBuffer(sEncDecBufferSize);
+
+// fudge to stop leak reporting
+#ifdef BOX_MEMORY_LEAK_TESTING
+namespace
+{
+ class leak_off
+ {
+ public:
+ leak_off()
+ {
+ MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer);
+ }
+ };
+ leak_off dont_report_as_leak;
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int)
+// Purpose: Private. Assigns the encoded filename string, encrypting.
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding)
+{
+ // Work out max size
+ int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4;
+
+ // Make sure encode/decode buffer has enough space
+ if(sEncDecBufferSize < maxOutSize)
+ {
+ TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, maxOutSize);
+ spEncDecBuffer.Resize(maxOutSize);
+ sEncDecBufferSize = maxOutSize;
+ }
+
+ // Pointer to buffer
+ uint8_t *buffer = spEncDecBuffer;
+ MEMLEAKFINDER_NOT_A_LEAK(buffer);
+
+ // Encode -- do entire block in one go
+ int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size());
+ // and add in header size
+ encSize += 2;
+
+ // Adjust header
+ BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding);
+
+ // Store the encoded string
+ assign((char*)buffer, encSize);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &)
+// Purpose: Decrypt the encoded filename using the cipher context
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const
+{
+ // Work out max size
+ int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(size()) + 4;
+
+ // Make sure encode/decode buffer has enough space
+ if(sEncDecBufferSize < maxOutSize)
+ {
+ TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, maxOutSize);
+ spEncDecBuffer.Resize(maxOutSize);
+ sEncDecBufferSize = maxOutSize;
+ }
+
+ // Pointer to buffer
+ uint8_t *buffer = spEncDecBuffer;
+ MEMLEAKFINDER_NOT_A_LEAK(buffer);
+
+ // Decrypt
+ const char *str = c_str() + 2;
+ int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, size() - 2);
+
+ // Assign to this
+ mClearFilename.assign((char*)buffer, sizeOut);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::EncodedFilenameChanged()
+// Purpose: The encoded filename stored has changed
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::EncodedFilenameChanged()
+{
+ BackupStoreFilename::EncodedFilenameChanged();
+
+ // Delete stored filename in clear
+ mClearFilename.erase();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int)
+// Purpose: Set the key used for Blowfish encryption of filenames
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength)
+{
+ // Initialisation vector not used. Can't use a different vector for each filename as
+ // that would stop comparisions on the server working.
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength);
+ sBlowfishEncrypt.SetIV(pIV);
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength);
+ sBlowfishDecrypt.SetIV(pIV);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetEncodingMethod(int)
+// Purpose: Set the encoding method used for filenames
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetEncodingMethod(int Method)
+{
+ sEncodeMethod = Method;
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFilenameClear.h b/lib/backupclient/BackupStoreFilenameClear.h
new file mode 100755
index 00000000..d4c45701
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilenameClear.h
@@ -0,0 +1,60 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilenameClear.h
+// Purpose: BackupStoreFilenames in the clear
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILENAMECLEAR__H
+#define BACKUPSTOREFILENAMECLEAR__H
+
+#include "BackupStoreFilename.h"
+
+class CipherContext;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFilenameClear
+// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreFilenameClear : public BackupStoreFilename
+{
+public:
+ BackupStoreFilenameClear();
+ BackupStoreFilenameClear(const std::string &rToEncode);
+ BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy);
+ BackupStoreFilenameClear(const BackupStoreFilename &rToCopy);
+ virtual ~BackupStoreFilenameClear();
+
+ // Because we need to use a different allocator for this class to avoid
+ // nasty things happening, can't return this as a reference. Which is a
+ // pity. But probably not too bad.
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+ const std::string GetClearFilename() const;
+#else
+ const std::string &GetClearFilename() const;
+#endif
+ void SetClearFilename(const std::string &rToEncode);
+
+ // Setup for encryption of filenames
+ static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength);
+ static void SetEncodingMethod(int Method);
+
+protected:
+ void MakeClearAvailable() const;
+ virtual void EncodedFilenameChanged();
+ void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding);
+ void DecryptEncoded(CipherContext &rCipherContext) const;
+
+private:
+ mutable BackupStoreFilename_base mClearFilename;
+};
+
+#endif // BACKUPSTOREFILENAMECLEAR__H
+
+
diff --git a/lib/backupclient/BackupStoreObjectDump.cpp b/lib/backupclient/BackupStoreObjectDump.cpp
new file mode 100644
index 00000000..caa0e82f
--- /dev/null
+++ b/lib/backupclient/BackupStoreObjectDump.cpp
@@ -0,0 +1,210 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreObjectDump.cpp
+// Purpose: Implementations of dumping objects to stdout/TRACE
+// Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <map>
+
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreFilename.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreObjectMagic.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static void OutputLine(FILE *, bool, const char *, ...)
+// Purpose: Output a line for the object dumping, to file and/or trace...
+// Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+static void OutputLine(FILE *file, bool ToTrace, const char *format, ...)
+{
+ char text[512];
+ int r = 0;
+ va_list ap;
+ va_start(ap, format);
+ r = vsnprintf(text, sizeof(text), format, ap);
+ va_end(ap);
+
+ if(file != 0)
+ {
+ ::fprintf(file, "%s", text);
+ }
+ if(ToTrace)
+ {
+ TRACE1("%s", text);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace)
+// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere)
+// Dump the contents to a file, or trace.
+// Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace)
+{
+ FILE *file = (FILE*)clibFileHandle;
+
+ OutputLine(file, ToTrace, "Directory object.\nObject ID: %llx\nContainer ID: %llx\nNumber entries: %d\n"\
+ "Attributes mod time: %llx\nAttributes size: %d\n", mObjectID, mContainerID, mEntries.size(),
+ mAttributesModTime, mAttributes.GetSize());
+
+ // So repeated filenames can be illustrated, even though they can't be decoded
+ std::map<BackupStoreFilename, int> nameNum;
+ int nameNumI = 0;
+
+ // Dump items
+ OutputLine(file, ToTrace, "Items:\nID Size AttrHash AtSz NSz NIdx Flags\n");
+ for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+ {
+ // Choose file name index number for this file
+ std::map<BackupStoreFilename, int>::iterator nn(nameNum.find((*i)->GetName()));
+ int ni = nameNumI;
+ if(nn != nameNum.end())
+ {
+ ni = nn->second;
+ }
+ else
+ {
+ nameNum[(*i)->GetName()] = nameNumI;
+ ++nameNumI;
+ }
+
+ // Do dependencies
+ char depends[128];
+ depends[0] = '\0';
+ int depends_l = 0;
+ if((*i)->GetDependsNewer() != 0)
+ {
+ depends_l += ::sprintf(depends + depends_l, " depNew(%llx)", (*i)->GetDependsNewer());
+ }
+ if((*i)->GetDependsOlder() != 0)
+ {
+ depends_l += ::sprintf(depends + depends_l, " depOld(%llx)", (*i)->GetDependsOlder());
+ }
+
+ // Output item
+ int16_t f = (*i)->GetFlags();
+ OutputLine(file, ToTrace, "%06llx %4lld %016llx %4d %3d %4d%s%s%s%s%s%s\n",
+ (*i)->GetObjectID(),
+ (*i)->GetSizeInBlocks(),
+ (*i)->GetAttributesHash(),
+ (*i)->GetAttributes().GetSize(),
+ (*i)->GetName().size(),
+ ni,
+ ((f & BackupStoreDirectory::Entry::Flags_File)?" file":""),
+ ((f & BackupStoreDirectory::Entry::Flags_Dir)?" dir":""),
+ ((f & BackupStoreDirectory::Entry::Flags_Deleted)?" del":""),
+ ((f & BackupStoreDirectory::Entry::Flags_OldVersion)?" old":""),
+ ((f & BackupStoreDirectory::Entry::Flags_RemoveASAP)?" removeASAP":""),
+ depends);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DumpFile(void *, bool, IOStream &)
+// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere)
+// Dump the contents to a file, or trace.
+// Created: 4/5/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile)
+{
+ FILE *file = (FILE*)clibFileHandle;
+
+ // Read header
+ file_StreamFormat hdr;
+ if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr),
+ 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Check and output header info
+ if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V0))
+ {
+ OutputLine(file, ToTrace, "File header doesn't have the correct magic, aborting dump\n");
+ return;
+ }
+
+ OutputLine(file, ToTrace, "File object.\nContainer ID: %llx\nModification time: %llx\n"\
+ "Max block clear size: %d\nOptions: %08x\nNum blocks: %d\n", ntoh64(hdr.mContainerID),
+ ntoh64(hdr.mModificationTime), ntohl(hdr.mMaxBlockClearSize), ntohl(hdr.mOptions),
+ ntoh64(hdr.mNumBlocks));
+
+ // Read the next two objects
+ BackupStoreFilename fn;
+ fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+ OutputLine(file, ToTrace, "Filename size: %d\n", fn.size());
+
+ BackupClientFileAttributes attr;
+ attr.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+ OutputLine(file, ToTrace, "Attributes size: %d\n", attr.GetSize());
+
+ // Dump the blocks
+ rFile.Seek(0, IOStream::SeekType_Absolute);
+ BackupStoreFile::MoveStreamPositionToBlockIndex(rFile);
+
+ // Read in header
+ file_BlockIndexHeader bhdr;
+ rFile.ReadFullBuffer(&bhdr, sizeof(bhdr), 0);
+ if(bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ && bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0))
+ {
+ OutputLine(file, ToTrace, "WARNING: Block header doesn't have the correct magic\n");
+ }
+ // number of blocks
+ int64_t nblocks = ntoh64(bhdr.mNumBlocks);
+ OutputLine(file, ToTrace, "Other file ID (for block refs): %llx\nNum blocks (in blk hdr): %lld\n",
+ ntoh64(bhdr.mOtherFileID), nblocks);
+
+ // Dump info about each block
+ OutputLine(file, ToTrace, "======== ===== ==========\n Index Where EncSz/Idx\n");
+ int64_t nnew = 0, nold = 0;
+ for(int64_t b = 0; b < nblocks; ++b)
+ {
+ file_BlockIndexEntry en;
+ if(!rFile.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ OutputLine(file, ToTrace, "Didn't manage to read block %lld from file\n", b);
+ continue;
+ }
+ int64_t s = ntoh64(en.mEncodedSize);
+ if(s > 0)
+ {
+ nnew++;
+ TRACE2("%8lld this s=%8lld\n", b, s);
+ }
+ else
+ {
+ nold++;
+ TRACE2("%8lld other i=%8lld\n", b, 0 - s);
+ }
+ }
+ TRACE0("======== ===== ==========\n");
+}
+
diff --git a/lib/backupclient/BackupStoreObjectMagic.h b/lib/backupclient/BackupStoreObjectMagic.h
new file mode 100755
index 00000000..7ee600a2
--- /dev/null
+++ b/lib/backupclient/BackupStoreObjectMagic.h
@@ -0,0 +1,31 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreObjectMagic.h
+// Purpose: Magic values for the start of objects in the backup store
+// Created: 19/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREOBJECTMAGIC__H
+#define BACKUPSTOREOBJECTMAGIC__H
+
+// Each of these values is the first 4 bytes of the object file.
+// Remember to swap from network to host byte order.
+
+// Magic value for file streams
+#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65
+// Do not use v0 in any new code!
+#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45
+
+// Magic for the block index at the file stream -- used to
+// ensure streams are reordered as expected
+#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478
+// Do not use v0 in any new code!
+#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B
+
+// Magic value for directory streams
+#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F
+
+#endif // BACKUPSTOREOBJECTMAGIC__H
+
diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra
new file mode 100755
index 00000000..66203e3c
--- /dev/null
+++ b/lib/backupclient/Makefile.extra
@@ -0,0 +1,16 @@
+
+MAKEPROTOCOL = ../../lib/server/makeprotocol.pl
+
+GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt
+
+# AUTOGEN SEEDING
+autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt
+ perl $(GEN_CMD_SRV)
+
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt
+ perl $(MAKEEXCEPTION) BackupStoreException.txt
+
diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp
new file mode 100755
index 00000000..91a6b758
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccountDatabase.cpp
@@ -0,0 +1,370 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccountDatabase.cpp
+// Purpose: Database of accounts for the backup store
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <string>
+#include <map>
+#include <stdio.h>
+#include <sys/stat.h>
+
+#include "BackupStoreAccountDatabase.h"
+#include "Guards.h"
+#include "FdGetLine.h"
+#include "BackupStoreException.h"
+#include "CommonException.h"
+#include "FileModificationTime.h"
+
+#include "MemLeakFindOn.h"
+
+class _BackupStoreAccountDatabase
+{
+public:
+ std::string mFilename;
+ std::map<int32_t, BackupStoreAccountDatabase::Entry> mDatabase;
+ box_time_t mModificationTime;
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *)
+// Purpose: Constructor
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *Filename)
+ : pImpl(new _BackupStoreAccountDatabase)
+{
+ pImpl->mFilename = Filename;
+ pImpl->mModificationTime = 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::~BackupStoreAccountDatabase()
+// Purpose: Destructor
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::~BackupStoreAccountDatabase()
+{
+ delete pImpl;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::Entry()
+// Purpose: Default constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry()
+ : mID(-1),
+ mDiscSet(-1)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::Entry(int32_t, int)
+// Purpose: Constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry(int32_t ID, int DiscSet)
+ : mID(ID),
+ mDiscSet(DiscSet)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::Entry(const Entry &)
+// Purpose: Copy constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry(const Entry &rEntry)
+ : mID(rEntry.mID),
+ mDiscSet(rEntry.mDiscSet)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::~Entry()
+// Purpose: Destructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::~Entry()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Read(const char *)
+// Purpose: Read in a database from disc
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreAccountDatabase> BackupStoreAccountDatabase::Read(const char *Filename)
+{
+ // Database object to use
+ std::auto_ptr<BackupStoreAccountDatabase> db(new BackupStoreAccountDatabase(Filename));
+
+ // Read in the file
+ db->ReadFile();
+
+ // Return to called
+ return db;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::ReadFile()
+// Purpose: Read the file off disc
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::ReadFile() const
+{
+ // Open file
+ FileHandleGuard<> file(pImpl->mFilename.c_str());
+
+ // Clear existing entries
+ pImpl->mDatabase.clear();
+
+ // Read in lines
+ FdGetLine getLine(file);
+
+ while(!getLine.IsEOF())
+ {
+ // Read and split up line
+ std::string l(getLine.GetLine(true));
+
+ if(!l.empty())
+ {
+ // Check...
+ int32_t id;
+ int discSet;
+ if(::sscanf(l.c_str(), "%x:%d", &id, &discSet) != 2)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadAccountDatabaseFile)
+ }
+
+ // Make a new entry
+ pImpl->mDatabase[id] = Entry(id, discSet);
+ }
+ }
+
+ // Store the modification time of the file
+ pImpl->mModificationTime = GetDBFileModificationTime();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::CheckUpToDate()
+// Purpose: Private. Ensure that the in memory database matches the one on disc
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::CheckUpToDate() const
+{
+ if(pImpl->mModificationTime != GetDBFileModificationTime())
+ {
+ // File has changed -- load it in again
+ ReadFile();
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::GetDBFileModificationTime()
+// Purpose: Get the current modification time of the database
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+box_time_t BackupStoreAccountDatabase::GetDBFileModificationTime() const
+{
+ struct stat st;
+ if(::stat(pImpl->mFilename.c_str(), &st) == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ return FileModificationTime(st);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Write()
+// Purpose: Write the database back to disc after modifying it
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::Write()
+{
+ // Open file for writing
+ // Would use this...
+ // FileHandleGuard<O_WRONLY | O_TRUNC> file(pImpl->mFilename.c_str());
+ // but gcc fails randomly on it on some platforms. Weird.
+
+ int file = ::open(pImpl->mFilename.c_str(), O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+ if(file == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileOpenError)
+ }
+
+ try
+ {
+ // Then write each entry
+ for(std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.begin());
+ i != pImpl->mDatabase.end(); ++i)
+ {
+ // Write out the entry
+ char line[256]; // more than enough for a couple of integers in string form
+ int s = ::sprintf(line, "%x:%d\n", i->second.GetID(), i->second.GetDiscSet());
+ if(::write(file, line, s) != s)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+
+ ::close(file);
+ }
+ catch(...)
+ {
+ ::close(file);
+ throw;
+ }
+
+ // Done.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::EntryExists(int32_t)
+// Purpose: Does an entry exist in the database?
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool BackupStoreAccountDatabase::EntryExists(int32_t ID) const
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ return pImpl->mDatabase.find(ID) != pImpl->mDatabase.end();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::GetEntry(int32_t)
+// Purpose: Retrieve an entry
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+const BackupStoreAccountDatabase::Entry &BackupStoreAccountDatabase::GetEntry(int32_t ID) const
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.find(ID));
+ if(i == pImpl->mDatabase.end())
+ {
+ THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry)
+ }
+
+ return i->second;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::AddEntry(int32_t, int)
+// Purpose: Add a new entry to the database
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::AddEntry(int32_t ID, int DiscSet)
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ pImpl->mDatabase[ID] = Entry(ID, DiscSet);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::DeleteEntry(int32_t)
+// Purpose: Delete an entry from the database
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::DeleteEntry(int32_t ID)
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.find(ID));
+ if(i == pImpl->mDatabase.end())
+ {
+ THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry)
+ }
+
+ pImpl->mDatabase.erase(i);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t>)
+// Purpose:
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t> &rIDsOut)
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ // Delete everything in the output list
+ rIDsOut.clear();
+
+ std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.begin());
+ for(; i != pImpl->mDatabase.end(); ++i)
+ {
+ rIDsOut.push_back(i->first);
+ }
+}
+
+
+
+
diff --git a/lib/backupstore/BackupStoreAccountDatabase.h b/lib/backupstore/BackupStoreAccountDatabase.h
new file mode 100755
index 00000000..8d6e7ad8
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccountDatabase.h
@@ -0,0 +1,75 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccountDatabase.h
+// Purpose: Database of accounts for the backup store
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREACCOUNTDATABASE__H
+#define BACKUPSTOREACCOUNTDATABASE__H
+
+#include <memory>
+#include <vector>
+
+#include "BoxTime.h"
+
+class _BackupStoreAccountDatabase;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreAccountDatabase
+// Purpose: Database of accounts for the backup store
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+class BackupStoreAccountDatabase
+{
+public:
+ friend class _BackupStoreAccountDatabase; // to stop compiler warnings
+ ~BackupStoreAccountDatabase();
+private:
+ BackupStoreAccountDatabase(const char *Filename);
+ BackupStoreAccountDatabase(const BackupStoreAccountDatabase &);
+public:
+
+ static std::auto_ptr<BackupStoreAccountDatabase> Read(const char *Filename);
+ void Write();
+
+ class Entry
+ {
+ public:
+ Entry();
+ Entry(int32_t ID, int DiscSet);
+ Entry(const Entry &rEntry);
+ ~Entry();
+
+ int32_t GetID() const {return mID;}
+ int GetDiscSet() const {return mDiscSet;}
+
+ private:
+ int32_t mID;
+ int mDiscSet;
+ };
+
+ bool EntryExists(int32_t ID) const;
+ const Entry &GetEntry(int32_t ID) const;
+ void AddEntry(int32_t ID, int DiscSet);
+ void DeleteEntry(int32_t ID);
+
+ // This interface should change in the future. But for now it'll do.
+ void GetAllAccountIDs(std::vector<int32_t> &rIDsOut);
+
+private:
+ void ReadFile() const; // const in concept only
+ void CheckUpToDate() const; // const in concept only
+ box_time_t GetDBFileModificationTime() const;
+
+private:
+ mutable _BackupStoreAccountDatabase *pImpl;
+};
+
+#endif // BACKUPSTOREACCOUNTDATABASE__H
+
diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp
new file mode 100755
index 00000000..36d9cad3
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccounts.cpp
@@ -0,0 +1,162 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccounts.cpp
+// Purpose: Account management for backup store server
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "BoxPortsAndFiles.h"
+#include "BackupStoreAccounts.h"
+#include "BackupStoreAccountDatabase.h"
+#include "RaidFileWrite.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+#include "UnixUser.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &)
+// Purpose: Constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase)
+ : mrDatabase(rDatabase)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::~BackupStoreAccounts()
+// Purpose: Destructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccounts::~BackupStoreAccounts()
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::Create(int32_t, int, int64_t, int64_t, const std::string &)
+// Purpose: Create a new account on the specified disc set.
+// If rAsUsername is not empty, then the account information will be written under the
+// username specified.
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername)
+{
+ {
+ // Become the user specified in the config file?
+ std::auto_ptr<UnixUser> user;
+ if(!rAsUsername.empty())
+ {
+ // Username specified, change...
+ user.reset(new UnixUser(rAsUsername.c_str()));
+ user->ChangeProcessUser(true /* temporary */);
+ // Change will be undone at the end of this function
+ }
+
+ // Get directory name
+ std::string dirName(MakeAccountRootDir(ID, DiscSet));
+
+ // Create a directory on disc
+ RaidFileWrite::CreateDirectory(DiscSet, dirName, true /* recursive */);
+
+ // Create an info file
+ BackupStoreInfo::CreateNew(ID, dirName, DiscSet, SizeSoftLimit, SizeHardLimit);
+
+ // And an empty directory
+ BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID);
+ int64_t rootDirSize = 0;
+ // Write it, knowing the directory scheme
+ {
+ RaidFileWrite rf(DiscSet, dirName + "o01");
+ rf.Open();
+ rootDir.WriteToStream(rf);
+ rootDirSize = rf.GetDiscUsageInBlocks();
+ rf.Commit(true);
+ }
+
+ // Update the store info to reflect the size of the root directory
+ std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, dirName, DiscSet, false /* ReadWrite */));
+ info->ChangeBlocksUsed(rootDirSize);
+ info->ChangeBlocksInDirectories(rootDirSize);
+
+ // Save it back
+ info->Save();
+ }
+
+ // As the original user...
+
+ // Create the entry in the database
+ mrDatabase.AddEntry(ID, DiscSet);
+
+ // Write the database back
+ mrDatabase.Write();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::GetAccountRoot(int32_t, std::string &, int &)
+// Purpose: Gets the root of an account, returning the info via references
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const
+{
+ // Find the account
+ const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID));
+
+ rRootDirOut = MakeAccountRootDir(ID, en.GetDiscSet());
+ rDiscSetOut = en.GetDiscSet();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::MakeAccountRootDir(int32_t, int)
+// Purpose: Private. Generates a root directory name for the account
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const
+{
+ char accid[64]; // big enough!
+ ::sprintf(accid, "%08x/", ID);
+ return std::string(std::string(BOX_RAIDFILE_ROOT_BBSTORED DIRECTORY_SEPARATOR) + accid);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::AccountExists(int32_t)
+// Purpose: Does an account exist?
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool BackupStoreAccounts::AccountExists(int32_t ID)
+{
+ return mrDatabase.EntryExists(ID);
+}
+
+
diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h
new file mode 100755
index 00000000..0c3dd103
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccounts.h
@@ -0,0 +1,47 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccounts.h
+// Purpose: Account management for backup store server
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREACCOUNTS__H
+#define BACKUPSTOREACCOUNTS__H
+
+#include <string>
+
+class BackupStoreAccountDatabase;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreAccounts
+// Purpose: Account management for backup store server
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+class BackupStoreAccounts
+{
+public:
+ BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase);
+ ~BackupStoreAccounts();
+private:
+ BackupStoreAccounts(const BackupStoreAccounts &rToCopy);
+
+public:
+ void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername);
+
+ bool AccountExists(int32_t ID);
+ void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const;
+
+private:
+ std::string MakeAccountRootDir(int32_t ID, int DiscSet) const;
+
+private:
+ BackupStoreAccountDatabase &mrDatabase;
+};
+
+#endif // BACKUPSTOREACCOUNTS__H
+
diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp
new file mode 100644
index 00000000..fb48c3da
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck.cpp
@@ -0,0 +1,745 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheck.cpp
+// Purpose: Check a store for consistency
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "BackupStoreCheck.h"
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::BackupStoreCheck(const std::string &, int, int32_t, bool, bool)
+// Purpose: Constructor
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet)
+ : mStoreRoot(rStoreRoot),
+ mDiscSetNumber(DiscSetNumber),
+ mAccountID(AccountID),
+ mFixErrors(FixErrors),
+ mQuiet(Quiet),
+ mNumberErrorsFound(0),
+ mLastIDInInfo(0),
+ mpInfoLastBlock(0),
+ mInfoLastBlockEntries(0),
+ mLostDirNameSerial(0),
+ mLostAndFoundDirectoryID(0),
+ mBlocksUsed(0),
+ mBlocksInOldFiles(0),
+ mBlocksInDeletedFiles(0),
+ mBlocksInDirectories(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::~BackupStoreCheck()
+// Purpose: Destructor
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::~BackupStoreCheck()
+{
+ // Clean up
+ FreeInfo();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::Check()
+// Purpose: Perform the check on the given account
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::Check()
+{
+ // Lock the account
+ {
+ std::string writeLockFilename;
+ StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename);
+
+ bool gotLock = false;
+ int triesLeft = 8;
+ do
+ {
+ gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */);
+
+ if(!gotLock)
+ {
+ --triesLeft;
+ ::sleep(1);
+ }
+ } while(!gotLock && triesLeft > 0);
+
+ if(!gotLock)
+ {
+ // Couldn't lock the account -- just stop now
+ if(!mQuiet)
+ {
+ ::printf("Couldn't lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server.\n");
+ }
+ THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount)
+ }
+ }
+
+ if(!mQuiet && mFixErrors)
+ {
+ ::printf("NOTE: Will fix errors encountered during checking.\n");
+ }
+
+ // Phase 1, check objects
+ if(!mQuiet)
+ {
+ ::printf("Check store account ID %08x\nPhase 1, check objects...\n", mAccountID);
+ }
+ CheckObjects();
+
+ // Phase 2, check directories
+ if(!mQuiet)
+ {
+ ::printf("Phase 2, check directories...\n");
+ }
+ CheckDirectories();
+
+ // Phase 3, check root
+ if(!mQuiet)
+ {
+ ::printf("Phase 3, check root...\n");
+ }
+ CheckRoot();
+
+ // Phase 4, check unattached objects
+ if(!mQuiet)
+ {
+ ::printf("Phase 4, fix unattached objects...\n");
+ }
+ CheckUnattachedObjects();
+
+ // Phase 5, fix bad info
+ if(!mQuiet)
+ {
+ ::printf("Phase 5, fix unrecovered inconsistencies...\n");
+ }
+ FixDirsWithWrongContainerID();
+ FixDirsWithLostDirs();
+
+ // Phase 6, regenerate store info
+ if(!mQuiet)
+ {
+ ::printf("Phase 6, regenerate store info...\n");
+ }
+ WriteNewStoreInfo();
+
+// DUMP_OBJECT_INFO
+
+ if(mNumberErrorsFound > 0)
+ {
+ ::printf("%lld errors found\n", mNumberErrorsFound);
+ if(!mFixErrors)
+ {
+ ::printf("NOTE: No changes to the store account have been made.\n");
+ }
+ if(!mFixErrors && mNumberErrorsFound > 0)
+ {
+ ::printf("Run again with fix option to fix these errors\n");
+ }
+ if(mNumberErrorsFound > 0)
+ {
+ ::printf("You should now use bbackupquery on the client machine to examine the store.\n");
+ if(mLostAndFoundDirectoryID != 0)
+ {
+ ::printf("A lost+found directory was created in the account root.\n"\
+ "This contains files and directories which could not be matched to existing directories.\n"\
+ "bbackupd will delete this directory in a few days time.\n");
+ }
+ }
+ }
+ else
+ {
+ ::printf("Store account checked, no errors found.\n");
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static TwoDigitHexToInt(const char *, int &)
+// Purpose: Convert a two digit hex string to an int, returning whether it's valid or not
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+static inline bool TwoDigitHexToInt(const char *String, int &rNumberOut)
+{
+ int n = 0;
+ // Char 0
+ if(String[0] >= '0' && String[0] <= '9')
+ {
+ n = (String[0] - '0') << 4;
+ }
+ else if(String[0] >= 'a' && String[0] <= 'f')
+ {
+ n = ((String[0] - 'a') + 0xa) << 4;
+ }
+ else
+ {
+ return false;
+ }
+ // Char 1
+ if(String[1] >= '0' && String[1] <= '9')
+ {
+ n |= String[1] - '0';
+ }
+ else if(String[1] >= 'a' && String[1] <= 'f')
+ {
+ n |= (String[1] - 'a') + 0xa;
+ }
+ else
+ {
+ return false;
+ }
+
+ // Return a valid number
+ rNumberOut = n;
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckObjects()
+// Purpose: Read in the contents of the directory, recurse to other levels,
+// checking objects for sanity and readability
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckObjects()
+{
+ // Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything
+ int64_t maxDir = 0;
+
+ // Find the maximum directory starting ID
+ {
+ // Make sure the starting root dir doesn't end with '/'.
+ std::string start(mStoreRoot);
+ if(start.size() > 0 && start[start.size() - 1] == '/')
+ {
+ start.resize(start.size() - 1);
+ }
+
+ maxDir = CheckObjectsScanDir(0, 1, mStoreRoot);
+ TRACE1("Max dir starting ID is %llx\n", maxDir);
+ }
+
+ // Then go through and scan all the objects within those directories
+ for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH))
+ {
+ CheckObjectsDir(d);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckObjectsScanDir(int64_t, int, int, const std::string &)
+// Purpose: Read in the contents of the directory, recurse to other levels,
+// return the maximum starting ID of any directory found.
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName)
+{
+ //TRACE2("Scan directory for max dir starting ID %s, StartID %lld\n", rDirName.c_str(), StartID);
+
+ int64_t maxID = StartID;
+
+ // Read in all the directories, and recurse downwards
+ {
+ std::vector<std::string> dirs;
+ RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName,
+ RaidFileRead::DirReadType_DirsOnly, dirs);
+
+ for(std::vector<std::string>::const_iterator i(dirs.begin()); i != dirs.end(); ++i)
+ {
+ // Check to see if it's the right name
+ int n = 0;
+ if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n)
+ && n < (1<<STORE_ID_SEGMENT_LENGTH))
+ {
+ // Next level down
+ int64_t mi = CheckObjectsScanDir(StartID | (n << (Level * STORE_ID_SEGMENT_LENGTH)), Level + 1,
+ rDirName + DIRECTORY_SEPARATOR + *i);
+ // Found a greater starting ID?
+ if(mi > maxID)
+ {
+ maxID = mi;
+ }
+ }
+ else
+ {
+ ::printf("Spurious or invalid directory %s/%s found%s -- delete manually\n", rDirName.c_str(), (*i).c_str(), mFixErrors?", deleting":"");
+ ++mNumberErrorsFound;
+ }
+ }
+ }
+
+ return maxID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckObjectsDir(int64_t)
+// Purpose: Check all the files within this directory which has the given starting ID.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
+{
+ // Make directory name -- first generate the filename of an entry in it
+ std::string dirName;
+ StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */);
+ // Check expectations
+ ASSERT(dirName.size() > 4 && dirName[dirName.size() - 4] == '/');
+ // Remove the filename from it
+ dirName.resize(dirName.size() - 4); // four chars for "/o00"
+
+ // Check directory exists
+ if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName))
+ {
+ TRACE1("RaidFile dir %s does not exist\n", dirName.c_str());
+ return;
+ }
+
+ // Read directory contents
+ std::vector<std::string> files;
+ RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName,
+ RaidFileRead::DirReadType_FilesOnly, files);
+
+ // Array of things present
+ bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)];
+ for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l)
+ {
+ idsPresent[l] = false;
+ }
+
+ // Parse each entry, building up a list of object IDs which are present in the dir.
+ // This is done so that whatever order is retured from the directory, objects are scanned
+ // in order.
+ // Filename must begin with a 'o' and be three characters long, otherwise it gets deleted.
+ for(std::vector<std::string>::const_iterator i(files.begin()); i != files.end(); ++i)
+ {
+ bool fileOK = true;
+ int n = 0;
+ if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n)
+ && n < (1<<STORE_ID_SEGMENT_LENGTH))
+ {
+ // Filename is valid, mark as existing
+ idsPresent[n] = true;
+ }
+ else
+ {
+ // info file in root dir is OK!
+ if(StartID != 0 || ::strcmp("info", (*i).c_str()) != 0)
+ {
+ fileOK = false;
+ }
+ }
+
+ if(!fileOK)
+ {
+ // Unexpected or bad file, delete it
+ ::printf("Spurious file %s/%s found%s\n", dirName.c_str(), (*i).c_str(), mFixErrors?", deleting":"");
+ ++mNumberErrorsFound;
+ if(mFixErrors)
+ {
+ RaidFileWrite del(mDiscSetNumber, dirName + DIRECTORY_SEPARATOR + *i);
+ del.Delete();
+ }
+ }
+ }
+
+ // Check all the objects found in this directory
+ for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i)
+ {
+ if(idsPresent[i])
+ {
+ // Check the object is OK, and add entry
+ char leaf[8];
+ ::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i);
+ if(!CheckAndAddObject(StartID | i, dirName + leaf))
+ {
+ // File was bad, delete it
+ ::printf("Corrupted file %s%s found%s\n", dirName.c_str(), leaf, mFixErrors?", deleting":"");
+ ++mNumberErrorsFound;
+ if(mFixErrors)
+ {
+ RaidFileWrite del(mDiscSetNumber, dirName + leaf);
+ del.Delete();
+ }
+ }
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckAndAddObject(int64_t, const std::string &)
+// Purpose: Check a specific object and add it to the list if it's OK -- if
+// there are any errors with the reading, return false and it'll be deleted.
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename)
+{
+ // Info on object...
+ bool isFile = true;
+ int64_t containerID = -1;
+ int64_t size = -1;
+
+ try
+ {
+ // Open file
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, rFilename));
+ size = file->GetDiscUsageInBlocks();
+
+ // Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile
+ uint32_t signature;
+ if(file->Read(&signature, sizeof(signature)) != sizeof(signature))
+ {
+ // Too short, can't read signature from it
+ return false;
+ }
+ // Seek back to beginning
+ file->Seek(0, IOStream::SeekType_Absolute);
+
+ // Then... check depending on the type
+ switch(ntohl(signature))
+ {
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
+#endif
+ // File... check
+ containerID = CheckFile(ObjectID, *file);
+ break;
+
+ case OBJECTMAGIC_DIR_MAGIC_VALUE:
+ isFile = false;
+ containerID = CheckDirInitial(ObjectID, *file);
+ break;
+
+ default:
+ // Unknown signature. Bad file. Very bad file.
+ return false;
+ break;
+ }
+
+ // Add to usage counts
+ int64_t s = file->GetDiscUsageInBlocks();
+ mBlocksUsed += s;
+ if(!isFile)
+ {
+ mBlocksInDirectories += s;
+ }
+ }
+ catch(...)
+ {
+ // Error caught, not a good file then, let it be deleted
+ return false;
+ }
+
+ // Got a container ID? (ie check was successful)
+ if(containerID == -1)
+ {
+ return false;
+ }
+
+ // Add to list of IDs known about
+ AddID(ObjectID, containerID, size, isFile);
+
+ // Report success
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckFile(int64_t, IOStream &)
+// Purpose: Do check on file, return original container ID if OK, or -1 on error
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream)
+{
+ // Check that it's not the root directory ID. Having a file as the root directory would be bad.
+ if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
+ {
+ // Get that dodgy thing deleted!
+ ::printf("Have file as root directory. This is bad.\n");
+ return -1;
+ }
+
+ // Check the format of the file, and obtain the container ID
+ int64_t originalContainerID = -1;
+ if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */,
+ &originalContainerID))
+ {
+ // Didn't verify
+ return -1;
+ }
+
+ return originalContainerID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckDirInitial(int64_t, IOStream &)
+// Purpose: Do initial check on directory, return container ID if OK, or -1 on error
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream)
+{
+ // Simply attempt to read in the directory
+ BackupStoreDirectory dir;
+ dir.ReadFromStream(rStream, IOStream::TimeOutInfinite);
+
+ // Check object ID
+ if(dir.GetObjectID() != ObjectID)
+ {
+ // Wrong object ID
+ return -1;
+ }
+
+ // Return container ID
+ return dir.GetContainerID();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckDirectories()
+// Purpose: Check the directories
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckDirectories()
+{
+ // Phase 1 did this:
+ // Checked that all the directories are readable
+ // Built a list of all directories and files which exist on the store
+ //
+ // This phase will check all the files in the directories, make
+ // a note of all directories which are missing, and do initial fixing.
+
+ // Scan all objects
+ for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ IDBlock *pblock = i->second;
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+
+ for(int e = 0; e < bentries; ++e)
+ {
+ uint8_t flags = GetFlags(pblock, e);
+ if(flags & Flags_IsDir)
+ {
+ // Found a directory. Read it in.
+ std::string filename;
+ StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */);
+ BackupStoreDirectory dir;
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Flag for modifications
+ bool isModified = false;
+
+ // Check for validity
+ if(dir.CheckAndFix())
+ {
+ // Wasn't quite right, and has been modified
+ ::printf("Directory ID %llx has bad structure\n", pblock->mID[e]);
+ ++mNumberErrorsFound;
+ isModified = true;
+ }
+
+ // Go through, and check that everything in that directory exists and is valid
+ std::vector<int64_t> toDelete;
+
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next()) != 0)
+ {
+ // Lookup the item
+ int32_t iIndex;
+ IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
+ bool badEntry = false;
+ if(piBlock != 0)
+ {
+ // Found. Get flags
+ uint8_t iflags = GetFlags(piBlock, iIndex);
+
+ // Is the type the same?
+ if(((iflags & Flags_IsDir) == Flags_IsDir)
+ != ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir))
+ {
+ // Entry is of wrong type
+ ::printf("Directory ID %llx references object %llx which has a different type than expected.\n", pblock->mID[e], en->GetObjectID());
+ badEntry = true;
+ }
+ else
+ {
+ // Check that the entry is not already contained.
+ if(iflags & Flags_IsContained)
+ {
+ badEntry = true;
+ ::printf("Directory ID %llx references object %llx which is already contained.\n", pblock->mID[e], en->GetObjectID());
+ }
+ else
+ {
+ // Not already contained -- mark as contained
+ SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
+
+ // Check that the container ID of the object is correct
+ if(piBlock->mContainer[iIndex] != pblock->mID[e])
+ {
+ // Needs fixing...
+ if(iflags & Flags_IsDir)
+ {
+ // Add to will fix later list
+ ::printf("Directory ID %llx has wrong container ID.\n", en->GetObjectID());
+ mDirsWithWrongContainerID.push_back(en->GetObjectID());
+ }
+ else
+ {
+ // This is OK for files, they might move
+ ::printf("File ID %llx has different container ID, probably moved\n", en->GetObjectID());
+ }
+
+ // Fix entry for now
+ piBlock->mContainer[iIndex] = pblock->mID[e];
+ }
+ }
+ }
+
+ // Check the object size, if it's OK and a file
+ if(!badEntry && !((iflags & Flags_IsDir) == Flags_IsDir))
+ {
+ if(en->GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[iIndex])
+ {
+ // Correct
+ en->SetSizeInBlocks(piBlock->mObjectSizeInBlocks[iIndex]);
+ // Mark as changed
+ isModified = true;
+ // Tell user
+ ::printf("Directory ID %llx has wrong size for object %llx\n", pblock->mID[e], en->GetObjectID());
+ }
+ }
+ }
+ else
+ {
+ // Item can't be found. Is it a directory?
+ if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir)
+ {
+ // Store the directory for later attention
+ mDirsWhichContainLostDirs[en->GetObjectID()] = pblock->mID[e];
+ }
+ else
+ {
+ // Just remove the entry
+ badEntry = true;
+ ::printf("Directory ID %llx references object %llx which does not exist.\n", pblock->mID[e], en->GetObjectID());
+ }
+ }
+
+ // Is this entry worth keeping?
+ if(badEntry)
+ {
+ toDelete.push_back(en->GetObjectID());
+ }
+ else
+ {
+ // Add to sizes?
+ if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion)
+ {
+ mBlocksInOldFiles += en->GetSizeInBlocks();
+ }
+ if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted)
+ {
+ mBlocksInDeletedFiles += en->GetSizeInBlocks();
+ }
+ }
+ }
+
+ if(toDelete.size() > 0)
+ {
+ // Delete entries from directory
+ for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d)
+ {
+ dir.DeleteEntry(*d);
+ }
+
+ // Mark as modified
+ isModified = true;
+
+ // Check the directory again, now that entries have been removed
+ dir.CheckAndFix();
+
+ // Errors found
+ ++mNumberErrorsFound;
+ }
+
+ if(isModified && mFixErrors)
+ {
+ ::printf("Fixing directory ID %llx\n", pblock->mID[e]);
+
+ // Save back to disc
+ RaidFileWrite fixed(mDiscSetNumber, filename);
+ fixed.Open(true /* allow overwriting */);
+ dir.WriteToStream(fixed);
+ // Commit it
+ fixed.Commit(true /* convert to raid representation now */);
+ }
+ }
+ }
+ }
+
+}
+
+
diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h
new file mode 100644
index 00000000..3f48312a
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck.h
@@ -0,0 +1,199 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheck.h
+// Purpose: Check a store for consistency
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECHECK__H
+#define BACKUPSTORECHECK__H
+
+#include <string>
+#include <map>
+#include <vector>
+#include <set>
+
+#include "NamedLock.h"
+class IOStream;
+class BackupStoreFilename;
+
+/*
+
+The following problems can be fixed:
+
+ * Spurious files deleted
+ * Corrupted files deleted
+ * Root ID as file, deleted
+ * Dirs with wrong object id inside, deleted
+ * Direcetory entries pointing to non-existant files, deleted
+ * Doubly references files have second reference deleted
+ * Wrong directory container IDs fixed
+ * Missing root recreated
+ * Reattach files which exist, but aren't referenced
+ - files go into directory original directory, if it still exists
+ - missing directories are inferred, and recreated
+ - or if all else fails, go into lost+found
+ - file dir entries take the original name and mod time
+ - directories go into lost+found
+ * Container IDs on directories corrected
+ * Inside directories,
+ - only one object per name has old version clear
+ - IDs aren't duplicated
+ * Bad store info files regenerated
+ * Bad sizes of files in directories fixed
+
+*/
+
+
+// Size of blocks in the list of IDs
+#ifdef NDEBUG
+ #define BACKUPSTORECHECK_BLOCK_SIZE (64*1024)
+#else
+ #define BACKUPSTORECHECK_BLOCK_SIZE 8
+#endif
+
+// The object ID type -- can redefine to uint32_t to produce a lower memory version for smaller stores
+typedef int64_t BackupStoreCheck_ID_t;
+// Can redefine the size type for lower memory usage too
+typedef int64_t BackupStoreCheck_Size_t;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreCheck
+// Purpose: Check a store for consistency
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+class BackupStoreCheck
+{
+public:
+ BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet);
+ ~BackupStoreCheck();
+private:
+ // no copying
+ BackupStoreCheck(const BackupStoreCheck &);
+ BackupStoreCheck &operator=(const BackupStoreCheck &);
+public:
+
+ // Do the exciting things
+ void Check();
+
+ bool ErrorsFound() {return mNumberErrorsFound > 0;}
+
+private:
+ enum
+ {
+ // Bit mask
+ Flags_IsDir = 1,
+ Flags_IsContained = 2,
+ // Mask
+ Flags__MASK = 3,
+ // Number of bits
+ Flags__NumFlags = 2,
+ // Items per uint8_t
+ Flags__NumItemsPerEntry = 4 // ie 8 / 2
+ };
+
+ typedef struct
+ {
+ // Note use arrays within the block, rather than the more obvious array of
+ // objects, to be more memory efficient -- think alignment of the byte values.
+ uint8_t mFlags[BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry];
+ BackupStoreCheck_ID_t mID[BACKUPSTORECHECK_BLOCK_SIZE];
+ BackupStoreCheck_ID_t mContainer[BACKUPSTORECHECK_BLOCK_SIZE];
+ BackupStoreCheck_Size_t mObjectSizeInBlocks[BACKUPSTORECHECK_BLOCK_SIZE];
+ } IDBlock;
+
+ // Phases of the check
+ void CheckObjects();
+ void CheckDirectories();
+ void CheckRoot();
+ void CheckUnattachedObjects();
+ void FixDirsWithWrongContainerID();
+ void FixDirsWithLostDirs();
+ void WriteNewStoreInfo();
+
+ // Checking functions
+ int64_t CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName);
+ void CheckObjectsDir(int64_t StartID);
+ bool CheckAndAddObject(int64_t ObjectID, const std::string &rFilename);
+ int64_t CheckFile(int64_t ObjectID, IOStream &rStream);
+ int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream);
+
+ // Fixing functions
+ bool TryToRecreateDirectory(int64_t MissingDirectoryID);
+ void InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory);
+ int64_t GetLostAndFoundDirID();
+ void CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID);
+
+ // Data handling
+ void FreeInfo();
+ void AddID(BackupStoreCheck_ID_t ID, BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile);
+ IDBlock *LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut);
+ inline void SetFlags(IDBlock *pBlock, int32_t Index, uint8_t Flags)
+ {
+ ASSERT(pBlock != 0);
+ ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE);
+ ASSERT(Flags < (1 << Flags__NumFlags));
+
+ pBlock->mFlags[Index / Flags__NumItemsPerEntry]
+ |= (Flags << ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags));
+ }
+ inline uint8_t GetFlags(IDBlock *pBlock, int32_t Index)
+ {
+ ASSERT(pBlock != 0);
+ ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE);
+
+ return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK;
+ }
+
+#ifndef NDEBUG
+ void DumpObjectInfo();
+ #define DUMP_OBJECT_INFO DumpObjectInfo();
+#else
+ #define DUMP_OBJECT_INFO
+#endif
+
+private:
+ std::string mStoreRoot;
+ int mDiscSetNumber;
+ int32_t mAccountID;
+ bool mFixErrors;
+ bool mQuiet;
+
+ int64_t mNumberErrorsFound;
+
+ // Lock for the store account
+ NamedLock mAccountLock;
+
+ // Storage for ID data
+ typedef std::map<BackupStoreCheck_ID_t, IDBlock*> Info_t;
+ Info_t mInfo;
+ BackupStoreCheck_ID_t mLastIDInInfo;
+ IDBlock *mpInfoLastBlock;
+ int32_t mInfoLastBlockEntries;
+
+ // List of stuff to fix
+ std::vector<BackupStoreCheck_ID_t> mDirsWithWrongContainerID;
+ // This is a map of lost dir ID -> existing dir ID
+ std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t> mDirsWhichContainLostDirs;
+
+ // Set of extra directories added
+ std::set<BackupStoreCheck_ID_t> mDirsAdded;
+
+ // Misc stuff
+ int32_t mLostDirNameSerial;
+ int64_t mLostAndFoundDirectoryID;
+
+ // Usage
+ int64_t mBlocksUsed;
+ int64_t mBlocksInOldFiles;
+ int64_t mBlocksInDeletedFiles;
+ int64_t mBlocksInDirectories;
+};
+
+#endif // BACKUPSTORECHECK__H
+
diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp
new file mode 100644
index 00000000..fe91d00d
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck2.cpp
@@ -0,0 +1,841 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheck2.cpp
+// Purpose: More backup store checking
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "BackupStoreCheck.h"
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreInfo.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckRoot()
+// Purpose: Check the root directory exists.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckRoot()
+{
+ int32_t index = 0;
+ IDBlock *pblock = LookupID(BACKUPSTORE_ROOT_DIRECTORY_ID, index);
+
+ if(pblock != 0)
+ {
+ // Found it. Which is lucky. Mark it as contained.
+ SetFlags(pblock, index, Flags_IsContained);
+ }
+ else
+ {
+ ::printf("Root directory doesn't exist\n");
+
+ ++mNumberErrorsFound;
+
+ if(mFixErrors)
+ {
+ // Create a new root directory
+ CreateBlankDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID);
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CreateBlankDirectory(int64_t, int64_t)
+// Purpose: Creates a blank directory
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID)
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ BackupStoreDirectory dir(DirectoryID, ContainingDirID);
+
+ // Serialise to disc
+ std::string filename;
+ StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */);
+ RaidFileWrite obj(mDiscSetNumber, filename);
+ obj.Open(false /* don't allow overwriting */);
+ dir.WriteToStream(obj);
+ int64_t size = obj.GetDiscUsageInBlocks();
+ obj.Commit(true /* convert to raid now */);
+
+ // Record the fact we've done this
+ mDirsAdded.insert(DirectoryID);
+
+ // Add to sizes
+ mBlocksUsed += size;
+ mBlocksInDirectories += size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckUnattachedObjects()
+// Purpose: Check for objects which aren't attached to anything
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckUnattachedObjects()
+{
+ // Scan all objects, finding ones which have no container
+ for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ IDBlock *pblock = i->second;
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+
+ for(int e = 0; e < bentries; ++e)
+ {
+ uint8_t flags = GetFlags(pblock, e);
+ if((flags & Flags_IsContained) == 0)
+ {
+ // Unattached object...
+ ::printf("Object %llx is unattached.\n", pblock->mID[e]);
+ ++mNumberErrorsFound;
+
+ // What's to be done?
+ int64_t putIntoDirectoryID = 0;
+
+ if((flags & Flags_IsDir) == Flags_IsDir)
+ {
+ // Directory. Just put into lost and found.
+ putIntoDirectoryID = GetLostAndFoundDirID();
+ }
+ else
+ {
+ // File. Only attempt to attach it somewhere if it isn't a patch
+ {
+ int64_t diffFromObjectID = 0;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */);
+ // The easiest way to do this is to verify it again. Not such a bad penalty, because
+ // this really shouldn't be done very often.
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ BackupStoreFile::VerifyEncodedFileFormat(*file, &diffFromObjectID);
+ }
+
+ // If not zero, then it depends on another file, which may or may not be available.
+ // Just delete it to be safe.
+ if(diffFromObjectID != 0)
+ {
+ ::printf("Object %llx is unattached, and is a patch. Deleting, cannot reliably recover.\n", pblock->mID[e]);
+
+ // Delete this object instead
+ if(mFixErrors)
+ {
+ RaidFileWrite del(mDiscSetNumber, filename);
+ del.Delete();
+ }
+
+ // Move on to next item
+ continue;
+ }
+ }
+
+ // Files contain their original filename, so perhaps the orginal directory still exists,
+ // or we can infer the existance of a directory?
+ // Look for a matching entry in the mDirsWhichContainLostDirs map.
+ // Can't do this with a directory, because the name just wouldn't be known, which is
+ // pretty useless as bbackupd would just delete it. So better to put it in lost+found
+ // where the admin can do something about it.
+ int32_t dirindex;
+ IDBlock *pdirblock = LookupID(pblock->mContainer[e], dirindex);
+ if(pdirblock != 0)
+ {
+ // Something with that ID has been found. Is it a directory?
+ if(GetFlags(pdirblock, dirindex) & Flags_IsDir)
+ {
+ // Directory exists, add to that one
+ putIntoDirectoryID = pblock->mContainer[e];
+ }
+ else
+ {
+ // Not a directory. Use lost and found dir
+ putIntoDirectoryID = GetLostAndFoundDirID();
+ }
+ }
+ else if(mDirsAdded.find(pblock->mContainer[e]) != mDirsAdded.end()
+ || TryToRecreateDirectory(pblock->mContainer[e]))
+ {
+ // The directory reappeared, or was created somehow elsewhere
+ putIntoDirectoryID = pblock->mContainer[e];
+ }
+ else
+ {
+ putIntoDirectoryID = GetLostAndFoundDirID();
+ }
+ }
+ ASSERT(putIntoDirectoryID != 0);
+
+ // Add it to the directory
+ InsertObjectIntoDirectory(pblock->mID[e], putIntoDirectoryID,
+ ((flags & Flags_IsDir) == Flags_IsDir));
+ }
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::TryToRecreateDirectory(int64_t)
+// Purpose: Recreate a missing directory
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID)
+{
+ // During the directory checking phase, a map of "missing directory" to
+ // containing directory was built. If we can find it here, then it's
+ // something which can be recreated!
+ std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator missing(
+ mDirsWhichContainLostDirs.find(MissingDirectoryID));
+ if(missing == mDirsWhichContainLostDirs.end())
+ {
+ // Not a missing directory, can't recreate.
+ return false;
+ }
+
+ // Can recreate this! Wooo!
+ if(!mFixErrors)
+ {
+ ::printf("Missing directory %llx could be recreated\n", MissingDirectoryID);
+ mDirsAdded.insert(MissingDirectoryID);
+ return true;
+ }
+ ::printf("Recreating missing directory %llx\n", MissingDirectoryID);
+
+ // Create a blank directory
+ BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */);
+ // Note that this directory already contains a directory entry pointing to
+ // this dir, so it doesn't have to be added.
+
+ // Serialise to disc
+ std::string filename;
+ StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */);
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(false /* don't allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+
+ // Record the fact we've done this
+ mDirsAdded.insert(MissingDirectoryID);
+
+ // Remove the entry from the map, so this doesn't happen again
+ mDirsWhichContainLostDirs.erase(missing);
+
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::GetLostAndFoundDirID()
+// Purpose: Returns the ID of the lost and found directory, creating it if necessary
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::GetLostAndFoundDirID()
+{
+ // Already allocated it?
+ if(mLostAndFoundDirectoryID != 0)
+ {
+ return mLostAndFoundDirectoryID;
+ }
+
+ if(!mFixErrors)
+ {
+ // The result will never be used anyway if errors aren't being fixed
+ return 1;
+ }
+
+ // Load up the root directory
+ BackupStoreDirectory dir;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(BACKUPSTORE_ROOT_DIRECTORY_ID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Find a suitable name
+ BackupStoreFilename lostAndFound;
+ int n = 0;
+ while(true)
+ {
+ char name[32];
+ ::sprintf(name, "lost+found%d", n++);
+ lostAndFound.SetAsClearFilename(name);
+ if(!dir.NameInUse(lostAndFound))
+ {
+ // Found a name which can be used
+ ::printf("Lost and found dir has name %s\n", name);
+ break;
+ }
+ }
+
+ // Allocate an ID
+ int64_t id = mLastIDInInfo + 1;
+
+ // Create a blank directory
+ CreateBlankDirectory(id, BACKUPSTORE_ROOT_DIRECTORY_ID);
+
+ // Add an entry for it
+ dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0);
+
+ // Write out root dir
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+
+ // Store
+ mLostAndFoundDirectoryID = id;
+
+ // Tell caller
+ return mLostAndFoundDirectoryID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::InsertObjectIntoDirectory(int64_t, int64_t, bool)
+// Purpose:
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory)
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ // Data for the object
+ BackupStoreFilename objectStoreFilename;
+ int64_t modTime = 100; // something which isn't zero or a special time
+ int32_t sizeInBlocks = 0; // suitable for directories
+
+ if(IsDirectory)
+ {
+ // Directory -- simply generate a name for it.
+ char name[32];
+ ::sprintf(name, "dir%08x", mLostDirNameSerial++);
+ objectStoreFilename.SetAsClearFilename(name);
+ }
+ else
+ {
+ // Files require a little more work...
+ // Open file
+ std::string fileFilename;
+ StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mDiscSetNumber, fileFilename, false /* don't make sure the dir exists */);
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, fileFilename));
+ // Fill in size information
+ sizeInBlocks = file->GetDiscUsageInBlocks();
+ // Read in header
+ file_StreamFormat hdr;
+ if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ ))
+ {
+ // This should never happen, everything has been checked before.
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+ // This tells us nice things
+ modTime = ntoh64(hdr.mModificationTime);
+ // And the filename comes next
+ objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Directory object
+ BackupStoreDirectory dir;
+
+ // Generate filename
+ std::string filename;
+ StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+
+ // Read it in
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Add a new entry in an appropraite place
+ dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks,
+ IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File));
+
+ // Fix any flags which have been broken, which there's a good change of going
+ dir.CheckAndFix();
+
+ // Write it out
+ if(mFixErrors)
+ {
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::FixDirsWithWrongContainerID()
+// Purpose: Rewrites container IDs where required
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FixDirsWithWrongContainerID()
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ // Run through things which need fixing
+ for(std::vector<BackupStoreCheck_ID_t>::iterator i(mDirsWithWrongContainerID.begin());
+ i != mDirsWithWrongContainerID.end(); ++i)
+ {
+ int32_t index = 0;
+ IDBlock *pblock = LookupID(*i, index);
+ if(pblock == 0) continue;
+
+ // Load in
+ BackupStoreDirectory dir;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(*i, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Adjust container ID
+ dir.SetContainerID(pblock->mContainer[index]);
+
+ // Write it out
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::FixDirsWithLostDirs()
+// Purpose: Fix directories
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FixDirsWithLostDirs()
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ // Run through things which need fixing
+ for(std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator i(mDirsWhichContainLostDirs.begin());
+ i != mDirsWhichContainLostDirs.end(); ++i)
+ {
+ int32_t index = 0;
+ IDBlock *pblock = LookupID(i->second, index);
+ if(pblock == 0) continue;
+
+ // Load in
+ BackupStoreDirectory dir;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(i->second, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Delete the dodgy entry
+ dir.DeleteEntry(i->first);
+
+ // Fix it up
+ dir.CheckAndFix();
+
+ // Write it out
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::WriteNewStoreInfo()
+// Purpose: Regenerate store info
+// Created: 23/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::WriteNewStoreInfo()
+{
+ // Attempt to load the existing store info file
+ std::auto_ptr<BackupStoreInfo> poldInfo;
+ try
+ {
+ poldInfo.reset(BackupStoreInfo::Load(mAccountID, mStoreRoot, mDiscSetNumber, true /* read only */).release());
+ }
+ catch(...)
+ {
+ ::printf("Load of existing store info failed, regenerating.\n");
+ ++mNumberErrorsFound;
+ }
+
+ // Minimum soft and hard limits
+ int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024;
+ int64_t minHard = ((minSoft * 11) / 10) + 1024;
+
+ // Need to do anything?
+ if(poldInfo.get() != 0 && mNumberErrorsFound == 0 && poldInfo->GetAccountID() == mAccountID)
+ {
+ // Leave the store info as it is, no need to alter it because nothing really changed,
+ // and the only essential thing was that the account ID was correct, which is was.
+ return;
+ }
+
+ // NOTE: We will always build a new store info, so the client store marker gets changed.
+
+ // Work out the new limits
+ int64_t softLimit = minSoft;
+ int64_t hardLimit = minHard;
+ if(poldInfo.get() != 0 && poldInfo->GetBlocksSoftLimit() > minSoft)
+ {
+ softLimit = poldInfo->GetBlocksSoftLimit();
+ }
+ else
+ {
+ ::printf("NOTE: Soft limit for account changed to ensure housekeeping doesn't delete files on next run\n");
+ }
+ if(poldInfo.get() != 0 && poldInfo->GetBlocksHardLimit() > minHard)
+ {
+ hardLimit = poldInfo->GetBlocksHardLimit();
+ }
+ else
+ {
+ ::printf("NOTE: Hard limit for account changed to ensure housekeeping doesn't delete files on next run\n");
+ }
+
+ // Object ID
+ int64_t lastObjID = mLastIDInInfo;
+ if(mLostAndFoundDirectoryID != 0)
+ {
+ mLastIDInInfo++;
+ }
+
+ // Build a new store info
+ std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::CreateForRegeneration(
+ mAccountID,
+ mStoreRoot,
+ mDiscSetNumber,
+ lastObjID,
+ mBlocksUsed,
+ mBlocksInOldFiles,
+ mBlocksInDeletedFiles,
+ mBlocksInDirectories,
+ softLimit,
+ hardLimit));
+
+ // Save to disc?
+ if(mFixErrors)
+ {
+ info->Save();
+ ::printf("New store info file written successfully.\n");
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::CheckAndFix()
+// Purpose: Check the directory for obvious logical problems, and fix them.
+// Return true if the directory was changed.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDirectory::CheckAndFix()
+{
+ bool changed = false;
+
+ // Check that if a file depends on a new version, that version is in this directory
+ {
+ std::vector<Entry*>::iterator i(mEntries.begin());
+ for(; i != mEntries.end(); ++i)
+ {
+ int64_t dependsNewer = (*i)->GetDependsNewer();
+ if(dependsNewer != 0)
+ {
+ BackupStoreDirectory::Entry *newerEn = FindEntryByID(dependsNewer);
+ if(newerEn == 0)
+ {
+ // Depends on something, but it isn't there.
+ TRACE2("Entry id %llx removed because depends on newer version %llx which doesn't exist\n", (*i)->GetObjectID(), dependsNewer);
+
+ // Remove
+ delete *i;
+ mEntries.erase(i);
+
+ // Start again at the beginning of the vector, the iterator is now invalid
+ i = mEntries.begin();
+
+ // Mark as changed
+ changed = true;
+ }
+ else
+ {
+ // Check that newerEn has it marked
+ if(newerEn->GetDependsOlder() != (*i)->GetObjectID())
+ {
+ // Wrong entry
+ TRACE3("Entry id %llx, correcting DependsOlder to %llx, was %llx\n", dependsNewer, (*i)->GetObjectID(), newerEn->GetDependsOlder());
+ newerEn->SetDependsOlder((*i)->GetObjectID());
+ // Mark as changed
+ changed = true;
+ }
+ }
+ }
+ }
+ }
+
+ // Check that if a file has a dependency marked, it exists, and remove it if it doesn't
+ {
+ std::vector<Entry*>::iterator i(mEntries.begin());
+ for(; i != mEntries.end(); ++i)
+ {
+ int64_t dependsOlder = (*i)->GetDependsOlder();
+ if(dependsOlder != 0 && FindEntryByID(dependsOlder) == 0)
+ {
+ // Has an older version marked, but this doesn't exist. Remove this mark
+ TRACE2("Entry id %llx was marked that %llx depended on it, which doesn't exist, dependency info cleaered\n", (*i)->GetObjectID(), dependsOlder);
+
+ (*i)->SetDependsOlder(0);
+
+ // Mark as changed
+ changed = true;
+ }
+ }
+ }
+
+ bool ch = false;
+ do
+ {
+ // Reset change marker
+ ch = false;
+
+ // Search backwards -- so see newer versions first
+ std::vector<Entry*>::iterator i(mEntries.end());
+ if(i == mEntries.begin())
+ {
+ // Directory is empty, stop now
+ return changed; // changed flag
+ }
+
+ // Records of things seen
+ std::set<int64_t> idsEncountered;
+ std::set<BackupStoreFilename> filenamesEncountered;
+
+ do
+ {
+ // Look at previous
+ --i;
+
+ bool removeEntry = false;
+ if((*i) == 0)
+ {
+ TRACE0("Remove because null pointer found\n");
+ removeEntry = true;
+ }
+ else
+ {
+ bool isDir = (((*i)->GetFlags() & Entry::Flags_Dir) == Entry::Flags_Dir);
+
+ // Check mutually exclusive flags
+ if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File))
+ {
+ // Bad! Unset the file flag
+ TRACE1("Entry %llx: File flag set when dir flag set\n", (*i)->GetObjectID());
+ (*i)->RemoveFlags(Entry::Flags_File);
+ changed = true;
+ }
+
+ // Check...
+ if(idsEncountered.find((*i)->GetObjectID()) != idsEncountered.end())
+ {
+ // ID already seen, or type doesn't match
+ TRACE1("Entry %llx: Remove because ID already seen\n", (*i)->GetObjectID());
+ removeEntry = true;
+ }
+ else
+ {
+ // Haven't already seen this ID, remember it
+ idsEncountered.insert((*i)->GetObjectID());
+
+ // Check to see if the name has already been encountered -- if not, then it
+ // needs to have the old version flag set
+ if(filenamesEncountered.find((*i)->GetName()) != filenamesEncountered.end())
+ {
+ // Seen before -- check old version flag set
+ if(((*i)->GetFlags() & Entry::Flags_OldVersion) != Entry::Flags_OldVersion
+ && ((*i)->GetFlags() & Entry::Flags_Deleted) == 0)
+ {
+ // Not set, set it
+ TRACE1("Entry %llx: Set old flag\n", (*i)->GetObjectID());
+ (*i)->AddFlags(Entry::Flags_OldVersion);
+ changed = true;
+ }
+ }
+ else
+ {
+ // Check old version flag NOT set
+ if(((*i)->GetFlags() & Entry::Flags_OldVersion) == Entry::Flags_OldVersion)
+ {
+ // Set, unset it
+ TRACE1("Entry %llx: Old flag unset\n", (*i)->GetObjectID());
+ (*i)->RemoveFlags(Entry::Flags_OldVersion);
+ changed = true;
+ }
+
+ // Remember filename
+ filenamesEncountered.insert((*i)->GetName());
+ }
+ }
+ }
+
+ if(removeEntry)
+ {
+ // Mark something as changed, in loop
+ ch = true;
+
+ // Mark something as globally changed
+ changed = true;
+
+ // erase the thing from the list
+ Entry *pentry = (*i);
+ mEntries.erase(i);
+
+ // And delete the entry object
+ delete pentry;
+
+ // Stop going around this loop, as the iterator is now invalid
+ break;
+ }
+ } while(i != mEntries.begin());
+
+ } while(ch != false);
+
+ return changed;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddUnattactedObject(...)
+// Purpose: Adds an object which is currently unattached. Assume that CheckAndFix() will be called afterwards.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::AddUnattactedObject(const BackupStoreFilename &rName,
+ box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags)
+{
+ Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags,
+ ModificationTime /* use as attr mod time too */);
+ try
+ {
+ // Want to order this just before the first object which has a higher ID,
+ // which is the place it's most likely to be correct.
+ std::vector<Entry*>::iterator i(mEntries.begin());
+ for(; i != mEntries.end(); ++i)
+ {
+ if((*i)->GetObjectID() > ObjectID)
+ {
+ // Found a good place to insert it
+ break;
+ }
+ }
+ if(i == mEntries.end())
+ {
+ mEntries.push_back(pnew);
+ }
+ else
+ {
+ mEntries.insert(i, 1 /* just the one copy */, pnew);
+ }
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::NameInUse(const BackupStoreFilename &)
+// Purpose: Returns true if the name is currently in use in the directory
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDirectory::NameInUse(const BackupStoreFilename &rName)
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+ {
+ if((*i)->GetName() == rName)
+ {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+
diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp
new file mode 100644
index 00000000..f22c8339
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheckData.cpp
@@ -0,0 +1,205 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheckData.cpp
+// Purpose: Data handling for store checking
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <memory>
+
+#include "BackupStoreCheck.h"
+#include "autogen_BackupStoreException.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::FreeInfo()
+// Purpose: Free all the data stored
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FreeInfo()
+{
+ // Free all the blocks
+ for(Info_t::iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ ::free(i->second);
+ }
+
+ // Clear the contents of the map
+ mInfo.clear();
+
+ // Reset the last ID, just in case
+ mpInfoLastBlock = 0;
+ mInfoLastBlockEntries = 0;
+ mLastIDInInfo = 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::AddID(BackupStoreCheck_ID_t, BackupStoreCheck_ID_t, bool)
+// Purpose: Add an ID to the list
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::AddID(BackupStoreCheck_ID_t ID,
+ BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile)
+{
+ // Check ID is OK.
+ if(ID <= mLastIDInInfo)
+ {
+ THROW_EXCEPTION(BackupStoreException, InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing)
+ }
+
+ // Can this go in the current block?
+ if(mpInfoLastBlock == 0 || mInfoLastBlockEntries >= BACKUPSTORECHECK_BLOCK_SIZE)
+ {
+ // No. Allocate a new one
+ IDBlock *pblk = (IDBlock*)::malloc(sizeof(IDBlock));
+ if(pblk == 0)
+ {
+ throw std::bad_alloc();
+ }
+ // Zero all the flags entries
+ for(int z = 0; z < (BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry); ++z)
+ {
+ pblk->mFlags[z] = 0;
+ }
+ // Store in map
+ mInfo[ID] = pblk;
+ // Allocated and stored OK, setup for use
+ mpInfoLastBlock = pblk;
+ mInfoLastBlockEntries = 0;
+ }
+ ASSERT(mpInfoLastBlock != 0 && mInfoLastBlockEntries < BACKUPSTORECHECK_BLOCK_SIZE);
+
+ // Add to block
+ mpInfoLastBlock->mID[mInfoLastBlockEntries] = ID;
+ mpInfoLastBlock->mContainer[mInfoLastBlockEntries] = Container;
+ mpInfoLastBlock->mObjectSizeInBlocks[mInfoLastBlockEntries] = ObjectSize;
+ SetFlags(mpInfoLastBlock, mInfoLastBlockEntries, IsFile?(0):(Flags_IsDir));
+
+ // Increment size
+ ++mInfoLastBlockEntries;
+
+ // Store last ID
+ mLastIDInInfo = ID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::LookupID(BackupStoreCheck_ID_t, int32_t
+// Purpose: Look up an ID. Return the block it's in, or zero if not found, and the
+// index within that block if the thing is found.
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut)
+{
+ IDBlock *pblock = 0;
+
+ // Find the lower matching block who's first entry is not less than ID
+ Info_t::const_iterator ib(mInfo.lower_bound(ID));
+
+ // Was there a block
+ if(ib == mInfo.end())
+ {
+ // Block wasn't found... could be in last block
+ pblock = mpInfoLastBlock;
+ }
+ else
+ {
+ // Found it as first entry?
+ if(ib->first == ID)
+ {
+ rIndexOut = 0;
+ return ib->second;
+ }
+
+ // Go back one block as it's not the first entry in this one
+ if(ib == mInfo.begin())
+ {
+ // Was first block, can't go back
+ return 0;
+ }
+ // Go back...
+ --ib;
+
+ // So, the ID will be in this block, if it's in anything
+ pblock = ib->second;
+ }
+
+ ASSERT(pblock != 0);
+ if(pblock == 0) return 0;
+
+ // How many entries are there in the block
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+
+ // Do binary search within block
+ int high = bentries;
+ int low = -1;
+ while(high - low > 1)
+ {
+ int i = (high + low) / 2;
+ if(ID <= pblock->mID[i])
+ {
+ high = i;
+ }
+ else
+ {
+ low = i;
+ }
+ }
+ if(ID == pblock->mID[high])
+ {
+ // Found
+ rIndexOut = high;
+ return pblock;
+ }
+
+ // Not found
+ return 0;
+}
+
+
+#ifndef NDEBUG
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::DumpObjectInfo()
+// Purpose: Debug only. Trace out all object info.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::DumpObjectInfo()
+{
+ for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ IDBlock *pblock = i->second;
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+ TRACE2("BLOCK @ 0x%08x, %d entries\n", pblock, bentries);
+
+ for(int e = 0; e < bentries; ++e)
+ {
+ uint8_t flags = GetFlags(pblock, e);
+ TRACE4("id %llx, c %llx, %s, %s\n",
+ pblock->mID[e], pblock->mContainer[e],
+ (flags & Flags_IsDir)?"dir":"file",
+ (flags & Flags_IsContained)?"contained":"unattached");
+ }
+ }
+}
+#endif
+
diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp
new file mode 100755
index 00000000..6fa05d06
--- /dev/null
+++ b/lib/backupstore/BackupStoreConfigVerify.cpp
@@ -0,0 +1,48 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreConfigVerify.h
+// Purpose: Configuration definition for the backup store server
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreConfigVerify.h"
+#include "ServerTLS.h"
+#include "BoxPortsAndFiles.h"
+
+#include "MemLeakFindOn.h"
+
+static const ConfigurationVerifyKey verifyserverkeys[] =
+{
+ SERVERTLS_VERIFY_SERVER_KEYS(0) // no default listen addresses
+};
+
+static const ConfigurationVerify verifyserver[] =
+{
+ {
+ "Server",
+ 0,
+ verifyserverkeys,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+ }
+};
+
+static const ConfigurationVerifyKey verifyrootkeys[] =
+{
+ {"AccountDatabase", 0, ConfigTest_Exists, 0},
+ {"TimeBetweenHousekeeping", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file
+ {"RaidFileConf", BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 0}
+};
+
+const ConfigurationVerify BackupConfigFileVerify =
+{
+ "root",
+ verifyserver,
+ verifyrootkeys,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+};
diff --git a/lib/backupstore/BackupStoreConfigVerify.h b/lib/backupstore/BackupStoreConfigVerify.h
new file mode 100755
index 00000000..815cfaed
--- /dev/null
+++ b/lib/backupstore/BackupStoreConfigVerify.h
@@ -0,0 +1,18 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreConfigVerify.h
+// Purpose: Configuration definition for the backup store server
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECONFIGVERIFY__H
+#define BACKUPSTORECONFIGVERIFY__H
+
+#include "Configuration.h"
+
+extern const ConfigurationVerify BackupConfigFileVerify;
+
+#endif // BACKUPSTORECONFIGVERIFY__H
+
diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp
new file mode 100755
index 00000000..a9effe00
--- /dev/null
+++ b/lib/backupstore/BackupStoreInfo.cpp
@@ -0,0 +1,592 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreInfo.cpp
+// Purpose: Main backup store information storage
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <algorithm>
+
+#include "BackupStoreInfo.h"
+#include "BackupStoreException.h"
+#include "RaidFileWrite.h"
+#include "RaidFileRead.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// ******************
+// make sure the defaults in CreateNew are modified!
+// ******************
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int32_t mAccountID;
+ int64_t mClientStoreMarker;
+ int64_t mLastObjectIDUsed;
+ int64_t mBlocksUsed;
+ int64_t mBlocksInOldFiles;
+ int64_t mBlocksInDeletedFiles;
+ int64_t mBlocksInDirectories;
+ int64_t mBlocksSoftLimit;
+ int64_t mBlocksHardLimit;
+ uint32_t mCurrentMarkNumber;
+ uint32_t mOptionsPresent; // bit mask of optional elements present
+ int64_t mNumberDeletedDirectories;
+ // Then loads of int64_t IDs for the deleted directories
+} info_StreamFormat;
+
+#define INFO_MAGIC_VALUE 0x34832476
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+#ifdef NDEBUG
+ #define NUM_DELETED_DIRS_BLOCK 256
+#else
+ #define NUM_DELETED_DIRS_BLOCK 2
+#endif
+
+#define INFO_FILENAME "info"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::BackupStoreInfo()
+// Purpose: Default constructor
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreInfo::BackupStoreInfo()
+ : mAccountID(-1),
+ mDiscSet(-1),
+ mReadOnly(true),
+ mIsModified(false),
+ mClientStoreMarker(0),
+ mLastObjectIDUsed(-1),
+ mBlocksUsed(0),
+ mBlocksInOldFiles(0),
+ mBlocksInDeletedFiles(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::~BackupStoreInfo
+// Purpose: Destructor
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreInfo::~BackupStoreInfo()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::CreateNew(int32_t, const std::string &, int)
+// Purpose: Create a new info file on disc
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+ // Initial header (is entire file)
+ info_StreamFormat hdr = {
+ htonl(INFO_MAGIC_VALUE), // mMagicValue
+ htonl(AccountID), // mAccountID
+ 0, // mClientStoreMarker
+ hton64(1), // mLastObjectIDUsed (which is the root directory)
+ 0, // mBlocksUsed
+ 0, // mBlocksInOldFiles
+ 0, // mBlocksInDeletedFiles
+ 0, // mBlocksInDirectories
+ hton64(BlockSoftLimit), // mBlocksSoftLimit
+ hton64(BlockHardLimit), // mBlocksHardLimit
+ 0, // mCurrentMarkNumber
+ 0, // mOptionsPresent
+ 0 // mNumberDeletedDirectories
+ };
+
+ // Generate the filename
+ ASSERT(rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR);
+ std::string fn(rRootDir + INFO_FILENAME);
+
+ // Open the file for writing
+ RaidFileWrite rf(DiscSet, fn);
+ rf.Open(false); // no overwriting, as this is a new file
+
+ // Write header
+ rf.Write(&hdr, sizeof(hdr));
+
+ // Commit it to disc, converting it to RAID now
+ rf.Commit(true);
+
+ // Done.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::Load(int32_t, const std::string &, int, bool)
+// Purpose: Loads the info from disc, given the root information. Can be marked as read only.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID)
+{
+ // Generate the filename
+ std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME);
+
+ // Open the file for reading (passing on optional request for revision ID)
+ std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID));
+
+ // Read in a header
+ info_StreamFormat hdr;
+ if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+ }
+
+ // Check it
+ if(ntohl(hdr.mMagicValue) != INFO_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != AccountID)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+ }
+
+ // Make new object
+ std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo);
+
+ // Put in basic location info
+ info->mAccountID = AccountID;
+ info->mDiscSet = DiscSet;
+ info->mFilename = fn;
+ info->mReadOnly = ReadOnly;
+
+ // Insert info from file
+ info->mClientStoreMarker = ntoh64(hdr.mClientStoreMarker);
+ info->mLastObjectIDUsed = ntoh64(hdr.mLastObjectIDUsed);
+ info->mBlocksUsed = ntoh64(hdr.mBlocksUsed);
+ info->mBlocksInOldFiles = ntoh64(hdr.mBlocksInOldFiles);
+ info->mBlocksInDeletedFiles = ntoh64(hdr.mBlocksInDeletedFiles);
+ info->mBlocksInDirectories = ntoh64(hdr.mBlocksInDirectories);
+ info->mBlocksSoftLimit = ntoh64(hdr.mBlocksSoftLimit);
+ info->mBlocksHardLimit = ntoh64(hdr.mBlocksHardLimit);
+
+ // Load up array of deleted objects
+ int64_t numDelObj = ntoh64(hdr.mNumberDeletedDirectories);
+
+ // Then load them in
+ if(numDelObj > 0)
+ {
+ int64_t objs[NUM_DELETED_DIRS_BLOCK];
+
+ int64_t toload = numDelObj;
+ while(toload > 0)
+ {
+ // How many in this one?
+ int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload));
+
+ if(!rf->ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+ }
+
+ // Add them
+ for(int t = 0; t < b; ++t)
+ {
+ info->mDeletedDirectories.push_back(ntoh64(objs[t]));
+ }
+
+ // Number loaded
+ toload -= b;
+ }
+ }
+
+ // Final check
+ if(static_cast<int64_t>(info->mDeletedDirectories.size()) != numDelObj)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+ }
+
+ // return it to caller
+ return info;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::CreateForRegeneration(...)
+// Purpose: Return an object which can be used to save for regeneration.
+// Created: 23/4/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration(int32_t AccountID, const std::string &rRootDir,
+ int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles,
+ int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+ // Generate the filename
+ std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME);
+
+ // Make new object
+ std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo);
+
+ // Put in basic info
+ info->mAccountID = AccountID;
+ info->mDiscSet = DiscSet;
+ info->mFilename = fn;
+ info->mReadOnly = false;
+
+ // Insert info starting info
+ info->mClientStoreMarker = 0;
+ info->mLastObjectIDUsed = LastObjectID;
+ info->mBlocksUsed = BlocksUsed;
+ info->mBlocksInOldFiles = BlocksInOldFiles;
+ info->mBlocksInDeletedFiles = BlocksInDeletedFiles;
+ info->mBlocksInDirectories = BlocksInDirectories;
+ info->mBlocksSoftLimit = BlockSoftLimit;
+ info->mBlocksHardLimit = BlockHardLimit;
+
+ // return it to caller
+ return info;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::Save()
+// Purpose: Save modified info back to disc
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::Save()
+{
+ // Make sure we're initialised (although should never come to this)
+ if(mFilename.empty() || mAccountID == -1 || mDiscSet == -1)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Can we do this?
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ // Then... open a write file
+ RaidFileWrite rf(mDiscSet, mFilename);
+ rf.Open(true); // allow overwriting
+
+ // Make header
+ info_StreamFormat hdr;
+ hdr.mMagicValue = htonl(INFO_MAGIC_VALUE);
+ hdr.mAccountID = htonl(mAccountID);
+ hdr.mClientStoreMarker = hton64(mClientStoreMarker);
+ hdr.mLastObjectIDUsed = hton64(mLastObjectIDUsed);
+ hdr.mBlocksUsed = hton64(mBlocksUsed);
+ hdr.mBlocksInOldFiles = hton64(mBlocksInOldFiles);
+ hdr.mBlocksInDeletedFiles = hton64(mBlocksInDeletedFiles);
+ hdr.mBlocksInDirectories = hton64(mBlocksInDirectories);
+ hdr.mBlocksSoftLimit = hton64(mBlocksSoftLimit);
+ hdr.mBlocksHardLimit = hton64(mBlocksHardLimit);
+ hdr.mCurrentMarkNumber = 0;
+ hdr.mOptionsPresent = 0;
+ hdr.mNumberDeletedDirectories = hton64(mDeletedDirectories.size());
+
+ // Write header
+ rf.Write(&hdr, sizeof(hdr));
+
+ // Write the deleted object list
+ if(mDeletedDirectories.size() > 0)
+ {
+ int64_t objs[NUM_DELETED_DIRS_BLOCK];
+
+ int tosave = mDeletedDirectories.size();
+ std::vector<int64_t>::iterator i(mDeletedDirectories.begin());
+ while(tosave > 0)
+ {
+ // How many in this one?
+ int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave));
+
+ // Add them
+ for(int t = 0; t < b; ++t)
+ {
+ ASSERT(i != mDeletedDirectories.end());
+ objs[t] = hton64((*i));
+ i++;
+ }
+
+ // Write
+ rf.Write(objs, b * sizeof(int64_t));
+
+ // Number saved
+ tosave -= b;
+ }
+ }
+
+ // Commit it to disc, converting it to RAID now
+ rf.Commit(true);
+
+ // Mark is as not modified
+ mIsModified = false;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksUsed(int32_t)
+// Purpose: Change number of blocks used, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksUsed + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksUsed += Delta;
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksInOldFiles(int32_t)
+// Purpose: Change number of blocks in old files, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksInOldFiles + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksInOldFiles += Delta;
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksInDeletedFiles(int32_t)
+// Purpose: Change number of blocks in deleted files, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksInDeletedFiles + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksInDeletedFiles += Delta;
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksInDirectories(int32_t)
+// Purpose: Change number of blocks in directories, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksInDirectories + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksInDirectories += Delta;
+
+ mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::CorrectAllUsedValues(int64_t, int64_t, int64_t, int64_t)
+// Purpose: Set all the usage counts to specific values -- use for correcting in housekeeping
+// if something when wrong during the backup connection, and the store info wasn't
+// saved back to disc.
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ // Set the values
+ mBlocksUsed = Used;
+ mBlocksInOldFiles = InOldFiles;
+ mBlocksInDeletedFiles = InDeletedFiles;
+ mBlocksInDirectories = InDirectories;
+
+ mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::AddDeletedDirectory(int64_t)
+// Purpose: Add a directory ID to the deleted list
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::AddDeletedDirectory(int64_t DirID)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ mDeletedDirectories.push_back(DirID);
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::RemovedDeletedDirectory(int64_t)
+// Purpose: Remove a directory from the deleted list
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::RemovedDeletedDirectory(int64_t DirID)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ std::vector<int64_t>::iterator i(std::find(mDeletedDirectories.begin(), mDeletedDirectories.end(), DirID));
+ if(i == mDeletedDirectories.end())
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoDirNotInList)
+ }
+ mDeletedDirectories.erase(i);
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeLimits(int64_t, int64_t)
+// Purpose: Change the soft and hard limits
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ mBlocksSoftLimit = BlockSoftLimit;
+ mBlocksHardLimit = BlockHardLimit;
+
+ mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::AllocateObjectID()
+// Purpose: Allocate an ID for a new object in the store.
+// Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreInfo::AllocateObjectID()
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if(mLastObjectIDUsed < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotInitialised)
+ }
+
+ // Return the next object ID
+ return ++mLastObjectIDUsed;
+
+ mIsModified = true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::SetClientStoreMarker(int64_t)
+// Purpose: Sets the client store marker
+// Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ mClientStoreMarker = ClientStoreMarker;
+
+ mIsModified = true;
+}
+
+
+
diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h
new file mode 100755
index 00000000..1aba5edb
--- /dev/null
+++ b/lib/backupstore/BackupStoreInfo.h
@@ -0,0 +1,111 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreInfo.h
+// Purpose: Main backup store information storage
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREINFO__H
+#define BACKUPSTOREINFO__H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+class BackupStoreCheck;
+
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreInfo
+// Purpose: Main backup store information storage
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class BackupStoreInfo
+{
+ friend class BackupStoreCheck;
+public:
+ ~BackupStoreInfo();
+private:
+ // Creation through static functions only
+ BackupStoreInfo();
+ // No copying allowed
+ BackupStoreInfo(const BackupStoreInfo &);
+
+public:
+ // Create a New account, saving a blank info object to the disc
+ static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit);
+
+ // Load it from the store
+ static std::auto_ptr<BackupStoreInfo> Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0);
+
+ // Has info been modified?
+ bool IsModified() const {return mIsModified;}
+
+ // Save modified infomation back to store
+ void Save();
+
+ // Data access functions
+ int32_t GetAccountID() const {return mAccountID;}
+ int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;}
+ int64_t GetBlocksUsed() const {return mBlocksUsed;}
+ int64_t GetBlocksInOldFiles() const {return mBlocksInOldFiles;}
+ int64_t GetBlocksInDeletedFiles() const {return mBlocksInDeletedFiles;}
+ int64_t GetBlocksInDirectories() const {return mBlocksInDirectories;}
+ const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;}
+ int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;}
+ int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;}
+ bool IsReadOnly() const {return mReadOnly;}
+ int GetDiscSetNumber() const {return mDiscSet;}
+
+ // Data modification functions
+ void ChangeBlocksUsed(int64_t Delta);
+ void ChangeBlocksInOldFiles(int64_t Delta);
+ void ChangeBlocksInDeletedFiles(int64_t Delta);
+ void ChangeBlocksInDirectories(int64_t Delta);
+ void CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories);
+ void AddDeletedDirectory(int64_t DirID);
+ void RemovedDeletedDirectory(int64_t DirID);
+ void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit);
+
+ // Object IDs
+ int64_t AllocateObjectID();
+
+ // Client marker set and get
+ int64_t GetClientStoreMarker() {return mClientStoreMarker;}
+ void SetClientStoreMarker(int64_t ClientStoreMarker);
+
+private:
+ static std::auto_ptr<BackupStoreInfo> CreateForRegeneration(int32_t AccountID, const std::string &rRootDir,
+ int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles,
+ int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit);
+
+private:
+ // Location information
+ int32_t mAccountID;
+ int mDiscSet;
+ std::string mFilename;
+ bool mReadOnly;
+ bool mIsModified;
+
+ // Client infomation
+ int64_t mClientStoreMarker;
+
+ // Account information
+ int64_t mLastObjectIDUsed;
+ int64_t mBlocksUsed;
+ int64_t mBlocksInOldFiles;
+ int64_t mBlocksInDeletedFiles;
+ int64_t mBlocksInDirectories;
+ int64_t mBlocksSoftLimit;
+ int64_t mBlocksHardLimit;
+ std::vector<int64_t> mDeletedDirectories;
+};
+
+
+#endif // BACKUPSTOREINFO__H
+
+
diff --git a/lib/backupstore/StoreStructure.cpp b/lib/backupstore/StoreStructure.cpp
new file mode 100755
index 00000000..45a1ce91
--- /dev/null
+++ b/lib/backupstore/StoreStructure.cpp
@@ -0,0 +1,95 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: StoreStructure.cpp
+// Purpose:
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "RaidFileController.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: StoreStructure::MakeObjectFilename(int64_t, const std::string &, int, std::string &, bool)
+// Purpose: Builds the object filename for a given object, given a root. Optionally ensure that the
+// directory exists.
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void StoreStructure::MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists)
+{
+ const static char *hex = "0123456789abcdef";
+
+ // Set output to root string
+ rFilenameOut = rStoreRoot;
+
+ // get the id value from the stored object ID so we can do
+ // bitwise operations on it.
+ uint64_t id = (uint64_t)ObjectID;
+
+ // get leafname, shift the bits which make up the leafname off
+ unsigned int leafname(id & STORE_ID_SEGMENT_MASK);
+ id >>= STORE_ID_SEGMENT_LENGTH;
+
+ // build pathname
+ while(id != 0)
+ {
+ // assumes that the segments are no bigger than 8 bits
+ int v = id & STORE_ID_SEGMENT_MASK;
+ rFilenameOut += hex[(v & 0xf0) >> 4];
+ rFilenameOut += hex[v & 0xf];
+ rFilenameOut += DIRECTORY_SEPARATOR_ASCHAR;
+
+ // shift the bits we used off the pathname
+ id >>= STORE_ID_SEGMENT_LENGTH;
+ }
+
+ // Want to make sure this exists?
+ if(EnsureDirectoryExists)
+ {
+ if(!RaidFileRead::DirectoryExists(DiscSet, rFilenameOut))
+ {
+ // Create it
+ RaidFileWrite::CreateDirectory(DiscSet, rFilenameOut, true /* recusive */);
+ }
+ }
+
+ // append the filename
+ rFilenameOut += 'o';
+ rFilenameOut += hex[(leafname & 0xf0) >> 4];
+ rFilenameOut += hex[leafname & 0xf];
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: StoreStructure::MakeWriteLockFilename(const std::string &, int, std::string &)
+// Purpose: Generate the on disc filename of the write lock file
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void StoreStructure::MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut)
+{
+ // Find the disc set
+ RaidFileController &rcontroller(RaidFileController::GetController());
+ RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(DiscSet));
+
+ // Make the filename
+ std::string writeLockFile(rdiscSet[0] + DIRECTORY_SEPARATOR + rStoreRoot + "write.lock");
+
+ // Return it to the caller
+ rFilenameOut = writeLockFile;
+}
+
+
diff --git a/lib/backupstore/StoreStructure.h b/lib/backupstore/StoreStructure.h
new file mode 100755
index 00000000..094c0deb
--- /dev/null
+++ b/lib/backupstore/StoreStructure.h
@@ -0,0 +1,32 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: StoreStructure.h
+// Purpose: Functions for placing files in the store
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef STORESTRUCTURE__H
+#define STORESTRUCTURE__H
+
+#include <string>
+
+#ifdef NDEBUG
+ #define STORE_ID_SEGMENT_LENGTH 8
+ #define STORE_ID_SEGMENT_MASK 0xff
+#else
+ // Debug we'll use lots and lots of directories to stress things
+ #define STORE_ID_SEGMENT_LENGTH 2
+ #define STORE_ID_SEGMENT_MASK 0x03
+#endif
+
+
+namespace StoreStructure
+{
+ void MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists);
+ void MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut);
+};
+
+#endif // STORESTRUCTURE__H
+
diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h
new file mode 100755
index 00000000..8c471171
--- /dev/null
+++ b/lib/common/BannerText.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BannerText.h
+// Purpose: Banner text for daemons and utilities
+// Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BANNERTEXT__H
+#define BANNERTEXT__H
+
+#define BANNER_TEXT(UtilityName) \
+ "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers 2003, 2004\n"
+
+#endif // BANNERTEXT__H
+
diff --git a/lib/common/BeginStructPackForWire.h b/lib/common/BeginStructPackForWire.h
new file mode 100644
index 00000000..f9f8f616
--- /dev/null
+++ b/lib/common/BeginStructPackForWire.h
@@ -0,0 +1,23 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BeginStructPackForWire.h
+// Purpose: Begin structure packing for wire
+// Created: 25/11/03
+//
+// --------------------------------------------------------------------------
+
+// No header guard -- this is intentional
+
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+
+#pragma pack(1)
+
+#else
+
+ logical error -- check BoxPlatform.h and including file
+
+#endif
+
+
+
diff --git a/lib/common/Box.h b/lib/common/Box.h
new file mode 100755
index 00000000..19e78ada
--- /dev/null
+++ b/lib/common/Box.h
@@ -0,0 +1,180 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: Box.h
+// Purpose: Main header file for the Box project
+// Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOX__H
+#define BOX__H
+
+// Use the same changes as gcc3 for gcc4
+#ifdef PLATFORM_GCC4
+ #define PLATFORM_GCC3
+#endif
+
+#include "BoxPlatform.h"
+
+// uncomment this line to enable full memory leak finding on all malloc-ed blocks (at least, ones used by the STL)
+//#define MEMLEAKFINDER_FULL_MALLOC_MONITORING
+
+#ifndef NDEBUG
+ // not available on OpenBSD... oh well.
+ //#define SHOW_BACKTRACE_ON_EXCEPTION
+#endif
+
+#ifdef SHOW_BACKTRACE_ON_EXCEPTION
+ // include "Utils.h"
+ #define OPTIONAL_DO_BACKTRACE DumpStackBacktrace();
+#else
+ #define OPTIONAL_DO_BACKTRACE
+#endif
+
+#include "CommonException.h"
+
+#ifndef NDEBUG
+
+ extern bool AssertFailuresToSyslog;
+ #define ASSERT_FAILS_TO_SYSLOG_ON {AssertFailuresToSyslog = true;}
+ void BoxDebugAssertFailed(char *cond, char *file, int line);
+ #define ASSERT(cond) {if(!(cond)) {BoxDebugAssertFailed(#cond, __FILE__, __LINE__); THROW_EXCEPTION(CommonException, AssertFailed)}}
+
+ // Note that syslog tracing is independent of BoxDebugTraceOn, but stdout tracing is not
+ extern bool BoxDebugTraceToSyslog;
+ #define TRACE_TO_SYSLOG(x) {BoxDebugTraceToSyslog = x;}
+ extern bool BoxDebugTraceToStdout;
+ #define TRACE_TO_STDOUT(x) {BoxDebugTraceToStdout = x;}
+
+ extern bool BoxDebugTraceOn;
+ int BoxDebug_printf(const char *format, ...);
+ int BoxDebugTrace(const char *format, ...);
+ #define TRACE0(msg) {BoxDebugTrace("%s", msg);}
+ #define TRACE1(msg, a0) {BoxDebugTrace(msg, a0);}
+ #define TRACE2(msg, a0, a1) {BoxDebugTrace(msg, a0, a1);}
+ #define TRACE3(msg, a0, a1, a2) {BoxDebugTrace(msg, a0, a1, a2);}
+ #define TRACE4(msg, a0, a1, a2, a3) {BoxDebugTrace(msg, a0, a1, a2, a3);}
+ #define TRACE5(msg, a0, a1, a2, a3, a4) {BoxDebugTrace(msg, a0, a1, a2, a3, a4);}
+ #define TRACE6(msg, a0, a1, a2, a3, a4, a5) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5);}
+ #define TRACE7(msg, a0, a1, a2, a3, a4, a5, a6) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6);}
+ #define TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6, a7);}
+
+ #ifndef PLATFORM_DISABLE_MEM_LEAK_TESTING
+ #define BOX_MEMORY_LEAK_TESTING
+ #endif
+
+ // Exception names
+ #define EXCEPTION_CODENAMES_EXTENDED
+
+#else
+ #define ASSERT_FAILS_TO_SYSLOG_ON
+ #define ASSERT(cond)
+
+ #define TRACE_TO_SYSLOG(x) {}
+ #define TRACE_TO_STDOUT(x) {}
+
+ #define TRACE0(msg)
+ #define TRACE1(msg, a0)
+ #define TRACE2(msg, a0, a1)
+ #define TRACE3(msg, a0, a1, a2)
+ #define TRACE4(msg, a0, a1, a2, a3)
+ #define TRACE5(msg, a0, a1, a2, a3, a4)
+ #define TRACE6(msg, a0, a1, a2, a3, a4, a5)
+ #define TRACE7(msg, a0, a1, a2, a3, a4, a5, a6)
+ #define TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7)
+
+ // Box Backup builds release get extra information for exception logging
+ #define EXCEPTION_CODENAMES_EXTENDED
+ #define EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION
+
+#endif
+
+#ifdef BOX_MEMORY_LEAK_TESTING
+ // Memory leak testing
+ #include "MemLeakFinder.h"
+ #define MEMLEAKFINDER_NOT_A_LEAK(x) memleakfinder_notaleak(x);
+ #define MEMLEAKFINDER_START {memleakfinder_global_enable = true;}
+ #define MEMLEAKFINDER_STOP {memleakfinder_global_enable = false;}
+#else
+ #define DEBUG_NEW new
+ #define MEMLEAKFINDER_NOT_A_LEAK(x)
+ #define MEMLEAKFINDER_START
+ #define MEMLEAKFINDER_STOP
+#endif
+
+
+#define THROW_EXCEPTION(type, subtype) \
+ { \
+ OPTIONAL_DO_BACKTRACE \
+ TRACE1("Exception thrown: " #type "(" #subtype ") at " __FILE__ "(%d)\n", __LINE__) \
+ throw type(type::subtype); \
+ }
+
+// extra macros for converting to network byte order
+
+// Always define a swap64 function, as it's useful.
+inline uint64_t box_swap64(uint64_t x)
+{
+ return ((x & 0xff) << 56 |
+ (x & 0xff00LL) << 40 |
+ (x & 0xff0000LL) << 24 |
+ (x & 0xff000000LL) << 8 |
+ (x & 0xff00000000LL) >> 8 |
+ (x & 0xff0000000000LL) >> 24 |
+ (x & 0xff000000000000LL) >> 40 |
+ (x & 0xff00000000000000LL) >> 56);
+}
+
+// Does the platform provide a built in SWAP64 we can use?
+#ifdef PLATFORM_NO_BUILT_IN_SWAP64
+
+ #define hton64(x) box_swap64(x)
+ #define ntoh64(x) box_swap64(x)
+
+#else
+
+ #if BYTE_ORDER == BIG_ENDIAN
+
+ // Less hassle than trying to find some working things
+ // on Darwin PPC
+ #define hton64(x) (x)
+ #define ntoh64(x) (x)
+
+ #else
+
+ #ifdef PLATFORM_LINUX
+ // On Linux, use some internal kernal stuff to do this
+ #include <asm/byteorder.h>
+ #define hton64 __cpu_to_be64
+ #define ntoh64 __be64_to_cpu
+ #else
+ #define hton64 htobe64
+ #define ntoh64 betoh64
+ #endif
+
+ // hack to make some of these work
+ // version in /usr/include/sys/endian.h doesn't include the 'LL' at the end of the constants
+ // provoking complaints from the compiler
+ #ifdef __GNUC__
+ #undef __swap64gen
+ #define __swap64gen(x) __extension__({ \
+ u_int64_t __swap64gen_x = (x); \
+ \
+ (u_int64_t)((__swap64gen_x & 0xff) << 56 | \
+ (__swap64gen_x & 0xff00LL) << 40 | \
+ (__swap64gen_x & 0xff0000LL) << 24 | \
+ (__swap64gen_x & 0xff000000LL) << 8 | \
+ (__swap64gen_x & 0xff00000000LL) >> 8 | \
+ (__swap64gen_x & 0xff0000000000LL) >> 24 | \
+ (__swap64gen_x & 0xff000000000000LL) >> 40 | \
+ (__swap64gen_x & 0xff00000000000000LL) >> 56); \
+ })
+ #endif // __GNUC__
+
+ #endif // n BYTE_ORDER == BIG_ENDIAN
+
+#endif // PLATFORM_NO_BUILT_IN_SWAP64
+
+#endif // BOX__H
+
diff --git a/lib/common/BoxException.cpp b/lib/common/BoxException.cpp
new file mode 100755
index 00000000..2503ca63
--- /dev/null
+++ b/lib/common/BoxException.cpp
@@ -0,0 +1,21 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxException.cpp
+// Purpose: Exception
+// Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BoxException.h"
+
+#include "MemLeakFindOn.h"
+
+BoxException::BoxException()
+{
+}
+
+BoxException::~BoxException() throw ()
+{
+}
diff --git a/lib/common/BoxException.h b/lib/common/BoxException.h
new file mode 100755
index 00000000..eb992f57
--- /dev/null
+++ b/lib/common/BoxException.h
@@ -0,0 +1,37 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxException.h
+// Purpose: Exception
+// Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXEXCEPTION__H
+#define BOXEXCEPTION__H
+
+#include <exception>
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BoxException
+// Purpose: Exception
+// Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+class BoxException : public std::exception
+{
+public:
+ BoxException();
+ ~BoxException() throw ();
+
+ virtual unsigned int GetType() const throw() = 0;
+ virtual unsigned int GetSubType() const throw() = 0;
+
+private:
+};
+
+
+#endif // BOXEXCEPTION__H
+
diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h
new file mode 100755
index 00000000..716366d2
--- /dev/null
+++ b/lib/common/BoxPlatform.h
@@ -0,0 +1,236 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxPlatform.h
+// Purpose: Specifies what each platform supports in more detail, and includes
+// extra files to get basic support for types.
+// Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXPLATFORM__H
+#define BOXPLATFORM__H
+
+#define DIRECTORY_SEPARATOR "/"
+#define DIRECTORY_SEPARATOR_ASCHAR '/'
+
+#define PLATFORM_DEV_NULL "/dev/null"
+
+
+// Other flags which might be useful...
+//
+// #define PLATFORM_BERKELEY_DB_NOT_SUPPORTED
+// -- dbopen etc not on this platform
+//
+// #define PLATFORM_REGEX_NOT_SUPPORTED
+// -- regex support not available on this platform
+
+
+#ifdef PLATFORM_OPENBSD
+
+ #include <sys/types.h>
+
+ #define PLATFORM_HAVE_setproctitle
+
+ #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp"
+
+ #define PLATFORM_HAVE_getpeereid
+
+ #define PLATFORM_RANDOM_DEVICE "/dev/arandom"
+
+#endif // PLATFORM_OPENBSD
+
+#ifdef PLATFORM_NETBSD
+
+ #include <sys/types.h>
+
+ #define PLATFORM_HAVE_setproctitle
+
+ #define PLATFORM_NO_BUILT_IN_SWAP64
+
+ #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp"
+
+ #define PLATFORM_KQUEUE_NOT_SUPPORTED
+
+ #define PLATFORM_RANDOM_DEVICE "/dev/urandom"
+
+#endif
+
+#ifdef PLATFORM_FREEBSD
+
+ #include <sys/types.h>
+ #include <netinet/in.h>
+
+ #define PLATFORM_HAVE_setproctitle
+
+ #define PLATFORM_NO_BUILT_IN_SWAP64
+
+ #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp"
+
+ #define PLATFORM_HAVE_getpeereid
+
+ #define PLATFORM_RANDOM_DEVICE "/dev/urandom"
+
+#endif // PLATFORM_FREEBSD
+
+#ifdef PLATFORM_DARWIN
+
+ #include <sys/types.h>
+ #include <netdb.h>
+
+ // types 'missing'
+ #ifndef _SOCKLEN_T
+ typedef int socklen_t;
+ #endif
+ typedef u_int8_t uint8_t;
+ typedef signed char int8_t;
+ typedef u_int64_t uint64_t;
+ typedef u_int32_t uint32_t;
+ typedef u_int16_t uint16_t;
+
+ // poll() emulator on Darwin misses this declaration
+ #define INFTIM -1
+
+ #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp"
+
+ #define PLATFORM_LCHOWN_NOT_SUPPORTED
+
+ #define PLATFORM_READLINE_NOT_SUPPORTED
+
+ #define PLATFORM_RANDOM_DEVICE "/dev/random"
+
+#endif // PLATFORM_DARWIN
+
+#ifdef PLATFORM_LINUX
+
+ #include <sys/types.h>
+
+ // for ntohl etc...
+ #include <netinet/in.h>
+
+ // types 'missing'
+ typedef u_int8_t uint8_t;
+ typedef signed char int8_t;
+ typedef u_int32_t uint32_t;
+ typedef u_int16_t uint16_t;
+ typedef u_int64_t uint64_t;
+
+ // not defined in Linux, a BSD thing
+ #define INFTIM -1
+
+ #define LLONG_MAX 9223372036854775807LL
+ #define LLONG_MIN (-LLONG_MAX - 1LL)
+
+ #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp"
+
+ #define PLATFORM_HAVE_getsockopt_SO_PEERCRED
+
+ // load in installation specific linux configuration
+ #include "../../local/_linux_platform.h"
+
+ #define PLATFORM_KQUEUE_NOT_SUPPORTED
+ #define PLATFORM_dirent_BROKEN_d_type
+ #define PLATFORM_stat_SHORT_mtime
+ #define PLATFORM_stat_NO_st_flags
+ #define PLATFORM_USES_MTAB_FILE_FOR_MOUNTS
+ #define PLATFORM_open_NO_O_EXLOCK
+ #define PLATFORM_sockaddr_NO_len
+
+ #define PLATFORM_RANDOM_DEVICE "/dev/urandom"
+
+ // If large file support is on, can't do the intercepts in the test/raidfile
+ #if _FILE_OFFSET_BITS == 64
+ #define PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE
+ #endif
+
+#endif // PLATFORM_LINUX
+
+#ifdef PLATFORM_CYGWIN
+
+ #define PLATFORM_BERKELEY_DB_NOT_SUPPORTED
+
+ #define PLATFORM_KQUEUE_NOT_SUPPORTED
+ #define PLATFORM_dirent_BROKEN_d_type
+ #define PLATFORM_stat_SHORT_mtime
+ #define PLATFORM_stat_NO_st_flags
+ #define PLATFORM_USES_MTAB_FILE_FOR_MOUNTS
+ #define PLATFORM_open_NO_O_EXLOCK
+ #define PLATFORM_sockaddr_NO_len
+ #define PLATFORM_NO_BUILT_IN_SWAP64
+
+ #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp"
+
+ #define PLATFORM_READLINE_NOT_SUPPORTED
+
+ #define LLONG_MAX 9223372036854775807LL
+ #define LLONG_MIN (-LLONG_MAX - 1LL)
+
+ #define INFTIM -1
+
+ // File listing canonical interesting mount points.
+ #define MNTTAB _PATH_MNTTAB
+
+ // File listing currently active mount points.
+ #define MOUNTED _PATH_MOUNTED
+
+ #define __need_FILE
+
+ // Extra includes
+ #include <stdint.h>
+ #include <stdlib.h>
+ #include <netinet/in.h>
+ #include <sys/socket.h>
+ #include <sys/stat.h>
+ #include <sys/types.h>
+ #include <dirent.h>
+ #include <stdio.h>
+ #include <paths.h>
+
+ // No easy random entropy source
+ #define PLATFORM_RANDOM_DEVICE_NONE
+
+#endif // PLATFORM_CYGWIN
+
+
+// Find out if credentials on UNIX sockets can be obtained
+#ifndef PLATFORM_HAVE_getpeereid
+ #ifndef PLATFORM_HAVE_getsockopt_SO_PEERCRED
+ #define PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET
+ #endif
+#endif
+
+
+// Compiler issues
+#ifdef __GNUC__
+
+ #ifdef PLATFORM_GCC3
+
+ // GCC v3 doesn't like pragmas in #defines
+ #define STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+
+ // But fortunately, the STL allocations are much better behaved.
+
+ #else
+
+ // Force STL to use malloc() for memory allocation
+ // -- slower, but doesn't gradually use more and more memory
+ // HOWEVER -- this 'fix' is broken on some platforms. Lots of fun!
+ #ifndef PLATFORM_STL_USE_MALLOC_BROKEN
+ #define __USE_MALLOC
+ #endif
+
+ // set packing to one bytes (can't use push/pop on gcc)
+ #define BEGIN_STRUCTURE_PACKING_FOR_WIRE #pragma pack(1)
+
+ // Use default packing
+ #define END_STRUCTURE_PACKING_FOR_WIRE #pragma pack()
+
+ #endif
+
+#else
+ compiler not supported!
+#endif
+
+
+#endif // BOXPLATFORM__H
+
diff --git a/lib/common/BoxPortsAndFiles.h b/lib/common/BoxPortsAndFiles.h
new file mode 100755
index 00000000..7788537e
--- /dev/null
+++ b/lib/common/BoxPortsAndFiles.h
@@ -0,0 +1,31 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxPortsAndFiles.h
+// Purpose: Central list of which tcp/ip ports and hardcoded file locations
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXPORTSANDFILES__H
+#define BOXPORTSANDFILES__H
+
+#define BOX_PORT_BASE 2200
+
+
+// Backup store daemon
+#define BOX_PORT_BBSTORED (BOX_PORT_BASE+1)
+#define BOX_FILE_BBSTORED_DEFAULT_CONFIG "/etc/box/bbstored.conf"
+// directory within the RAIDFILE root for the backup store daemon
+#define BOX_RAIDFILE_ROOT_BBSTORED "backup"
+
+// Backup client daemon
+#define BOX_FILE_BBACKUPD_DEFAULT_CONFIG "/etc/box/bbackupd.conf"
+
+
+// RaidFile conf location efault
+#define BOX_FILE_RAIDFILE_DEFAULT_CONFIG "/etc/box/raidfile.conf"
+
+
+#endif // BOXPORTSANDFILES__H
+
diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp
new file mode 100755
index 00000000..feada309
--- /dev/null
+++ b/lib/common/BoxTime.cpp
@@ -0,0 +1,32 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxTime.cpp
+// Purpose: Time for the box
+// Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <time.h>
+
+#include "BoxTime.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: GetCurrentBoxTime()
+// Purpose: Returns the current time as a box time. (1 sec precision)
+// Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+box_time_t GetCurrentBoxTime()
+{
+ ASSERT(sizeof(uint32_t) == sizeof(time_t));
+ return SecondsToBoxTime((uint32_t)time(0));
+}
+
+
diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h
new file mode 100755
index 00000000..a7a25ac6
--- /dev/null
+++ b/lib/common/BoxTime.h
@@ -0,0 +1,44 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxTime.h
+// Purpose: How time is represented
+// Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXTIME__H
+#define BOXTIME__H
+
+// Time is presented as an unsigned 64 bit integer, in microseconds
+typedef uint64_t box_time_t;
+
+#define NANO_SEC_IN_SEC (1000000000LL)
+#define NANO_SEC_IN_USEC (1000)
+#define NANO_SEC_IN_USEC_LL (1000LL)
+#define MICRO_SEC_IN_SEC (1000000)
+#define MICRO_SEC_IN_SEC_LL (1000000LL)
+#define MILLI_SEC_IN_NANO_SEC (1000)
+#define MILLI_SEC_IN_NANO_SEC_LL (1000LL)
+
+box_time_t GetCurrentBoxTime();
+
+inline box_time_t SecondsToBoxTime(uint32_t Seconds)
+{
+ return ((box_time_t)Seconds * MICRO_SEC_IN_SEC_LL);
+}
+inline box_time_t SecondsToBoxTime(uint64_t Seconds)
+{
+ return ((box_time_t)Seconds * MICRO_SEC_IN_SEC_LL);
+}
+inline int64_t BoxTimeToSeconds(box_time_t Time)
+{
+ return Time / MICRO_SEC_IN_SEC_LL;
+}
+inline int64_t BoxTimeToMilliSeconds(box_time_t Time)
+{
+ return Time / MILLI_SEC_IN_NANO_SEC_LL;
+}
+
+#endif // BOXTIME__H
+
diff --git a/lib/common/BoxTimeToText.cpp b/lib/common/BoxTimeToText.cpp
new file mode 100755
index 00000000..94b0d152
--- /dev/null
+++ b/lib/common/BoxTimeToText.cpp
@@ -0,0 +1,41 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxTimeToText.cpp
+// Purpose: Convert box time to text
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <time.h>
+#include <stdio.h>
+
+#include "BoxTimeToText.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BoxTimeToISO8601String(box_time_t)
+// Purpose: Convert a 64 bit box time to a ISO 8601 complient string
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+std::string BoxTimeToISO8601String(box_time_t Time)
+{
+ time_t timeInSecs = (time_t)BoxTimeToSeconds(Time);
+ struct tm time;
+ gmtime_r(&timeInSecs, &time);
+
+ char str[128]; // more than enough space
+ sprintf(str, "%04d-%02d-%02dT%02d:%02d:%02d", time.tm_year + 1900,
+ time.tm_mon + 1, time.tm_mday, time.tm_hour, time.tm_min, time.tm_sec);
+
+ return std::string(str);
+}
+
+
diff --git a/lib/common/BoxTimeToText.h b/lib/common/BoxTimeToText.h
new file mode 100755
index 00000000..a25c70b6
--- /dev/null
+++ b/lib/common/BoxTimeToText.h
@@ -0,0 +1,19 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxTimeToText.h
+// Purpose: Convert box time to text
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXTIMETOTEXT__H
+#define BOXTIMETOTEXT__H
+
+#include <string>
+#include "BoxTime.h"
+
+std::string BoxTimeToISO8601String(box_time_t Time);
+
+#endif // BOXTIMETOTEXT__H
+
diff --git a/lib/common/BoxTimeToUnix.h b/lib/common/BoxTimeToUnix.h
new file mode 100755
index 00000000..17e57e27
--- /dev/null
+++ b/lib/common/BoxTimeToUnix.h
@@ -0,0 +1,30 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxTimeToUnix.h
+// Purpose: Convert times in 64 bit values to UNIX structures
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#ifndef FILEMODIFICATIONTIMETOTIMEVAL__H
+#define FILEMODIFICATIONTIMETOTIMEVAL__H
+
+#include <sys/time.h>
+
+#include "BoxTime.h"
+
+inline void BoxTimeToTimeval(box_time_t Time, struct timeval &tv)
+{
+ tv.tv_sec = (long)(Time / MICRO_SEC_IN_SEC_LL);
+ tv.tv_usec = (long)(Time % MICRO_SEC_IN_SEC_LL);
+}
+
+inline void BoxTimeToTimespec(box_time_t Time, struct timespec &tv)
+{
+ tv.tv_sec = (time_t)(Time / MICRO_SEC_IN_SEC_LL);
+ tv.tv_nsec = ((long)(Time % MICRO_SEC_IN_SEC_LL)) * NANO_SEC_IN_USEC;
+}
+
+#endif // FILEMODIFICATIONTIMETOTIMEVAL__H
+
diff --git a/lib/common/CollectInBufferStream.cpp b/lib/common/CollectInBufferStream.cpp
new file mode 100755
index 00000000..90e2e7bc
--- /dev/null
+++ b/lib/common/CollectInBufferStream.cpp
@@ -0,0 +1,274 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: CollectInBufferStream.cpp
+// Purpose: Collect data in a buffer, and then read it out.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "CollectInBufferStream.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+#define INITIAL_BUFFER_SIZE 1024
+#define MAX_BUFFER_ADDITION (1024*64)
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::CollectInBufferStream()
+// Purpose: Constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+CollectInBufferStream::CollectInBufferStream()
+ : mBuffer(INITIAL_BUFFER_SIZE),
+ mBufferSize(INITIAL_BUFFER_SIZE),
+ mBytesInBuffer(0),
+ mReadPosition(0),
+ mInWritePhase(true)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::~CollectInBufferStream()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+CollectInBufferStream::~CollectInBufferStream()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::Read(void *, int, int)
+// Purpose: As interface. But only works in read phase
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+int CollectInBufferStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+
+ // Adjust to number of bytes left
+ if(NBytes > (mBytesInBuffer - mReadPosition))
+ {
+ NBytes = (mBytesInBuffer - mReadPosition);
+ }
+ ASSERT(NBytes >= 0);
+ if(NBytes <= 0) return 0; // careful now
+
+ // Copy in the requested number of bytes and adjust the read pointer
+ ::memcpy(pBuffer, ((char*)mBuffer) + mReadPosition, NBytes);
+ mReadPosition += NBytes;
+
+ return NBytes;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::BytesLeftToRead()
+// Purpose: As interface. But only works in read phase
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type CollectInBufferStream::BytesLeftToRead()
+{
+ if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+
+ return (mBytesInBuffer - mReadPosition);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::Write(void *, int)
+// Purpose: As interface. But only works in write phase
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::Write(const void *pBuffer, int NBytes)
+{
+ if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+
+ // Enough space in the buffer
+ if((mBytesInBuffer + NBytes) > mBufferSize)
+ {
+ // Need to reallocate... what's the block size we'll use?
+ int allocateBlockSize = mBufferSize;
+ if(allocateBlockSize > MAX_BUFFER_ADDITION)
+ {
+ allocateBlockSize = MAX_BUFFER_ADDITION;
+ }
+
+ // Write it the easy way. Although it's not the most efficient...
+ int newSize = mBufferSize;
+ while(newSize < (mBytesInBuffer + NBytes))
+ {
+ newSize += allocateBlockSize;
+ }
+
+ // Reallocate buffer
+ mBuffer.Resize(newSize);
+
+ // Store new size
+ mBufferSize = newSize;
+ }
+
+ // Copy in data and adjust counter
+ ::memcpy(((char*)mBuffer) + mBytesInBuffer, pBuffer, NBytes);
+ mBytesInBuffer += NBytes;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::GetPosition()
+// Purpose: In write phase, returns the number of bytes written, in read
+// phase, the number of bytes to go
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type CollectInBufferStream::GetPosition() const
+{
+ return mInWritePhase?mBytesInBuffer:mReadPosition;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::Seek(pos_type, int)
+// Purpose: As interface. But read phase only.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::Seek(pos_type Offset, int SeekType)
+{
+ if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+
+ int newPos = 0;
+ switch(SeekType)
+ {
+ case IOStream::SeekType_Absolute:
+ newPos = Offset;
+ break;
+ case IOStream::SeekType_Relative:
+ newPos = mReadPosition + Offset;
+ break;
+ case IOStream::SeekType_End:
+ newPos = mBytesInBuffer + Offset;
+ break;
+ default:
+ THROW_EXCEPTION(CommonException, IOStreamBadSeekType)
+ break;
+ }
+
+ // Make sure it doesn't go over
+ if(newPos > mBytesInBuffer)
+ {
+ newPos = mBytesInBuffer;
+ }
+ // or under
+ if(newPos < 0)
+ {
+ newPos = 0;
+ }
+
+ // Set the new read position
+ mReadPosition = newPos;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::StreamDataLeft()
+// Purpose: As interface
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool CollectInBufferStream::StreamDataLeft()
+{
+ return mInWritePhase?(false):(mReadPosition < mBytesInBuffer);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::StreamClosed()
+// Purpose: As interface
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool CollectInBufferStream::StreamClosed()
+{
+ return !mInWritePhase;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::SetForReading()
+// Purpose: Switch to read phase, after all data written
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::SetForReading()
+{
+ if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+
+ // Move to read phase
+ mInWritePhase = false;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::GetBuffer()
+// Purpose: Returns the buffer
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void *CollectInBufferStream::GetBuffer() const
+{
+ return mBuffer.GetPtr();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::GetSize()
+// Purpose: Returns the buffer size
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+int CollectInBufferStream::GetSize() const
+{
+ return mBytesInBuffer;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::Reset()
+// Purpose: Reset the stream, so it is empty and ready to be written to.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::Reset()
+{
+ mInWritePhase = true;
+ mBytesInBuffer = 0;
+ mReadPosition = 0;
+}
+
diff --git a/lib/common/CollectInBufferStream.h b/lib/common/CollectInBufferStream.h
new file mode 100755
index 00000000..d73af8db
--- /dev/null
+++ b/lib/common/CollectInBufferStream.h
@@ -0,0 +1,60 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: CollectInBufferStream.h
+// Purpose: Collect data in a buffer, and then read it out.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef COLLECTINBUFFERSTREAM__H
+#define COLLECTINBUFFERSTREAM__H
+
+#include "IOStream.h"
+#include "Guards.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: CollectInBufferStream
+// Purpose: Collect data in a buffer, and then read it out.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class CollectInBufferStream : public IOStream
+{
+public:
+ CollectInBufferStream();
+ ~CollectInBufferStream();
+private:
+ // No copying
+ CollectInBufferStream(const CollectInBufferStream &);
+ CollectInBufferStream(const IOStream &);
+public:
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+ virtual pos_type BytesLeftToRead();
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual pos_type GetPosition() const;
+ virtual void Seek(pos_type Offset, int SeekType);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+ void SetForReading();
+
+ void Reset();
+
+ void *GetBuffer() const;
+ int GetSize() const;
+ bool IsSetForReading() const {return !mInWritePhase;}
+
+private:
+ MemoryBlockGuard<char*> mBuffer;
+ int mBufferSize;
+ int mBytesInBuffer;
+ int mReadPosition;
+ bool mInWritePhase;
+};
+
+#endif // COLLECTINBUFFERSTREAM__H
+
diff --git a/lib/common/CommonException.h b/lib/common/CommonException.h
new file mode 100755
index 00000000..a0eb3bf5
--- /dev/null
+++ b/lib/common/CommonException.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: CommonException.h
+// Purpose: Exception
+// Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef COMMONEXCEPTION__H
+#define COMMONEXCEPTION__H
+
+// Compatibility header with old non-autogen exception scheme
+#include "autogen_CommonException.h"
+
+#endif // COMMONEXCEPTION__H
+
diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt
new file mode 100644
index 00000000..3875ed71
--- /dev/null
+++ b/lib/common/CommonException.txt
@@ -0,0 +1,44 @@
+
+# NOTE: Exception descriptions are for public distributions of Box Backup only -- do not rely for other applications.
+
+
+EXCEPTION Common 1
+
+Internal 0
+AssertFailed 1
+OSFileOpenError 2 Can't open a file -- attempted to load a non-existant config file or bad file referenced within?
+OSFileCloseError 3
+FileAlreadyClosed 4
+BadArguments 5
+ConfigNoKey 6
+ConfigNoSubConfig 7
+GetLineNoHandle 8
+OSFileError 9 Error accessing a file. Check permissions.
+GetLineEOF 10
+ConfigBadIntValue 11
+GetLineTooLarge 12 Protects against very large lines using up lots of memory.
+NotSupported 13
+OSFileReadError 14
+OSFileWriteError 15
+FileClosed 16
+IOStreamBadSeekType 17
+CantWriteToPartialReadStream 18
+CollectInBufferStreamNotInCorrectPhase 19
+NamedLockAlreadyLockingSomething 20
+NamedLockNotHeld 21
+StreamableMemBlockIncompleteRead 22
+MemBlockStreamNotSupported 23
+StreamDoesntHaveRequiredProperty 24
+CannotWriteToReadGatherStream 25
+ReadGatherStreamAddingBadBlock 26
+CouldNotLookUpUsername 27
+CouldNotRestoreProcessUser 28
+CouldNotChangeProcessUser 29
+RegexNotSupportedOnThisPlatform 30 Your platform does not have built in regular expression libraries.
+BadRegularExpression 31
+CouldNotCreateKQueue 32
+KEventErrorAdd 33
+KEventErrorWait 34
+KEventErrorRemove 35
+KQueueNotSupportedOnThisPlatform 36
+IOStreamGetLineNotEnoughDataToIgnore 37 Bad value passed to IOStreamGetLine::IgnoreBufferedData()
diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp
new file mode 100755
index 00000000..def93571
--- /dev/null
+++ b/lib/common/Configuration.cpp
@@ -0,0 +1,741 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: Configuration.cpp
+// Purpose: Reading configuration files
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <limits.h>
+
+#include "Configuration.h"
+#include "CommonException.h"
+#include "Guards.h"
+#include "FdGetLine.h"
+
+#include "MemLeakFindOn.h"
+
+// utility whitespace function
+inline bool iw(int c)
+{
+ return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded
+}
+
+// boolean values
+static const char *sValueBooleanStrings[] = {"yes", "true", "no", "false", 0};
+static const bool sValueBooleanValue[] = {true, true, false, false};
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::Configuration(const std::string &)
+// Purpose: Constructor
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+Configuration::Configuration(const std::string &rName)
+ : mName(rName)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::Configuration(const Configuration &)
+// Purpose: Copy constructor
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+Configuration::Configuration(const Configuration &rToCopy)
+ : mName(rToCopy.mName),
+ mSubConfigurations(rToCopy.mSubConfigurations),
+ mKeys(rToCopy.mKeys)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::~Configuration()
+// Purpose: Destructor
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+Configuration::~Configuration()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::LoadAndVerify(const std::string &, const ConfigurationVerify *, std::string &)
+// Purpose: Loads a configuration file from disc, checks it. Returns NULL if it was faulting, in which
+// case they'll be an error message.
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<Configuration> Configuration::LoadAndVerify(const char *Filename, const ConfigurationVerify *pVerify, std::string &rErrorMsg)
+{
+ // Check arguments
+ if(Filename == 0)
+ {
+ THROW_EXCEPTION(CommonException, BadArguments)
+ }
+
+ // Just to make sure
+ rErrorMsg.erase();
+
+ // Open the file
+ FileHandleGuard<O_RDONLY> file(Filename);
+
+ // GetLine object
+ FdGetLine getline(file);
+
+ // Object to create
+ Configuration *pconfig = new Configuration(std::string("<root>"));
+
+ try
+ {
+ // Load
+ LoadInto(*pconfig, getline, rErrorMsg, true);
+
+ if(!rErrorMsg.empty())
+ {
+ // An error occured, return now
+ //TRACE1("Error message from LoadInto: %s", rErrorMsg.c_str());
+ TRACE0("Error at Configuration::LoadInfo\n");
+ delete pconfig;
+ pconfig = 0;
+ return std::auto_ptr<Configuration>(0);
+ }
+
+ // Verify?
+ if(pVerify)
+ {
+ if(!Verify(*pconfig, *pVerify, std::string(), rErrorMsg))
+ {
+ //TRACE1("Error message from Verify: %s", rErrorMsg.c_str());
+ TRACE0("Error at Configuration::Verify\n");
+ delete pconfig;
+ pconfig = 0;
+ return std::auto_ptr<Configuration>(0);
+ }
+ }
+ }
+ catch(...)
+ {
+ // Clean up
+ delete pconfig;
+ pconfig = 0;
+ throw;
+ }
+
+ // Success. Return result.
+ return std::auto_ptr<Configuration>(pconfig);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: LoadInto(Configuration &, FdGetLine &, std::string &, bool)
+// Purpose: Private. Load configuration information from the file into the config object.
+// Returns 'abort' flag, if error, will be appended to rErrorMsg.
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel)
+{
+ bool startBlockExpected = false;
+ std::string blockName;
+
+ //TRACE1("BLOCK: |%s|\n", rConfig.mName.c_str());
+
+ while(!rGetLine.IsEOF())
+ {
+ std::string line(rGetLine.GetLine(true)); /* preprocess out whitespace and comments */
+
+ if(line.empty())
+ {
+ // Ignore blank lines
+ continue;
+ }
+
+ // Line an open block string?
+ if(line == "{")
+ {
+ if(startBlockExpected)
+ {
+ // New config object
+ Configuration config(blockName);
+
+ // Continue processing into this block
+ if(!LoadInto(config, rGetLine, rErrorMsg, false))
+ {
+ // Abort error
+ return false;
+ }
+
+ startBlockExpected = false;
+
+ // Store...
+ rConfig.mSubConfigurations.push_back(std::pair<std::string, Configuration>(blockName, config));
+ }
+ else
+ {
+ rErrorMsg += "Unexpected start block in " + rConfig.mName + "\n";
+ }
+ }
+ else
+ {
+ // Close block?
+ if(line == "}")
+ {
+ if(RootLevel)
+ {
+ // error -- root level doesn't have a close
+ rErrorMsg += "Root level has close block -- forget to terminate subblock?\n";
+ // but otherwise ignore
+ }
+ else
+ {
+ //TRACE0("ENDBLOCK\n");
+ return true; // All very good and nice
+ }
+ }
+ // Either a key, or a sub block beginning
+ else
+ {
+ // Can't be a start block
+ if(startBlockExpected)
+ {
+ rErrorMsg += "Block " + blockName + " wasn't started correctly (no '{' on line of it's own)\n";
+ startBlockExpected = false;
+ }
+
+ // Has the line got an = in it?
+ unsigned int equals = 0;
+ for(; equals < line.size(); ++equals)
+ {
+ if(line[equals] == '=')
+ {
+ // found!
+ break;
+ }
+ }
+ if(equals < line.size())
+ {
+ // Make key value pair
+ unsigned int keyend = equals;
+ while(keyend > 0 && iw(line[keyend-1]))
+ {
+ keyend--;
+ }
+ unsigned int valuestart = equals+1;
+ while(valuestart < line.size() && iw(line[valuestart]))
+ {
+ valuestart++;
+ }
+ if(keyend > 0 && valuestart <= line.size())
+ {
+ std::string key(line.substr(0, keyend));
+ std::string value(line.substr(valuestart));
+ //TRACE2("KEY: |%s|=|%s|\n", key.c_str(), value.c_str());
+
+ // Check for duplicate values
+ if(rConfig.mKeys.find(key) != rConfig.mKeys.end())
+ {
+ // Multi-values allowed here, but checked later on
+ rConfig.mKeys[key] += MultiValueSeparator;
+ rConfig.mKeys[key] += value;
+ }
+ else
+ {
+ // Store
+ rConfig.mKeys[key] = value;
+ }
+ }
+ else
+ {
+ rErrorMsg += "Invalid key in block "+rConfig.mName+"\n";
+ }
+ }
+ else
+ {
+ // Start of sub block
+ blockName = line;
+ startBlockExpected = true;
+ }
+ }
+ }
+ }
+
+ // End of file?
+ if(!RootLevel && rGetLine.IsEOF())
+ {
+ // Error if EOF and this isn't the root level
+ rErrorMsg += "File ended without terminating all subblocks\n";
+ }
+
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::KeyExists(const char *)
+// Purpose: Checks to see if a key exists
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+bool Configuration::KeyExists(const char *pKeyName) const
+{
+ if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+ return mKeys.find(pKeyName) != mKeys.end();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::GetKeyValue(const char *)
+// Purpose: Returns the value of a configuration variable
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+const std::string &Configuration::GetKeyValue(const char *pKeyName) const
+{
+ if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+ std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName));
+
+ if(i == mKeys.end())
+ {
+ THROW_EXCEPTION(CommonException, ConfigNoKey)
+ }
+ else
+ {
+ return i->second;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::GetKeyValueInt(const char *)
+// Purpose: Gets a key value as an integer
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+int Configuration::GetKeyValueInt(const char *pKeyName) const
+{
+ if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+ std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName));
+
+ if(i == mKeys.end())
+ {
+ THROW_EXCEPTION(CommonException, ConfigNoKey)
+ }
+ else
+ {
+ long value = ::strtol((i->second).c_str(), NULL, 0 /* C style handling */);
+ if(value == LONG_MAX || value == LONG_MIN)
+ {
+ THROW_EXCEPTION(CommonException, ConfigBadIntValue)
+ }
+ return (int)value;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::GetKeyValueBool(const char *) const
+// Purpose: Gets a key value as a boolean
+// Created: 17/2/04
+//
+// --------------------------------------------------------------------------
+bool Configuration::GetKeyValueBool(const char *pKeyName) const
+{
+ if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+ std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName));
+
+ if(i == mKeys.end())
+ {
+ THROW_EXCEPTION(CommonException, ConfigNoKey)
+ }
+ else
+ {
+ bool value = false;
+
+ // Anything this is called for should have been verified as having a correct
+ // string in the verification section. However, this does default to false
+ // if it isn't in the string table.
+
+ for(int l = 0; sValueBooleanStrings[l] != 0; ++l)
+ {
+ if(::strcasecmp((i->second).c_str(), sValueBooleanStrings[l]) == 0)
+ {
+ // Found.
+ value = sValueBooleanValue[l];
+ break;
+ }
+ }
+
+ return value;
+ }
+
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::GetKeyNames()
+// Purpose: Returns list of key names
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+std::vector<std::string> Configuration::GetKeyNames() const
+{
+ std::map<std::string, std::string>::const_iterator i(mKeys.begin());
+
+ std::vector<std::string> r;
+
+ for(; i != mKeys.end(); ++i)
+ {
+ r.push_back(i->first);
+ }
+
+ return r;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::SubConfigurationExists(const char *)
+// Purpose: Checks to see if a sub configuration exists
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+bool Configuration::SubConfigurationExists(const char *pSubName) const
+{
+ if(pSubName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+ // Attempt to find it...
+ std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin());
+
+ for(; i != mSubConfigurations.end(); ++i)
+ {
+ // This the one?
+ if(i->first == pSubName)
+ {
+ // Yes.
+ return true;
+ }
+ }
+
+ // didn't find it.
+ return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::GetSubConfiguration(const char *)
+// Purpose: Gets a sub configuration
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+const Configuration &Configuration::GetSubConfiguration(const char *pSubName) const
+{
+ if(pSubName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+ // Attempt to find it...
+ std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin());
+
+ for(; i != mSubConfigurations.end(); ++i)
+ {
+ // This the one?
+ if(i->first == pSubName)
+ {
+ // Yes.
+ return i->second;
+ }
+ }
+
+ THROW_EXCEPTION(CommonException, ConfigNoSubConfig)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::GetSubConfigurationNames()
+// Purpose: Return list of sub configuration names
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+std::vector<std::string> Configuration::GetSubConfigurationNames() const
+{
+ std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin());
+
+ std::vector<std::string> r;
+
+ for(; i != mSubConfigurations.end(); ++i)
+ {
+ r.push_back(i->first);
+ }
+
+ return r;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: Configuration::Verify(const Configuration &, const ConfigurationVerify &, const std::string &, std::string &)
+// Purpose: Return list of sub configuration names
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg)
+{
+ bool ok = true;
+
+ // First... check the keys
+ if(rVerify.mpKeys != 0)
+ {
+ const ConfigurationVerifyKey *pvkey = rVerify.mpKeys;
+
+ bool todo = true;
+ do
+ {
+ // Can the key be found?
+ ASSERT(pvkey->mpName);
+ if(rConfig.KeyExists(pvkey->mpName))
+ {
+ // Get value
+ const std::string &rval = rConfig.GetKeyValue(pvkey->mpName);
+ const char *val = rval.c_str();
+
+ // Check it's a number?
+ if((pvkey->Tests & ConfigTest_IsInt) == ConfigTest_IsInt)
+ {
+ // Test it...
+ char *end;
+ long r = ::strtol(val, &end, 0);
+ if(r == LONG_MIN || r == LONG_MAX || end != (val + rval.size()))
+ {
+ // not a good value
+ ok = false;
+ rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) is not a valid integer.\n";
+ }
+ }
+
+ // Check it's a bool?
+ if((pvkey->Tests & ConfigTest_IsBool) == ConfigTest_IsBool)
+ {
+ // See if it's one of the allowed strings.
+ bool found = false;
+ for(int l = 0; sValueBooleanStrings[l] != 0; ++l)
+ {
+ if(::strcasecmp(val, sValueBooleanStrings[l]) == 0)
+ {
+ // Found.
+ found = true;
+ break;
+ }
+ }
+
+ // Error if it's not one of them.
+ if(!found)
+ {
+ ok = false;
+ rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) is not a valid boolean value.\n";
+ }
+ }
+
+ // Check for multi valued statments where they're not allowed
+ if((pvkey->Tests & ConfigTest_MultiValueAllowed) == 0)
+ {
+ // Check to see if this key is a multi-value -- it shouldn't be
+ if(rval.find(MultiValueSeparator) != rval.npos)
+ {
+ ok = false;
+ rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) multi value not allowed (duplicated key?).\n";
+ }
+ }
+ }
+ else
+ {
+ // Is it required to exist?
+ if((pvkey->Tests & ConfigTest_Exists) == ConfigTest_Exists)
+ {
+ // Should exist, but doesn't.
+ ok = false;
+ rErrorMsg += rLevel + rConfig.mName + "." + pvkey->mpName + " (key) is missing.\n";
+ }
+ else if(pvkey->mpDefaultValue)
+ {
+ rConfig.mKeys[std::string(pvkey->mpName)] = std::string(pvkey->mpDefaultValue);
+ }
+ }
+
+ if((pvkey->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry)
+ {
+ // No more!
+ todo = false;
+ }
+
+ // next
+ pvkey++;
+
+ } while(todo);
+
+ // Check for additional keys
+ for(std::map<std::string, std::string>::const_iterator i = rConfig.mKeys.begin();
+ i != rConfig.mKeys.end(); ++i)
+ {
+ // Is the name in the list?
+ const ConfigurationVerifyKey *scan = rVerify.mpKeys;
+ bool found = false;
+ while(scan)
+ {
+ if(scan->mpName == i->first)
+ {
+ found = true;
+ break;
+ }
+
+ // Next?
+ if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry)
+ {
+ break;
+ }
+ scan++;
+ }
+
+ if(!found)
+ {
+ // Shouldn't exist, but does.
+ ok = false;
+ rErrorMsg += rLevel + rConfig.mName + "." + i->first + " (key) is not a known key. Check spelling and placement.\n";
+ }
+ }
+ }
+
+ // Then the sub configurations
+ if(rVerify.mpSubConfigurations)
+ {
+ // Find the wildcard entry, if it exists, and check that required subconfigs are there
+ const ConfigurationVerify *wildcardverify = 0;
+
+ const ConfigurationVerify *scan = rVerify.mpSubConfigurations;
+ while(scan)
+ {
+ ASSERT(scan->mpName);
+ if(scan->mpName[0] == '*')
+ {
+ wildcardverify = scan;
+ }
+
+ // Required?
+ if((scan->Tests & ConfigTest_Exists) == ConfigTest_Exists)
+ {
+ if(scan->mpName[0] == '*')
+ {
+ // Check something exists
+ if(rConfig.mSubConfigurations.size() < 1)
+ {
+ // A sub config should exist, but doesn't.
+ ok = false;
+ rErrorMsg += rLevel + rConfig.mName + ".* (block) is missing (a block must be present).\n";
+ }
+ }
+ else
+ {
+ // Check real thing exists
+ if(!rConfig.SubConfigurationExists(scan->mpName))
+ {
+ // Should exist, but doesn't.
+ ok = false;
+ rErrorMsg += rLevel + rConfig.mName + "." + scan->mpName + " (block) is missing.\n";
+ }
+ }
+ }
+
+ // Next?
+ if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry)
+ {
+ break;
+ }
+ scan++;
+ }
+
+ // Go through the sub configurations, one by one
+ for(std::list<std::pair<std::string, Configuration> >::const_iterator i(rConfig.mSubConfigurations.begin());
+ i != rConfig.mSubConfigurations.end(); ++i)
+ {
+ // Can this be found?
+ const ConfigurationVerify *subverify = 0;
+
+ const ConfigurationVerify *scan = rVerify.mpSubConfigurations;
+ const char *name = i->first.c_str();
+ ASSERT(name);
+ while(scan)
+ {
+ if(strcmp(scan->mpName, name) == 0)
+ {
+ // found it!
+ subverify = scan;
+ }
+
+ // Next?
+ if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry)
+ {
+ break;
+ }
+ scan++;
+ }
+
+ // Use wildcard?
+ if(subverify == 0)
+ {
+ subverify = wildcardverify;
+ }
+
+ // Verify
+ if(subverify)
+ {
+ // override const-ness here...
+ if(!Verify((Configuration&)i->second, *subverify, rConfig.mName + '.', rErrorMsg))
+ {
+ ok = false;
+ }
+ }
+ }
+ }
+
+ return ok;
+}
+
+
diff --git a/lib/common/Configuration.h b/lib/common/Configuration.h
new file mode 100755
index 00000000..4c455b0f
--- /dev/null
+++ b/lib/common/Configuration.h
@@ -0,0 +1,98 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: Configuration
+// Purpose: Reading configuration files
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+
+#ifndef CONFIGURATION__H
+#define CONFIGURATION__H
+
+#include <map>
+#include <list>
+#include <vector>
+#include <string>
+#include <memory>
+
+// For defining tests
+enum
+{
+ ConfigTest_LastEntry = 1,
+ ConfigTest_Exists = 2,
+ ConfigTest_IsInt = 4,
+ ConfigTest_MultiValueAllowed = 8,
+ ConfigTest_IsBool = 16
+};
+
+class ConfigurationVerifyKey
+{
+public:
+ const char *mpName; // "*" for all other keys (not implemented yet)
+ const char *mpDefaultValue; // default for when it's not present
+ int Tests;
+ void *TestFunction; // set to zero for now, will implement later
+};
+
+class ConfigurationVerify
+{
+public:
+ const char *mpName; // "*" for all other sub config names
+ const ConfigurationVerify *mpSubConfigurations;
+ const ConfigurationVerifyKey *mpKeys;
+ int Tests;
+ void *TestFunction; // set to zero for now, will implement later
+};
+
+class FdGetLine;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: Configuration
+// Purpose: Loading, checking, and representing configuration files
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+class Configuration
+{
+private:
+ Configuration(const std::string &rName);
+public:
+ Configuration(const Configuration &rToCopy);
+ ~Configuration();
+
+ enum
+ {
+ // The character to separate multi-values
+ MultiValueSeparator = '\x01'
+ };
+
+ static std::auto_ptr<Configuration> LoadAndVerify(const char *Filename, const ConfigurationVerify *pVerify, std::string &rErrorMsg);
+ static std::auto_ptr<Configuration> Load(const char *Filename, std::string &rErrorMsg) { return LoadAndVerify(Filename, 0, rErrorMsg); }
+
+ bool KeyExists(const char *pKeyName) const;
+ const std::string &GetKeyValue(const char *pKeyName) const;
+ int GetKeyValueInt(const char *pKeyName) const;
+ bool GetKeyValueBool(const char *pKeyName) const;
+ std::vector<std::string> GetKeyNames() const;
+
+ bool SubConfigurationExists(const char *pSubName) const;
+ const Configuration &GetSubConfiguration(const char *pSubName) const;
+ std::vector<std::string> GetSubConfigurationNames() const;
+
+ std::string mName;
+ // Order of sub blocks preserved
+ typedef std::list<std::pair<std::string, Configuration> > SubConfigListType;
+ SubConfigListType mSubConfigurations;
+ // Order of keys, not preserved
+ std::map<std::string, std::string> mKeys;
+
+private:
+ static bool LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel);
+ static bool Verify(Configuration &rConfig, const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg);
+};
+
+#endif // CONFIGURATION__H
+
diff --git a/lib/common/Conversion.h b/lib/common/Conversion.h
new file mode 100644
index 00000000..cba5bb08
--- /dev/null
+++ b/lib/common/Conversion.h
@@ -0,0 +1,98 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: Conversion.h
+// Purpose: Convert between various types
+// Created: 9/4/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef CONVERSION__H
+#define CONVERSION__H
+
+#include <string>
+
+namespace BoxConvert
+{
+ // --------------------------------------------------------------------------
+ //
+ // Function
+ // Name: BoxConvert::Convert<to_type, from_type>(to_type &, from_type)
+ // Purpose: Convert from types to types
+ // Created: 9/4/04
+ //
+ // --------------------------------------------------------------------------
+ template<typename to_type, typename from_type>
+ inline to_type Convert(from_type From)
+ {
+ // Default conversion, simply use C++ conversion
+ return From;
+ }
+
+ // Specialise for string -> integer
+ int32_t _ConvertStringToInt(const char *pString, int Size);
+ template<>
+ inline int32_t Convert<int32_t, const std::string &>(const std::string &rFrom)
+ {
+ return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 32);
+ }
+ template<>
+ inline int16_t Convert<int16_t, const std::string &>(const std::string &rFrom)
+ {
+ return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 16);
+ }
+ template<>
+ inline int8_t Convert<int8_t, const std::string &>(const std::string &rFrom)
+ {
+ return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 8);
+ }
+ template<>
+ inline int32_t Convert<int32_t, const char *>(const char *pFrom)
+ {
+ return BoxConvert::_ConvertStringToInt(pFrom, 32);
+ }
+ template<>
+ inline int16_t Convert<int16_t, const char *>(const char *pFrom)
+ {
+ return BoxConvert::_ConvertStringToInt(pFrom, 16);
+ }
+ template<>
+ inline int8_t Convert<int8_t, const char *>(const char *pFrom)
+ {
+ return BoxConvert::_ConvertStringToInt(pFrom, 8);
+ }
+
+ // Specialise for integer -> string
+ void _ConvertIntToString(std::string &rTo, int32_t From);
+ template<>
+ inline std::string Convert<std::string, int32_t>(int32_t From)
+ {
+ std::string r;
+ BoxConvert::_ConvertIntToString(r, From);
+ return r;
+ }
+ template<>
+ inline std::string Convert<std::string, int16_t>(int16_t From)
+ {
+ std::string r;
+ BoxConvert::_ConvertIntToString(r, From);
+ return r;
+ }
+ template<>
+ inline std::string Convert<std::string, int8_t>(int8_t From)
+ {
+ std::string r;
+ BoxConvert::_ConvertIntToString(r, From);
+ return r;
+ }
+
+ // Specialise for bool -> string
+ template<>
+ inline std::string Convert<std::string, bool>(bool From)
+ {
+ return std::string(From?"true":"false");
+ }
+};
+
+#endif // CONVERSION__H
+
diff --git a/lib/common/ConversionException.txt b/lib/common/ConversionException.txt
new file mode 100644
index 00000000..91b5fa9a
--- /dev/null
+++ b/lib/common/ConversionException.txt
@@ -0,0 +1,8 @@
+
+EXCEPTION Conversion 12
+
+Internal 0
+CannotConvertEmptyStringToInt 1
+BadStringRepresentationOfInt 2
+IntOverflowInConvertFromString 3
+BadIntSize 4
diff --git a/lib/common/ConversionString.cpp b/lib/common/ConversionString.cpp
new file mode 100644
index 00000000..9e237a77
--- /dev/null
+++ b/lib/common/ConversionString.cpp
@@ -0,0 +1,123 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: ConversionString.cpp
+// Purpose: Conversions to and from strings
+// Created: 9/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+
+#include "Conversion.h"
+#include "autogen_ConversionException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BoxConvert::_ConvertStringToInt(const char *, int)
+// Purpose: Convert from string to integer, with range checking.
+// Always does signed -- no point in unsigned as C++ type checking
+// isn't up to handling it properly.
+// If a null pointer is passed in, then returns 0.
+// Created: 9/4/04
+//
+// --------------------------------------------------------------------------
+int32_t BoxConvert::_ConvertStringToInt(const char *pString, int Size)
+{
+ // Handle null strings gracefully.
+ if(pString == 0)
+ {
+ return 0;
+ }
+
+ // Check for initial validity
+ if(*pString == '\0')
+ {
+ THROW_EXCEPTION(ConversionException, CannotConvertEmptyStringToInt)
+ }
+
+ // Convert.
+ char *numEnd = 0;
+ errno = 0; // Some platforms don't reset it.
+ long r = ::strtol(pString, &numEnd, 0);
+
+ // Check that all the characters were used
+ if(*numEnd != '\0')
+ {
+ THROW_EXCEPTION(ConversionException, BadStringRepresentationOfInt)
+ }
+
+ // Error check
+ if(r == 0 && errno == EINVAL)
+ {
+ THROW_EXCEPTION(ConversionException, BadStringRepresentationOfInt)
+ }
+
+ // Range check from strtol
+ if((r == LONG_MIN || r == LONG_MAX) && errno == ERANGE)
+ {
+ THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString)
+ }
+
+ // Check range for size of integer
+ switch(Size)
+ {
+ case 32:
+ {
+ // No extra checking needed, if this assert holds true
+ ASSERT(sizeof(long) == sizeof(int32_t));
+ }
+ break;
+
+ case 16:
+ {
+ if(r <= (0 - 0x7fff) || r > 0x7fff)
+ {
+ THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString)
+ }
+ break;
+ }
+
+ case 8:
+ {
+ if(r <= (0 - 0x7f) || r > 0x7f)
+ {
+ THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString)
+ }
+ break;
+ }
+
+ default:
+ {
+ THROW_EXCEPTION(ConversionException, BadIntSize)
+ break;
+ }
+ }
+
+ // Return number
+ return r;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BoxConvert::_ConvertIntToString(std::string &, int32_t)
+// Purpose: Convert signed interger to a string
+// Created: 9/4/04
+//
+// --------------------------------------------------------------------------
+void BoxConvert::_ConvertIntToString(std::string &rTo, int32_t From)
+{
+ char text[64]; // size more than enough
+ ::sprintf(text, "%d", From);
+ rTo = text;
+}
+
diff --git a/lib/common/DebugAssertFailed.cpp b/lib/common/DebugAssertFailed.cpp
new file mode 100755
index 00000000..2acf4602
--- /dev/null
+++ b/lib/common/DebugAssertFailed.cpp
@@ -0,0 +1,32 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: AssertFailed.cpp
+// Purpose: Assert failure code
+// Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef NDEBUG
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <syslog.h>
+
+#include "MemLeakFindOn.h"
+
+bool AssertFailuresToSyslog = false;
+
+void BoxDebugAssertFailed(char *cond, char *file, int line)
+{
+ printf("ASSERT FAILED: [%s] at %s(%d)\n", cond, file, line);
+ if(AssertFailuresToSyslog)
+ {
+ ::syslog(LOG_ERR, "ASSERT FAILED: [%s] at %s(%d)\n", cond, file, line);
+ }
+}
+
+
+#endif // NDEBUG
+
diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp
new file mode 100755
index 00000000..e1877a13
--- /dev/null
+++ b/lib/common/DebugMemLeakFinder.cpp
@@ -0,0 +1,377 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: MemLeakFinder.cpp
+// Purpose: Memory leak finder
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+
+#ifndef NDEBUG
+
+#include "Box.h"
+
+#undef malloc
+#undef realloc
+#undef free
+
+#include <map>
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <set>
+
+bool memleakfinder_global_enable = false;
+
+typedef struct
+{
+ size_t size;
+ const char *file;
+ int line;
+} MallocBlockInfo;
+
+typedef struct
+{
+ size_t size;
+ const char *file;
+ int line;
+ bool array;
+} ObjectInfo;
+
+namespace
+{
+ static std::map<void *, MallocBlockInfo> sMallocBlocks;
+ static std::map<void *, ObjectInfo> sObjectBlocks;
+
+ static bool sTrackMallocInSection = false;
+ static std::set<void *> sSectionMallocBlocks;
+ static bool sTrackObjectsInSection = false;
+ static std::map<void *, ObjectInfo> sSectionObjectBlocks;
+
+ static std::set<void *> sNotLeaks;
+
+ void *sNotLeaksPre[1024];
+ int sNotLeaksPreNum = 0;
+}
+
+void memleakfinder_malloc_add_block(void *b, size_t size, const char *file, int line)
+{
+ if(b != 0)
+ {
+ MallocBlockInfo i;
+ i.size = size;
+ i.file = file;
+ i.line = line;
+ sMallocBlocks[b] = i;
+
+ if(sTrackMallocInSection)
+ {
+ sSectionMallocBlocks.insert(b);
+ }
+ }
+}
+
+
+void *memleakfinder_malloc(size_t size, const char *file, int line)
+{
+ void *b = ::malloc(size);
+ if(!memleakfinder_global_enable) return b;
+
+ memleakfinder_malloc_add_block(b, size, file, line);
+
+ //TRACE4("malloc(), %d, %s, %d, %08x\n", size, file, line, b);
+ return b;
+}
+
+void *memleakfinder_realloc(void *ptr, size_t size)
+{
+ if(!memleakfinder_global_enable)
+ {
+ return ::realloc(ptr, size);
+ }
+
+ // Check it's been allocated
+ std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr));
+ if(i == sMallocBlocks.end())
+ {
+ TRACE1("Block %x realloc(), but not in list. Error? Or allocated in startup static objects?\n", ptr);
+ void *b = ::realloc(ptr, size);
+ memleakfinder_malloc_add_block(b, size, "FOUND-IN-REALLOC", 0);
+ return b;
+ }
+
+ void *b = ::realloc(ptr, size);
+
+ // Worked?
+ if(b != 0)
+ {
+ // Update map
+ MallocBlockInfo inf = i->second;
+ inf.size = size;
+ sMallocBlocks.erase(i);
+ sMallocBlocks[b] = inf;
+
+ if(sTrackMallocInSection)
+ {
+ std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr));
+ if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si);
+ sSectionMallocBlocks.insert(b);
+ }
+ }
+
+ //TRACE3("realloc(), %d, %08x->%08x\n", size, ptr, b);
+ return b;
+}
+
+void memleakfinder_free(void *ptr)
+{
+ if(memleakfinder_global_enable)
+ {
+ // Check it's been allocated
+ std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr));
+ if(i != sMallocBlocks.end())
+ {
+ sMallocBlocks.erase(i);
+ }
+ else
+ {
+ TRACE1("Block %x freed, but not known. Error? Or allocated in startup static allocation?\n", ptr);
+ }
+
+ if(sTrackMallocInSection)
+ {
+ std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr));
+ if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si);
+ }
+ }
+
+ //TRACE1("free(), %08x\n", ptr);
+ ::free(ptr);
+}
+
+
+void memleakfinder_notaleak_insert_pre()
+{
+ if(!memleakfinder_global_enable) return;
+ for(int l = 0; l < sNotLeaksPreNum; l++)
+ {
+ sNotLeaks.insert(sNotLeaksPre[l]);
+ }
+ sNotLeaksPreNum = 0;
+}
+
+bool is_leak(void *ptr)
+{
+ memleakfinder_notaleak_insert_pre();
+ return sNotLeaks.find(ptr) == sNotLeaks.end();
+}
+
+void memleakfinder_notaleak(void *ptr)
+{
+ memleakfinder_notaleak_insert_pre();
+ if(memleakfinder_global_enable)
+ {
+ sNotLeaks.insert(ptr);
+ }
+ else
+ {
+ sNotLeaksPre[sNotLeaksPreNum++] = ptr;
+ }
+/* {
+ std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr));
+ if(i != sMallocBlocks.end()) sMallocBlocks.erase(i);
+ }
+ {
+ std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr));
+ if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si);
+ }
+ {
+ std::map<void *, ObjectInfo>::iterator i(sObjectBlocks.find(ptr));
+ if(i != sObjectBlocks.end()) sObjectBlocks.erase(i);
+ }*/
+}
+
+
+
+// start monitoring a section of code
+void memleakfinder_startsectionmonitor()
+{
+ sTrackMallocInSection = true;
+ sSectionMallocBlocks.clear();
+ sTrackObjectsInSection = true;
+ sSectionObjectBlocks.clear();
+}
+
+// trace all blocks allocated and still allocated since memleakfinder_startsectionmonitor() called
+void memleakfinder_traceblocksinsection()
+{
+ std::set<void *>::iterator s(sSectionMallocBlocks.begin());
+ for(; s != sSectionMallocBlocks.end(); ++s)
+ {
+ std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.find(*s));
+ if(i == sMallocBlocks.end())
+ {
+ TRACE0("Logical error in section block finding\n");
+ }
+ else
+ {
+ TRACE4("Block 0x%08x size %d allocated at %s:%d\n", (int)i->first, i->second.size, i->second.file, i->second.line);
+ }
+ }
+ for(std::map<void *, ObjectInfo>::const_iterator i(sSectionObjectBlocks.begin()); i != sSectionObjectBlocks.end(); ++i)
+ {
+ TRACE5("Object%s 0x%08x size %d allocated at %s:%d\n", i->second.array?" []":"", (int)i->first, i->second.size, i->second.file, i->second.line);
+ }
+}
+
+int memleakfinder_numleaks()
+{
+ int n = 0;
+
+ for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i)
+ {
+ if(is_leak(i->first)) ++n;
+ }
+
+ for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i)
+ {
+ if(is_leak(i->first)) ++n;
+ }
+
+ return n;
+}
+
+void memleakfinder_reportleaks_file(FILE *file)
+{
+ for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i)
+ {
+ if(is_leak(i->first)) ::fprintf(file, "Block 0x%08x size %d allocated at %s:%d\n", (int)i->first, i->second.size, i->second.file, i->second.line);
+ }
+ for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i)
+ {
+ if(is_leak(i->first)) ::fprintf(file, "Object%s 0x%08x size %d allocated at %s:%d\n", i->second.array?" []":"", (int)i->first, i->second.size, i->second.file, i->second.line);
+ }
+}
+
+void memleakfinder_reportleaks()
+{
+ // report to stdout
+ memleakfinder_reportleaks_file(stdout);
+}
+
+void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext)
+{
+ FILE *file = ::fopen(filename, "a");
+ if(file != 0)
+ {
+ if(memleakfinder_numleaks() > 0)
+ {
+ fprintf(file, "MEMORY LEAKS FROM PROCESS %d (%s)\n", getpid(), markertext);
+ memleakfinder_reportleaks_file(file);
+ }
+
+ ::fclose(file);
+ }
+ else
+ {
+ printf("WARNING: Couldn't open memory leak results file %s for appending\n", filename);
+ }
+}
+
+static char atexit_filename[512];
+static char atexit_markertext[512];
+static bool atexit_registered = false;
+
+void memleakfinder_atexit()
+{
+ memleakfinder_reportleaks_appendfile(atexit_filename, atexit_markertext);
+}
+
+void memleakfinder_setup_exit_report(const char *filename, const char *markertext)
+{
+ ::strcpy(atexit_filename, filename);
+ ::strcpy(atexit_markertext, markertext);
+ if(!atexit_registered)
+ {
+ atexit(memleakfinder_atexit);
+ atexit_registered = true;
+ }
+}
+
+
+
+
+void add_object_block(void *block, size_t size, const char *file, int line, bool array)
+{
+ if(!memleakfinder_global_enable) return;
+
+ if(block != 0)
+ {
+ ObjectInfo i;
+ i.size = size;
+ i.file = file;
+ i.line = line;
+ i.array = array;
+ sObjectBlocks[block] = i;
+
+ if(sTrackObjectsInSection)
+ {
+ sSectionObjectBlocks[block] = i;
+ }
+ }
+}
+
+void remove_object_block(void *block)
+{
+ if(!memleakfinder_global_enable) return;
+
+ std::map<void *, ObjectInfo>::iterator i(sObjectBlocks.find(block));
+ if(i != sObjectBlocks.end())
+ {
+ sObjectBlocks.erase(i);
+ }
+
+ if(sTrackObjectsInSection)
+ {
+ std::map<void *, ObjectInfo>::iterator i(sSectionObjectBlocks.find(block));
+ if(i != sSectionObjectBlocks.end())
+ {
+ sSectionObjectBlocks.erase(i);
+ }
+ }
+
+ // If it's not in the list, just ignore it, as lots of stuff goes this way...
+}
+
+void *operator new(size_t size, const char *file, int line)
+{
+ void *r = ::malloc(size);
+ add_object_block(r, size, file, line, false);
+ //TRACE4("new(), %d, %s, %d, %08x\n", size, file, line, r);
+ return r;
+}
+
+void *operator new[](size_t size, const char *file, int line)
+{
+ void *r = ::malloc(size);
+ add_object_block(r, size, file, line, true);
+ //TRACE4("new[](), %d, %s, %d, %08x\n", size, file, line, r);
+ return r;
+}
+
+void operator delete[](void *ptr) throw ()
+{
+ ::free(ptr);
+ remove_object_block(ptr);
+ //TRACE1("delete[]() called, %08x\n", ptr);
+}
+
+void operator delete(void *ptr) throw ()
+{
+ ::free(ptr);
+ remove_object_block(ptr);
+ //TRACE1("delete() called, %08x\n", ptr);
+}
+
+#endif // NDEBUG
diff --git a/lib/common/DebugPrintf.cpp b/lib/common/DebugPrintf.cpp
new file mode 100755
index 00000000..02c25496
--- /dev/null
+++ b/lib/common/DebugPrintf.cpp
@@ -0,0 +1,72 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: DebugPrintf.cpp
+// Purpose: Implementation of a printf function, to avoid a stdio.h include in Box.h
+// Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+
+#ifndef NDEBUG
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <syslog.h>
+
+#include "MemLeakFindOn.h"
+
+// Use this apparently superflous printf function to avoid having to
+// include stdio.h in every file in the project.
+
+int BoxDebug_printf(const char *format, ...)
+{
+ va_list ap;
+ va_start(ap, format);
+ int r = vprintf(format, ap);
+ va_end(ap);
+ return r;
+}
+
+
+bool BoxDebugTraceOn = true;
+bool BoxDebugTraceToStdout = true;
+bool BoxDebugTraceToSyslog = false;
+
+int BoxDebugTrace(const char *format, ...)
+{
+ char text[512];
+ int r = 0;
+ if(BoxDebugTraceOn || BoxDebugTraceToSyslog)
+ {
+ va_list ap;
+ va_start(ap, format);
+ r = vsnprintf(text, sizeof(text), format, ap);
+ va_end(ap);
+ }
+
+ // Send to stdout if trace is on and std out is enabled
+ if(BoxDebugTraceOn && BoxDebugTraceToStdout)
+ {
+ printf("%s", text);
+ }
+
+ // But tracing to syslog is independent of tracing being on or not
+ if(BoxDebugTraceToSyslog)
+ {
+ // Remove trailing '\n', if it's there
+ if(r > 0 && text[r] == '\n')
+ {
+ text[r] = '\0';
+ --r;
+ }
+ // Log it
+ ::syslog(LOG_INFO, "TRACE: %s", text);
+ }
+
+ return r;
+}
+
+
+#endif // NDEBUG
diff --git a/lib/common/EndStructPackForWire.h b/lib/common/EndStructPackForWire.h
new file mode 100644
index 00000000..c9655cd7
--- /dev/null
+++ b/lib/common/EndStructPackForWire.h
@@ -0,0 +1,23 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: EndStructPackForWire.h
+// Purpose: End structure packing for wire
+// Created: 25/11/03
+//
+// --------------------------------------------------------------------------
+
+// No header guard -- this is intentional
+
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+
+#pragma pack()
+
+#else
+
+ logical error -- check BoxPlatform.h and including file
+
+#endif
+
+
+
diff --git a/lib/common/EventWatchFilesystemObject.cpp b/lib/common/EventWatchFilesystemObject.cpp
new file mode 100644
index 00000000..1611d5bd
--- /dev/null
+++ b/lib/common/EventWatchFilesystemObject.cpp
@@ -0,0 +1,99 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: EventWatchFilesystemObject.cpp
+// Purpose: WaitForEvent compatible object for watching directories
+// Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "EventWatchFilesystemObject.h"
+#include "autogen_CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: EventWatchFilesystemObject::EventWatchFilesystemObject(const char *)
+// Purpose: Constructor -- opens the file object
+// Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename)
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+ : mDescriptor(::open(Filename, O_RDONLY /*O_EVTONLY*/, 0))
+#endif
+{
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+ if(mDescriptor == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileOpenError)
+ }
+#else
+ THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform)
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: EventWatchFilesystemObject::~EventWatchFilesystemObject()
+// Purpose: Destructor
+// Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+EventWatchFilesystemObject::~EventWatchFilesystemObject()
+{
+ if(mDescriptor != -1)
+ {
+ ::close(mDescriptor);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesystemObject &)
+// Purpose: Copy constructor
+// Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy)
+ : mDescriptor(::dup(rToCopy.mDescriptor))
+{
+ if(mDescriptor == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+}
+
+
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: EventWatchFilesystemObject::FillInKEvent(struct kevent &, int)
+// Purpose: For WaitForEvent
+// Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+void EventWatchFilesystemObject::FillInKEvent(struct kevent &rEvent, int Flags) const
+{
+ EV_SET(&rEvent, mDescriptor, EVFILT_VNODE, EV_CLEAR, NOTE_DELETE | NOTE_WRITE, 0, (void*)this);
+}
+#else
+void EventWatchFilesystemObject::FillInPoll(int &fd, short &events, int Flags) const
+{
+ THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform)
+}
+#endif
+
diff --git a/lib/common/EventWatchFilesystemObject.h b/lib/common/EventWatchFilesystemObject.h
new file mode 100644
index 00000000..d8a7245b
--- /dev/null
+++ b/lib/common/EventWatchFilesystemObject.h
@@ -0,0 +1,48 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: EventWatchFilesystemObject.h
+// Purpose: WaitForEvent compatible object for watching directories
+// Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef EVENTWATCHFILESYSTEMOBJECT__H
+#define EVENTWATCHFILESYSTEMOBJECT__H
+
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+ #include <sys/event.h>
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: EventWatchFilesystemObject
+// Purpose: WaitForEvent compatible object for watching files and directories
+// Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+class EventWatchFilesystemObject
+{
+public:
+ EventWatchFilesystemObject(const char *Filename);
+ ~EventWatchFilesystemObject();
+ EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy);
+private:
+ // Assignment not allowed
+ EventWatchFilesystemObject &operator=(const EventWatchFilesystemObject &);
+public:
+
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+ void FillInKEvent(struct kevent &rEvent, int Flags = 0) const;
+#else
+ void FillInPoll(int &fd, short &events, int Flags = 0) const;
+#endif
+
+private:
+ int mDescriptor;
+};
+
+#endif // EventWatchFilesystemObject__H
+
diff --git a/lib/common/ExcludeList.cpp b/lib/common/ExcludeList.cpp
new file mode 100755
index 00000000..6350869d
--- /dev/null
+++ b/lib/common/ExcludeList.cpp
@@ -0,0 +1,219 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: ExcludeList.cpp
+// Purpose: General purpose exclusion list
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+ #include <regex.h>
+ #define EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED
+#endif
+
+#include "ExcludeList.h"
+#include "Utils.h"
+#include "Configuration.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ExcludeList::ExcludeList()
+// Purpose: Constructor. Generates an exclude list which will allow everything
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+ExcludeList::ExcludeList()
+ : mpAlwaysInclude(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ExcludeList::~ExcludeList()
+// Purpose: Destructor
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+ExcludeList::~ExcludeList()
+{
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+ // free regex memory
+ while(mRegex.size() > 0)
+ {
+ regex_t *pregex = mRegex.back();
+ mRegex.pop_back();
+ // Free regex storage, and the structure itself
+ ::regfree(pregex);
+ delete pregex;
+ }
+#endif
+
+ // Clean up exceptions list
+ if(mpAlwaysInclude != 0)
+ {
+ delete mpAlwaysInclude;
+ mpAlwaysInclude = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ExcludeList::AddDefiniteEntries(const std::string &)
+// Purpose: Adds a number of definite entries to the exclude list -- ones which
+// will be excluded if and only if the test string matches exactly.
+// Uses the Configuration classes' multi-value conventions, with
+// multiple entires in one string separated by Configuration::MultiValueSeparator
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+void ExcludeList::AddDefiniteEntries(const std::string &rEntries)
+{
+ // Split strings up
+ std::vector<std::string> ens;
+ SplitString(rEntries, Configuration::MultiValueSeparator, ens);
+
+ // Add to set of excluded strings
+ for(std::vector<std::string>::const_iterator i(ens.begin()); i != ens.end(); ++i)
+ {
+ if(i->size() > 0)
+ {
+ mDefinite.insert(*i);
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ExcludeList::AddRegexEntries(const std::string &)
+// Purpose: Adds a number of regular expression entries to the exclude list --
+// if the test expression matches any of these regex, it will be excluded.
+// Uses the Configuration classes' multi-value conventions, with
+// multiple entires in one string separated by Configuration::MultiValueSeparator
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+void ExcludeList::AddRegexEntries(const std::string &rEntries)
+{
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+
+ // Split strings up
+ std::vector<std::string> ens;
+ SplitString(rEntries, Configuration::MultiValueSeparator, ens);
+
+ // Create and add new regular expressions
+ for(std::vector<std::string>::const_iterator i(ens.begin()); i != ens.end(); ++i)
+ {
+ if(i->size() > 0)
+ {
+ // Allocate memory
+ regex_t *pregex = new regex_t;
+
+ try
+ {
+ // Compile
+ if(::regcomp(pregex, i->c_str(), REG_EXTENDED | REG_NOSUB) != 0)
+ {
+ THROW_EXCEPTION(CommonException, BadRegularExpression)
+ }
+
+ // Store in list of regular expressions
+ mRegex.push_back(pregex);
+ }
+ catch(...)
+ {
+ delete pregex;
+ throw;
+ }
+ }
+ }
+
+#else
+ THROW_EXCEPTION(CommonException, RegexNotSupportedOnThisPlatform)
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ExcludeList::IsExcluded(const std::string &)
+// Purpose: Returns true if the entry should be excluded
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+bool ExcludeList::IsExcluded(const std::string &rTest) const
+{
+ // Check against the always include list
+ if(mpAlwaysInclude != 0)
+ {
+ if(mpAlwaysInclude->IsExcluded(rTest))
+ {
+ // Because the "always include" list says it's 'excluded'
+ // this means it should actually be included.
+ return false;
+ }
+ }
+
+ // Is it in the set of definite entries?
+ if(mDefinite.find(rTest) != mDefinite.end())
+ {
+ return true;
+ }
+
+ // Check against regular expressions
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+ for(std::vector<regex_t *>::const_iterator i(mRegex.begin()); i != mRegex.end(); ++i)
+ {
+ // Test against this expression
+ if(regexec(*i, rTest.c_str(), 0, 0 /* no match information required */, 0 /* no flags */) == 0)
+ {
+ // match happened
+ return true;
+ }
+ // In all other cases, including an error, just continue to the next expression
+ }
+#endif
+
+ return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ExcludeList::SetAlwaysIncludeList(ExcludeList *)
+// Purpose: Takes ownership of the list, deletes any pre-existing list.
+// NULL is acceptable to delete the list.
+// The AlwaysInclude list is a list of exceptions to the exclusions.
+// Created: 19/2/04
+//
+// --------------------------------------------------------------------------
+void ExcludeList::SetAlwaysIncludeList(ExcludeList *pAlwaysInclude)
+{
+ // Delete old list
+ if(mpAlwaysInclude != 0)
+ {
+ delete mpAlwaysInclude;
+ mpAlwaysInclude = 0;
+ }
+
+ // Store the pointer
+ mpAlwaysInclude = pAlwaysInclude;
+}
+
+
+
+
+
diff --git a/lib/common/ExcludeList.h b/lib/common/ExcludeList.h
new file mode 100755
index 00000000..a1954044
--- /dev/null
+++ b/lib/common/ExcludeList.h
@@ -0,0 +1,65 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: ExcludeList.h
+// Purpose: General purpose exclusion list
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef EXCLUDELIST__H
+#define EXCLUDELIST__H
+
+#include <string>
+#include <set>
+#include <vector>
+
+// avoid including regex.h in lots of places
+#ifndef EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED
+ typedef int regex_t;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: ExcludeList
+// Purpose: General purpose exclusion list
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+class ExcludeList
+{
+public:
+ ExcludeList();
+ ~ExcludeList();
+
+ void AddDefiniteEntries(const std::string &rEntries);
+ void AddRegexEntries(const std::string &rEntries);
+
+ // Add exceptions to the exclusions (takes ownership)
+ void SetAlwaysIncludeList(ExcludeList *pAlwaysInclude);
+
+ // Test function
+ bool IsExcluded(const std::string &rTest) const;
+
+ // Mainly for tests
+ unsigned int SizeOfDefiniteList() const {return mDefinite.size();}
+ unsigned int SizeOfRegexList() const
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+ {return mRegex.size();}
+#else
+ {return 0;}
+#endif
+
+private:
+ std::set<std::string> mDefinite;
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+ std::vector<regex_t *> mRegex;
+#endif
+
+ // For exceptions to the excludes
+ ExcludeList *mpAlwaysInclude;
+};
+
+#endif // EXCLUDELIST__H
+
diff --git a/lib/common/FdGetLine.cpp b/lib/common/FdGetLine.cpp
new file mode 100755
index 00000000..dee02604
--- /dev/null
+++ b/lib/common/FdGetLine.cpp
@@ -0,0 +1,211 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: FdGetLine.cpp
+// Purpose: Line based file descriptor reading
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "FdGetLine.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// utility whitespace function
+inline bool iw(int c)
+{
+ return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FdGetLine::FdGetLine(int)
+// Purpose: Constructor, taking file descriptor
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+FdGetLine::FdGetLine(int fd)
+ : mFileHandle(fd),
+ mLineNumber(0),
+ mBufferBegin(0),
+ mBytesInBuffer(0),
+ mPendingEOF(false),
+ mEOF(false)
+{
+ if(mFileHandle < 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+ //printf("FdGetLine buffer size = %d\n", sizeof(mBuffer));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FdGetLine::~FdGetLine()
+// Purpose: Destructor
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+FdGetLine::~FdGetLine()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FdGetLine::GetLine(bool)
+// Purpose: Returns a file from the file. If Preprocess is true, leading
+// and trailing whitespace is removed, and comments (after #)
+// are deleted.
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+std::string FdGetLine::GetLine(bool Preprocess)
+{
+ if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)}
+
+ // EOF?
+ if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)}
+
+ std::string r;
+
+ bool foundLineEnd = false;
+
+ while(!foundLineEnd && !mEOF)
+ {
+ // Use any bytes left in the buffer
+ while(mBufferBegin < mBytesInBuffer)
+ {
+ int c = mBuffer[mBufferBegin++];
+ if(c == '\r')
+ {
+ // Ignore nasty Windows line ending extra chars
+ }
+ else if(c == '\n')
+ {
+ // Line end!
+ foundLineEnd = true;
+ break;
+ }
+ else
+ {
+ // Add to string
+ r += c;
+ }
+
+ // Implicit line ending at EOF
+ if(mBufferBegin >= mBytesInBuffer && mPendingEOF)
+ {
+ foundLineEnd = true;
+ }
+ }
+
+ // Check size
+ if(r.size() > FDGETLINE_MAX_LINE_SIZE)
+ {
+ THROW_EXCEPTION(CommonException, GetLineTooLarge)
+ }
+
+ // Read more in?
+ if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF)
+ {
+ int bytes = ::read(mFileHandle, mBuffer, sizeof(mBuffer));
+
+ // Error?
+ if(bytes == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ // Adjust buffer info
+ mBytesInBuffer = bytes;
+ mBufferBegin = 0;
+
+ // EOF / closed?
+ if(bytes == 0)
+ {
+ mPendingEOF = true;
+ }
+ }
+
+ // EOF?
+ if(mPendingEOF && mBufferBegin >= mBytesInBuffer)
+ {
+ // File is EOF, and now we've depleted the buffer completely, so tell caller as well.
+ mEOF = true;
+ }
+ }
+
+ if(!Preprocess)
+ {
+ return r;
+ }
+ else
+ {
+ // Check for comment char, but char before must be whitespace
+ int end = 0;
+ int size = r.size();
+ while(end < size)
+ {
+ if(r[end] == '#' && (end == 0 || (iw(r[end-1]))))
+ {
+ break;
+ }
+ end++;
+ }
+
+ // Remove whitespace
+ int begin = 0;
+ while(begin < size && iw(r[begin]))
+ {
+ begin++;
+ }
+ if(!iw(r[end])) end--;
+ while(end > begin && iw(r[end]))
+ {
+ end--;
+ }
+
+ // Return a sub string
+ return r.substr(begin, end - begin + 1);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FdGetLine::DetachFile()
+// Purpose: Detaches the file handle, setting the file pointer correctly.
+// Probably not good for sockets...
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+void FdGetLine::DetachFile()
+{
+ if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)}
+
+ // Adjust file pointer
+ int bytesOver = mBufferBegin - mBufferBegin;
+ ASSERT(bytesOver >= 0);
+ if(bytesOver > 0)
+ {
+ if(::lseek(mFileHandle, 0 - bytesOver, SEEK_CUR) == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+
+ // Unset file pointer
+ mFileHandle = -1;
+}
+
+
diff --git a/lib/common/FdGetLine.h b/lib/common/FdGetLine.h
new file mode 100755
index 00000000..fecb0371
--- /dev/null
+++ b/lib/common/FdGetLine.h
@@ -0,0 +1,61 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: FdGetLine.h
+// Purpose: Line based file descriptor reading
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+
+#ifndef FDGETLINE__H
+#define FDGETLINE__H
+
+#include <string>
+
+#ifdef NDEBUG
+ #define FDGETLINE_BUFFER_SIZE 1024
+#else
+ #define FDGETLINE_BUFFER_SIZE 4
+#endif
+
+// Just a very large upper bound for line size to avoid
+// people sending lots of data over sockets and causing memory problems.
+#define FDGETLINE_MAX_LINE_SIZE (1024*256)
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: FdGetLine
+// Purpose: Line based file descriptor reading
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+class FdGetLine
+{
+public:
+ FdGetLine(int fd);
+ ~FdGetLine();
+private:
+ FdGetLine(const FdGetLine &rToCopy);
+
+public:
+ std::string GetLine(bool Preprocess = false);
+ bool IsEOF() {return mEOF;}
+ int GetLineNumber() {return mLineNumber;}
+
+ // Call to detach, setting file pointer correctly to last bit read.
+ // Only works for lseek-able file descriptors.
+ void DetachFile();
+
+private:
+ char mBuffer[FDGETLINE_BUFFER_SIZE];
+ int mFileHandle;
+ int mLineNumber;
+ int mBufferBegin;
+ int mBytesInBuffer;
+ bool mPendingEOF;
+ bool mEOF;
+};
+
+#endif // FDGETLINE__H
+
diff --git a/lib/common/FileModificationTime.h b/lib/common/FileModificationTime.h
new file mode 100755
index 00000000..78f5c115
--- /dev/null
+++ b/lib/common/FileModificationTime.h
@@ -0,0 +1,57 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: FileModificationTime.h
+// Purpose: Function for getting file modification time.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef FILEMODIFICATIONTIME__H
+#define FILEMODIFICATIONTIME__H
+
+#include <sys/stat.h>
+
+#include "BoxTime.h"
+
+inline box_time_t FileModificationTime(struct stat &st)
+{
+#ifdef PLATFORM_stat_SHORT_mtime
+ box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL);
+#else
+ box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
+ + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
+#endif
+
+ return datamodified;
+}
+
+inline box_time_t FileAttrModificationTime(struct stat &st)
+{
+#ifdef PLATFORM_stat_SHORT_mtime
+ box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL);
+#else
+ box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
+ + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
+#endif
+
+ return statusmodified;
+}
+
+inline box_time_t FileModificationTimeMaxModAndAttr(struct stat &st)
+{
+#ifdef PLATFORM_stat_SHORT_mtime
+ box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL);
+ box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL);
+#else
+ box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
+ + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
+ box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
+ + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
+#endif
+
+ return (datamodified > statusmodified)?datamodified:statusmodified;
+}
+
+#endif // FILEMODIFICATIONTIME__H
+
diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp
new file mode 100755
index 00000000..ca6894ae
--- /dev/null
+++ b/lib/common/FileStream.cpp
@@ -0,0 +1,245 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: FileStream.cpp
+// Purpose: IOStream interface to files
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "FileStream.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::FileStream(const char *, int, int)
+// Purpose: Constructor, opens file
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+FileStream::FileStream(const char *Filename, int flags, int mode)
+ : mOSFileHandle(::open(Filename, flags, mode)),
+ mIsEOF(false)
+{
+ if(mOSFileHandle < 0)
+ {
+ MEMLEAKFINDER_NOT_A_LEAK(this);
+ THROW_EXCEPTION(CommonException, OSFileOpenError)
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::FileStream(int)
+// Purpose: Constructor, using existing file descriptor
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+FileStream::FileStream(int FileDescriptor)
+ : mOSFileHandle(FileDescriptor),
+ mIsEOF(false)
+{
+ if(mOSFileHandle < 0)
+ {
+ MEMLEAKFINDER_NOT_A_LEAK(this);
+ THROW_EXCEPTION(CommonException, OSFileOpenError)
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::FileStream(const FileStream &)
+// Purpose: Copy constructor, creates a duplicate of the file handle
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+FileStream::FileStream(const FileStream &rToCopy)
+ : mOSFileHandle(::dup(rToCopy.mOSFileHandle)),
+ mIsEOF(rToCopy.mIsEOF)
+{
+ if(mOSFileHandle < 0)
+ {
+ MEMLEAKFINDER_NOT_A_LEAK(this);
+ THROW_EXCEPTION(CommonException, OSFileOpenError)
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::~FileStream()
+// Purpose: Destructor, closes file
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+FileStream::~FileStream()
+{
+ if(mOSFileHandle >= 0)
+ {
+ Close();
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::Read(void *, int)
+// Purpose: Reads bytes from the file
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+int FileStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)}
+ int r = ::read(mOSFileHandle, pBuffer, NBytes);
+ if(r == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileReadError)
+ }
+ if(r == 0)
+ {
+ mIsEOF = true;
+ }
+
+ return r;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::BytesLeftToRead()
+// Purpose: Returns number of bytes to read (may not be most efficient function ever)
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type FileStream::BytesLeftToRead()
+{
+ struct stat st;
+ if(::fstat(mOSFileHandle, &st) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ return st.st_size - GetPosition();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::Write(void *, int)
+// Purpose: Writes bytes to the file
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void FileStream::Write(const void *pBuffer, int NBytes)
+{
+ if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)}
+ if(::write(mOSFileHandle, pBuffer, NBytes) != NBytes)
+ {
+ THROW_EXCEPTION(CommonException, OSFileWriteError)
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::GetPosition()
+// Purpose: Get position in stream
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type FileStream::GetPosition() const
+{
+ if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)}
+ off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR);
+ if(p == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ return (IOStream::pos_type)p;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::Seek(pos_type, int)
+// Purpose: Seeks within file, as lseek
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void FileStream::Seek(IOStream::pos_type Offset, int SeekType)
+{
+ if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)}
+ if(::lseek(mOSFileHandle, Offset, ConvertSeekTypeToOSWhence(SeekType)) == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ // Not end of file any more!
+ mIsEOF = false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::Close()
+// Purpose: Closes the underlying file
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void FileStream::Close()
+{
+ if(mOSFileHandle < 0)
+ {
+ THROW_EXCEPTION(CommonException, FileAlreadyClosed)
+ }
+ if(::close(mOSFileHandle) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileCloseError)
+ }
+ mOSFileHandle = -1;
+ mIsEOF = true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::StreamDataLeft()
+// Purpose: Any data left to write?
+// Created: 2003/08/02
+//
+// --------------------------------------------------------------------------
+bool FileStream::StreamDataLeft()
+{
+ return !mIsEOF;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: FileStream::StreamClosed()
+// Purpose: Is the stream closed?
+// Created: 2003/08/02
+//
+// --------------------------------------------------------------------------
+bool FileStream::StreamClosed()
+{
+ return mIsEOF;
+}
+
diff --git a/lib/common/FileStream.h b/lib/common/FileStream.h
new file mode 100755
index 00000000..bb3cc459
--- /dev/null
+++ b/lib/common/FileStream.h
@@ -0,0 +1,46 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: FileStream.h
+// Purpose: FileStream interface to files
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef FILESTREAM__H
+#define FILESTREAM__H
+
+#include "IOStream.h"
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+class FileStream : public IOStream
+{
+public:
+ FileStream(const char *Filename, int flags = O_RDONLY, int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH));
+ FileStream(int FileDescriptor);
+ FileStream(const FileStream &rToCopy);
+ virtual ~FileStream();
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+ virtual pos_type BytesLeftToRead();
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual pos_type GetPosition() const;
+ virtual void Seek(IOStream::pos_type Offset, int SeekType);
+ virtual void Close();
+
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+private:
+ int mOSFileHandle;
+ bool mIsEOF;
+};
+
+
+#endif // FILESTREAM__H
+
+
diff --git a/lib/common/Guards.h b/lib/common/Guards.h
new file mode 100755
index 00000000..6efc5614
--- /dev/null
+++ b/lib/common/Guards.h
@@ -0,0 +1,112 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: Guards.h
+// Purpose: Classes which ensure things are closed/deleted properly when
+// going out of scope. Easy exception proof code, etc
+// Created: 2003/07/12
+//
+// --------------------------------------------------------------------------
+
+#ifndef GUARDS__H
+#define GUARDS__H
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <new>
+
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+template <int flags = O_RDONLY, int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)>
+class FileHandleGuard
+{
+public:
+ FileHandleGuard(const char *filename)
+ : mOSFileHandle(::open(filename, flags, mode))
+ {
+ if(mOSFileHandle < 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileOpenError)
+ }
+ }
+
+ ~FileHandleGuard()
+ {
+ if(mOSFileHandle >= 0)
+ {
+ Close();
+ }
+ }
+
+ void Close()
+ {
+ if(mOSFileHandle < 0)
+ {
+ THROW_EXCEPTION(CommonException, FileAlreadyClosed)
+ }
+ if(::close(mOSFileHandle) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileCloseError)
+ }
+ mOSFileHandle = -1;
+ }
+
+ operator int() const
+ {
+ return mOSFileHandle;
+ }
+
+private:
+ int mOSFileHandle;
+};
+
+template<typename type>
+class MemoryBlockGuard
+{
+public:
+ MemoryBlockGuard(int BlockSize)
+ : mpBlock(::malloc(BlockSize))
+ {
+ if(mpBlock == 0)
+ {
+ throw std::bad_alloc();
+ }
+ }
+
+ ~MemoryBlockGuard()
+ {
+ free(mpBlock);
+ }
+
+ operator type() const
+ {
+ return (type)mpBlock;
+ }
+
+ type GetPtr() const
+ {
+ return (type)mpBlock;
+ }
+
+ void Resize(int NewSize)
+ {
+ void *ptrn = ::realloc(mpBlock, NewSize);
+ if(ptrn == 0)
+ {
+ throw std::bad_alloc();
+ }
+ mpBlock = ptrn;
+ }
+
+private:
+ void *mpBlock;
+};
+
+#include "MemLeakFindOff.h"
+
+#endif // GUARDS__H
+
diff --git a/lib/common/IOStream.cpp b/lib/common/IOStream.cpp
new file mode 100755
index 00000000..024eefcc
--- /dev/null
+++ b/lib/common/IOStream.cpp
@@ -0,0 +1,229 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: IOStream.cpp
+// Purpose: I/O Stream abstraction
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "IOStream.h"
+#include "CommonException.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::IOStream()
+// Purpose: Constructor
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+IOStream::IOStream()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::IOStream(const IOStream &)
+// Purpose: Copy constructor (exceptions)
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+IOStream::IOStream(const IOStream &rToCopy)
+{
+ THROW_EXCEPTION(CommonException, NotSupported)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::~IOStream()
+// Purpose: Destructor
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+IOStream::~IOStream()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::Close()
+// Purpose: Close the stream
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void IOStream::Close()
+{
+ // Do nothing by default -- let the destructor clear everything up.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::Seek(int, int)
+// Purpose: Seek in stream (if supported)
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void IOStream::Seek(IOStream::pos_type Offset, int SeekType)
+{
+ THROW_EXCEPTION(CommonException, NotSupported)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::GetPosition()
+// Purpose: Returns current position in stream (if supported)
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type IOStream::GetPosition() const
+{
+ THROW_EXCEPTION(CommonException, NotSupported)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::ConvertSeekTypeToOSWhence(int)
+// Purpose: Return an whence arg for lseek given a IOStream seek type
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+int IOStream::ConvertSeekTypeToOSWhence(int SeekType)
+{
+ // Should be nicely optimised out as values are choosen in header file to match OS values.
+ int ostype = SEEK_SET;
+ switch(SeekType)
+ {
+ case SeekType_Absolute:
+ ostype = SEEK_SET;
+ break;
+ case SeekType_Relative:
+ ostype = SEEK_CUR;
+ break;
+ case SeekType_End:
+ ostype = SEEK_END;
+ break;
+
+ default:
+ THROW_EXCEPTION(CommonException, IOStreamBadSeekType)
+ }
+
+ return ostype;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::ReadFullBuffer(void *, int, int)
+// Purpose: Reads bytes into buffer, returning whether or not it managed to
+// get all the bytes required. Exception and abort use of stream
+// if this returns false.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool IOStream::ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout)
+{
+ int bytesToGo = NBytes;
+ char *buffer = (char*)pBuffer;
+ if(pNBytesRead) (*pNBytesRead) = 0;
+
+ while(bytesToGo > 0)
+ {
+ int bytesRead = Read(buffer, bytesToGo, Timeout);
+ if(bytesRead == 0)
+ {
+ // Timeout or something
+ return false;
+ }
+ // Increment things
+ bytesToGo -= bytesRead;
+ buffer += bytesRead;
+ if(pNBytesRead) (*pNBytesRead) += bytesRead;
+ }
+
+ // Got everything
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::WriteAllBuffered()
+// Purpose: Ensures that any data which has been buffered is written to the stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void IOStream::WriteAllBuffered()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::BytesLeftToRead()
+// Purpose: Numbers of bytes left to read in the stream, or
+// IOStream::SizeOfStreamUnknown if this isn't known.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type IOStream::BytesLeftToRead()
+{
+ return IOStream::SizeOfStreamUnknown;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStream::CopyStreamTo(IOStream &, int Timeout)
+// Purpose: Copies the entire stream to another stream (reading from this,
+// writing to rCopyTo). Returns whether the copy completed (ie
+// StreamDataLeft() returns false)
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool IOStream::CopyStreamTo(IOStream &rCopyTo, int Timeout, int BufferSize)
+{
+ // Make sure there's something to do before allocating that buffer
+ if(!StreamDataLeft())
+ {
+ return true; // complete, even though nothing happened
+ }
+
+ // Buffer
+ MemoryBlockGuard<char*> buffer(BufferSize);
+
+ // Get copying!
+ while(StreamDataLeft())
+ {
+ // Read some data
+ int bytes = Read(buffer, BufferSize, Timeout);
+ if(bytes == 0 && StreamDataLeft())
+ {
+ return false; // incomplete, timed out
+ }
+
+ // Write some data
+ if(bytes != 0)
+ {
+ rCopyTo.Write(buffer, bytes);
+ }
+ }
+
+ return true; // completed
+}
+
+
diff --git a/lib/common/IOStream.h b/lib/common/IOStream.h
new file mode 100755
index 00000000..042ccca4
--- /dev/null
+++ b/lib/common/IOStream.h
@@ -0,0 +1,67 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: IOStream.h
+// Purpose: I/O Stream abstraction
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef IOSTREAM__H
+#define IOSTREAM__H
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: IOStream
+// Purpose: Abstract interface to streams of data
+// Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+class IOStream
+{
+public:
+ IOStream();
+ IOStream(const IOStream &rToCopy);
+ virtual ~IOStream();
+
+ enum
+ {
+ TimeOutInfinite = -1,
+ SizeOfStreamUnknown = -1
+ };
+
+ enum
+ {
+ SeekType_Absolute = 0,
+ SeekType_Relative = 1,
+ SeekType_End = 2
+ };
+
+ // Timeout in milliseconds
+ // Read may return 0 -- does not mean end of stream.
+ typedef int64_t pos_type;
+ virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite) = 0;
+ virtual pos_type BytesLeftToRead(); // may return IOStream::SizeOfStreamUnknown (and will for most stream types)
+ virtual void Write(const void *pBuffer, int NBytes) = 0;
+ virtual void WriteAllBuffered();
+ virtual pos_type GetPosition() const;
+ virtual void Seek(pos_type Offset, int SeekType);
+ virtual void Close();
+
+ // Has all data that can be read been read?
+ virtual bool StreamDataLeft() = 0;
+ // Has the stream been closed (writing not possible)
+ virtual bool StreamClosed() = 0;
+
+ // Utility functions
+ bool ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout = IOStream::TimeOutInfinite);
+ bool CopyStreamTo(IOStream &rCopyTo, int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024);
+
+ static int ConvertSeekTypeToOSWhence(int SeekType);
+};
+
+
+#endif // IOSTREAM__H
+
+
diff --git a/lib/common/IOStreamGetLine.cpp b/lib/common/IOStreamGetLine.cpp
new file mode 100755
index 00000000..27a77c29
--- /dev/null
+++ b/lib/common/IOStreamGetLine.cpp
@@ -0,0 +1,227 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: IOStreamGetLine.cpp
+// Purpose: Line based file descriptor reading
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "IOStreamGetLine.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// utility whitespace function
+inline bool iw(int c)
+{
+ return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStreamGetLine::IOStreamGetLine(int)
+// Purpose: Constructor, taking file descriptor
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+IOStreamGetLine::IOStreamGetLine(IOStream &Stream)
+ : mrStream(Stream),
+ mLineNumber(0),
+ mBufferBegin(0),
+ mBytesInBuffer(0),
+ mPendingEOF(false),
+ mEOF(false)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStreamGetLine::~IOStreamGetLine()
+// Purpose: Destructor
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+IOStreamGetLine::~IOStreamGetLine()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStreamGetLine::GetLine(std::string &, bool, int)
+// Purpose: Gets a line from the file, returning it in rOutput. If Preprocess is true, leading
+// and trailing whitespace is removed, and comments (after #)
+// are deleted.
+// Returns true if a line is available now, false if retrying may get a line (eg timeout, signal),
+// and exceptions if it's EOF.
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+bool IOStreamGetLine::GetLine(std::string &rOutput, bool Preprocess, int Timeout)
+{
+ // EOF?
+ if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)}
+
+ // Initialise string to stored into
+ std::string r(mPendingString);
+ mPendingString.erase();
+
+ bool foundLineEnd = false;
+
+ while(!foundLineEnd && !mEOF)
+ {
+ // Use any bytes left in the buffer
+ while(mBufferBegin < mBytesInBuffer)
+ {
+ int c = mBuffer[mBufferBegin++];
+ if(c == '\r')
+ {
+ // Ignore nasty Windows line ending extra chars
+ }
+ else if(c == '\n')
+ {
+ // Line end!
+ foundLineEnd = true;
+ break;
+ }
+ else
+ {
+ // Add to string
+ r += c;
+ }
+
+ // Implicit line ending at EOF
+ if(mBufferBegin >= mBytesInBuffer && mPendingEOF)
+ {
+ foundLineEnd = true;
+ }
+ }
+
+ // Check size
+ if(r.size() > IOSTREAMGETLINE_MAX_LINE_SIZE)
+ {
+ THROW_EXCEPTION(CommonException, GetLineTooLarge)
+ }
+
+ // Read more in?
+ if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF)
+ {
+ int bytes = mrStream.Read(mBuffer, sizeof(mBuffer), Timeout);
+
+ // Adjust buffer info
+ mBytesInBuffer = bytes;
+ mBufferBegin = 0;
+
+ // EOF / closed?
+ if(!mrStream.StreamDataLeft())
+ {
+ mPendingEOF = true;
+ }
+
+ // No data returned?
+ if(bytes == 0 && mrStream.StreamDataLeft())
+ {
+ // store string away
+ mPendingString = r;
+ // Return false;
+ return false;
+ }
+ }
+
+ // EOF?
+ if(mPendingEOF && mBufferBegin >= mBytesInBuffer)
+ {
+ // File is EOF, and now we've depleted the buffer completely, so tell caller as well.
+ mEOF = true;
+ }
+ }
+
+ if(!Preprocess)
+ {
+ rOutput = r;
+ return true;
+ }
+ else
+ {
+ // Check for comment char, but char before must be whitespace
+ int end = 0;
+ int size = r.size();
+ while(end < size)
+ {
+ if(r[end] == '#' && (end == 0 || (iw(r[end-1]))))
+ {
+ break;
+ }
+ end++;
+ }
+
+ // Remove whitespace
+ int begin = 0;
+ while(begin < size && iw(r[begin]))
+ {
+ begin++;
+ }
+ if(!iw(r[end])) end--;
+ while(end > begin && iw(r[end]))
+ {
+ end--;
+ }
+
+ // Return a sub string
+ rOutput = r.substr(begin, end - begin + 1);
+ return true;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStreamGetLine::DetachFile()
+// Purpose: Detaches the file handle, setting the file pointer correctly.
+// Probably not good for sockets...
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+void IOStreamGetLine::DetachFile()
+{
+ // Adjust file pointer
+ int bytesOver = mBytesInBuffer - mBufferBegin;
+ ASSERT(bytesOver >= 0);
+ if(bytesOver > 0)
+ {
+ mrStream.Seek(0 - bytesOver, IOStream::SeekType_Relative);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: IOStreamGetLine::IgnoreBufferedData(int)
+// Purpose: Ignore buffered bytes (effectively removing them from the
+// beginning of the buffered data.)
+// Cannot remove more bytes than are currently in the buffer.
+// Be careful when this is used!
+// Created: 22/12/04
+//
+// --------------------------------------------------------------------------
+void IOStreamGetLine::IgnoreBufferedData(int BytesToIgnore)
+{
+ int bytesInBuffer = mBytesInBuffer - mBufferBegin;
+ if(BytesToIgnore < 0 || BytesToIgnore > bytesInBuffer)
+ {
+ THROW_EXCEPTION(CommonException, IOStreamGetLineNotEnoughDataToIgnore)
+ }
+ mBufferBegin += BytesToIgnore;
+}
+
+
+
diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h
new file mode 100755
index 00000000..cf152e5a
--- /dev/null
+++ b/lib/common/IOStreamGetLine.h
@@ -0,0 +1,71 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: IOStreamGetLine.h
+// Purpose: Line based file descriptor reading
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+
+#ifndef IOSTREAMGETLINE__H
+#define IOSTREAMGETLINE__H
+
+#include <string>
+
+#include "IOStream.h"
+
+#ifdef NDEBUG
+ #define IOSTREAMGETLINE_BUFFER_SIZE 1024
+#else
+ #define IOSTREAMGETLINE_BUFFER_SIZE 4
+#endif
+
+// Just a very large upper bound for line size to avoid
+// people sending lots of data over sockets and causing memory problems.
+#define IOSTREAMGETLINE_MAX_LINE_SIZE (1024*256)
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: IOStreamGetLine
+// Purpose: Line based stream reading
+// Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+class IOStreamGetLine
+{
+public:
+ IOStreamGetLine(IOStream &Stream);
+ ~IOStreamGetLine();
+private:
+ IOStreamGetLine(const IOStreamGetLine &rToCopy);
+
+public:
+ bool GetLine(std::string &rOutput, bool Preprocess = false, int Timeout = IOStream::TimeOutInfinite);
+ bool IsEOF() {return mEOF;}
+ int GetLineNumber() {return mLineNumber;}
+
+ // Call to detach, setting file pointer correctly to last bit read.
+ // Only works for lseek-able file descriptors.
+ void DetachFile();
+
+ // For doing interesting stuff with the remaining data...
+ // Be careful with this!
+ const void *GetBufferedData() const {return mBuffer + mBufferBegin;}
+ int GetSizeOfBufferedData() const {return mBytesInBuffer - mBufferBegin;}
+ void IgnoreBufferedData(int BytesToIgnore);
+ IOStream &GetUnderlyingStream() {return mrStream;}
+
+private:
+ char mBuffer[IOSTREAMGETLINE_BUFFER_SIZE];
+ IOStream &mrStream;
+ int mLineNumber;
+ int mBufferBegin;
+ int mBytesInBuffer;
+ bool mPendingEOF;
+ bool mEOF;
+ std::string mPendingString;
+};
+
+#endif // IOSTREAMGETLINE__H
+
diff --git a/lib/common/LinuxWorkaround.cpp b/lib/common/LinuxWorkaround.cpp
new file mode 100755
index 00000000..7900fa6e
--- /dev/null
+++ b/lib/common/LinuxWorkaround.cpp
@@ -0,0 +1,73 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: LinuxWorkaround.cpp
+// Purpose: Workarounds for Linux
+// Created: 2003/10/31
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/stat.h>
+#include <unistd.h>
+#include <dirent.h>
+
+#include <string>
+
+#include "LinuxWorkaround.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+#ifdef PLATFORM_LINUX
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: LinuxWorkaround_FinishDirentStruct(struct dirent *, const char *)
+// Purpose: Finishes off filling in a dirent structure, which Linux leaves incomplete.
+// Created: 2003/10/31
+//
+// --------------------------------------------------------------------------
+void LinuxWorkaround_FinishDirentStruct(struct dirent *entry, const char *DirectoryName)
+{
+ // From man readdir under Linux:
+ //
+ // BUGS
+ // Field d_type is not implemented as of libc6 2.1 and will always return
+ // DT_UNKNOWN (0).
+ //
+ // What kind of an OS is this?
+
+
+ // Build filename of this entry
+ std::string fn(DirectoryName);
+ fn += '/';
+ fn += entry->d_name;
+
+ // Do a stat on it
+ struct stat st;
+ if(::lstat(fn.c_str(), &st) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ // Fill in the d_type field.
+ if(S_ISREG(st.st_mode))
+ {
+ entry->d_type = DT_REG;
+ }
+ else if(S_ISDIR(st.st_mode))
+ {
+ entry->d_type = DT_DIR;
+ }
+ else if(S_ISLNK(st.st_mode))
+ {
+ entry->d_type = DT_LNK;
+ }
+ // otherwise leave it as we found it
+}
+
+#endif // PLATFORM_LINUX
+
diff --git a/lib/common/LinuxWorkaround.h b/lib/common/LinuxWorkaround.h
new file mode 100755
index 00000000..bcd27495
--- /dev/null
+++ b/lib/common/LinuxWorkaround.h
@@ -0,0 +1,20 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: LinuxWorkaround.h
+// Purpose: Workarounds for Linux
+// Created: 2003/10/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef LINUXWORKAROUND__H
+#define LINUXWORKAROUND__H
+
+#ifdef PLATFORM_LINUX
+
+void LinuxWorkaround_FinishDirentStruct(struct dirent *entry, const char *DirectoryName);
+
+#endif // PLATFORM_LINUX
+
+#endif // LINUXWORKAROUND__H
+
diff --git a/lib/common/MainHelper.h b/lib/common/MainHelper.h
new file mode 100755
index 00000000..ab75af96
--- /dev/null
+++ b/lib/common/MainHelper.h
@@ -0,0 +1,42 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: MainHelper.h
+// Purpose: Helper stuff for main() programs
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+
+#ifndef MAINHELPER__H
+#define MAINHELPER__H
+
+#include <stdio.h>
+
+#include "BoxException.h"
+
+#define MAINHELPER_START \
+ if(argc == 2 && ::strcmp(argv[1], "--version") == 0) \
+ { printf(BOX_VERSION "\n"); return 0; } \
+ MEMLEAKFINDER_START \
+ try {
+#define MAINHELPER_END \
+ } catch(BoxException &e) { \
+ printf("Exception: %s (%d/%d)\n", e.what(), e.GetType(), e.GetSubType()); \
+ return 1; \
+ } catch(std::exception &e) { \
+ printf("Exception: %s\n", e.what()); \
+ return 1; \
+ } catch(...) { \
+ printf("Exception: <UNKNOWN>\n"); \
+ return 1; }
+
+#ifdef BOX_MEMORY_LEAK_TESTING
+ #define MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, marker) \
+ memleakfinder_setup_exit_report(file, marker);
+#else
+ #define MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, marker)
+#endif // BOX_MEMORY_LEAK_TESTING
+
+
+#endif // MAINHELPER__H
+
diff --git a/lib/common/Makefile.extra b/lib/common/Makefile.extra
new file mode 100755
index 00000000..92e6fbb3
--- /dev/null
+++ b/lib/common/Makefile.extra
@@ -0,0 +1,11 @@
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_CommonException.h autogen_CommonException.cpp: $(MAKEEXCEPTION) CommonException.txt
+ perl $(MAKEEXCEPTION) CommonException.txt
+
+# AUTOGEN SEEDING
+autogen_ConversionException.h autogen_ConversionException.cpp: $(MAKEEXCEPTION) ConversionException.txt
+ perl $(MAKEEXCEPTION) ConversionException.txt
+
diff --git a/lib/common/MemBlockStream.cpp b/lib/common/MemBlockStream.cpp
new file mode 100755
index 00000000..538a7ef8
--- /dev/null
+++ b/lib/common/MemBlockStream.cpp
@@ -0,0 +1,235 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: MemBlockStream.cpp
+// Purpose: Stream out data from any memory block
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "MemBlockStream.h"
+#include "CommonException.h"
+#include "StreamableMemBlock.h"
+#include "CollectInBufferStream.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::MemBlockStream()
+// Purpose: Constructor (doesn't copy block, careful with lifetimes)
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+MemBlockStream::MemBlockStream(const void *pBuffer, int Size)
+ : mpBuffer((char*)pBuffer),
+ mBytesInBuffer(Size),
+ mReadPosition(0)
+{
+ ASSERT(pBuffer != 0);
+ ASSERT(Size >= 0);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::MemBlockStream(const StreamableMemBlock &)
+// Purpose: Constructor (doesn't copy block, careful with lifetimes)
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+MemBlockStream::MemBlockStream(const StreamableMemBlock &rBlock)
+ : mpBuffer((char*)rBlock.GetBuffer()),
+ mBytesInBuffer(rBlock.GetSize()),
+ mReadPosition(0)
+{
+ ASSERT(mpBuffer != 0);
+ ASSERT(mBytesInBuffer >= 0);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::MemBlockStream(const StreamableMemBlock &)
+// Purpose: Constructor (doesn't copy block, careful with lifetimes)
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+MemBlockStream::MemBlockStream(const CollectInBufferStream &rBuffer)
+ : mpBuffer((char*)rBuffer.GetBuffer()),
+ mBytesInBuffer(rBuffer.GetSize()),
+ mReadPosition(0)
+{
+ ASSERT(mpBuffer != 0);
+ ASSERT(mBytesInBuffer >= 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::MemBlockStream(const MemBlockStream &)
+// Purpose: Copy constructor
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+MemBlockStream::MemBlockStream(const MemBlockStream &rToCopy)
+ : mpBuffer(rToCopy.mpBuffer),
+ mBytesInBuffer(rToCopy.mBytesInBuffer),
+ mReadPosition(0)
+{
+ ASSERT(mpBuffer != 0);
+ ASSERT(mBytesInBuffer >= 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::~MemBlockStream()
+// Purpose: Destructor
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+MemBlockStream::~MemBlockStream()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::Read(void *, int, int)
+// Purpose: As interface. But only works in read phase
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+int MemBlockStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Adjust to number of bytes left
+ if(NBytes > (mBytesInBuffer - mReadPosition))
+ {
+ NBytes = (mBytesInBuffer - mReadPosition);
+ }
+ ASSERT(NBytes >= 0);
+ if(NBytes <= 0) return 0; // careful now
+
+ // Copy in the requested number of bytes and adjust the read pointer
+ ::memcpy(pBuffer, mpBuffer + mReadPosition, NBytes);
+ mReadPosition += NBytes;
+
+ return NBytes;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::BytesLeftToRead()
+// Purpose: As interface. But only works in read phase
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type MemBlockStream::BytesLeftToRead()
+{
+ return (mBytesInBuffer - mReadPosition);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::Write(void *, int)
+// Purpose: As interface. But only works in write phase
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void MemBlockStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(CommonException, MemBlockStreamNotSupported)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::GetPosition()
+// Purpose: In write phase, returns the number of bytes written, in read
+// phase, the number of bytes to go
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type MemBlockStream::GetPosition() const
+{
+ return mReadPosition;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::Seek(pos_type, int)
+// Purpose: As interface.
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void MemBlockStream::Seek(pos_type Offset, int SeekType)
+{
+ int newPos = 0;
+ switch(SeekType)
+ {
+ case IOStream::SeekType_Absolute:
+ newPos = Offset;
+ break;
+ case IOStream::SeekType_Relative:
+ newPos = mReadPosition + Offset;
+ break;
+ case IOStream::SeekType_End:
+ newPos = mBytesInBuffer + Offset;
+ break;
+ default:
+ THROW_EXCEPTION(CommonException, IOStreamBadSeekType)
+ break;
+ }
+
+ // Make sure it doesn't go over
+ if(newPos > mBytesInBuffer)
+ {
+ newPos = mBytesInBuffer;
+ }
+ // or under
+ if(newPos < 0)
+ {
+ newPos = 0;
+ }
+
+ // Set the new read position
+ mReadPosition = newPos;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::StreamDataLeft()
+// Purpose: As interface
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+bool MemBlockStream::StreamDataLeft()
+{
+ return mReadPosition < mBytesInBuffer;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: MemBlockStream::StreamClosed()
+// Purpose: As interface
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+bool MemBlockStream::StreamClosed()
+{
+ return true;
+}
+
diff --git a/lib/common/MemBlockStream.h b/lib/common/MemBlockStream.h
new file mode 100755
index 00000000..f78ff8e6
--- /dev/null
+++ b/lib/common/MemBlockStream.h
@@ -0,0 +1,52 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: MemBlockStream.h
+// Purpose: Stream out data from any memory block
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+
+#ifndef MEMBLOCKSTREAM__H
+#define MEMBLOCKSTREAM__H
+
+#include "IOStream.h"
+
+class StreamableMemBlock;
+class CollectInBufferStream;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: MemBlockStream
+// Purpose: Stream out data from any memory block -- be careful the lifetime
+// of the block is greater than the lifetime of this stream.
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+class MemBlockStream : public IOStream
+{
+public:
+ MemBlockStream(const void *pBuffer, int Size);
+ MemBlockStream(const StreamableMemBlock &rBlock);
+ MemBlockStream(const CollectInBufferStream &rBuffer);
+ MemBlockStream(const MemBlockStream &rToCopy);
+ ~MemBlockStream();
+public:
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+ virtual pos_type BytesLeftToRead();
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual pos_type GetPosition() const;
+ virtual void Seek(pos_type Offset, int SeekType);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+private:
+ const char *mpBuffer;
+ int mBytesInBuffer;
+ int mReadPosition;
+};
+
+#endif // MEMBLOCKSTREAM__H
+
diff --git a/lib/common/MemLeakFindOff.h b/lib/common/MemLeakFindOff.h
new file mode 100755
index 00000000..1cc98bac
--- /dev/null
+++ b/lib/common/MemLeakFindOff.h
@@ -0,0 +1,27 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: MemLeakFindOff.h
+// Purpose: Switch memory leak finding off
+// Created: 13/1/04
+//
+// --------------------------------------------------------------------------
+
+// no header guard
+
+#ifdef BOX_MEMORY_LEAK_TESTING
+
+#undef new
+
+#ifndef MEMLEAKFINDER_FULL_MALLOC_MONITORING
+ #ifdef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED
+ #undef malloc
+ #undef realloc
+ #undef free
+ #undef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED
+ #endif
+#endif
+
+#undef MEMLEAKFINDER_ENABLED
+
+#endif
diff --git a/lib/common/MemLeakFindOn.h b/lib/common/MemLeakFindOn.h
new file mode 100755
index 00000000..c20fe25a
--- /dev/null
+++ b/lib/common/MemLeakFindOn.h
@@ -0,0 +1,25 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: MemLeakFindOn.h
+// Purpose: Switch memory leak finding on
+// Created: 13/1/04
+//
+// --------------------------------------------------------------------------
+
+// no header guard
+
+#ifdef BOX_MEMORY_LEAK_TESTING
+
+#define new DEBUG_NEW
+
+#ifndef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED
+ #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__)
+ #define realloc memleakfinder_realloc
+ #define free memleakfinder_free
+ #define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED
+#endif
+
+#define MEMLEAKFINDER_ENABLED
+
+#endif
diff --git a/lib/common/MemLeakFinder.h b/lib/common/MemLeakFinder.h
new file mode 100755
index 00000000..f5887dac
--- /dev/null
+++ b/lib/common/MemLeakFinder.h
@@ -0,0 +1,60 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: MemLeakFinder.h
+// Purpose: Memory leak finder
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef MEMLEAKFINDER__H
+#define MEMLEAKFINDER__H
+
+#define DEBUG_NEW new(__FILE__,__LINE__)
+
+#ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING
+ // include stdlib now, to avoid problems with having the macros defined already
+ #include <stdlib.h>
+#endif
+
+// global enable flag
+extern bool memleakfinder_global_enable;
+
+extern "C"
+{
+ void *memleakfinder_malloc(size_t size, const char *file, int line);
+ void *memleakfinder_realloc(void *ptr, size_t size);
+ void memleakfinder_free(void *ptr);
+}
+
+int memleakfinder_numleaks();
+
+void memleakfinder_reportleaks();
+
+void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext);
+
+void memleakfinder_setup_exit_report(const char *filename, const char *markertext);
+
+void memleakfinder_startsectionmonitor();
+
+void memleakfinder_traceblocksinsection();
+
+void memleakfinder_notaleak(void *ptr);
+
+void *operator new(size_t size, const char *file, int line);
+void *operator new[](size_t size, const char *file, int line);
+
+void operator delete(void *ptr) throw ();
+void operator delete[](void *ptr) throw ();
+
+// define the malloc functions now, if required
+#ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING
+ #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__)
+ #define realloc memleakfinder_realloc
+ #define free memleakfinder_free
+ #define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED
+#endif
+
+
+#endif // MEMLEAKFINDER__H
+
diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp
new file mode 100755
index 00000000..1f6e038a
--- /dev/null
+++ b/lib/common/NamedLock.cpp
@@ -0,0 +1,144 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: NamedLock.cpp
+// Purpose: A global named lock, implemented as a lock file in file system
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <fcntl.h>
+#include <errno.h>
+#include <unistd.h>
+#ifdef PLATFORM_LINUX
+ #include <sys/file.h>
+#endif // PLATFORM_LINUX
+#ifdef PLATFORM_CYGWIN
+ #include <sys/file.h>
+#endif // PLATFORM_CYGWIN
+
+#include "NamedLock.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: NamedLock::NamedLock()
+// Purpose: Constructor
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+NamedLock::NamedLock()
+ : mFileDescriptor(-1)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: NamedLock::~NamedLock()
+// Purpose: Destructor (automatically unlocks if locked)
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+NamedLock::~NamedLock()
+{
+ if(mFileDescriptor != -1)
+ {
+ ReleaseLock();
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: NamedLock::TryAndGetLock(const char *, int)
+// Purpose: Trys to get a lock on the name in the file system.
+// IMPORTANT NOTE: If a file exists with this name, it will be deleted.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+bool NamedLock::TryAndGetLock(const char *Filename, int mode)
+{
+ // Check
+ if(mFileDescriptor != -1)
+ {
+ THROW_EXCEPTION(CommonException, NamedLockAlreadyLockingSomething)
+ }
+
+ // See if the lock can be got
+#ifdef PLATFORM_open_NO_O_EXLOCK
+ int fd = ::open(Filename, O_WRONLY | O_CREAT | O_TRUNC, mode);
+ if(fd == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ if(::flock(fd, LOCK_EX | LOCK_NB) != 0)
+ {
+ ::close(fd);
+ if(errno == EWOULDBLOCK)
+ {
+ return false;
+ }
+ else
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+
+ // Success
+ mFileDescriptor = fd;
+
+ return true;
+#else
+ int fd = ::open(Filename, O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode);
+ if(fd != -1)
+ {
+ // Got a lock, lovely
+ mFileDescriptor = fd;
+ return true;
+ }
+
+ // Failed. Why?
+ if(errno != EWOULDBLOCK)
+ {
+ // Not the expected error
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ return false;
+#endif
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: NamedLock::ReleaseLock()
+// Purpose: Release the lock. Exceptions if the lock is not held
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void NamedLock::ReleaseLock()
+{
+ // Got a lock?
+ if(mFileDescriptor == -1)
+ {
+ THROW_EXCEPTION(CommonException, NamedLockNotHeld)
+ }
+
+ // Close the file
+ if(::close(mFileDescriptor) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ // Mark as unlocked
+ mFileDescriptor = -1;
+}
+
+
+
+
diff --git a/lib/common/NamedLock.h b/lib/common/NamedLock.h
new file mode 100755
index 00000000..5eac712a
--- /dev/null
+++ b/lib/common/NamedLock.h
@@ -0,0 +1,41 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: NamedLock.h
+// Purpose: A global named lock, implemented as a lock file in file system
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef NAMEDLOCK__H
+#define NAMEDLOCK__H
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: NamedLock
+// Purpose: A global named lock, implemented as a lock file in file system
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class NamedLock
+{
+public:
+ NamedLock();
+ ~NamedLock();
+private:
+ // No copying allowed
+ NamedLock(const NamedLock &);
+
+public:
+ bool TryAndGetLock(const char *Filename, int mode = 0755);
+ bool GotLock() {return mFileDescriptor != -1;}
+ void ReleaseLock();
+
+
+private:
+ int mFileDescriptor;
+};
+
+#endif // NAMEDLOCK__H
+
diff --git a/lib/common/PartialReadStream.cpp b/lib/common/PartialReadStream.cpp
new file mode 100755
index 00000000..0b5c4cf6
--- /dev/null
+++ b/lib/common/PartialReadStream.cpp
@@ -0,0 +1,135 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: PartialReadStream.h
+// Purpose: Read part of another stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "PartialReadStream.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: PartialReadStream::PartialReadStream(IOStream &, int)
+// Purpose: Constructor, taking another stream and the number of bytes
+// to be read from it.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+PartialReadStream::PartialReadStream(IOStream &rSource, int BytesToRead)
+ : mrSource(rSource),
+ mBytesLeft(BytesToRead)
+{
+ ASSERT(BytesToRead > 0);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: PartialReadStream::~PartialReadStream()
+// Purpose: Destructor. Won't absorb any unread bytes.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+PartialReadStream::~PartialReadStream()
+{
+ // Warn in debug mode
+ if(mBytesLeft != 0)
+ {
+ TRACE1("PartialReadStream::~PartialReadStream when mBytesLeft = %d\n", mBytesLeft);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: PartialReadStream::Read(void *, int, int)
+// Purpose: As interface.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+int PartialReadStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Finished?
+ if(mBytesLeft <= 0)
+ {
+ return 0;
+ }
+
+ // Asking for more than is allowed?
+ if(NBytes > mBytesLeft)
+ {
+ // Adjust downwards
+ NBytes = mBytesLeft;
+ }
+
+ // Route the request to the source
+ int read = mrSource.Read(pBuffer, NBytes, Timeout);
+ ASSERT(read <= mBytesLeft);
+
+ // Adjust the count
+ mBytesLeft -= read;
+
+ // Return the number read
+ return read;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: PartialReadStream::BytesLeftToRead()
+// Purpose: As interface.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type PartialReadStream::BytesLeftToRead()
+{
+ return mBytesLeft;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: PartialReadStream::Write(const void *, int)
+// Purpose: As interface. But will exception.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void PartialReadStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(CommonException, CantWriteToPartialReadStream)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: PartialReadStream::StreamDataLeft()
+// Purpose: As interface.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool PartialReadStream::StreamDataLeft()
+{
+ return mBytesLeft != 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: PartialReadStream::StreamClosed()
+// Purpose: As interface.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool PartialReadStream::StreamClosed()
+{
+ // always closed
+ return true;
+}
+
diff --git a/lib/common/PartialReadStream.h b/lib/common/PartialReadStream.h
new file mode 100755
index 00000000..42cb7aeb
--- /dev/null
+++ b/lib/common/PartialReadStream.h
@@ -0,0 +1,46 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: PartialReadStream.h
+// Purpose: Read part of another stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef PARTIALREADSTREAM__H
+#define PARTIALREADSTREAM__H
+
+#include "IOStream.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: PartialReadStream
+// Purpose: Read part of another stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class PartialReadStream : public IOStream
+{
+public:
+ PartialReadStream(IOStream &rSource, int BytesToRead);
+ ~PartialReadStream();
+private:
+ // no copying allowed
+ PartialReadStream(const IOStream &);
+ PartialReadStream(const PartialReadStream &);
+
+public:
+ virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+ virtual pos_type BytesLeftToRead();
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+private:
+ IOStream &mrSource;
+ int mBytesLeft;
+};
+
+#endif // PARTIALREADSTREAM__H
+
diff --git a/lib/common/ReadGatherStream.cpp b/lib/common/ReadGatherStream.cpp
new file mode 100755
index 00000000..9ccc3a54
--- /dev/null
+++ b/lib/common/ReadGatherStream.cpp
@@ -0,0 +1,262 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: ReadGatherStream.cpp
+// Purpose: Build a stream (for reading only) out of a number of other streams.
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "ReadGatherStream.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ReadGatherStream::ReadGatherStream(bool)
+// Purpose: Constructor. Args says whether or not all the component streams will be deleted when this
+// object is deleted.
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+ReadGatherStream::ReadGatherStream(bool DeleteComponentStreamsOnDestruction)
+ : mDeleteComponentStreamsOnDestruction(DeleteComponentStreamsOnDestruction),
+ mCurrentPosition(0),
+ mTotalSize(0),
+ mCurrentBlock(0),
+ mPositionInCurrentBlock(0),
+ mSeekDoneForCurrent(false)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ReadGatherStream::~ReadGatherStream()
+// Purpose: Destructor. Will delete all the stream objects, if required.
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+ReadGatherStream::~ReadGatherStream()
+{
+ // Delete compoenent streams?
+ if(mDeleteComponentStreamsOnDestruction)
+ {
+ for(unsigned int l = 0; l < mComponents.size(); ++l)
+ {
+ delete mComponents[l];
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ReadGatherStream::AddComponent(IOStream *)
+// Purpose: Add a component to this stream, returning the index of this component
+// in the internal list. Use this with AddBlock()
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+int ReadGatherStream::AddComponent(IOStream *pStream)
+{
+ ASSERT(pStream != 0);
+
+ // Just add the component to the list, returning it's index.
+ int index = mComponents.size();
+ mComponents.push_back(pStream);
+ return index;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ReadGatherStream::AddBlock(int, pos_type, bool, pos_type)
+// Purpose: Add a block to the list of blocks being gathered into one stream.
+// Length is length of block to read from this component, Seek == true
+// if a seek is required, and if true, SeekTo is the position (absolute)
+// in the stream to be seeked to when this block is required.
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+void ReadGatherStream::AddBlock(int Component, pos_type Length, bool Seek, pos_type SeekTo)
+{
+ // Check block
+ if(Component < 0 || Component >= (int)mComponents.size() || Length < 0 || SeekTo < 0)
+ {
+ THROW_EXCEPTION(CommonException, ReadGatherStreamAddingBadBlock);
+ }
+
+ // Add to list
+ Block b;
+ b.mLength = Length;
+ b.mSeekTo = SeekTo;
+ b.mComponent = Component;
+ b.mSeek = Seek;
+
+ mBlocks.push_back(b);
+
+ // And update the total size
+ mTotalSize += Length;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: ReadGatherStream::Read(void *, int, int)
+// Purpose: As interface.
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+int ReadGatherStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ int bytesToRead = NBytes;
+ uint8_t *buffer = (uint8_t*)pBuffer;
+
+ while(bytesToRead > 0)
+ {
+ // Done?
+ if(mCurrentBlock >= mBlocks.size())
+ {
+ // Stop now, as have finished the last block
+ return NBytes - bytesToRead;
+ }
+
+ // Seek?
+ if(mPositionInCurrentBlock == 0 && mBlocks[mCurrentBlock].mSeek && !mSeekDoneForCurrent)
+ {
+ // Do seeks in this manner so that seeks are done regardless of whether the block
+ // has length > 0, and it will only be done once, and at as late a stage as possible.
+
+ mComponents[mBlocks[mCurrentBlock].mComponent]->Seek(mBlocks[mCurrentBlock].mSeekTo, IOStream::SeekType_Absolute);
+
+ mSeekDoneForCurrent = true;
+ }
+
+ // Anything in the current block?
+ if(mPositionInCurrentBlock < mBlocks[mCurrentBlock].mLength)
+ {
+ // Read!
+ int s = mBlocks[mCurrentBlock].mLength - mPositionInCurrentBlock;
+ if(s > bytesToRead) s = bytesToRead;
+
+ int r = mComponents[mBlocks[mCurrentBlock].mComponent]->Read(buffer, s, Timeout);
+
+ // update variables
+ mPositionInCurrentBlock += r;
+ buffer += r;
+ bytesToRead -= r;
+ mCurrentPosition += r;
+
+ if(r != s)
+ {
+ // Stream returned less than requested. To avoid blockin