summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorReinhard Tartler <siretart@tauware.de>2007-02-14 09:01:45 +0100
committerReinhard Tartler <siretart@tauware.de>2007-02-14 09:01:45 +0100
commitaa2943800f9c00823720af98da036813ebf5cd2c (patch)
tree099cc0264d32a36ab89aa3f48cbf34612c3cd225 /lib
initial commit
Diffstat (limited to 'lib')
-rw-r--r--lib/backupclient/BackupClientCryptoKeys.cpp105
-rw-r--r--lib/backupclient/BackupClientCryptoKeys.h93
-rw-r--r--lib/backupclient/BackupClientFileAttributes.cpp1040
-rw-r--r--lib/backupclient/BackupClientFileAttributes.h111
-rw-r--r--lib/backupclient/BackupClientMakeExcludeList.cpp113
-rw-r--r--lib/backupclient/BackupClientMakeExcludeList.h86
-rw-r--r--lib/backupclient/BackupClientRestore.cpp509
-rw-r--r--lib/backupclient/BackupClientRestore.h64
-rw-r--r--lib/backupclient/BackupDaemonConfigVerify.cpp144
-rw-r--r--lib/backupclient/BackupDaemonConfigVerify.h56
-rw-r--r--lib/backupclient/BackupStoreConstants.h91
-rw-r--r--lib/backupclient/BackupStoreDirectory.cpp601
-rw-r--r--lib/backupclient/BackupStoreDirectory.h306
-rw-r--r--lib/backupclient/BackupStoreException.h55
-rw-r--r--lib/backupclient/BackupStoreException.txt70
-rw-r--r--lib/backupclient/BackupStoreFile.cpp1568
-rw-r--r--lib/backupclient/BackupStoreFile.h259
-rw-r--r--lib/backupclient/BackupStoreFileCmbDiff.cpp364
-rw-r--r--lib/backupclient/BackupStoreFileCmbIdx.cpp362
-rw-r--r--lib/backupclient/BackupStoreFileCombine.cpp448
-rw-r--r--lib/backupclient/BackupStoreFileCryptVar.cpp69
-rw-r--r--lib/backupclient/BackupStoreFileCryptVar.h77
-rw-r--r--lib/backupclient/BackupStoreFileDiff.cpp1094
-rw-r--r--lib/backupclient/BackupStoreFileEncodeStream.cpp713
-rw-r--r--lib/backupclient/BackupStoreFileEncodeStream.h165
-rw-r--r--lib/backupclient/BackupStoreFileRevDiff.cpp296
-rw-r--r--lib/backupclient/BackupStoreFileWire.h112
-rw-r--r--lib/backupclient/BackupStoreFilename.cpp317
-rw-r--r--lib/backupclient/BackupStoreFilename.h123
-rw-r--r--lib/backupclient/BackupStoreFilenameClear.cpp367
-rw-r--r--lib/backupclient/BackupStoreFilenameClear.h98
-rw-r--r--lib/backupclient/BackupStoreObjectDump.cpp256
-rw-r--r--lib/backupclient/BackupStoreObjectMagic.h69
-rw-r--r--lib/backupclient/Makefile.extra16
-rw-r--r--lib/backupstore/BackupStoreAccountDatabase.cpp408
-rw-r--r--lib/backupstore/BackupStoreAccountDatabase.h113
-rw-r--r--lib/backupstore/BackupStoreAccounts.cpp200
-rw-r--r--lib/backupstore/BackupStoreAccounts.h85
-rw-r--r--lib/backupstore/BackupStoreCheck.cpp783
-rw-r--r--lib/backupstore/BackupStoreCheck.h237
-rw-r--r--lib/backupstore/BackupStoreCheck2.cpp879
-rw-r--r--lib/backupstore/BackupStoreCheckData.cpp243
-rw-r--r--lib/backupstore/BackupStoreConfigVerify.cpp86
-rw-r--r--lib/backupstore/BackupStoreConfigVerify.h56
-rw-r--r--lib/backupstore/BackupStoreInfo.cpp630
-rw-r--r--lib/backupstore/BackupStoreInfo.h149
-rw-r--r--lib/backupstore/StoreStructure.cpp133
-rw-r--r--lib/backupstore/StoreStructure.h70
-rw-r--r--lib/common/Archive.h199
-rw-r--r--lib/common/BannerText.h55
-rw-r--r--lib/common/BeginStructPackForWire.h61
-rw-r--r--lib/common/Box.h191
-rw-r--r--lib/common/BoxConfig.h.in397
-rw-r--r--lib/common/BoxException.cpp59
-rw-r--r--lib/common/BoxException.h75
-rw-r--r--lib/common/BoxPlatform.h190
-rw-r--r--lib/common/BoxPortsAndFiles.h74
-rw-r--r--lib/common/BoxTime.cpp69
-rw-r--r--lib/common/BoxTime.h78
-rw-r--r--lib/common/BoxTimeToText.cpp100
-rw-r--r--lib/common/BoxTimeToText.h57
-rw-r--r--lib/common/BoxTimeToUnix.h72
-rw-r--r--lib/common/CollectInBufferStream.cpp312
-rw-r--r--lib/common/CollectInBufferStream.h98
-rw-r--r--lib/common/CommonException.h55
-rw-r--r--lib/common/CommonException.txt46
-rw-r--r--lib/common/Configuration.cpp779
-rw-r--r--lib/common/Configuration.h136
-rw-r--r--lib/common/Conversion.h136
-rw-r--r--lib/common/ConversionException.txt8
-rw-r--r--lib/common/ConversionString.cpp167
-rw-r--r--lib/common/DebugAssertFailed.cpp75
-rw-r--r--lib/common/DebugMemLeakFinder.cpp426
-rw-r--r--lib/common/DebugPrintf.cpp121
-rw-r--r--lib/common/EndStructPackForWire.h61
-rw-r--r--lib/common/EventWatchFilesystemObject.cpp140
-rw-r--r--lib/common/EventWatchFilesystemObject.h86
-rw-r--r--lib/common/ExcludeList.cpp436
-rw-r--r--lib/common/ExcludeList.h109
-rw-r--r--lib/common/FdGetLine.cpp266
-rw-r--r--lib/common/FdGetLine.h99
-rw-r--r--lib/common/FileModificationTime.h95
-rw-r--r--lib/common/FileStream.cpp381
-rw-r--r--lib/common/FileStream.h107
-rw-r--r--lib/common/Guards.h153
-rw-r--r--lib/common/IOStream.cpp279
-rw-r--r--lib/common/IOStream.h105
-rw-r--r--lib/common/IOStreamGetLine.cpp265
-rw-r--r--lib/common/IOStreamGetLine.h109
-rw-r--r--lib/common/MainHelper.h80
-rw-r--r--lib/common/Makefile.extra11
-rw-r--r--lib/common/MemBlockStream.cpp273
-rw-r--r--lib/common/MemBlockStream.h90
-rw-r--r--lib/common/MemLeakFindOff.h65
-rw-r--r--lib/common/MemLeakFindOn.h63
-rw-r--r--lib/common/MemLeakFinder.h98
-rw-r--r--lib/common/NamedLock.cpp204
-rw-r--r--lib/common/NamedLock.h79
-rw-r--r--lib/common/PartialReadStream.cpp173
-rw-r--r--lib/common/PartialReadStream.h84
-rw-r--r--lib/common/ReadGatherStream.cpp300
-rw-r--r--lib/common/ReadGatherStream.h105
-rw-r--r--lib/common/StreamableMemBlock.cpp402
-rw-r--r--lib/common/StreamableMemBlock.h109
-rw-r--r--lib/common/TemporaryDirectory.h84
-rw-r--r--lib/common/Test.h318
-rw-r--r--lib/common/UnixUser.cpp164
-rw-r--r--lib/common/UnixUser.h75
-rw-r--r--lib/common/Utils.cpp199
-rw-r--r--lib/common/Utils.h74
-rw-r--r--lib/common/WaitForEvent.cpp235
-rw-r--r--lib/common/WaitForEvent.h186
-rwxr-xr-xlib/common/makeexception.pl315
-rw-r--r--lib/compress/Compress.h233
-rw-r--r--lib/compress/CompressException.h55
-rw-r--r--lib/compress/CompressException.txt12
-rw-r--r--lib/compress/CompressStream.cpp463
-rw-r--r--lib/compress/CompressStream.h100
-rw-r--r--lib/compress/Makefile.extra7
-rw-r--r--lib/crypto/CipherAES.cpp201
-rw-r--r--lib/crypto/CipherAES.h88
-rw-r--r--lib/crypto/CipherBlowfish.cpp258
-rw-r--r--lib/crypto/CipherBlowfish.h98
-rw-r--r--lib/crypto/CipherContext.cpp653
-rw-r--r--lib/crypto/CipherContext.h121
-rw-r--r--lib/crypto/CipherDescription.cpp111
-rw-r--r--lib/crypto/CipherDescription.h97
-rw-r--r--lib/crypto/CipherException.h55
-rw-r--r--lib/crypto/CipherException.txt18
-rw-r--r--lib/crypto/MD5Digest.cpp120
-rw-r--r--lib/crypto/MD5Digest.h95
-rw-r--r--lib/crypto/Makefile.extra7
-rw-r--r--lib/crypto/Random.cpp165
-rw-r--r--lib/crypto/Random.h63
-rw-r--r--lib/crypto/RollingChecksum.cpp100
-rw-r--r--lib/crypto/RollingChecksum.h145
-rw-r--r--lib/raidfile/Makefile.extra7
-rw-r--r--lib/raidfile/RaidFileController.cpp255
-rw-r--r--lib/raidfile/RaidFileController.h145
-rw-r--r--lib/raidfile/RaidFileException.h55
-rw-r--r--lib/raidfile/RaidFileException.txt25
-rw-r--r--lib/raidfile/RaidFileRead.cpp1735
-rw-r--r--lib/raidfile/RaidFileRead.h110
-rw-r--r--lib/raidfile/RaidFileUtil.cpp226
-rw-r--r--lib/raidfile/RaidFileUtil.h135
-rw-r--r--lib/raidfile/RaidFileWrite.cpp859
-rw-r--r--lib/raidfile/RaidFileWrite.h104
-rwxr-xr-xlib/raidfile/raidfile-config135
-rw-r--r--lib/server/ConnectionException.txt27
-rw-r--r--lib/server/Daemon.cpp704
-rw-r--r--lib/server/Daemon.h124
-rw-r--r--lib/server/LocalProcessStream.cpp151
-rw-r--r--lib/server/LocalProcessStream.h57
-rw-r--r--lib/server/Makefile.extra11
-rw-r--r--lib/server/Protocol.cpp1181
-rw-r--r--lib/server/Protocol.h239
-rw-r--r--lib/server/ProtocolObject.cpp163
-rw-r--r--lib/server/ProtocolObject.h79
-rw-r--r--lib/server/ProtocolUncertainStream.cpp227
-rw-r--r--lib/server/ProtocolUncertainStream.h85
-rw-r--r--lib/server/ProtocolWire.h81
-rw-r--r--lib/server/SSLLib.cpp123
-rw-r--r--lib/server/SSLLib.h74
-rw-r--r--lib/server/ServerException.h84
-rw-r--r--lib/server/ServerException.txt39
-rw-r--r--lib/server/ServerStream.h381
-rw-r--r--lib/server/ServerTLS.h118
-rw-r--r--lib/server/Socket.cpp216
-rw-r--r--lib/server/Socket.h92
-rw-r--r--lib/server/SocketListen.h319
-rw-r--r--lib/server/SocketStream.cpp475
-rw-r--r--lib/server/SocketStream.h109
-rw-r--r--lib/server/SocketStreamTLS.cpp514
-rw-r--r--lib/server/SocketStreamTLS.h98
-rw-r--r--lib/server/TLSContext.cpp158
-rw-r--r--lib/server/TLSContext.h79
-rwxr-xr-xlib/server/makeprotocol.pl1035
-rw-r--r--lib/win32/WinNamedPipeStream.cpp350
-rw-r--r--lib/win32/WinNamedPipeStream.h98
-rw-r--r--lib/win32/config.h.win32396
-rw-r--r--lib/win32/emu.cpp1267
-rw-r--r--lib/win32/emu.h520
182 files changed, 41532 insertions, 0 deletions
diff --git a/lib/backupclient/BackupClientCryptoKeys.cpp b/lib/backupclient/BackupClientCryptoKeys.cpp
new file mode 100644
index 00000000..f4ecfec4
--- /dev/null
+++ b/lib/backupclient/BackupClientCryptoKeys.cpp
@@ -0,0 +1,105 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientCryptoKeys.cpp
+// Purpose: function for setting up all the backup client keys
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "BackupClientCryptoKeys.h"
+#include "FileStream.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupStoreException.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreFile.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientCryptoKeys_Setup(const char *)
+// Purpose: Read in the key material file, and set keys to all the backup elements required.
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename)
+{
+ // Read in the key material
+ unsigned char KeyMaterial[BACKUPCRYPTOKEYS_FILE_SIZE];
+
+ // Open the file
+ FileStream file(KeyMaterialFilename);
+ // Read in data
+ if(!file.ReadFullBuffer(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntLoadClientKeyMaterial)
+ }
+
+ // Tell the filename how to encrypt
+ BackupStoreFilenameClear::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH,
+ KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH);
+ BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish);
+
+ // Tell the attributes how to encrypt
+ BackupClientFileAttributes::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH);
+ // and the secret for hashing
+ BackupClientFileAttributes::SetAttributeHashSecret(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH);
+
+ // Tell the files how to encrypt
+ BackupStoreFile::SetBlowfishKeys(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH,
+ KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH);
+#ifndef HAVE_OLD_SSL
+ // Use AES where available
+ BackupStoreFile::SetAESKey(KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH);
+#endif
+
+ // Wipe the key material from memory
+ ::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE);
+}
+
+
+
diff --git a/lib/backupclient/BackupClientCryptoKeys.h b/lib/backupclient/BackupClientCryptoKeys.h
new file mode 100644
index 00000000..87cb2dc9
--- /dev/null
+++ b/lib/backupclient/BackupClientCryptoKeys.h
@@ -0,0 +1,93 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientCryptoKeys.h
+// Purpose: Format of crypto keys file, and function for setting everything up
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTCRYTOKEYS__H
+#define BACKUPCLIENTCRYTOKEYS__H
+
+
+// All keys are the maximum size that Blowfish supports. Since only the
+// setup time is affected by key length (encryption same speed whatever)
+// there is no disadvantage to using long keys as they are never
+// transmitted and are static over long periods of time.
+
+
+// All sizes in bytes. Some gaps deliberately left in the used material.
+
+// How long the key material file is expected to be
+#define BACKUPCRYPTOKEYS_FILE_SIZE 1024
+
+// key for encrypting filenames (448 bits)
+#define BACKUPCRYPTOKEYS_FILENAME_KEY_START 0
+#define BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH 56
+#define BACKUPCRYPTOKEYS_FILENAME_IV_START (0 + BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH)
+#define BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH 8
+
+// key for encrypting attributes (448 bits)
+#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START (BACKUPCRYPTOKEYS_FILENAME_KEY_START+64)
+#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH 56
+
+// Blowfish key for encrypting file data (448 bits (max blowfish key length))
+#define BACKUPCRYPTOKEYS_FILE_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START+64)
+#define BACKUPCRYPTOKEYS_FILE_KEY_LENGTH 56
+
+// key for encrypting file block index entries
+#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START (BACKUPCRYPTOKEYS_FILE_KEY_START+64)
+#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH 56
+
+// Secret for hashing attributes
+#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START (BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START+64)
+#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH 128
+
+// AES key for encrypting file data (256 bits (max AES key length))
+#define BACKUPCRYPTOKEYS_FILE_AES_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START+128)
+#define BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH 32
+
+
+void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename);
+
+#endif // BACKUPCLIENTCRYTOKEYS__H
+
diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp
new file mode 100644
index 00000000..6e875cda
--- /dev/null
+++ b/lib/backupclient/BackupClientFileAttributes.cpp
@@ -0,0 +1,1040 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientFileAttributes.cpp
+// Purpose: Storage of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#ifdef HAVE_UNISTD_H
+ #include <unistd.h>
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <limits.h>
+#include <algorithm>
+#include <new>
+#include <vector>
+#ifdef HAVE_SYS_XATTR_H
+#include <cerrno>
+#include <sys/xattr.h>
+#endif
+
+#include "BackupClientFileAttributes.h"
+#include "CommonException.h"
+#include "FileModificationTime.h"
+#include "BoxTimeToUnix.h"
+#include "BackupStoreException.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "MD5Digest.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+#define ATTRIBUTETYPE_GENERIC_UNIX 1
+
+#define ATTRIBUTE_ENCODING_BLOWFISH 2
+
+typedef struct
+{
+ int32_t AttributeType;
+ u_int32_t UID;
+ u_int32_t GID;
+ u_int64_t ModificationTime;
+ u_int64_t AttrModificationTime;
+ u_int32_t UserDefinedFlags;
+ u_int32_t FileGenerationNumber;
+ u_int16_t Mode;
+ // Symbolic link filename may follow
+ // Extended attribute (xattr) information may follow, format is:
+ // u_int32_t Size of extended attribute block (excluding this word)
+ // For each of NumberOfAttributes (sorted by AttributeName):
+ // u_int16_t AttributeNameLength
+ // char AttributeName[AttributeNameLength]
+ // u_int32_t AttributeValueLength
+ // unsigned char AttributeValue[AttributeValueLength]
+ // AttributeName is 0 terminated, AttributeValue is not (and may be binary data)
+} attr_StreamFormat;
+
+// This has wire packing so it's compatible across platforms
+// Use wider than necessary sizes, just to be careful.
+typedef struct
+{
+ int32_t uid, gid, mode;
+} attributeHashData;
+
+// Use default packing
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+
+#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256
+
+// Hide private static variables from the rest of the world
+// -- don't put them as static class variables to avoid openssl/evp.h being
+// included all over the project.
+namespace
+{
+ CipherContext sBlowfishEncrypt;
+ CipherContext sBlowfishDecrypt;
+ uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH];
+ int sAttributeHashSecretLength = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::BackupClientFileAttributes()
+// Purpose: Default constructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::BackupClientFileAttributes()
+ : mpClearAttributes(0)
+{
+ ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &)
+// Purpose: Copy constructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy)
+ : StreamableMemBlock(rToCopy), // base class does the hard work
+ mpClearAttributes(0)
+{
+}
+BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy)
+ : StreamableMemBlock(rToCopy), // base class does the hard work
+ mpClearAttributes(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::~BackupClientFileAttributes()
+// Purpose: Destructor
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::~BackupClientFileAttributes()
+{
+ if(mpClearAttributes)
+ {
+ delete mpClearAttributes;
+ mpClearAttributes = 0;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &)
+// Purpose: Assignment operator
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr)
+{
+ StreamableMemBlock::Set(rAttr);
+ RemoveClear(); // make sure no decrypted version held
+ return *this;
+}
+// Assume users play nice
+BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr)
+{
+ StreamableMemBlock::Set(rAttr);
+ RemoveClear(); // make sure no decrypted version held
+ return *this;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &)
+// Purpose: Comparison operator
+// Created: 2003/10/09
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const
+{
+ EnsureClearAvailable();
+ rAttr.EnsureClearAvailable();
+
+ return mpClearAttributes->operator==(*rAttr.mpClearAttributes);
+}
+// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data.
+/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const
+{
+ StreamableMemBlock *pDecoded = 0;
+
+ try
+ {
+ EnsureClearAvailable();
+ StreamableMemBlock *pDecoded = MakeClear(rAttr);
+
+ // Compare using clear version
+ bool compared = mpClearAttributes->operator==(rAttr);
+
+ // Delete temporary
+ delete pDecoded;
+
+ return compared;
+ }
+ catch(...)
+ {
+ delete pDecoded;
+ throw;
+ }
+}*/
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool)
+// Purpose: Compare, optionally ignoring the attribute modification time and/or modification time, and some data which is
+// irrelevant in practise (eg file generation number)
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime, bool IgnoreModTime) const
+{
+ EnsureClearAvailable();
+ rAttr.EnsureClearAvailable();
+
+ // Check sizes are the same, as a first check
+ if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize())
+ {
+ return false;
+ }
+
+ // Then check the elements of the two things
+ // Bytes are checked in network order, but this doesn't matter as we're only checking for equality.
+ attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+ attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer();
+
+ if(a1->AttributeType != a2->AttributeType
+ || a1->UID != a2->UID
+ || a1->GID != a2->GID
+ || a1->UserDefinedFlags != a2->UserDefinedFlags
+ || a1->Mode != a2->Mode)
+ {
+ return false;
+ }
+
+ if(!IgnoreModTime)
+ {
+ if(a1->ModificationTime != a2->ModificationTime)
+ {
+ return false;
+ }
+ }
+
+ if(!IgnoreAttrModTime)
+ {
+ if(a1->AttrModificationTime != a2->AttrModificationTime)
+ {
+ return false;
+ }
+ }
+
+ // Check symlink string?
+ unsigned int size = mpClearAttributes->GetSize();
+ if(size > sizeof(attr_StreamFormat))
+ {
+ // Symlink strings don't match. This also compares xattrs
+ if(::memcmp(a1 + 1, a2 + 1, size - sizeof(attr_StreamFormat)) != 0)
+ {
+ return false;
+ }
+ }
+
+ // Passes all test, must be OK
+ return true;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadAttributes(const char *)
+// Purpose: Read the attributes of the file, and store them ready for streaming.
+// Optionally retrieve the modification time and attribute modification time.
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroModificationTimes, box_time_t *pModTime,
+ box_time_t *pAttrModTime, int64_t *pFileSize, InodeRefType *pInodeNumber, bool *pHasMultipleLinks)
+{
+ StreamableMemBlock *pnewAttr = 0;
+ try
+ {
+ struct stat st;
+ if(::lstat(Filename, &st) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ // Modification times etc
+ if(pModTime) {*pModTime = FileModificationTime(st);}
+ if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);}
+ if(pFileSize) {*pFileSize = st.st_size;}
+ if(pInodeNumber) {*pInodeNumber = st.st_ino;}
+ if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);}
+
+ pnewAttr = new StreamableMemBlock;
+
+ FillAttributes(*pnewAttr, Filename, st, ZeroModificationTimes);
+
+#ifndef WIN32
+ // Is it a link?
+ if((st.st_mode & S_IFMT) == S_IFLNK)
+ {
+ FillAttributesLink(*pnewAttr, Filename, st);
+ }
+#endif
+
+ FillExtendedAttr(*pnewAttr, Filename);
+
+#ifdef WIN32
+ //this is to catch those problems with invalid time stamps stored...
+ //need to find out the reason why - but also a catch as well.
+
+ attr_StreamFormat *pattr =
+ (attr_StreamFormat*)pnewAttr->GetBuffer();
+ ASSERT(pattr != 0);
+
+ // __time64_t winTime = BoxTimeToSeconds(
+ // pnewAttr->ModificationTime);
+
+ u_int64_t modTime = box_ntoh64(pattr->ModificationTime);
+ box_time_t modSecs = BoxTimeToSeconds(modTime);
+ __time64_t winTime = modSecs;
+
+ // _MAX__TIME64_T doesn't seem to be defined, but the code below
+ // will throw an assertion failure if we exceed it :-)
+ // Microsoft says dates up to the year 3000 are valid, which
+ // is a bit more than 15 * 2^32. Even that doesn't seem
+ // to be true (still aborts), but it can at least hold 2^32.
+ if (winTime >= 0x100000000LL || _gmtime64(&winTime) == 0)
+ {
+ ::syslog(LOG_ERR, "Invalid Modification Time "
+ "caught for file: %s", Filename);
+ pattr->ModificationTime = 0;
+ }
+
+ modTime = box_ntoh64(pattr->AttrModificationTime);
+ modSecs = BoxTimeToSeconds(modTime);
+ winTime = modSecs;
+
+ if (winTime > 0x100000000LL || _gmtime64(&winTime) == 0)
+ {
+ ::syslog(LOG_ERR, "Invalid Attribute Modification "
+ "Time caught for file: %s", Filename);
+ pattr->AttrModificationTime = 0;
+ }
+#endif
+
+ // Attributes ready. Encrypt into this block
+ EncryptAttr(*pnewAttr);
+
+ // Store the new attributes
+ RemoveClear();
+ mpClearAttributes = pnewAttr;
+ pnewAttr = 0;
+ }
+ catch(...)
+ {
+ // clean up
+ delete pnewAttr;
+ pnewAttr = 0;
+ throw;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadAttributesLink()
+// Purpose: Private function, handles standard attributes for all objects
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st, bool ZeroModificationTimes)
+{
+ outputBlock.ResizeBlock(sizeof(attr_StreamFormat));
+ attr_StreamFormat *pattr = (attr_StreamFormat*)outputBlock.GetBuffer();
+ ASSERT(pattr != 0);
+
+ // Fill in the entries
+ pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX);
+ pattr->UID = htonl(st.st_uid);
+ pattr->GID = htonl(st.st_gid);
+ if(ZeroModificationTimes)
+ {
+ pattr->ModificationTime = 0;
+ pattr->AttrModificationTime = 0;
+ }
+ else
+ {
+ pattr->ModificationTime = box_hton64(FileModificationTime(st));
+ pattr->AttrModificationTime = box_hton64(FileAttrModificationTime(st));
+ }
+ pattr->Mode = htons(st.st_mode);
+
+#ifndef HAVE_STRUCT_STAT_ST_FLAGS
+ pattr->UserDefinedFlags = 0;
+ pattr->FileGenerationNumber = 0;
+#else
+ pattr->UserDefinedFlags = htonl(st.st_flags);
+ pattr->FileGenerationNumber = htonl(st.st_gen);
+#endif
+}
+#ifndef WIN32
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadAttributesLink()
+// Purpose: Private function, handles the case where a symbolic link is needed
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st)
+{
+ // Make sure we're only called for symbolic links
+ ASSERT((st.st_mode & S_IFMT) == S_IFLNK);
+
+ // Get the filename the link is linked to
+ char linkedTo[PATH_MAX+4];
+ int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX);
+ if(linkedToSize == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+
+ int oldSize = outputBlock.GetSize();
+ outputBlock.ResizeBlock(oldSize+linkedToSize+1);
+ char* buffer = static_cast<char*>(outputBlock.GetBuffer());
+
+ // Add the path name for the symbolic link, and add 0 termination
+ std::memcpy(buffer+oldSize, linkedTo, linkedToSize);
+ buffer[oldSize+linkedToSize] = '\0';
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::ReadExtendedAttr(const char *, unsigned char**)
+// Purpose: Private function, read the extended attributes of the file into the block
+// Created: 2005/06/12
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename)
+{
+#ifdef HAVE_SYS_XATTR_H
+ int listBufferSize = 1000;
+ char* list = new char[listBufferSize];
+
+ try
+ {
+ // This returns an unordered list of attribute names, each 0 terminated,
+ // concatenated together
+ int listSize = ::llistxattr(Filename, list, listBufferSize);
+
+ if(listSize>listBufferSize)
+ {
+ delete[] list, list = NULL;
+ list = new char[listSize];
+ listSize = ::llistxattr(Filename, list, listSize);
+ }
+
+ if(listSize>0)
+ {
+ // Extract list of attribute names so we can sort them
+ std::vector<std::string> attrKeys;
+ for(int i = 0; i<listSize; ++i)
+ {
+ std::string attrKey(list+i);
+ i += attrKey.size();
+ attrKeys.push_back(attrKey);
+ }
+ sort(attrKeys.begin(), attrKeys.end());
+
+ // Make initial space in block
+ int xattrSize = outputBlock.GetSize();
+ int xattrBufferSize = (xattrSize+listSize)>500 ? (xattrSize+listSize)*2 : 1000;
+ outputBlock.ResizeBlock(xattrBufferSize);
+ unsigned char* buffer = static_cast<unsigned char*>(outputBlock.GetBuffer());
+
+ // Leave space for attr block size later
+ int xattrBlockSizeOffset = xattrSize;
+ xattrSize += sizeof(u_int32_t);
+
+ // Loop for each attribute
+ for(std::vector<std::string>::const_iterator attrKeyI = attrKeys.begin(); attrKeyI!=attrKeys.end(); ++attrKeyI)
+ {
+ std::string attrKey(*attrKeyI);
+
+ if(xattrSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t)>static_cast<unsigned int>(xattrBufferSize))
+ {
+ xattrBufferSize = (xattrBufferSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t))*2;
+ outputBlock.ResizeBlock(xattrBufferSize);
+ buffer = static_cast<unsigned char*>(outputBlock.GetBuffer());
+ }
+
+ // Store length and text for attibute name
+ u_int16_t keyLength = htons(attrKey.size()+1);
+ std::memcpy(buffer+xattrSize, &keyLength, sizeof(u_int16_t));
+ xattrSize += sizeof(u_int16_t);
+ std::memcpy(buffer+xattrSize, attrKey.c_str(), attrKey.size()+1);
+ xattrSize += attrKey.size()+1;
+
+ // Leave space for value size
+ int valueSizeOffset = xattrSize;
+ xattrSize += sizeof(u_int32_t);
+
+ // Find size of attribute (must call with buffer and length 0 on some platforms,
+ // as -1 is returned if the data doesn't fit.)
+ int valueSize = ::lgetxattr(Filename, attrKey.c_str(), 0, 0);
+ if(valueSize<0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+
+ // Resize block, if needed
+ if(xattrSize+valueSize>xattrBufferSize)
+ {
+ xattrBufferSize = (xattrBufferSize+valueSize)*2;
+ outputBlock.ResizeBlock(xattrBufferSize);
+ buffer = static_cast<unsigned char*>(outputBlock.GetBuffer());
+ }
+
+ // This gets the attribute value (may be text or binary), no termination
+ valueSize = ::lgetxattr(Filename, attrKey.c_str(), buffer+xattrSize, xattrBufferSize-xattrSize);
+ if(valueSize<0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+ xattrSize += valueSize;
+
+ // Fill in value size
+ u_int32_t valueLength = htonl(valueSize);
+ std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(u_int32_t));
+ }
+
+ // Fill in attribute block size
+ u_int32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(u_int32_t));
+ std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(u_int32_t));
+
+ outputBlock.ResizeBlock(xattrSize);
+ }
+ else if(listSize<0 && errno!=EOPNOTSUPP && errno!=EACCES)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+ }
+ catch(...)
+ {
+ delete[] list;
+ throw;
+ }
+ delete[] list;
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::WriteAttributes(const char *)
+// Purpose: Apply the stored attributes to the file
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::WriteAttributes(const char *Filename) const
+{
+ // Got something loaded
+ if(GetSize() <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Make sure there are clear attributes to use
+ EnsureClearAvailable();
+ ASSERT(mpClearAttributes != 0);
+
+ // Check if the decrypted attributes are small enough, and the type of attributes stored
+ if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+ }
+ int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+ ASSERT(type != 0);
+ if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX)
+ {
+ // Don't know what to do with these
+ THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+ }
+
+ // Check there is enough space for an attributes block
+ if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat))
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Get pointer to structure
+ attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+ int xattrOffset = sizeof(attr_StreamFormat);
+
+ // is it a symlink?
+ int16_t mode = ntohs(pattr->Mode);
+ if((mode & S_IFMT) == S_IFLNK)
+ {
+ // Check things are sensible
+ if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1)
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+#ifdef WIN32
+ ::syslog(LOG_WARNING,
+ "Cannot create symbolic links on Windows: %s",
+ Filename);
+#else
+ // Make a symlink, first deleting anything in the way
+ ::unlink(Filename);
+ if(::symlink((char*)(pattr + 1), Filename) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+#endif
+
+ xattrOffset += std::strlen(reinterpret_cast<char*>(pattr+1))+1;
+ }
+
+ // If working as root, set user IDs
+ if(::geteuid() == 0)
+ {
+ #ifndef HAVE_LCHOWN
+ // only if not a link, can't set their owner on this platform
+ if((mode & S_IFMT) != S_IFLNK)
+ {
+ // Not a link, use normal chown
+ if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+ #else
+ if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) // use the version which sets things on symlinks
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ #endif
+ }
+
+ if(static_cast<int>(xattrOffset+sizeof(u_int32_t))<=mpClearAttributes->GetSize())
+ {
+ WriteExtendedAttr(Filename, xattrOffset);
+ }
+
+ // Stop now if symlink, because otherwise it'll just be applied to the target
+ if((mode & S_IFMT) == S_IFLNK)
+ {
+ return;
+ }
+
+ // Set modification time?
+ box_time_t modtime = box_ntoh64(pattr->ModificationTime);
+ if(modtime != 0)
+ {
+ // Work out times as timevals
+ struct timeval times[2];
+ BoxTimeToTimeval(modtime, times[1]);
+ // Copy access time as well, why not, got to set it to something
+ times[0] = times[1];
+ // Attr modification time will be changed anyway, nothing that can be done about it
+
+ // Try to apply
+ if(::utimes(Filename, times) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+
+ // Apply everything else... (allowable mode flags only)
+ if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID | S_ISGID | S_ISVTX)) != 0) // mode must be done last (think setuid)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::IsSymLink()
+// Purpose: Do these attributes represent a symbolic link?
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::IsSymLink() const
+{
+ EnsureClearAvailable();
+
+ // Got the right kind of thing?
+ if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ // Get the type of attributes stored
+ int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+ ASSERT(type != 0);
+ if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat))
+ {
+ // Check link
+ attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+ return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK;
+ }
+
+ return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::RemoveClear()
+// Purpose: Private. Deletes any clear version of the attributes that may be held
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::RemoveClear() const
+{
+ if(mpClearAttributes)
+ {
+ delete mpClearAttributes;
+ }
+ mpClearAttributes = 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::EnsureClearAvailable()
+// Purpose: Private. Makes sure the clear version is available
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::EnsureClearAvailable() const
+{
+ if(mpClearAttributes == 0)
+ {
+ mpClearAttributes = MakeClear(*this);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset)
+// Purpose: Private function, apply the stored extended attributes to the file
+// Created: 2005/06/13
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) const
+{
+#ifdef HAVE_SYS_XATTR_H
+ const char* buffer = static_cast<char*>(mpClearAttributes->GetBuffer());
+
+ u_int32_t xattrBlockLength = 0;
+ std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(u_int32_t));
+ int xattrBlockSize = ntohl(xattrBlockLength);
+ xattrOffset += sizeof(u_int32_t);
+
+ int xattrEnd = xattrOffset+xattrBlockSize;
+ if(xattrEnd>mpClearAttributes->GetSize())
+ {
+ // Too small
+ THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+ }
+
+ while(xattrOffset<xattrEnd)
+ {
+ u_int16_t keyLength = 0;
+ std::memcpy(&keyLength, buffer+xattrOffset, sizeof(u_int16_t));
+ int keySize = ntohs(keyLength);
+ xattrOffset += sizeof(u_int16_t);
+
+ const char* key = buffer+xattrOffset;
+ xattrOffset += keySize;
+
+ u_int32_t valueLength = 0;
+ std::memcpy(&valueLength, buffer+xattrOffset, sizeof(u_int32_t));
+ int valueSize = ntohl(valueLength);
+ xattrOffset += sizeof(u_int32_t);
+
+ // FIXME: Warn on EOPNOTSUPP
+ if(::lsetxattr(Filename, key, buffer+xattrOffset, valueSize, 0)!=0 && errno!=EOPNOTSUPP)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+
+ xattrOffset += valueSize;
+ }
+
+ ASSERT(xattrOffset==xattrEnd);
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &)
+// Purpose: Static. Decrypts stored attributes.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted)
+{
+ // New block
+ StreamableMemBlock *pdecrypted = 0;
+
+ try
+ {
+ // Check the block is big enough for IV and header
+ int ivSize = sBlowfishEncrypt.GetIVLength();
+ if(rEncrypted.GetSize() <= (ivSize + 1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes);
+ }
+
+ // How much space is needed for the output?
+ int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize);
+
+ // Allocate it
+ pdecrypted = new StreamableMemBlock(maxDecryptedSize);
+
+ // ptr to block
+ uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer();
+
+ // Check that the header has right type
+ if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH)
+ {
+ THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding);
+ }
+
+ // Set IV
+ sBlowfishDecrypt.SetIV(encBlock + 1);
+
+ // Decrypt
+ int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1));
+
+ // Resize block to fit
+ pdecrypted->ResizeBlock(decryptedSize);
+ }
+ catch(...)
+ {
+ delete pdecrypted;
+ pdecrypted = 0;
+ }
+
+ return pdecrypted;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int)
+// Purpose: Static. Sets the key to use for encryption and decryption.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength)
+{
+ // IVs set later
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &)
+// Purpose: Private. Encrypt the given attributes into this block.
+// Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt)
+{
+ // Free any existing block
+ FreeBlock();
+
+ // Work out the maximum amount of space we need
+ int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize());
+ // And the size of the IV
+ int ivSize = sBlowfishEncrypt.GetIVLength();
+
+ // Allocate this space
+ AllocateBlock(maxEncryptedSize + ivSize + 1);
+
+ // Store the encoding byte
+ uint8_t *block = (uint8_t*)GetBuffer();
+ block[0] = ATTRIBUTE_ENCODING_BLOWFISH;
+
+ // Generate and store an IV for this attribute block
+ int ivSize2 = 0;
+ const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2);
+ ASSERT(ivSize == ivSize2);
+
+ // Copy into the encrypted block
+ ::memcpy(block + 1, iv, ivSize);
+
+ // Do the transform
+ int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize());
+
+ // Resize this block
+ ResizeBlock(encrytedSize + ivSize + 1);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int)
+// Purpose: Set the secret for the filename attribute hash
+// Created: 25/4/04
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength)
+{
+ if(SecretLength > (int)sizeof(sAttributeHashSecret))
+ {
+ SecretLength = sizeof(sAttributeHashSecret);
+ }
+ if(SecretLength < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Copy
+ ::memcpy(sAttributeHashSecret, pSecret, SecretLength);
+ sAttributeHashSecretLength = SecretLength;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientFileAttributes::GenerateAttributeHash(struct stat &, const std::string &, const std::string &)
+// Purpose: Generate a 64 bit hash from the attributes, used to detect changes.
+// Include filename in the hash, so that it changes from one file to another,
+// so don't reveal identical attributes.
+// Created: 25/4/04
+//
+// --------------------------------------------------------------------------
+uint64_t BackupClientFileAttributes::GenerateAttributeHash(struct stat &st, const std::string &filename, const std::string &leafname)
+{
+ if(sAttributeHashSecretLength == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet)
+ }
+
+ // Assemble stuff we're interested in
+ attributeHashData hashData;
+ memset(&hashData, 0, sizeof(hashData));
+ // Use network byte order and large sizes to be cross platform
+ hashData.uid = htonl(st.st_uid);
+ hashData.gid = htonl(st.st_gid);
+ hashData.mode = htonl(st.st_mode);
+
+ StreamableMemBlock xattr;
+ FillExtendedAttr(xattr, filename.c_str());
+
+ // Create a MD5 hash of the data, filename, and secret
+ MD5Digest digest;
+ digest.Add(&hashData, sizeof(hashData));
+ digest.Add(xattr.GetBuffer(), xattr.GetSize());
+ digest.Add(leafname.c_str(), leafname.size());
+ digest.Add(sAttributeHashSecret, sAttributeHashSecretLength);
+ digest.Finish();
+
+ // Return the first 64 bits of the hash
+ uint64_t result = *((uint64_t *)(digest.DigestAsData()));
+ return result;
+}
diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h
new file mode 100644
index 00000000..c11aaf83
--- /dev/null
+++ b/lib/backupclient/BackupClientFileAttributes.h
@@ -0,0 +1,111 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientFileAttributes.h
+// Purpose: Storage of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTFILEATTRIBUTES__H
+#define BACKUPCLIENTFILEATTRIBUTES__H
+
+#include <string>
+
+#include "StreamableMemBlock.h"
+#include "BoxTime.h"
+
+struct stat;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupClientFileAttributes
+// Purpose: Storage, streaming and application of file attributes
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+class BackupClientFileAttributes : public StreamableMemBlock
+{
+public:
+ BackupClientFileAttributes();
+ BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy);
+ BackupClientFileAttributes(const StreamableMemBlock &rToCopy);
+ ~BackupClientFileAttributes();
+ BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr);
+ BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr);
+ bool operator==(const BackupClientFileAttributes &rAttr) const;
+// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous?
+
+ bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const;
+
+ // Prevent access to base class members accidently
+ void Set();
+
+ void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false,
+ box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0,
+ InodeRefType *pInodeNumber = 0, bool *pHasMultipleLinks = 0);
+ void WriteAttributes(const char *Filename) const;
+
+ bool IsSymLink() const;
+
+ static void SetBlowfishKey(const void *pKey, int KeyLength);
+ static void SetAttributeHashSecret(const void *pSecret, int SecretLength);
+
+ static uint64_t GenerateAttributeHash(struct stat &st, const std::string &filename, const std::string &leafname);
+ static void FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename);
+
+private:
+ static void FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st, bool ZeroModificationTimes);
+ static void FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st);
+ void WriteExtendedAttr(const char *Filename, int xattrOffset) const;
+
+ void RemoveClear() const;
+ void EnsureClearAvailable() const;
+ static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted);
+ void EncryptAttr(const StreamableMemBlock &rToEncrypt);
+
+private:
+ mutable StreamableMemBlock *mpClearAttributes;
+};
+
+#endif // BACKUPCLIENTFILEATTRIBUTES__H
+
diff --git a/lib/backupclient/BackupClientMakeExcludeList.cpp b/lib/backupclient/BackupClientMakeExcludeList.cpp
new file mode 100644
index 00000000..93bf074f
--- /dev/null
+++ b/lib/backupclient/BackupClientMakeExcludeList.cpp
@@ -0,0 +1,113 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientMakeExcludeList.cpp
+// Purpose: Makes exclude lists from bbbackupd config location entries
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupClientMakeExcludeList.h"
+#include "Configuration.h"
+#include "ExcludeList.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientMakeExcludeList(const Configuration &, const char *, const char *)
+// Purpose: Given a Configuration object corresponding to a bbackupd Location, and the
+// two names of the keys for definite and regex entries, return a ExcludeList.
+// Or 0 if it isn't required.
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName,
+ const char *AlwaysIncludeDefiniteName, const char *AlwaysIncludeRegexName)
+{
+ // Check that at least one of the entries exists
+ if(!rConfig.KeyExists(DefiniteName) && !rConfig.KeyExists(RegexName))
+ {
+ // Neither exists -- return 0 as an Exclude list isn't required.
+ return 0;
+ }
+
+ // Create the exclude list
+ ExcludeList *pexclude = new ExcludeList;
+
+ try
+ {
+ // Definite names to add?
+ if(rConfig.KeyExists(DefiniteName))
+ {
+ pexclude->AddDefiniteEntries(rConfig.GetKeyValue(DefiniteName));
+ }
+ // Regular expressions to add?
+ if(rConfig.KeyExists(RegexName))
+ {
+ pexclude->AddRegexEntries(rConfig.GetKeyValue(RegexName));
+ }
+
+ // Add a "always include" list?
+ if(AlwaysIncludeDefiniteName != 0 && AlwaysIncludeRegexName != 0)
+ {
+ // This will accept NULL as a valid argument, so safe to do this.
+ pexclude->SetAlwaysIncludeList(
+ BackupClientMakeExcludeList(rConfig, AlwaysIncludeDefiniteName, AlwaysIncludeRegexName)
+ );
+ }
+ }
+ catch(...)
+ {
+ // Clean up
+ delete pexclude;
+ throw;
+ }
+
+ return pexclude;
+}
+
+
+
diff --git a/lib/backupclient/BackupClientMakeExcludeList.h b/lib/backupclient/BackupClientMakeExcludeList.h
new file mode 100644
index 00000000..b5a6c680
--- /dev/null
+++ b/lib/backupclient/BackupClientMakeExcludeList.h
@@ -0,0 +1,86 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientMakeExcludeList.h
+// Purpose: Makes exclude lists from bbbackupd config location entries
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTMAKEEXCLUDELIST__H
+#define BACKUPCLIENTMAKEEXCLUDELIST__H
+
+class ExcludeList;
+class Configuration;
+
+ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName,
+ const char *AlwaysIncludeDefiniteName = 0, const char *AlwaysIncludeRegexName = 0);
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientMakeExcludeList_Files(const Configuration &)
+// Purpose: Create a exclude list from config file entries for files. May return 0.
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+inline ExcludeList *BackupClientMakeExcludeList_Files(const Configuration &rConfig)
+{
+ return BackupClientMakeExcludeList(rConfig, "ExcludeFile", "ExcludeFilesRegex", "AlwaysIncludeFile", "AlwaysIncludeFilesRegex");
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientMakeExcludeList_Dirs(const Configuration &)
+// Purpose: Create a exclude list from config file entries for directories. May return 0.
+// Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+inline ExcludeList *BackupClientMakeExcludeList_Dirs(const Configuration &rConfig)
+{
+ return BackupClientMakeExcludeList(rConfig, "ExcludeDir", "ExcludeDirsRegex", "AlwaysIncludeDir", "AlwaysIncludeDirsRegex");
+}
+
+
+#endif // BACKUPCLIENTMAKEEXCLUDELIST__H
+
diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp
new file mode 100644
index 00000000..4e657d5d
--- /dev/null
+++ b/lib/backupclient/BackupClientRestore.cpp
@@ -0,0 +1,509 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientRestore.cpp
+// Purpose:
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#ifdef HAVE_UNISTD_H
+ #include <unistd.h>
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string>
+#include <set>
+#include <limits.h>
+#include <stdio.h>
+
+#include "BackupClientRestore.h"
+#include "autogen_BackupProtocolClient.h"
+#include "CommonException.h"
+#include "BackupClientFileAttributes.h"
+#include "IOStream.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "CollectInBufferStream.h"
+#include "FileStream.h"
+#include "Utils.h"
+
+#include "MemLeakFindOn.h"
+
+#define MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES (128*1024)
+
+class RestoreResumeInfo
+{
+public:
+ // constructor
+ RestoreResumeInfo()
+ : mNextLevelID(0),
+ mpNextLevel(0)
+ {
+ }
+
+ // destructor
+ ~RestoreResumeInfo()
+ {
+ delete mpNextLevel;
+ mpNextLevel = 0;
+ }
+
+ // Get a next level object
+ RestoreResumeInfo &AddLevel(int64_t ID, const std::string &rLocalName)
+ {
+ ASSERT(mpNextLevel == 0 && mNextLevelID == 0);
+ mpNextLevel = new RestoreResumeInfo;
+ mNextLevelID = ID;
+ mNextLevelLocalName = rLocalName;
+ return *mpNextLevel;
+ }
+
+ // Remove the next level info
+ void RemoveLevel()
+ {
+ ASSERT(mpNextLevel != 0 && mNextLevelID != 0);
+ delete mpNextLevel;
+ mpNextLevel = 0;
+ mNextLevelID = 0;
+ mNextLevelLocalName.erase();
+ }
+
+ void Save(const std::string &rFilename) const
+ {
+ // TODO: use proper buffered streams when they're done
+ // Build info in memory buffer
+ CollectInBufferStream write;
+
+ // Save this level
+ SaveLevel(write);
+
+ // Store in file
+ write.SetForReading();
+ FileStream file(rFilename.c_str(), O_WRONLY | O_CREAT);
+ write.CopyStreamTo(file, IOStream::TimeOutInfinite, 8*1024 /* large buffer */);
+ }
+
+ void SaveLevel(IOStream &rWrite) const
+ {
+ // Write the restored objects
+ int64_t numObjects = mRestoredObjects.size();
+ rWrite.Write(&numObjects, sizeof(numObjects));
+ for(std::set<int64_t>::const_iterator i(mRestoredObjects.begin()); i != mRestoredObjects.end(); ++i)
+ {
+ int64_t id = *i;
+ rWrite.Write(&id, sizeof(id));
+ }
+
+ // Next level?
+ if(mpNextLevel != 0)
+ {
+ // ID
+ rWrite.Write(&mNextLevelID, sizeof(mNextLevelID));
+ // Name string
+ int32_t nsize = mNextLevelLocalName.size();
+ rWrite.Write(&nsize, sizeof(nsize));
+ rWrite.Write(mNextLevelLocalName.c_str(), nsize);
+ // And then the level itself
+ mpNextLevel->SaveLevel(rWrite);
+ }
+ else
+ {
+ // Just write a zero
+ int64_t zero = 0;
+ rWrite.Write(&zero, sizeof(zero));
+ }
+ }
+
+ // Not written to be efficient -- shouldn't be called very often.
+ bool Load(const std::string &rFilename)
+ {
+ // Delete and reset if necessary
+ if(mpNextLevel != 0)
+ {
+ RemoveLevel();
+ }
+
+ // Open file
+ FileStream file(rFilename.c_str());
+
+ // Load this level
+ return LoadLevel(file);
+ }
+
+ #define CHECKED_READ(x, s) if(!rRead.ReadFullBuffer(x, s, 0)) {return false;}
+ bool LoadLevel(IOStream &rRead)
+ {
+ // Load the restored objects list
+ mRestoredObjects.clear();
+ int64_t numObjects = 0;
+ CHECKED_READ(&numObjects, sizeof(numObjects));
+ for(int64_t o = 0; o < numObjects; ++o)
+ {
+ int64_t id;
+ CHECKED_READ(&id, sizeof(id));
+ mRestoredObjects.insert(id);
+ }
+
+ // ID of next level?
+ int64_t nextID = 0;
+ CHECKED_READ(&nextID, sizeof(nextID));
+ if(nextID != 0)
+ {
+ // Load the next level!
+ std::string name;
+ int32_t nsize = 0;
+ CHECKED_READ(&nsize, sizeof(nsize));
+ char n[PATH_MAX];
+ if(nsize > PATH_MAX) return false;
+ CHECKED_READ(n, nsize);
+ name.assign(n, nsize);
+
+ // Create a new level
+ mpNextLevel = new RestoreResumeInfo;
+ mNextLevelID = nextID;
+ mNextLevelLocalName = name;
+
+ // And ask it to read itself in
+ if(!mpNextLevel->LoadLevel(rRead))
+ {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ // List of objects at this level which have been done already
+ std::set<int64_t> mRestoredObjects;
+ // Next level ID
+ int64_t mNextLevelID;
+ // Pointer to next level
+ RestoreResumeInfo *mpNextLevel;
+ // Local filename of next level
+ std::string mNextLevelLocalName;
+};
+
+// parameters structure
+typedef struct
+{
+ bool PrintDots;
+ bool RestoreDeleted;
+ std::string mRestoreResumeInfoFilename;
+ RestoreResumeInfo mResumeInfo;
+} RestoreParams;
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientRestoreDir(BackupProtocolClient &, int64_t, const char *, bool)
+// Purpose: Restore a directory
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t DirectoryID, std::string &rLocalDirectoryName,
+ RestoreParams &Params, RestoreResumeInfo &rLevel)
+{
+ // If we're resuming... check that we haven't got a next level to look at
+ if(rLevel.mpNextLevel != 0)
+ {
+ // Recurse immediately
+ std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + rLevel.mNextLevelLocalName);
+ BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, localDirname, Params, *rLevel.mpNextLevel);
+
+ // Add it to the list of done itmes
+ rLevel.mRestoredObjects.insert(rLevel.mNextLevelID);
+
+ // Remove the level for the recursed directory
+ rLevel.RemoveLevel();
+ }
+
+ // Save the resumption information
+ Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+
+ // Create the local directory (if not already done) -- path and owner set later, just use restrictive owner mode
+ switch(ObjectExists(rLocalDirectoryName.c_str()))
+ {
+ case ObjectExists_Dir:
+ // Do nothing
+ break;
+ case ObjectExists_File:
+ {
+ // File exists with this name, which is fun. Get rid of it.
+ ::printf("WARNING: File present with name '%s', removing out of the way of restored directory. Use specific restore with ID to restore this object.", rLocalDirectoryName.c_str());
+ if(::unlink(rLocalDirectoryName.c_str()) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+ TRACE1("In restore, directory name collision with file %s", rLocalDirectoryName.c_str());
+ }
+ // follow through to... (no break)
+ case ObjectExists_NoObject:
+ if(::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError);
+ }
+ break;
+ default:
+ ASSERT(false);
+ break;
+ }
+
+ // Fetch the directory listing from the server -- getting a list of files which is approparite to the restore type
+ rConnection.QueryListDirectory(
+ DirectoryID,
+ Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING),
+ BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)),
+ true /* want attributes */);
+
+ // Retrieve the directory from the stream following
+ BackupStoreDirectory dir;
+ std::auto_ptr<IOStream> dirstream(rConnection.ReceiveStream());
+ dir.ReadFromStream(*dirstream, rConnection.GetTimeout());
+
+ // Apply attributes to the directory
+ const StreamableMemBlock &dirAttrBlock(dir.GetAttributes());
+ BackupClientFileAttributes dirAttr(dirAttrBlock);
+ dirAttr.WriteAttributes(rLocalDirectoryName.c_str());
+
+ int64_t bytesWrittenSinceLastRestoreInfoSave = 0;
+
+ // Process files
+ {
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+ {
+ // Check ID hasn't already been done
+ if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end())
+ {
+ // Local name
+ BackupStoreFilenameClear nm(en->GetName());
+ std::string localFilename(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename());
+
+ // Unlink anything which already exists -- for resuming restores, we can't overwrite files already there.
+ ::unlink(localFilename.c_str());
+
+ // Request it from the store
+ rConnection.QueryGetFile(DirectoryID, en->GetObjectID());
+
+ // Stream containing encoded file
+ std::auto_ptr<IOStream> objectStream(rConnection.ReceiveStream());
+
+ // Decode the file -- need to do different things depending on whether
+ // the directory entry has additional attributes
+ if(en->HasAttributes())
+ {
+ // Use these attributes
+ const StreamableMemBlock &storeAttr(en->GetAttributes());
+ BackupClientFileAttributes attr(storeAttr);
+ BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout(), &attr);
+ }
+ else
+ {
+ // Use attributes stored in file
+ BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout());
+ }
+
+ // Progress display?
+ if(Params.PrintDots)
+ {
+ printf(".");
+ fflush(stdout);
+ }
+
+ // Add it to the list of done itmes
+ rLevel.mRestoredObjects.insert(en->GetObjectID());
+
+ // Save restore info?
+ int64_t fileSize;
+ if(FileExists(localFilename.c_str(), &fileSize, true /* treat links as not existing */))
+ {
+ // File exists...
+ bytesWrittenSinceLastRestoreInfoSave += fileSize;
+
+ if(bytesWrittenSinceLastRestoreInfoSave > MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES)
+ {
+ // Save the restore info, in case it's needed later
+ Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+ bytesWrittenSinceLastRestoreInfoSave = 0;
+ }
+ }
+ }
+ }
+ }
+
+ // Make sure the restore info has been saved
+ if(bytesWrittenSinceLastRestoreInfoSave != 0)
+ {
+ // Save the restore info, in case it's needed later
+ Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+ bytesWrittenSinceLastRestoreInfoSave = 0;
+ }
+
+
+ // Recuse to directories
+ {
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
+ {
+ // Check ID hasn't already been done
+ if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end())
+ {
+ // Local name
+ BackupStoreFilenameClear nm(en->GetName());
+ std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename());
+
+ // Add the level for the next entry
+ RestoreResumeInfo &rnextLevel(rLevel.AddLevel(en->GetObjectID(), nm.GetClearFilename()));
+
+ // Recurse
+ BackupClientRestoreDir(rConnection, en->GetObjectID(), localDirname, Params, rnextLevel);
+
+ // Remove the level for the above call
+ rLevel.RemoveLevel();
+
+ // Add it to the list of done itmes
+ rLevel.mRestoredObjects.insert(en->GetObjectID());
+ }
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupClientRestore(BackupProtocolClient &, int64_t, const char *, bool, bool)
+// Purpose: Restore a directory on the server to a local directory on the disc.
+//
+// The local directory must not already exist.
+//
+// If a restore is aborted for any reason, then it may be resumed if
+// Resume == true. If Resume == false and resumption is possible, then
+// Restore_ResumePossible is returned.
+//
+// Set RestoreDeleted to restore a deleted directory. This may not give the
+// directory structure when it was deleted, because files may have been deleted
+// within it before it was deleted.
+//
+// Returns Restore_TargetExists if the target directory exists, but
+// there is no restore possible. (Won't attempt to overwrite things.)
+//
+// Returns Restore_Complete on success. (Exceptions on error.)
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName,
+ bool PrintDots, bool RestoreDeleted, bool UndeleteAfterRestoreDeleted, bool Resume)
+{
+ // Parameter block
+ RestoreParams params;
+ params.PrintDots = PrintDots;
+ params.RestoreDeleted = RestoreDeleted;
+ params.mRestoreResumeInfoFilename = LocalDirectoryName;
+ params.mRestoreResumeInfoFilename += ".boxbackupresume";
+
+ // Target exists?
+ int targetExistance = ObjectExists(LocalDirectoryName);
+
+ // Does any resumption information exist?
+ bool doingResume = false;
+ if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && targetExistance == ObjectExists_Dir)
+ {
+ if(!Resume)
+ {
+ // Caller didn't specify that resume should be done, so refuse to do it
+ // but say why.
+ return Restore_ResumePossible;
+ }
+
+ // Attempt to load the resume info file
+ if(!params.mResumeInfo.Load(params.mRestoreResumeInfoFilename))
+ {
+ // failed -- bad file, so things have gone a bit wrong
+ return Restore_TargetExists;
+ }
+
+ // Flag as doing resume so next check isn't actually performed
+ doingResume = true;
+ }
+
+ // Does the directory already exist?
+ if(targetExistance != ObjectExists_NoObject && !doingResume)
+ {
+ // Don't do anything in this case!
+ return Restore_TargetExists;
+ }
+
+ // Restore the directory
+ std::string localName(LocalDirectoryName);
+ BackupClientRestoreDir(rConnection, DirectoryID, localName, params, params.mResumeInfo);
+
+ // Undelete the directory on the server?
+ if(RestoreDeleted && UndeleteAfterRestoreDeleted)
+ {
+ // Send the command
+ rConnection.QueryUndeleteDirectory(DirectoryID);
+ }
+
+ // Finish progress display?
+ if(PrintDots)
+ {
+ printf("\n");
+ fflush(stdout);
+ }
+
+ // Delete the resume information file
+ ::unlink(params.mRestoreResumeInfoFilename.c_str());
+
+ return Restore_Complete;
+}
+
+
+
+
diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h
new file mode 100644
index 00000000..6421cc8a
--- /dev/null
+++ b/lib/backupclient/BackupClientRestore.h
@@ -0,0 +1,64 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupClientRestore.h
+// Purpose: Functions to restore files from a backup store
+// Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSCLIENTRESTORE_H
+#define BACKUPSCLIENTRESTORE__H
+
+class BackupProtocolClient;
+
+enum
+{
+ Restore_Complete = 0,
+ Restore_ResumePossible = 1,
+ Restore_TargetExists = 2
+};
+
+int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName,
+ bool PrintDots = false, bool RestoreDeleted = false, bool UndeleteAfterRestoreDeleted = false, bool Resume = false);
+
+#endif // BACKUPSCLIENTRESTORE__H
+
diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp
new file mode 100644
index 00000000..02df95e8
--- /dev/null
+++ b/lib/backupclient/BackupDaemonConfigVerify.cpp
@@ -0,0 +1,144 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupDaemonConfigVerify.cpp
+// Purpose: Configuration file definition for bbackupd
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupDaemonConfigVerify.h"
+#include "Daemon.h"
+#include "BoxPortsAndFiles.h"
+
+#include "MemLeakFindOn.h"
+
+
+static const ConfigurationVerifyKey backuplocationkeys[] =
+{
+ {"ExcludeFile", 0, ConfigTest_MultiValueAllowed, 0},
+ {"ExcludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"ExcludeDir", 0, ConfigTest_MultiValueAllowed, 0},
+ {"ExcludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeFile", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeDir", 0, ConfigTest_MultiValueAllowed, 0},
+ {"AlwaysIncludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0},
+ {"Path", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+static const ConfigurationVerify backuplocations[] =
+{
+ {
+ "*",
+ 0,
+ backuplocationkeys,
+ ConfigTest_LastEntry,
+ 0
+ }
+};
+
+static const ConfigurationVerifyKey verifyserverkeys[] =
+{
+ DAEMON_VERIFY_SERVER_KEYS
+};
+
+static const ConfigurationVerify verifyserver[] =
+{
+ {
+ "Server",
+ 0,
+ verifyserverkeys,
+ ConfigTest_Exists,
+ 0
+ },
+ {
+ "BackupLocations",
+ backuplocations,
+ 0,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+ }
+};
+
+static const ConfigurationVerifyKey verifyrootkeys[] =
+{
+ {"AccountNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+
+ {"UpdateStoreInterval", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"MinimumFileAge", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"MaxUploadWait", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"MaxFileTimeInFuture", "172800", ConfigTest_IsInt, 0}, // file is uploaded if the file is this much in the future (2 days default)
+
+ {"AutomaticBackup", "yes", ConfigTest_IsBool, 0},
+
+ {"SyncAllowScript", 0, 0, 0}, // optional script to run to see if the sync should be started now
+ // return "now" if it's allowed, or a number of seconds if it's not
+
+ {"MaximumDiffingTime", 0, ConfigTest_IsInt, 0},
+
+ {"FileTrackingSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"DiffingUploadSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"StoreHostname", 0, ConfigTest_Exists, 0},
+ {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file
+
+ {"CommandSocket", 0, 0, 0}, // not compulsory to have this
+ {"KeepAliveTime", 0, ConfigTest_IsInt, 0}, // optional
+ {"StoreObjectInfoFile", 0, 0, 0}, // optional
+
+ {"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full
+
+ {"CertificateFile", 0, ConfigTest_Exists, 0},
+ {"PrivateKeyFile", 0, ConfigTest_Exists, 0},
+ {"TrustedCAsFile", 0, ConfigTest_Exists, 0},
+ {"KeysFile", 0, ConfigTest_Exists, 0},
+ {"DataDirectory", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+const ConfigurationVerify BackupDaemonConfigVerify =
+{
+ "root",
+ verifyserver,
+ verifyrootkeys,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+};
diff --git a/lib/backupclient/BackupDaemonConfigVerify.h b/lib/backupclient/BackupDaemonConfigVerify.h
new file mode 100644
index 00000000..92d9f6c4
--- /dev/null
+++ b/lib/backupclient/BackupDaemonConfigVerify.h
@@ -0,0 +1,56 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupDaemonConfigVerify.h
+// Purpose: Configuration file definition for bbackupd
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPDAEMONCONFIGVERIFY__H
+#define BACKUPDAEMONCONFIGVERIFY__H
+
+#include "Configuration.h"
+
+extern const ConfigurationVerify BackupDaemonConfigVerify;
+
+#endif // BACKUPDAEMONCONFIGVERIFY__H
+
diff --git a/lib/backupclient/BackupStoreConstants.h b/lib/backupclient/BackupStoreConstants.h
new file mode 100644
index 00000000..b7b1483d
--- /dev/null
+++ b/lib/backupclient/BackupStoreConstants.h
@@ -0,0 +1,91 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreContants.h
+// Purpose: constants for the backup system
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECONSTANTS__H
+#define BACKUPSTORECONSTANTS__H
+
+#define BACKUPSTORE_ROOT_DIRECTORY_ID 1
+
+#define BACKUP_STORE_SERVER_VERSION 1
+
+// Minimum size for a chunk to be compressed
+#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256
+
+// min and max sizes for blocks
+#define BACKUP_FILE_MIN_BLOCK_SIZE 4096
+#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024)
+
+// Increase the block size if there are more than this number of blocks
+#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096
+
+// Avoid creating blocks smaller than this
+#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128
+
+// Maximum number of sizes to do an rsync-like scan for
+#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 64
+
+// When doing rsync scans, do not scan for blocks smaller than
+#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 128
+
+// A limit to stop diffing running out of control: If more than this
+// times the number of blocks in the original index are found, stop
+// looking. This stops really bad cases of diffing files containing
+// all the same byte using huge amounts of memory and processor time.
+// This is a multiple of the number of blocks in the diff from file.
+#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096
+
+// How many seconds to wait before deleting unused root directory entries?
+#ifndef NDEBUG
+ // Debug: 30 seconds (easier to test)
+ #define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER 30
+#else
+ // Release: 2 days (plenty of time for sysadmins to notice, or change their mind)
+ #define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER 172800
+#endif
+
+#endif // BACKUPSTORECONSTANTS__H
+
diff --git a/lib/backupclient/BackupStoreDirectory.cpp b/lib/backupclient/BackupStoreDirectory.cpp
new file mode 100644
index 00000000..b7aa4511
--- /dev/null
+++ b/lib/backupclient/BackupStoreDirectory.cpp
@@ -0,0 +1,601 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory.h
+// Purpose: Representation of a backup directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+
+#include "BackupStoreDirectory.h"
+#include "IOStream.h"
+#include "BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int32_t mNumEntries;
+ int64_t mObjectID; // this object ID
+ int64_t mContainerID; // ID of container
+ uint64_t mAttributesModTime;
+ int32_t mOptionsPresent; // bit mask of optional sections / features present
+ // Then a StreamableMemBlock for attributes
+} dir_StreamFormat;
+
+typedef enum
+{
+ Option_DependencyInfoPresent = 1
+} dir_StreamFormatOptions;
+
+typedef struct
+{
+ uint64_t mModificationTime;
+ int64_t mObjectID;
+ int64_t mSizeInBlocks;
+ uint64_t mAttributesHash;
+ int16_t mFlags; // order smaller items after bigger ones (for alignment)
+ // Then a BackupStoreFilename
+ // Then a StreamableMemBlock for attributes
+} en_StreamFormat;
+
+typedef struct
+{
+ int64_t mDependsNewer;
+ int64_t mDependsOlder;
+} en_StreamFormatDepends;
+
+// Use default packing
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::BackupStoreDirectory()
+// Purpose: Constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::BackupStoreDirectory()
+ : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0)
+{
+ ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t)
+// Purpose: Constructor giving object and container IDs
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID)
+ : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::~BackupStoreDirectory()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::~BackupStoreDirectory()
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+ {
+ delete (*i);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int)
+// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Get the header
+ dir_StreamFormat hdr;
+ if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic value...
+ if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat)
+ }
+
+ // Get data
+ mObjectID = box_ntoh64(hdr.mObjectID);
+ mContainerID = box_ntoh64(hdr.mContainerID);
+ mAttributesModTime = box_ntoh64(hdr.mAttributesModTime);
+
+ // Options
+ int32_t options = ntohl(hdr.mOptionsPresent);
+
+ // Get attributes
+ mAttributes.ReadFromStream(rStream, Timeout);
+
+ // Decode count
+ int count = ntohl(hdr.mNumEntries);
+
+ // Clear existing list
+ mEntries.clear();
+
+ // Read them in!
+ for(int c = 0; c < count; ++c)
+ {
+ Entry *pen = new Entry;
+ try
+ {
+ // Read from stream
+ pen->ReadFromStream(rStream, Timeout);
+
+ // Add to list
+ mEntries.push_back(pen);
+ }
+ catch(...)
+ {
+ delete pen;
+ throw;
+ }
+ }
+
+ // Read in dependency info?
+ if(options & Option_DependencyInfoPresent)
+ {
+ // Read in extra dependency data
+ for(int c = 0; c < count; ++c)
+ {
+ mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout);
+ }
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool)
+// Purpose: Writes a selection of entries to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const
+{
+ // Get count of entries
+ int32_t count = mEntries.size();
+ if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Need to count the entries
+ count = 0;
+ Iterator i(*this);
+ while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0)
+ {
+ count++;
+ }
+ }
+
+ // Check that sensible IDs have been set
+ ASSERT(mObjectID != 0);
+ ASSERT(mContainerID != 0);
+
+ // Need dependency info?
+ bool dependencyInfoRequired = false;
+ if(StreamDependencyInfo)
+ {
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ if(pen->HasDependencies())
+ {
+ dependencyInfoRequired = true;
+ }
+ }
+ }
+
+ // Options
+ int32_t options = 0;
+ if(dependencyInfoRequired) options |= Option_DependencyInfoPresent;
+
+ // Build header
+ dir_StreamFormat hdr;
+ hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE);
+ hdr.mNumEntries = htonl(count);
+ hdr.mObjectID = box_hton64(mObjectID);
+ hdr.mContainerID = box_hton64(mContainerID);
+ hdr.mAttributesModTime = box_hton64(mAttributesModTime);
+ hdr.mOptionsPresent = htonl(options);
+
+ // Write header
+ rStream.Write(&hdr, sizeof(hdr));
+
+ // Write the attributes?
+ if(StreamAttributes)
+ {
+ mAttributes.WriteToStream(rStream);
+ }
+ else
+ {
+ // Write a blank header instead
+ StreamableMemBlock::WriteEmptyBlockToStream(rStream);
+ }
+
+ // Then write all the entries
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ pen->WriteToStream(rStream);
+ }
+
+ // Write dependency info?
+ if(dependencyInfoRequired)
+ {
+ Iterator i(*this);
+ Entry *pen = 0;
+ while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+ {
+ pen->WriteToStreamDependencyInfo(rStream);
+ }
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddEntry(const Entry &)
+// Purpose: Adds entry to directory (no checking)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy)
+{
+ Entry *pnew = new Entry(rEntryToCopy);
+ try
+ {
+ mEntries.push_back(pnew);
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+
+ return pnew;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
+// Purpose: Adds entry to directory (no checking)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime)
+{
+ Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime);
+ try
+ {
+ mEntries.push_back(pnew);
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+
+ return pnew;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::DeleteEntry(int64_t)
+// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient)
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::DeleteEntry(int64_t ObjectID)
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin());
+ i != mEntries.end(); ++i)
+ {
+ if((*i)->mObjectID == ObjectID)
+ {
+ // Delete
+ delete (*i);
+ // Remove from list
+ mEntries.erase(i);
+ // Done
+ return;
+ }
+ }
+
+ // Not found
+ THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::FindEntryByID(int64_t)
+// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist.
+// Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const
+{
+ for(std::vector<Entry*>::const_iterator i(mEntries.begin());
+ i != mEntries.end(); ++i)
+ {
+ if((*i)->mObjectID == ObjectID)
+ {
+ // Found
+ return (*i);
+ }
+ }
+
+ // Not found
+ return 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry()
+// Purpose: Constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry()
+ : mModificationTime(0),
+ mObjectID(0),
+ mSizeInBlocks(0),
+ mFlags(0),
+ mAttributesHash(0),
+ mMinMarkNumber(0),
+ mMarkNumber(0),
+ mDependsNewer(0),
+ mDependsOlder(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::~Entry()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::~Entry()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry(const Entry &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry(const Entry &rToCopy)
+ : mName(rToCopy.mName),
+ mModificationTime(rToCopy.mModificationTime),
+ mObjectID(rToCopy.mObjectID),
+ mSizeInBlocks(rToCopy.mSizeInBlocks),
+ mFlags(rToCopy.mFlags),
+ mAttributesHash(rToCopy.mAttributesHash),
+ mAttributes(rToCopy.mAttributes),
+ mMinMarkNumber(rToCopy.mMinMarkNumber),
+ mMarkNumber(rToCopy.mMarkNumber),
+ mDependsNewer(rToCopy.mDependsNewer),
+ mDependsOlder(rToCopy.mDependsOlder)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
+// Purpose: Constructor from values
+// Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash)
+ : mName(rName),
+ mModificationTime(ModificationTime),
+ mObjectID(ObjectID),
+ mSizeInBlocks(SizeInBlocks),
+ mFlags(Flags),
+ mAttributesHash(AttributesHash),
+ mMinMarkNumber(0),
+ mMarkNumber(0),
+ mDependsNewer(0),
+ mDependsOlder(0)
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int)
+// Purpose: Read an entry from a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Grab the raw bytes from the stream which compose the header
+ en_StreamFormat entry;
+ if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Do reading first before modifying the variables, to be more exception safe
+
+ // Get the filename
+ BackupStoreFilename name;
+ name.ReadFromStream(rStream, Timeout);
+
+ // Get the attributes
+ mAttributes.ReadFromStream(rStream, Timeout);
+
+ // Store the rest of the bits
+ mModificationTime = box_ntoh64(entry.mModificationTime);
+ mObjectID = box_ntoh64(entry.mObjectID);
+ mSizeInBlocks = box_ntoh64(entry.mSizeInBlocks);
+ mAttributesHash = box_ntoh64(entry.mAttributesHash);
+ mFlags = ntohs(entry.mFlags);
+ mName = name;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &)
+// Purpose: Writes the entry to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const
+{
+ // Build a structure
+ en_StreamFormat entry;
+ entry.mModificationTime = box_hton64(mModificationTime);
+ entry.mObjectID = box_hton64(mObjectID);
+ entry.mSizeInBlocks = box_hton64(mSizeInBlocks);
+ entry.mAttributesHash = box_hton64(mAttributesHash);
+ entry.mFlags = htons(mFlags);
+
+ // Write it
+ rStream.Write(&entry, sizeof(entry));
+
+ // Write the filename
+ mName.WriteToStream(rStream);
+
+ // Write any attributes
+ mAttributes.WriteToStream(rStream);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int)
+// Purpose: Read the optional dependency info from a stream
+// Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout)
+{
+ // Grab the raw bytes from the stream which compose the header
+ en_StreamFormatDepends depends;
+ if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Store the data
+ mDependsNewer = box_ntoh64(depends.mDependsNewer);
+ mDependsOlder = box_ntoh64(depends.mDependsOlder);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &)
+// Purpose: Write the optional dependency info to a stream
+// Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const
+{
+ // Build structure
+ en_StreamFormatDepends depends;
+ depends.mDependsNewer = box_hton64(mDependsNewer);
+ depends.mDependsOlder = box_hton64(mDependsOlder);
+ // Write
+ rStream.Write(&depends, sizeof(depends));
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreDirectory.h b/lib/backupclient/BackupStoreDirectory.h
new file mode 100644
index 00000000..cb11e028
--- /dev/null
+++ b/lib/backupclient/BackupStoreDirectory.h
@@ -0,0 +1,306 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreDirectory.h
+// Purpose: Representation of a backup directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREDIRECTORY__H
+#define BACKUPSTOREDIRECTORY__H
+
+#include <string>
+#include <vector>
+
+#include "BackupStoreFilenameClear.h"
+#include "StreamableMemBlock.h"
+#include "BoxTime.h"
+
+class IOStream;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreDirectory
+// Purpose: In memory representation of a directory
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreDirectory
+{
+public:
+ BackupStoreDirectory();
+ BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID);
+private:
+ // Copying not allowed
+ BackupStoreDirectory(const BackupStoreDirectory &rToCopy);
+public:
+ ~BackupStoreDirectory();
+
+ class Entry
+ {
+ public:
+ friend class BackupStoreDirectory;
+
+ Entry();
+ ~Entry();
+ Entry(const Entry &rToCopy);
+ Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash);
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream) const;
+
+ const BackupStoreFilename &GetName() const {return mName;}
+ box_time_t GetModificationTime() const {return mModificationTime;}
+ int64_t GetObjectID() const {return mObjectID;}
+ int64_t GetSizeInBlocks() const {return mSizeInBlocks;}
+ int16_t GetFlags() const {return mFlags;}
+ void AddFlags(int16_t Flags) {mFlags |= Flags;}
+ void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;}
+
+ // Some things can be changed
+ void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;}
+ void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;}
+
+ // Attributes
+ bool HasAttributes() const {return !mAttributes.IsEmpty();}
+ void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;}
+ const StreamableMemBlock &GetAttributes() const {return mAttributes;}
+ uint64_t GetAttributesHash() const {return mAttributesHash;}
+
+ // Marks
+ // The lowest mark number a version of a file of this name has ever had
+ uint32_t GetMinMarkNumber() const {return mMinMarkNumber;}
+ // The mark number on this file
+ uint32_t GetMarkNumber() const {return mMarkNumber;}
+
+ // Make sure these flags are synced with those in backupprocotol.txt
+ // ListDirectory command
+ enum
+ {
+ Flags_INCLUDE_EVERYTHING = -1,
+ Flags_EXCLUDE_NOTHING = 0,
+ Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below!
+ Flags_File = 1,
+ Flags_Dir = 2,
+ Flags_Deleted = 4,
+ Flags_OldVersion = 8,
+ Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion
+ };
+ // characters for textual listing of files -- see bbackupquery/BackupQueries
+ #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR"
+
+ bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet)
+ {
+ return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet))
+ && ((mFlags & FlagsNotToBeSet) == 0);
+ };
+
+ // Get dependency info
+ // new version this depends on
+ int64_t GetDependsNewer() const {return mDependsNewer;}
+ void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;}
+ // older version which depends on this
+ int64_t GetDependsOlder() const {return mDependsOlder;}
+ void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;}
+
+ // Dependency info saving
+ bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;}
+ void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout);
+ void WriteToStreamDependencyInfo(IOStream &rStream) const;
+
+ private:
+ BackupStoreFilename mName;
+ box_time_t mModificationTime;
+ int64_t mObjectID;
+ int64_t mSizeInBlocks;
+ int16_t mFlags;
+ uint64_t mAttributesHash;
+ StreamableMemBlock mAttributes;
+ uint32_t mMinMarkNumber;
+ uint32_t mMarkNumber;
+
+ uint64_t mDependsNewer; // new version this depends on
+ uint64_t mDependsOlder; // older version which depends on this
+ };
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream,
+ int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING,
+ int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING,
+ bool StreamAttributes = true, bool StreamDependencyInfo = true) const;
+
+ Entry *AddEntry(const Entry &rEntryToCopy);
+ Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime);
+ void DeleteEntry(int64_t ObjectID);
+ Entry *FindEntryByID(int64_t ObjectID) const;
+
+ int64_t GetObjectID() const {return mObjectID;}
+ int64_t GetContainerID() const {return mContainerID;}
+
+ // Need to be able to update the container ID when moving objects
+ void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;}
+
+ // Purely for use of server -- not serialised into streams
+ int64_t GetRevisionID() const {return mRevisionID;}
+ void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;}
+
+ unsigned int GetNumberOfEntries() const {return mEntries.size();}
+
+ // User info -- not serialised into streams
+ int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;}
+ void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;}
+
+ // Attributes
+ bool HasAttributes() const {return !mAttributes.IsEmpty();}
+ void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;}
+ const StreamableMemBlock &GetAttributes() const {return mAttributes;}
+ box_time_t GetAttributesModTime() const {return mAttributesModTime;}
+
+ class Iterator
+ {
+ public:
+ Iterator(const BackupStoreDirectory &rDir)
+ : mrDir(rDir), i(rDir.mEntries.begin())
+ {
+ }
+
+ BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags
+ while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.end())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ // WARNING: This function is really very inefficient.
+ // Only use when you want to look up ONE filename, not in a loop looking up lots.
+ // In a looping situation, cache the decrypted filenames in another memory structure.
+ BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags or filename
+ while( (i != mrDir.mEntries.end())
+ && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) )
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.end())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ private:
+ const BackupStoreDirectory &mrDir;
+ std::vector<Entry*>::const_iterator i;
+ };
+
+ friend class Iterator;
+
+ class ReverseIterator
+ {
+ public:
+ ReverseIterator(const BackupStoreDirectory &rDir)
+ : mrDir(rDir), i(rDir.mEntries.rbegin())
+ {
+ }
+
+ BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+ {
+ // Skip over things which don't match the required flags
+ while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+ {
+ ++i;
+ }
+ // Not the last one?
+ if(i == mrDir.mEntries.rend())
+ {
+ return 0;
+ }
+ // Return entry, and increment
+ return (*(i++));
+ }
+
+ private:
+ const BackupStoreDirectory &mrDir;
+ std::vector<Entry*>::const_reverse_iterator i;
+ };
+
+ friend class ReverseIterator;
+
+ // For recovery of the store
+ // Implemented in BackupStoreCheck2.cpp
+ bool CheckAndFix();
+ void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags);
+ bool NameInUse(const BackupStoreFilename &rName);
+ // Don't use these functions in normal code!
+
+ // For testing
+ void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;}
+
+ // Debug and diagonistics
+ void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere
+
+private:
+ int64_t mRevisionID;
+ int64_t mObjectID;
+ int64_t mContainerID;
+ std::vector<Entry*> mEntries;
+ box_time_t mAttributesModTime;
+ StreamableMemBlock mAttributes;
+ int64_t mUserInfo1;
+};
+
+#endif // BACKUPSTOREDIRECTORY__H
+
diff --git a/lib/backupclient/BackupStoreException.h b/lib/backupclient/BackupStoreException.h
new file mode 100644
index 00000000..338bdfad
--- /dev/null
+++ b/lib/backupclient/BackupStoreException.h
@@ -0,0 +1,55 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreException.h
+// Purpose: Exception
+// Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREEXCEPTION__H
+#define BACKUPSTOREEXCEPTION__H
+
+// Compatibility
+#include "autogen_BackupStoreException.h"
+
+#endif // BACKUPSTOREEXCEPTION__H
+
diff --git a/lib/backupclient/BackupStoreException.txt b/lib/backupclient/BackupStoreException.txt
new file mode 100644
index 00000000..50c615b2
--- /dev/null
+++ b/lib/backupclient/BackupStoreException.txt
@@ -0,0 +1,70 @@
+EXCEPTION BackupStore 4
+
+Internal 0
+BadAccountDatabaseFile 1
+AccountDatabaseNoSuchEntry 2
+InvalidBackupStoreFilename 3
+UnknownFilenameEncoding 4
+CouldntReadEntireStructureFromStream 5
+BadDirectoryFormat 6
+CouldNotFindEntryInDirectory 7
+OutputFileAlreadyExists 8
+OSFileError 9
+StreamDoesntHaveRequiredFeatures 10
+BadBackupStoreFile 11
+CouldNotLoadStoreInfo 12
+BadStoreInfoOnLoad 13
+StoreInfoIsReadOnly 14
+StoreInfoDirNotInList 15
+StoreInfoBlockDeltaMakesValueNegative 16
+DirectoryHasBeenDeleted 17
+StoreInfoNotInitialised 18
+StoreInfoAlreadyLoaded 19
+StoreInfoNotLoaded 20
+ReadFileFromStreamTimedOut 21
+FileWrongSizeAfterBeingStored 22
+AddedFileDoesNotVerify 23
+StoreInfoForWrongAccount 24
+ContextIsReadOnly 25
+AttributesNotLoaded 26
+AttributesNotUnderstood 27
+WrongServerVersion 28 # client side
+ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account.
+NameAlreadyExistsInDirectory 30
+BerkelyDBFailure 31 # client side
+InodeMapIsReadOnly 32 # client side
+InodeMapNotOpen 33 # client side
+FilenameEncryptionKeyNotKnown 34
+FilenameEncryptionNoKeyForSpecifiedMethod 35
+FilenameEncryptionNotSetup 36
+CouldntLoadClientKeyMaterial 37
+BadEncryptedAttributes 38
+EncryptedAttributesHaveUnknownEncoding 39
+OutputSizeTooSmallForChunk 40
+BadEncodedChunk 41
+NotEnoughSpaceToDecodeChunk 42
+ChunkHasUnknownEncoding 43
+ChunkContainsBadCompressedData 44
+CantWriteToEncodedFileStream 45
+Temp_FileEncodeStreamDidntReadBuffer 46
+CantWriteToDecodedFileStream 47
+WhenDecodingExpectedToReadButCouldnt 48
+BackupStoreFileFailedIntegrityCheck 49
+ThereIsNoDataInASymLink 50
+IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51
+BlockEntryEncodingDidntGiveExpectedLength 52
+CouldNotFindUnusedIDDuringAllocation 53
+AddedFileExceedsStorageLimit 54
+CannotDiffAnIncompleteStoreFile 55
+CannotDecodeDiffedFilesWithoutCombining 56
+FailedToReadBlockOnCombine 57
+OnCombineFromFileIsIncomplete 58
+BadNotifySysadminEventCode 59
+InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60
+CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server?
+AttributeHashSecretNotSet 62
+AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES
+SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection.
+IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related
+DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory
+PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it.
diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp
new file mode 100644
index 00000000..4609dbc1
--- /dev/null
+++ b/lib/backupclient/BackupStoreFile.cpp
@@ -0,0 +1,1568 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFile.cpp
+// Purpose: Utils for manipulating files
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#ifdef HAVE_UNISTD_H
+ #include <unistd.h>
+#endif
+
+#include <sys/stat.h>
+#include <string.h>
+#include <new>
+#include <string.h>
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ #ifndef WIN32
+ #include <syslog.h>
+ #endif
+ #include <stdio.h>
+#endif
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFilename.h"
+#include "BackupStoreException.h"
+#include "IOStream.h"
+#include "Guards.h"
+#include "FileModificationTime.h"
+#include "FileStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreObjectMagic.h"
+#include "Compress.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "CipherAES.h"
+#include "BackupStoreConstants.h"
+#include "CollectInBufferStream.h"
+#include "RollingChecksum.h"
+#include "MD5Digest.h"
+#include "ReadGatherStream.h"
+#include "Random.h"
+#include "BackupStoreFileEncodeStream.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+
+// How big a buffer to use for copying files
+#define COPY_BUFFER_SIZE (8*1024)
+
+// Statistics
+BackupStoreFileStats BackupStoreFile::msStats = {0,0,0};
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool sWarnedAboutBackwardsCompatiblity = false;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &)
+// Purpose: Encode a file into something for storing on file server.
+// Requires a real filename so full info can be stored.
+//
+// Returns a stream. Most of the work is done by the stream
+// when data is actually requested -- the file will be held
+// open until the stream is deleted or the file finished.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime)
+{
+ // Create the stream
+ std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
+
+ // Do the initial setup
+ ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, 0 /* no recipe, just encode */,
+ ContainerID, rStoreFilename, pModificationTime);
+
+ // Return the stream for the caller
+ return stream;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &)
+// Purpose: Verify that an encoded file meets the format requirements.
+// Doesn't verify that the data is intact and can be decoded.
+// Optionally returns the ID of the file which it is diffed from,
+// and the (original) container ID.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut)
+{
+ // Get the size of the file
+ int64_t fileSize = rFile.BytesLeftToRead();
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Get the header...
+ file_StreamFormat hdr;
+ if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read header
+ return false;
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ return false;
+ }
+
+ // Get a filename, see if it loads OK
+ try
+ {
+ BackupStoreFilename fn;
+ fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+ }
+ catch(...)
+ {
+ // an error occured while reading it, so that's not good
+ return false;
+ }
+
+ // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not
+ try
+ {
+ int32_t size_s;
+ if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+ }
+ int size = ntohl(size_s);
+ // Skip forward the size
+ rFile.Seek(size, IOStream::SeekType_Relative);
+ }
+ catch(...)
+ {
+ // an error occured while reading it, so that's not good
+ return false;
+ }
+
+ // Get current position in file -- the end of the header
+ int64_t headerEnd = rFile.GetPosition();
+
+ // Get number of blocks
+ int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
+
+ // Calculate where the block index will be, check it's reasonable
+ int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+ if(blockIndexLoc < headerEnd)
+ {
+ // Not enough space left for the block index, let alone the blocks themselves
+ return false;
+ }
+
+ // Load the block index header
+ rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute);
+ file_BlockIndexHeader blkhdr;
+ if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read block index header -- assume bad file
+ return false;
+ }
+
+ // Check header
+ if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
+#endif
+ )
+ || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != numBlocks)
+ {
+ // Bad header -- either magic value or number of blocks is wrong
+ return false;
+ }
+
+ // Flag for recording whether a block is referenced from another file
+ bool blockFromOtherFileReferenced = false;
+
+ // Read the index, checking that the length values all make sense
+ int64_t currentBlockStart = headerEnd;
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read block entry
+ file_BlockIndexEntry blk;
+ if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */))
+ {
+ // Couldn't read block index entry -- assume bad file
+ return false;
+ }
+
+ // Check size and location
+ int64_t blkSize = box_ntoh64(blk.mEncodedSize);
+ if(blkSize <= 0)
+ {
+ // Mark that this file references another file
+ blockFromOtherFileReferenced = true;
+ }
+ else
+ {
+ // This block is actually in this file
+ if((currentBlockStart + blkSize) > blockIndexLoc)
+ {
+ // Encoded size makes the block run over the index
+ return false;
+ }
+
+ // Move the current block start ot the end of this block
+ currentBlockStart += blkSize;
+ }
+ }
+
+ // Check that there's no empty space
+ if(currentBlockStart != blockIndexLoc)
+ {
+ return false;
+ }
+
+ // Check that if another block is references, then the ID is there, and if one isn't there is no ID.
+ int64_t otherID = box_ntoh64(blkhdr.mOtherFileID);
+ if((otherID != 0 && blockFromOtherFileReferenced == false)
+ || (otherID == 0 && blockFromOtherFileReferenced == true))
+ {
+ // Doesn't look good!
+ return false;
+ }
+
+ // Does the caller want the other ID?
+ if(pDiffFromObjectIDOut)
+ {
+ *pDiffFromObjectIDOut = otherID;
+ }
+
+ // Does the caller want the container ID?
+ if(pContainerIDOut)
+ {
+ *pContainerIDOut = box_ntoh64(hdr.mContainerID);
+ }
+
+ // Passes all tests
+ return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeFile(IOStream &, const char *)
+// Purpose: Decode a file. Will set file attributes. File must not exist.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Does file exist?
+ struct stat st;
+ if(::stat(DecodedFilename, &st) == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists)
+ }
+
+ // Try, delete output file if error
+ try
+ {
+ // Make a stream for outputting this file
+ FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL);
+
+ // Get the decoding stream
+ std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr));
+
+ // Is it a symlink?
+ if(!stream->IsSymLink())
+ {
+ // Copy it out to the file
+ stream->CopyStreamTo(out);
+ }
+
+ // Write the attributes
+ stream->GetAttributes().WriteAttributes(DecodedFilename);
+ }
+ catch(...)
+ {
+ ::unlink(DecodedFilename);
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *)
+// Purpose: Return a stream which will decode the encrypted file data on the fly.
+// Accepts streams in block index first, or main header first, order. In the latter case,
+// the stream must be Seek()able.
+//
+// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow
+// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Create stream
+ std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout));
+
+ // Get it ready
+ stream->Setup(pAlterativeAttr);
+
+ // Return to caller
+ return stream;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int)
+// Purpose: Constructor
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout)
+ : mrEncodedFile(rEncodedFile),
+ mTimeout(Timeout),
+ mNumBlocks(0),
+ mpBlockIndex(0),
+ mpEncodedData(0),
+ mpClearData(0),
+ mClearDataSize(0),
+ mCurrentBlock(-1),
+ mCurrentBlockClearSize(0),
+ mPositionInCurrentBlock(0),
+ mEntryIVBase(42) // different to default value in the encoded stream!
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ , mIsOldVersion(false)
+#endif
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::~DecodedStream()
+// Purpose: Desctructor
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::DecodedStream::~DecodedStream()
+{
+ // Free any allocated memory
+ if(mpBlockIndex)
+ {
+ ::free(mpBlockIndex);
+ }
+ if(mpEncodedData)
+ {
+ BackupStoreFile::CodingChunkFree(mpEncodedData);
+ }
+ if(mpClearData)
+ {
+ ::free(mpClearData);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *)
+// Purpose: Get the stream ready to decode -- reads in headers
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr)
+{
+ // Get the size of the file
+ int64_t fileSize = mrEncodedFile.BytesLeftToRead();
+
+ // Get the magic number to work out which order the stream is in
+ int32_t magic;
+ if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read magic value
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ bool inFileOrder = true;
+ switch(ntohl(magic))
+ {
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
+ mIsOldVersion = true;
+ // control flows on
+#endif
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
+ inFileOrder = true;
+ break;
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0:
+ mIsOldVersion = true;
+ // control flows on
+#endif
+ case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1:
+ inFileOrder = false;
+ break;
+
+ default:
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // If not in file order, then the index list must be read now
+ if(!inFileOrder)
+ {
+ ReadBlockIndex(true /* have already read and verified the magic number */);
+ }
+
+ // Get header
+ file_StreamFormat hdr;
+ if(inFileOrder)
+ {
+ // Read the header, without the magic number
+ if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic),
+ 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ // Put in magic number
+ hdr.mMagicValue = magic;
+ }
+ else
+ {
+ // Not in file order, so need to read the full header
+ if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Get the filename
+ mFilename.ReadFromStream(mrEncodedFile, mTimeout);
+
+ // Get the attributes (either from stream, or supplied attributes)
+ if(pAlterativeAttr != 0)
+ {
+ // Read dummy attributes
+ BackupClientFileAttributes attr;
+ attr.ReadFromStream(mrEncodedFile, mTimeout);
+
+ // Set to supplied attributes
+ mAttributes = *pAlterativeAttr;
+ }
+ else
+ {
+ // Read the attributes from the stream
+ mAttributes.ReadFromStream(mrEncodedFile, mTimeout);
+ }
+
+ // If it is in file order, go and read the file attributes
+ // Requires that the stream can seek
+ if(inFileOrder)
+ {
+ // Make sure the file size is known
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Store current location (beginning of encoded blocks)
+ int64_t endOfHeaderPos = mrEncodedFile.GetPosition();
+
+ // Work out where the index is
+ int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
+ int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+
+ // Seek to that position
+ mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute);
+
+ // Read the block index
+ ReadBlockIndex(false /* magic number still to be read */);
+
+ // Seek back to the end of header position, ready for reading the chunks
+ mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute);
+ }
+
+ // Check view of blocks from block header and file header match
+ if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Need to allocate some memory for the two blocks for reading encoded data, and clear data
+ if(mNumBlocks > 0)
+ {
+ // Find the maximum encoded data size
+ int32_t maxEncodedDataSize = 0;
+ const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
+ ASSERT(entry != 0);
+ for(int64_t e = 0; e < mNumBlocks; e++)
+ {
+ // Get the clear and encoded size
+ int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize);
+ ASSERT(encodedSize > 0);
+
+ // Larger?
+ if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize;
+ }
+
+ // Allocate those blocks!
+ mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32);
+
+ // Allocate the block for the clear data, using the hint from the header.
+ // If this is wrong, things will exception neatly later on, so it can't be used
+ // to do anything more than cause an error on downloading.
+ mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32;
+ mpClearData = (uint8_t*)::malloc(mClearDataSize);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool)
+// Purpose: Read the block index from the stream, and store in internal buffer (minus header)
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead)
+{
+ // Header
+ file_BlockIndexHeader blkhdr;
+
+ // Read it in -- way depends on how whether the magic number has already been read
+ if(MagicAlreadyRead)
+ {
+ // Read the header, without the magic number
+ if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue),
+ 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+ else
+ {
+ // Magic not already read, so need to read the full header
+ if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Check magic value
+ if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ }
+
+ // Get the number of blocks out of the header
+ mNumBlocks = box_ntoh64(blkhdr.mNumBlocks);
+
+ // Read the IV base
+ mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase);
+
+ // Load the block entries in?
+ if(mNumBlocks > 0)
+ {
+ // How big is the index?
+ int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks;
+
+ // Allocate some memory
+ mpBlockIndex = ::malloc(indexSize);
+ if(mpBlockIndex == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Read it in
+ if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Read(void *, int, int)
+// Purpose: As interface. Reads decrpyted data.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Symlinks don't have data. So can't read it. Not even zero bytes.
+ if(IsSymLink())
+ {
+ // Don't allow reading in this case
+ THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink);
+ }
+
+ // Already finished?
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // At end of stream, nothing to do
+ return 0;
+ }
+
+ int bytesToRead = NBytes;
+ uint8_t *output = (uint8_t*)pBuffer;
+
+ while(bytesToRead > 0 && mCurrentBlock < mNumBlocks)
+ {
+ // Anything left in the current block?
+ if(mPositionInCurrentBlock < mCurrentBlockClearSize)
+ {
+ // Copy data out of this buffer
+ int s = mCurrentBlockClearSize - mPositionInCurrentBlock;
+ if(s > bytesToRead) s = bytesToRead; // limit to requested data
+
+ // Copy
+ ::memcpy(output, mpClearData + mPositionInCurrentBlock, s);
+
+ // Update positions
+ output += s;
+ mPositionInCurrentBlock += s;
+ bytesToRead -= s;
+ }
+
+ // Need to get some more data?
+ if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize)
+ {
+ // Number of next block
+ ++mCurrentBlock;
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // Stop now!
+ break;
+ }
+
+ // Get the size from the block index
+ const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
+ int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize);
+ if(encodedSize <= 0)
+ {
+ // The caller is attempting to decode a file which is the direct result of a diff
+ // operation, and so does not contain all the data.
+ // It needs to be combined with the previous version first.
+ THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining)
+ }
+
+ // Load in next block
+ if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Decode the data
+ mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize);
+
+ // Calculate IV for this entry
+ uint64_t iv = mEntryIVBase;
+ iv += mCurrentBlock;
+ // Convert to network byte order before encrypting with it, so that restores work on
+ // platforms with different endiannesses.
+ iv = box_hton64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Make sure this is the right size
+ if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
+ {
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ if(!mIsOldVersion)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the
+ // IV for the encrypted section. Try again, with the thing the other way round
+ iv = box_swap64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+ if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ else
+ {
+ // Warn and log this issue
+ if(!sWarnedAboutBackwardsCompatiblity)
+ {
+ ::printf("WARNING: Decoded one or more files using backwards compatibility mode for block index.\n");
+ ::syslog(LOG_ERR, "WARNING: Decoded one or more files using backwards compatibility mode for block index.\n");
+ sWarnedAboutBackwardsCompatiblity = true;
+ }
+ }
+#else
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+#endif
+ }
+
+ // Check the digest
+ MD5Digest md5;
+ md5.Add(mpClearData, mCurrentBlockClearSize);
+ md5.Finish();
+ if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum))
+ {
+ THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck)
+ }
+
+ // Set vars to say what's happening
+ mPositionInCurrentBlock = 0;
+ }
+ }
+
+ ASSERT(bytesToRead >= 0);
+ ASSERT(bytesToRead <= NBytes);
+
+ return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::IsSymLink()
+// Purpose: Is the unencoded file actually a symlink?
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::IsSymLink()
+{
+ // First, check in with the attributes
+ if(!mAttributes.IsSymLink())
+ {
+ return false;
+ }
+
+ // So the attributes think it is a symlink.
+ // Consistency check...
+ if(mNumBlocks != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::Write(const void *, int)
+// Purpose: As interface. Throws exception, as you can't write to this stream.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::StreamDataLeft()
+// Purpose: As interface. Any data left?
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::StreamDataLeft()
+{
+ return mCurrentBlock < mNumBlocks;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodedStream::StreamClosed()
+// Purpose: As interface. Always returns true, no writing allowed.
+// Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::StreamClosed()
+{
+ // Can't write to this stream!
+ return true;
+}
+
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::SetBlowfishKey(const void *, int)
+// Purpose: Static. Sets the key to use for encryption and decryption.
+// Created: 7/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength)
+{
+ // IVs set later
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+
+ sBlowfishEncryptBlockEntry.Reset();
+ sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
+ sBlowfishEncryptBlockEntry.UsePadding(false);
+ sBlowfishDecryptBlockEntry.Reset();
+ sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
+ sBlowfishDecryptBlockEntry.UsePadding(false);
+}
+
+
+#ifndef HAVE_OLD_SSL
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::SetAESKey(const void *, int)
+// Purpose: Sets the AES key to use for file data encryption. Will select AES as
+// the cipher to use when encrypting.
+// Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength)
+{
+ // Setup context
+ sAESEncrypt.Reset();
+ sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
+ sAESDecrypt.Reset();
+ sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
+
+ // Set encryption to use this key, instead of the "default" blowfish key
+ spEncrypt = &sAESEncrypt;
+ sEncryptCipherType = HEADER_AES_ENCODING;
+}
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int)
+// Purpose: The maximum output size of a block, given the chunk size
+// Created: 7/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize)
+{
+ // Calculate... the maximum size of output by first the largest it could be after compression,
+ // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck
+ // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing
+ // an problem where a rather non-compressable file didn't fit in a block buffer.)
+ return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1
+ + sBlowfishEncrypt.GetIVLength() + 128;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &)
+// Purpose: Encodes a chunk (encryption, possible compressed beforehand)
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput)
+{
+ ASSERT(spEncrypt != 0);
+
+ // Check there's some space in the output block
+ if(rOutput.mBufferSize < 256)
+ {
+ rOutput.Reallocate(256);
+ }
+
+ // Check alignment of the block
+ ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+
+ // Want to compress it?
+ bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE);
+
+ // Build header
+ uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT;
+ if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED;
+
+ // Store header
+ rOutput.mpBuffer[0] = header;
+ int outOffset = 1;
+
+ // Setup cipher, and store the IV
+ int ivLen = 0;
+ const void *iv = spEncrypt->SetRandomIV(ivLen);
+ ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen);
+ outOffset += ivLen;
+
+ // Start encryption process
+ spEncrypt->Begin();
+
+ #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \
+ { \
+ if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \
+ { \
+ rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \
+ } \
+ }
+
+ // Encode the chunk
+ if(compressChunk)
+ {
+ // buffer to compress into
+ uint8_t buffer[2048];
+
+ // Set compressor with all the chunk as an input
+ Compress<true> compress;
+ compress.Input(Chunk, ChunkSize);
+ compress.FinishInput();
+
+ // Get and encrypt output
+ while(!compress.OutputHasFinished())
+ {
+ int s = compress.Output(buffer, sizeof(buffer));
+ if(s > 0)
+ {
+ ENCODECHUNK_CHECK_SPACE(s)
+ outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s);
+ }
+ else
+ {
+ // Should never happen, as we put all the input in in one go.
+ // So if this happens, it means there's a logical problem somewhere
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+ }
+ ENCODECHUNK_CHECK_SPACE(16)
+ outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
+ }
+ else
+ {
+ // Straight encryption
+ ENCODECHUNK_CHECK_SPACE(ChunkSize)
+ outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize);
+ ENCODECHUNK_CHECK_SPACE(16)
+ outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
+ }
+
+ ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check
+
+ return outOffset;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int)
+// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find
+// the extra output buffer size needed before calling.
+// See notes in EncodeChunk() for notes re alignment of the
+// encoded data.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize)
+{
+ // Check alignment of the encoded block
+ ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+
+ // First check
+ if(EncodedSize < 1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
+ }
+
+ const uint8_t *input = (uint8_t*)Encoded;
+
+ // Get header, make checks, etc
+ uint8_t header = input[0];
+ bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED;
+ uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT);
+ if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING)
+ {
+ THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding)
+ }
+
+#ifndef HAVE_OLD_SSL
+ // Choose cipher
+ CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt);
+#else
+ // AES not supported with this version of OpenSSL
+ if(encodingType == HEADER_AES_ENCODING)
+ {
+ THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL)
+ }
+ CipherContext &cipher(sBlowfishDecrypt);
+#endif
+
+ // Check enough space for header, an IV and one byte of input
+ int ivLen = cipher.GetIVLength();
+ if(EncodedSize < (1 + ivLen + 1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
+ }
+
+ // Set IV in decrypt context, and start
+ cipher.SetIV(input + 1);
+ cipher.Begin();
+
+ // Setup vars for code
+ int inOffset = 1 + ivLen;
+ uint8_t *output = (uint8_t*)Output;
+ int outOffset = 0;
+
+ // Do action
+ if(chunkCompressed)
+ {
+ // Do things in chunks
+ uint8_t buffer[2048];
+ int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer));
+
+ // Decompressor
+ Compress<false> decompress;
+
+ while(inOffset < EncodedSize)
+ {
+ // Decrypt a block
+ int bl = inputBlockLen;
+ if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long
+ int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl);
+ inOffset += bl;
+
+ // Decompress the decrypted data
+ if(s > 0)
+ {
+ decompress.Input(buffer, s);
+ int os = 0;
+ do
+ {
+ os = decompress.Output(output + outOffset, OutputSize - outOffset);
+ outOffset += os;
+ } while(os > 0);
+
+ // Check that there's space left in the output buffer -- there always should be
+ if(outOffset >= OutputSize)
+ {
+ THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
+ }
+ }
+ }
+
+ // Get any compressed data remaining in the cipher context and compression
+ int s = cipher.Final(buffer, sizeof(buffer));
+ decompress.Input(buffer, s);
+ decompress.FinishInput();
+ while(!decompress.OutputHasFinished())
+ {
+ int os = decompress.Output(output + outOffset, OutputSize - outOffset);
+ outOffset += os;
+
+ // Check that there's space left in the output buffer -- there always should be
+ if(outOffset >= OutputSize)
+ {
+ THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
+ }
+ }
+ }
+ else
+ {
+ // Easy decryption
+ outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset);
+ outOffset += cipher.Final(output + outOffset, OutputSize - outOffset);
+ }
+
+ return outOffset;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool)
+// Purpose: Returns a stream which gives a Stream order version of the encoded file.
+// If TakeOwnership == true, then the input stream will be deleted when the
+// returned stream is deleted.
+// The input stream must be seekable.
+// Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership)
+{
+ ASSERT(pStream != 0);
+
+ // Get the size of the file
+ int64_t fileSize = pStream->BytesLeftToRead();
+ if(fileSize == IOStream::SizeOfStreamUnknown)
+ {
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+ }
+
+ // Read the header
+ int bytesRead = 0;
+ file_StreamFormat hdr;
+ bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead);
+
+ // Seek backwards to put the file pointer back where it was before we started this
+ pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative);
+
+ // Check we got a block
+ if(!readBlock)
+ {
+ // Couldn't read header -- assume file bad
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Get number of blocks
+ int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
+
+ // Calculate where the block index will be, check it's reasonable
+ int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+ int64_t blockIndexLoc = fileSize - blockIndexSize;
+ if(blockIndexLoc < 0)
+ {
+ // Doesn't look good!
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Build a reordered stream
+ std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership));
+
+ // Set it up...
+ ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get()));
+ int component = rreordered.AddComponent(pStream);
+ // Send out the block index
+ rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc);
+ // And then the rest of the file
+ rreordered.AddBlock(component, blockIndexLoc, true, 0);
+
+ return reordered;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ResetStats()
+// Purpose: Reset the gathered statistics
+// Created: 20/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::ResetStats()
+{
+ msStats.mBytesInEncodedFiles = 0;
+ msStats.mBytesAlreadyOnServer = 0;
+ msStats.mTotalFileStreamSize = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &)
+// Purpose: Compares the contents of a file against the checksums contained in the
+// block index. Returns true if the checksums match, meaning the file is
+// extremely likely to match the original. Will always consume the entire index.
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout)
+{
+ // is it a symlink?
+ bool sourceIsSymlink = false;
+ {
+ struct stat st;
+ if(::lstat(Filename, &st) == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ if((st.st_mode & S_IFMT) == S_IFLNK)
+ {
+ sourceIsSymlink = true;
+ }
+ }
+
+ // Open file, if it's not a symlink
+ std::auto_ptr<FileStream> in;
+ if(!sourceIsSymlink)
+ {
+ in.reset(new FileStream(Filename));
+ }
+
+ // Read header
+ file_BlockIndexHeader hdr;
+ if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic
+ if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0);
+#endif
+
+ // Get basic information
+ int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
+ uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase);
+
+ //TODO: Verify that these sizes look reasonable
+
+ // setup
+ void *data = 0;
+ int32_t dataSize = -1;
+ bool matches = true;
+ int64_t totalSizeInBlockIndex = 0;
+
+ try
+ {
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read an entry from the stream
+ file_BlockIndexEntry entry;
+ if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read entry
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Calculate IV for this entry
+ uint64_t iv = entryIVBase;
+ iv += b;
+ iv = box_hton64(iv);
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ if(isOldVersion)
+ {
+ // Reverse the IV for compatibility
+ iv = box_swap64(iv);
+ }
+#endif
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry.mEnEnc, sizeof(entry.mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Size of block
+ int32_t blockClearSize = ntohl(entryEnc.mSize);
+ if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ totalSizeInBlockIndex += blockClearSize;
+
+ // Make sure there's enough memory allocated to load the block in
+ if(dataSize < blockClearSize)
+ {
+ // Too small, free the block if it's already allocated
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+ // Allocate a block
+ data = ::malloc(blockClearSize + 128);
+ if(data == 0)
+ {
+ throw std::bad_alloc();
+ }
+ dataSize = blockClearSize + 128;
+ }
+
+ // Load in the block from the file, if it's not a symlink
+ if(!sourceIsSymlink)
+ {
+ if(in->Read(data, blockClearSize) != blockClearSize)
+ {
+ // Not enough data left in the file, can't possibly match
+ matches = false;
+ }
+ else
+ {
+ // Check the checksum
+ MD5Digest md5;
+ md5.Add(data, blockClearSize);
+ md5.Finish();
+ if(!md5.DigestMatches(entryEnc.mStrongChecksum))
+ {
+ // Checksum didn't match
+ matches = false;
+ }
+ }
+ }
+
+ // Keep on going regardless, to make sure the entire block index stream is read
+ // -- must always be consistent about what happens with the stream.
+ }
+ }
+ catch(...)
+ {
+ // clean up in case of errors
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+ throw;
+ }
+
+ // free block
+ if(data != 0)
+ {
+ ::free(data);
+ data = 0;
+ }
+
+ // Check for data left over if it's not a symlink
+ if(!sourceIsSymlink)
+ {
+ // Anything left to read in the file?
+ if(in->BytesLeftToRead() != 0)
+ {
+ // File has extra data at the end
+ matches = false;
+ }
+ }
+
+ // Symlinks must have zero size on server
+ if(sourceIsSymlink)
+ {
+ matches = (totalSizeInBlockIndex == 0);
+ }
+
+ return matches;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer()
+// Purpose: Constructor
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::EncodingBuffer::EncodingBuffer()
+ : mpBuffer(0),
+ mBufferSize(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer()
+// Purpose: Destructor
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::EncodingBuffer::~EncodingBuffer()
+{
+ if(mpBuffer != 0)
+ {
+ BackupStoreFile::CodingChunkFree(mpBuffer);
+ mpBuffer = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::Allocate(int)
+// Purpose: Do initial allocation of block
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::EncodingBuffer::Allocate(int Size)
+{
+ ASSERT(mpBuffer == 0);
+ uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ mpBuffer = buffer;
+ mBufferSize = Size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodingBuffer::Reallocate(int)
+// Purpose: Reallocate the block. Try not to call this, it has to copy
+// the entire contents as the block can't be reallocated straight.
+// Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize)
+{
+#ifndef WIN32
+ TRACE2("Reallocating EncodingBuffer from %d to %d\n", mBufferSize, NewSize);
+#endif
+ ASSERT(mpBuffer != 0);
+ uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ // Copy data
+ ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize);
+
+ // Free old
+ BackupStoreFile::CodingChunkFree(mpBuffer);
+
+ // Store new buffer
+ mpBuffer = buffer;
+ mBufferSize = NewSize;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: DiffTimer::DiffTimer();
+// Purpose: Constructor
+// Created: 2005/02/01
+//
+// --------------------------------------------------------------------------
+DiffTimer::DiffTimer()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: DiffTimer::DiffTimer();
+// Purpose: Destructor
+// Created: 2005/02/01
+//
+// --------------------------------------------------------------------------
+DiffTimer::~DiffTimer()
+{
+}
diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h
new file mode 100644
index 00000000..521db7f0
--- /dev/null
+++ b/lib/backupclient/BackupStoreFile.h
@@ -0,0 +1,259 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFile.h
+// Purpose: Utils for manipulating files
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILE__H
+#define BACKUPSTOREFILE__H
+
+#include "IOStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreFilename.h"
+
+#include <memory>
+
+typedef struct
+{
+ int64_t mBytesInEncodedFiles;
+ int64_t mBytesAlreadyOnServer;
+ int64_t mTotalFileStreamSize;
+} BackupStoreFileStats;
+
+// Uncomment to disable backwards compatibility
+//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+
+
+// Output buffer to EncodeChunk and input data to DecodeChunk must
+// have specific alignment, see function comments.
+#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16
+#define BACKUPSTOREFILE_CODING_OFFSET 15
+
+// Have some memory allocation commands, note closing "Off" at end of file.
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: DiffTimer
+// Purpose: Interface for classes that can keep track of diffing time,
+// and send SSL keepalive messages
+// Created: 2006/01/19
+//
+// --------------------------------------------------------------------------
+class DiffTimer
+{
+public:
+ DiffTimer();
+ virtual ~DiffTimer();
+public:
+ virtual void DoKeepAlive() = 0;
+ virtual time_t GetTimeMgmtEpoch() = 0;
+ virtual int GetMaximumDiffingTime() = 0;
+ virtual int GetKeepaliveTime() = 0;
+};
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFile
+// Purpose: Class to hold together utils for maniplating files.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class BackupStoreFile
+{
+public:
+ class DecodedStream : public IOStream
+ {
+ friend class BackupStoreFile;
+ private:
+ DecodedStream(IOStream &rEncodedFile, int Timeout);
+ DecodedStream(const DecodedStream &); // not allowed
+ DecodedStream &operator=(const DecodedStream &); // not allowed
+ public:
+ ~DecodedStream();
+
+ // Stream functions
+ virtual int Read(void *pBuffer, int NBytes, int Timeout);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+ // Accessor functions
+ const BackupClientFileAttributes &GetAttributes() {return mAttributes;}
+ const BackupStoreFilename &GetFilename() {return mFilename;}
+ int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests
+
+ bool IsSymLink();
+
+ private:
+ void Setup(const BackupClientFileAttributes *pAlterativeAttr);
+ void ReadBlockIndex(bool MagicAlreadyRead);
+
+ private:
+ IOStream &mrEncodedFile;
+ int mTimeout;
+ BackupClientFileAttributes mAttributes;
+ BackupStoreFilename mFilename;
+ int64_t mNumBlocks;
+ void *mpBlockIndex;
+ uint8_t *mpEncodedData;
+ uint8_t *mpClearData;
+ int mClearDataSize;
+ int mCurrentBlock;
+ int mCurrentBlockClearSize;
+ int mPositionInCurrentBlock;
+ uint64_t mEntryIVBase;
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ bool mIsOldVersion;
+#endif
+ };
+
+
+ // Main interface
+ static std::auto_ptr<IOStream> EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0);
+ static std::auto_ptr<IOStream> EncodeFileDiff
+ (
+ const char *Filename, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename,
+ int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
+ int Timeout,
+ DiffTimer *pDiffTimer,
+ int64_t *pModificationTime = 0,
+ bool *pIsCompletelyDifferent = 0
+ );
+ static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0);
+ static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut);
+ static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut);
+ static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0);
+ static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
+ static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
+ static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout);
+ static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false);
+
+ // Stream manipulation
+ static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership);
+ static void MoveStreamPositionToBlockIndex(IOStream &rStream);
+
+ // Crypto setup
+ static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength);
+#ifndef HAVE_OLD_SSL
+ static void SetAESKey(const void *pKey, int KeyLength);
+#endif
+
+ // Allocation of properly aligning chunks for decoding and encoding chunks
+ inline static void *CodingChunkAlloc(int Size)
+ {
+ uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3));
+ if(a == 0) return 0;
+ // Align to main block size
+ ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size
+ uint8_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE
+ - (uint8_t)(((unsigned long)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE);
+ uint8_t *b = (a + adjustment);
+ // Store adjustment
+ *b = adjustment;
+ // Return offset
+ return b + BACKUPSTOREFILE_CODING_OFFSET;
+ }
+ inline static void CodingChunkFree(void *Block)
+ {
+ // Check alignment is as expected
+ ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size
+ ASSERT((uint8_t)(((unsigned long)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+ uint8_t *a = (uint8_t*)Block;
+ a -= BACKUPSTOREFILE_CODING_OFFSET;
+ // Adjust downwards...
+ a -= *a;
+ free(a);
+ }
+
+ static void DiffTimerExpired();
+
+ // Building blocks
+ class EncodingBuffer
+ {
+ public:
+ EncodingBuffer();
+ ~EncodingBuffer();
+ private:
+ // No copying
+ EncodingBuffer(const EncodingBuffer &);
+ EncodingBuffer &operator=(const EncodingBuffer &);
+ public:
+ void Allocate(int Size);
+ void Reallocate(int NewSize);
+
+ uint8_t *mpBuffer;
+ int mBufferSize;
+ };
+ static int MaxBlockSizeForChunkSize(int ChunkSize);
+ static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput);
+
+ // Caller should know how big the output size is, but also allocate a bit more memory to cover various
+ // overheads allowed for in checks
+ static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize)
+ {
+ // Plenty big enough
+ return KnownChunkSize + 256;
+ }
+ static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize);
+
+ // Statisitics, not designed to be completely reliable
+ static void ResetStats();
+ static BackupStoreFileStats msStats;
+
+ // For debug
+#ifndef NDEBUG
+ static bool TraceDetailsOfDiffProcess;
+#endif
+
+ // For decoding encoded files
+ static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile);
+};
+
+#include "MemLeakFindOff.h"
+
+#endif // BACKUPSTOREFILE__H
diff --git a/lib/backupclient/BackupStoreFileCmbDiff.cpp b/lib/backupclient/BackupStoreFileCmbDiff.cpp
new file mode 100644
index 00000000..5b28fceb
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCmbDiff.cpp
@@ -0,0 +1,364 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCmbDiff.cpp
+// Purpose: Combine two diffs together
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <stdlib.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CombineDiffs(IOStream &, IOStream &, IOStream &rOut)
+// Purpose: Given two diffs, combine them into a single diff, to produce a diff
+// which, combined with the original file, creates the result of applying
+// rDiff, then rDiff2. Two opens of rDiff2 are required
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut)
+{
+ // Skip header of first diff, record where the data starts, and skip to the index
+ int64_t diff1DataStarts = 0;
+ {
+ // Read the header for the From file
+ file_StreamFormat diff1Hdr;
+ if(!rDiff1.ReadFullBuffer(&diff1Hdr, sizeof(diff1Hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diff1Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Skip over the filename and attributes of the From file
+ // BLOCK
+ {
+ BackupStoreFilename filename2;
+ filename2.ReadFromStream(rDiff1, IOStream::TimeOutInfinite);
+ int32_t size_s;
+ if(!rDiff1.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+ }
+ int size = ntohl(size_s);
+ // Skip forward the size
+ rDiff1.Seek(size, IOStream::SeekType_Relative);
+ }
+ // Record position
+ diff1DataStarts = rDiff1.GetPosition();
+ // Skip to index
+ rDiff1.Seek(0 - (((box_ntoh64(diff1Hdr.mNumBlocks)) * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+ }
+
+ // Read the index of the first diff
+ // Header first
+ file_BlockIndexHeader diff1IdxHdr;
+ if(!rDiff1.ReadFullBuffer(&diff1IdxHdr, sizeof(diff1IdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(diff1IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ int64_t diff1NumBlocks = box_ntoh64(diff1IdxHdr.mNumBlocks);
+ // Allocate some memory
+ int64_t *diff1BlockStartPositions = (int64_t*)::malloc((diff1NumBlocks + 1) * sizeof(int64_t));
+ if(diff1BlockStartPositions == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Buffer data
+ void *buffer = 0;
+ int bufferSize = 0;
+
+ try
+ {
+ // Then the entries:
+ // For each entry, want to know if it's in the file, and if so, how big it is.
+ // We'll store this as an array of file positions in the file, with an additioal
+ // entry on the end so that we can work out the length of the last block.
+ // If an entry isn't in the file, then store 0 - (position in other file).
+ int64_t diff1Position = diff1DataStarts;
+ for(int64_t b = 0; b < diff1NumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff1.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Where's the block?
+ int64_t blockEn = box_ntoh64(e.mEncodedSize);
+ if(blockEn <= 0)
+ {
+ // Just store the negated block number
+ diff1BlockStartPositions[b] = blockEn;
+ }
+ else
+ {
+ // Block is present in this file
+ diff1BlockStartPositions[b] = diff1Position;
+ diff1Position += blockEn;
+ }
+ }
+
+ // Finish off the list, so the last entry can have it's size calcuated.
+ diff1BlockStartPositions[diff1NumBlocks] = diff1Position;
+
+ // Now read the second diff's header, copying it to the out file
+ file_StreamFormat diff2Hdr;
+ if(!rDiff2.ReadFullBuffer(&diff2Hdr, sizeof(diff2Hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diff2Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Copy
+ rOut.Write(&diff2Hdr, sizeof(diff2Hdr));
+ // Copy over filename and attributes
+ // BLOCK
+ {
+ BackupStoreFilename filename;
+ filename.ReadFromStream(rDiff2, IOStream::TimeOutInfinite);
+ filename.WriteToStream(rOut);
+ StreamableMemBlock attr;
+ attr.ReadFromStream(rDiff2, IOStream::TimeOutInfinite);
+ attr.WriteToStream(rOut);
+ }
+
+ // Get to the index of rDiff2b, and read the header
+ MoveStreamPositionToBlockIndex(rDiff2b);
+ file_BlockIndexHeader diff2IdxHdr;
+ if(!rDiff2b.ReadFullBuffer(&diff2IdxHdr, sizeof(diff2IdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(diff2IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ int64_t diff2NumBlocks = box_ntoh64(diff2IdxHdr.mNumBlocks);
+ int64_t diff2IndexEntriesStart = rDiff2b.GetPosition();
+
+ // Then read all the entries
+ int64_t diff2FilePosition = rDiff2.GetPosition();
+ for(int64_t b = 0; b < diff2NumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // What do to next about copying data
+ bool copyBlock = false;
+ int copySize = 0;
+ int64_t copyFrom = 0;
+ bool fromFileDiff1 = false;
+
+ // Where's the block?
+ int64_t blockEn = box_ntoh64(e.mEncodedSize);
+ if(blockEn > 0)
+ {
+ // Block is present in this file -- copy to out
+ copyBlock = true;
+ copyFrom = diff2FilePosition;
+ copySize = (int)blockEn;
+
+ // Move pointer onwards
+ diff2FilePosition += blockEn;
+ }
+ else
+ {
+ // Block isn't present here -- is it present in the old one?
+ int64_t blockIndex = 0 - blockEn;
+ if(blockIndex < 0 || blockIndex > diff1NumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ if(diff1BlockStartPositions[blockIndex] > 0)
+ {
+ // Block is in the old diff file, copy it across
+ copyBlock = true;
+ copyFrom = diff1BlockStartPositions[blockIndex];
+ int nb = blockIndex + 1;
+ while(diff1BlockStartPositions[nb] <= 0)
+ {
+ // This is safe, because the last entry will terminate it properly!
+ ++nb;
+ ASSERT(nb <= diff1NumBlocks);
+ }
+ copySize = diff1BlockStartPositions[nb] - copyFrom;
+ fromFileDiff1 = true;
+ }
+ }
+ //TRACE4("%d %d %lld %d\n", copyBlock, copySize, copyFrom, fromFileDiff1);
+
+ // Copy data to the output file?
+ if(copyBlock)
+ {
+ // Allocate enough space
+ if(bufferSize < copySize || buffer == 0)
+ {
+ // Free old block
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ bufferSize = 0;
+ }
+ // Allocate new block
+ buffer = ::malloc(copySize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ bufferSize = copySize;
+ }
+ ASSERT(bufferSize >= copySize);
+
+ // Load in the data
+ if(fromFileDiff1)
+ {
+ rDiff1.Seek(copyFrom, IOStream::SeekType_Absolute);
+ if(!rDiff1.ReadFullBuffer(buffer, copySize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ }
+ else
+ {
+ rDiff2.Seek(copyFrom, IOStream::SeekType_Absolute);
+ if(!rDiff2.ReadFullBuffer(buffer, copySize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ }
+ // Write out data
+ rOut.Write(buffer, copySize);
+ }
+ }
+
+ // Write the modified header
+ diff2IdxHdr.mOtherFileID = diff1IdxHdr.mOtherFileID;
+ rOut.Write(&diff2IdxHdr, sizeof(diff2IdxHdr));
+
+ // Then we'll write out the index, reading the data again
+ rDiff2b.Seek(diff2IndexEntriesStart, IOStream::SeekType_Absolute);
+ for(int64_t b = 0; b < diff2NumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Where's the block?
+ int64_t blockEn = box_ntoh64(e.mEncodedSize);
+
+ // If it's not in this file, it needs modification...
+ if(blockEn <= 0)
+ {
+ int64_t blockIndex = 0 - blockEn;
+ // In another file. Need to translate this against the other diff
+ if(diff1BlockStartPositions[blockIndex] > 0)
+ {
+ // Block is in the first diff file, stick in size
+ int nb = blockIndex + 1;
+ while(diff1BlockStartPositions[nb] <= 0)
+ {
+ // This is safe, because the last entry will terminate it properly!
+ ++nb;
+ ASSERT(nb <= diff1NumBlocks);
+ }
+ int64_t size = diff1BlockStartPositions[nb] - diff1BlockStartPositions[blockIndex];
+ e.mEncodedSize = box_hton64(size);
+ }
+ else
+ {
+ // Block in the original file, use translated value
+ e.mEncodedSize = box_hton64(diff1BlockStartPositions[blockIndex]);
+ }
+ }
+
+ // Write entry
+ rOut.Write(&e, sizeof(e));
+ }
+ }
+ catch(...)
+ {
+ // clean up
+ ::free(diff1BlockStartPositions);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+ throw;
+ }
+
+ // Clean up allocated memory
+ ::free(diff1BlockStartPositions);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+}
+
diff --git a/lib/backupclient/BackupStoreFileCmbIdx.cpp b/lib/backupclient/BackupStoreFileCmbIdx.cpp
new file mode 100644
index 00000000..8343f83f
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCmbIdx.cpp
@@ -0,0 +1,362 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCmbIdx.cpp
+// Purpose: Combine indicies of a delta file and the file it's a diff from.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <string.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+// Hide from outside world
+namespace
+{
+
+class BSFCombinedIndexStream : public IOStream
+{
+public:
+ BSFCombinedIndexStream(IOStream *pDiff);
+ ~BSFCombinedIndexStream();
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+ virtual void Initialise(IOStream &rFrom);
+
+private:
+ IOStream *mpDiff;
+ bool mIsInitialised;
+ bool mHeaderWritten;
+ file_BlockIndexHeader mHeader;
+ int64_t mNumEntriesToGo;
+ int64_t mNumEntriesInFromFile;
+ int64_t *mFromBlockSizes; // NOTE: Entries in network byte order
+};
+
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CombineFileIndices(IOStream &, IOStream &, bool)
+// Purpose: Given a diff file and the file it's a diff from, return a stream from which
+// can be read the index of the combined file, without actually combining them.
+// The stream of the diff must have a lifetime greater than or equal to the
+// lifetime of the returned stream object. The full "from" file stream
+// only needs to exist during the actual function call.
+// If you pass in dodgy files which aren't related, then you will either
+// get an error or bad results. So don't do that.
+// If DiffIsIndexOnly is true, then rDiff is assumed to be a stream positioned
+// at the beginning of the block index. Similarly for FromIsIndexOnly.
+// WARNING: Reads of the returned streams with buffer sizes less than 64 bytes
+// will not return any data.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly, bool FromIsIndexOnly)
+{
+ // Reposition file pointers?
+ if(!DiffIsIndexOnly)
+ {
+ MoveStreamPositionToBlockIndex(rDiff);
+ }
+ if(!FromIsIndexOnly)
+ {
+ MoveStreamPositionToBlockIndex(rFrom);
+ }
+
+ // Create object
+ std::auto_ptr<IOStream> stream(new BSFCombinedIndexStream(&rDiff));
+
+ // Initialise it
+ ((BSFCombinedIndexStream *)stream.get())->Initialise(rFrom);
+
+ // And return the stream
+ return stream;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::BSFCombinedIndexStream()
+// Purpose: Private class. Constructor.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+BSFCombinedIndexStream::BSFCombinedIndexStream(IOStream *pDiff)
+ : mpDiff(pDiff),
+ mIsInitialised(false),
+ mHeaderWritten(false),
+ mNumEntriesToGo(0),
+ mNumEntriesInFromFile(0),
+ mFromBlockSizes(0)
+{
+ ASSERT(mpDiff != 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::~BSFCombinedIndexStream()
+// Purpose: Private class. Destructor.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+BSFCombinedIndexStream::~BSFCombinedIndexStream()
+{
+ if(mFromBlockSizes != 0)
+ {
+ ::free(mFromBlockSizes);
+ mFromBlockSizes = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::Initialise(IOStream &)
+// Purpose: Private class. Initalise from the streams (diff passed in constructor).
+// Both streams must have file pointer positioned at the block index.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+void BSFCombinedIndexStream::Initialise(IOStream &rFrom)
+{
+ // Paranoia is good.
+ if(mIsInitialised)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Look at the diff file: Read in the header
+ if(!mpDiff->ReadFullBuffer(&mHeader, sizeof(mHeader), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(mHeader.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Read relevant data.
+ mNumEntriesToGo = box_ntoh64(mHeader.mNumBlocks);
+
+ // Adjust a bit to reflect the fact it's no longer a diff
+ mHeader.mOtherFileID = box_hton64(0);
+
+ // Now look at the from file: Read header
+ file_BlockIndexHeader fromHdr;
+ if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Then... allocate memory for the list of sizes
+ mNumEntriesInFromFile = box_ntoh64(fromHdr.mNumBlocks);
+ mFromBlockSizes = (int64_t*)::malloc(mNumEntriesInFromFile * sizeof(int64_t));
+ if(mFromBlockSizes == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // And read them all in!
+ for(int64_t b = 0; b < mNumEntriesInFromFile; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check that the from file isn't a delta in itself
+ if(box_ntoh64(e.mEncodedSize) <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+ }
+
+ // Store size (in network byte order)
+ mFromBlockSizes[b] = e.mEncodedSize;
+ }
+
+ // Flag as initialised
+ mIsInitialised = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::Read(void *, int, int)
+// Purpose: Private class. As interface.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Paranoia is good.
+ if(!mIsInitialised || mFromBlockSizes == 0 || mpDiff == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ int written = 0;
+
+ // Header output yet?
+ if(!mHeaderWritten)
+ {
+ // Enough space?
+ if(NBytes < (int)sizeof(mHeader)) return 0;
+
+ // Copy in
+ ::memcpy(pBuffer, &mHeader, sizeof(mHeader));
+ NBytes -= sizeof(mHeader);
+ written += sizeof(mHeader);
+
+ // Flag it's done
+ mHeaderWritten = true;
+ }
+
+ // How many entries can be written?
+ int entriesToWrite = NBytes / sizeof(file_BlockIndexEntry);
+ if(entriesToWrite > mNumEntriesToGo)
+ {
+ entriesToWrite = mNumEntriesToGo;
+ }
+
+ // Setup ready to go
+ file_BlockIndexEntry *poutput = (file_BlockIndexEntry*)(((uint8_t*)pBuffer) + written);
+
+ // Write entries
+ for(int b = 0; b < entriesToWrite; ++b)
+ {
+ if(!mpDiff->ReadFullBuffer(&(poutput[b]), sizeof(file_BlockIndexEntry), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Does this need adjusting?
+ int s = box_ntoh64(poutput[b].mEncodedSize);
+ if(s <= 0)
+ {
+ // A reference to a block in the from file
+ int block = 0 - s;
+ ASSERT(block >= 0);
+ if(block >= mNumEntriesInFromFile)
+ {
+ // That's not good, the block doesn't exist
+ THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+ }
+
+ // Adjust the entry in the buffer
+ poutput[b].mEncodedSize = mFromBlockSizes[block]; // stored in network byte order, no translation necessary
+ }
+ }
+
+ // Update written count
+ written += entriesToWrite * sizeof(file_BlockIndexEntry);
+ mNumEntriesToGo -= entriesToWrite;
+
+ return written;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::Write(const void *, int)
+// Purpose: Private class. As interface.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::StreamDataLeft()
+// Purpose: Private class. As interface
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+bool BSFCombinedIndexStream::StreamDataLeft()
+{
+ return (!mHeaderWritten) || (mNumEntriesToGo > 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BSFCombinedIndexStream::StreamClosed()
+// Purpose: Private class. As interface.
+// Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+bool BSFCombinedIndexStream::StreamClosed()
+{
+ return true; // doesn't do writing
+}
+
diff --git a/lib/backupclient/BackupStoreFileCombine.cpp b/lib/backupclient/BackupStoreFileCombine.cpp
new file mode 100644
index 00000000..0a1704eb
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCombine.cpp
@@ -0,0 +1,448 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCombine.cpp
+// Purpose: File combining for BackupStoreFile
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+#include "FileStream.h"
+
+#include "MemLeakFindOn.h"
+
+typedef struct
+{
+ int64_t mFilePosition;
+} FromIndexEntry;
+
+static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries);
+static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut);
+static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut);
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::CombineFile(IOStream &, IOStream &, IOStream &)
+// Purpose: Where rDiff is a store file which is incomplete as a result of a
+// diffing operation, rFrom is the file it is diffed from, and
+// rOut is the stream in which to place the result, the old file
+// and new file are combined into a file containing all the data.
+// rDiff2 is the same file as rDiff, opened again to get two
+// independent streams to the same file.
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut)
+{
+ // Read and copy the header.
+ file_StreamFormat hdr;
+ if(!rDiff.ReadFullBuffer(&hdr, sizeof(hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Copy
+ rOut.Write(&hdr, sizeof(hdr));
+ // Copy over filename and attributes
+ // BLOCK
+ {
+ BackupStoreFilename filename;
+ filename.ReadFromStream(rDiff, IOStream::TimeOutInfinite);
+ filename.WriteToStream(rOut);
+ StreamableMemBlock attr;
+ attr.ReadFromStream(rDiff, IOStream::TimeOutInfinite);
+ attr.WriteToStream(rOut);
+ }
+
+ // Read the header for the From file
+ file_StreamFormat fromHdr;
+ if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Skip over the filename and attributes of the From file
+ // BLOCK
+ {
+ BackupStoreFilename filename2;
+ filename2.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ int32_t size_s;
+ if(!rFrom.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+ }
+ int size = ntohl(size_s);
+ // Skip forward the size
+ rFrom.Seek(size, IOStream::SeekType_Relative);
+ }
+
+ // Allocate memory for the block index of the From file
+ int64_t fromNumBlocks = box_ntoh64(fromHdr.mNumBlocks);
+ // NOTE: An extra entry is required so that the length of the last block can be calculated
+ FromIndexEntry *pFromIndex = (FromIndexEntry*)::malloc((fromNumBlocks+1) * sizeof(FromIndexEntry));
+ if(pFromIndex == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ try
+ {
+ // Load the index from the From file, calculating the offsets in the
+ // file as we go along, and enforce that everything should be present.
+ LoadFromIndex(rFrom, pFromIndex, fromNumBlocks);
+
+ // Read in the block index of the Diff file in small chunks, and output data
+ // for each block, either from this file, or the other file.
+ int64_t diffNumBlocks = box_ntoh64(hdr.mNumBlocks);
+ CopyData(rDiff /* positioned at start of data */, rDiff2, diffNumBlocks, rFrom, pFromIndex, fromNumBlocks, rOut);
+
+ // Read in the block index again, and output the new block index, simply
+ // filling in the sizes of blocks from the old file.
+ WriteNewIndex(rDiff, diffNumBlocks, pFromIndex, fromNumBlocks, rOut);
+
+ // Free buffers
+ ::free(pFromIndex);
+ pFromIndex = 0;
+ }
+ catch(...)
+ {
+ // Clean up
+ if(pFromIndex != 0)
+ {
+ ::free(pFromIndex);
+ pFromIndex = 0;
+ }
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static LoadFromIndex(IOStream &, FromIndexEntry *, int64_t)
+// Purpose: Static. Load the index from the From file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries)
+{
+ ASSERT(pIndex != 0);
+ ASSERT(NumEntries >= 0);
+
+ // Get the starting point in the file
+ int64_t filePos = rFrom.GetPosition();
+
+ // Jump to the end of the file to read the index
+ rFrom.Seek(0 - ((NumEntries * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+
+ // Read block index header
+ file_BlockIndexHeader blkhdr;
+ if(!rFrom.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != NumEntries)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // And then the block entries
+ for(int64_t b = 0; b < NumEntries; ++b)
+ {
+ // Read
+ file_BlockIndexEntry en;
+ if(!rFrom.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // Add to list
+ pIndex[b].mFilePosition = filePos;
+
+ // Encoded size?
+ int64_t encodedSize = box_ntoh64(en.mEncodedSize);
+ // Check that the block is actually there
+ if(encodedSize <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+ }
+
+ // Move file pointer on
+ filePos += encodedSize;
+ }
+
+ // Store the position in the very last entry, so the size of the last entry can be calculated
+ pIndex[NumEntries].mFilePosition = filePos;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static CopyData(IOStream &, IOStream &, int64_t, IOStream &, FromIndexEntry *, int64_t, IOStream &)
+// Purpose: Static. Copy data from the Diff and From file to the out file.
+// rDiffData is at beginning of data.
+// rDiffIndex at any position.
+// rFrom is at any position.
+// rOut is after the header, ready for data
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks,
+ IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut)
+{
+ // Jump to the end of the diff file to read the index
+ rDiffIndex.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+
+ // Read block index header
+ file_BlockIndexHeader diffBlkhdr;
+ if(!rDiffIndex.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || (int64_t)box_ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Record where the From file is
+ int64_t fromPos = rFrom.GetPosition();
+
+ // Buffer data
+ void *buffer = 0;
+ int bufferSize = 0;
+
+ try
+ {
+ // Read the blocks in!
+ for(int64_t b = 0; b < DiffNumBlocks; ++b)
+ {
+ // Read
+ file_BlockIndexEntry en;
+ if(!rDiffIndex.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // What's the size value stored in the entry
+ int64_t encodedSize = box_ntoh64(en.mEncodedSize);
+
+ // How much data will be read?
+ int32_t blockSize = 0;
+ if(encodedSize > 0)
+ {
+ // The block is actually in the diff file
+ blockSize = encodedSize;
+ }
+ else
+ {
+ // It's in the from file. First, check to see if it's valid
+ int64_t blockIdx = (0 - encodedSize);
+ if(blockIdx > FromNumBlocks)
+ {
+ // References a block which doesn't actually exist
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Calculate size. This operation is safe because of the extra entry at the end
+ blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition;
+ }
+ ASSERT(blockSize > 0);
+
+ // Make sure there's memory available to copy this
+ if(bufferSize < blockSize || buffer == 0)
+ {
+ // Free old block
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ bufferSize = 0;
+ }
+ // Allocate new block
+ buffer = ::malloc(blockSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ bufferSize = blockSize;
+ }
+ ASSERT(bufferSize >= blockSize);
+
+ // Load in data from one of the files
+ if(encodedSize > 0)
+ {
+ // Load from diff file
+ if(!rDiffData.ReadFullBuffer(buffer, blockSize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ }
+ else
+ {
+ // Locate and read the data from the from file
+ int64_t blockIdx = (0 - encodedSize);
+ // Seek if necessary
+ if(fromPos != pFromIndex[blockIdx].mFilePosition)
+ {
+ rFrom.Seek(pFromIndex[blockIdx].mFilePosition, IOStream::SeekType_Absolute);
+ fromPos = pFromIndex[blockIdx].mFilePosition;
+ }
+ // Read
+ if(!rFrom.ReadFullBuffer(buffer, blockSize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // Update fromPos to current position
+ fromPos += blockSize;
+ }
+
+ // Write data to out file
+ rOut.Write(buffer, blockSize);
+ }
+
+ // Free buffer, if allocated
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ }
+ }
+ catch(...)
+ {
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ }
+ throw;
+ }
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static WriteNewIndex(IOStream &, int64_t, FromIndexEntry *, int64_t, IOStream &)
+// Purpose: Write the index to the out file, just copying from the diff file and
+// adjusting the entries.
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut)
+{
+ // Jump to the end of the diff file to read the index
+ rDiff.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+
+ // Read block index header
+ file_BlockIndexHeader diffBlkhdr;
+ if(!rDiff.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || (int64_t)box_ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Write it out with a blanked out other file ID
+ diffBlkhdr.mOtherFileID = box_hton64(0);
+ rOut.Write(&diffBlkhdr, sizeof(diffBlkhdr));
+
+ // Rewrite the index
+ for(int64_t b = 0; b < DiffNumBlocks; ++b)
+ {
+ file_BlockIndexEntry en;
+ if(!rDiff.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+
+ // What's the size value stored in the entry
+ int64_t encodedSize = box_ntoh64(en.mEncodedSize);
+
+ // Need to adjust it?
+ if(encodedSize <= 0)
+ {
+ // This actually refers to a block in the from file. So rewrite this.
+ int64_t blockIdx = (0 - encodedSize);
+ if(blockIdx > FromNumBlocks)
+ {
+ // References a block which doesn't actually exist
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Calculate size. This operation is safe because of the extra entry at the end
+ int32_t blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition;
+ // Then replace entry
+ en.mEncodedSize = box_hton64(((uint64_t)blockSize));
+ }
+
+ // Write entry
+ rOut.Write(&en, sizeof(en));
+ }
+}
+
+
+
+
+
diff --git a/lib/backupclient/BackupStoreFileCryptVar.cpp b/lib/backupclient/BackupStoreFileCryptVar.cpp
new file mode 100644
index 00000000..15c818e9
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCryptVar.cpp
@@ -0,0 +1,69 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCryptVar.cpp
+// Purpose: Cryptographic keys for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFileWire.h"
+
+#include "MemLeakFindOn.h"
+
+CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt;
+CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt;
+
+#ifndef HAVE_OLD_SSL
+ CipherContext BackupStoreFileCryptVar::sAESEncrypt;
+ CipherContext BackupStoreFileCryptVar::sAESDecrypt;
+#endif
+
+// Default to blowfish
+CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt;
+uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING;
+
+CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry;
+CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry;
+
diff --git a/lib/backupclient/BackupStoreFileCryptVar.h b/lib/backupclient/BackupStoreFileCryptVar.h
new file mode 100644
index 00000000..314499f2
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCryptVar.h
@@ -0,0 +1,77 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileCryptVar.h
+// Purpose: Cryptographic keys for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILECRYPTVAR__H
+#define BACKUPSTOREFILECRYPTVAR__H
+
+#include "CipherContext.h"
+
+// Hide private static variables from the rest of the world by putting them
+// as static variables in a namespace.
+// -- don't put them as static class variables to avoid openssl/evp.h being
+// included all over the project.
+namespace BackupStoreFileCryptVar
+{
+ // Keys for the main file data
+ extern CipherContext sBlowfishEncrypt;
+ extern CipherContext sBlowfishDecrypt;
+ // Use AES when available
+#ifndef HAVE_OLD_SSL
+ extern CipherContext sAESEncrypt;
+ extern CipherContext sAESDecrypt;
+#endif
+ // How encoding will be done
+ extern CipherContext *spEncrypt;
+ extern uint8_t sEncryptCipherType;
+
+ // Keys for the block indicies
+ extern CipherContext sBlowfishEncryptBlockEntry;
+ extern CipherContext sBlowfishDecryptBlockEntry;
+}
+
+#endif // BACKUPSTOREFILECRYPTVAR__H
+
diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp
new file mode 100644
index 00000000..ecda26b2
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileDiff.cpp
@@ -0,0 +1,1094 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileDiff.cpp
+// Purpose: Functions relating to diffing BackupStoreFiles
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <map>
+
+#ifdef HAVE_TIME_H
+ #include <time.h>
+#elif HAVE_SYS_TIME_H
+ #include <sys/time.h>
+#endif
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreFileEncodeStream.h"
+#include "BackupStoreConstants.h"
+#include "FileStream.h"
+#include "RollingChecksum.h"
+#include "MD5Digest.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+using namespace BackupStoreFileCreation;
+
+// By default, don't trace out details of the diff as we go along -- would fill up logs significantly.
+// But it's useful for the test.
+#ifndef NDEBUG
+ bool BackupStoreFile::TraceDetailsOfDiffProcess = false;
+#endif
+
+static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis);
+static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]);
+static void SearchForMatchingBlocks(IOStream &rFile,
+ std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex,
+ int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES],
+ DiffTimer *pDiffTimer);
+static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable);
+static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum &fastSum, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber,
+BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks);
+static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile);
+
+// sDiffTimerExpired flags when the diff timer has expired. When true, the
+// diff routine should check the wall clock as soon as possible, to determine
+// whether it's time for a keepalive to be sent, or whether the diff has been
+// running for too long and should be terminated.
+static bool sDiffTimerExpired = false;
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DiffTimerExpired()
+// Purpose: Notifies BackupStoreFile object that the diff operation
+// timer has expired, which may mean that a keepalive should
+// be sent, or the diff should be terminated. Called from an
+// external timer, so it should not do more than set a flag.
+//
+// Created: 19/1/06
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DiffTimerExpired()
+{
+ sDiffTimerExpired = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &)
+// Purpose: Move the file pointer in this stream to just before the block index.
+// Assumes that the stream is at the beginning, seekable, and
+// reading from the stream is OK.
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream)
+{
+ // Size of file
+ int64_t fileSize = rStream.BytesLeftToRead();
+
+ // Get header
+ file_StreamFormat hdr;
+
+ // Read the header
+ if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Check magic number
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ )
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Work out where the index is
+ int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
+ int64_t blockHeaderPosFromEnd = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+
+ // Sanity check
+ if(blockHeaderPosFromEnd > static_cast<int64_t>(fileSize - sizeof(file_StreamFormat)))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Seek to that position
+ rStream.Seek(0 - blockHeaderPosFromEnd, IOStream::SeekType_End);
+
+ // Done. Stream now in right position (as long as the file is formatted correctly)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *)
+// Purpose: Similar to EncodeFile, but takes the object ID of the file it's
+// diffing from, and the index of the blocks in a stream. It'll then
+// calculate which blocks can be reused from that old file.
+// The timeout is the timeout value for reading the diff block index.
+// If pIsCompletelyDifferent != 0, it will be set to true if the
+// the two files are completely different (do not share any block), false otherwise.
+//
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff
+(
+ const char *Filename, int64_t ContainerID,
+ const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID,
+ IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer,
+ int64_t *pModificationTime, bool *pIsCompletelyDifferent)
+{
+ // Is it a symlink?
+ {
+ struct stat st;
+ if(::lstat(Filename, &st) != 0)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ if((st.st_mode & S_IFLNK) == S_IFLNK)
+ {
+ // Don't do diffs for symlinks
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = true;
+ }
+ return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime);
+ }
+ }
+
+ // Load in the blocks
+ BlocksAvailableEntry *pindex = 0;
+ int64_t blocksInIndex = 0;
+ bool canDiffFromThis = false;
+ LoadIndex(rDiffFromBlockIndex, DiffFromObjectID, &pindex, blocksInIndex, Timeout, canDiffFromThis);
+ //TRACE1("Diff: Blocks in index: %lld\n", blocksInIndex);
+
+ if(!canDiffFromThis)
+ {
+ // Don't do diffing...
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = true;
+ }
+ return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime);
+ }
+
+ // Pointer to recipe we're going to create
+ BackupStoreFileEncodeStream::Recipe *precipe = 0;
+
+ try
+ {
+ // Find which sizes should be scanned
+ int32_t sizesToScan[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+ FindMostUsedSizes(pindex, blocksInIndex, sizesToScan);
+
+ // Flag for reporting to the user
+ bool completelyDifferent;
+
+ // BLOCK
+ {
+ // Search the file to find matching blocks
+ std::map<int64_t, int64_t> foundBlocks; // map of offset in file to index in block index
+ int64_t sizeOfInputFile = 0;
+ // BLOCK
+ {
+ FileStream file(Filename);
+ // Get size of file
+ sizeOfInputFile = file.BytesLeftToRead();
+ // Find all those lovely matching blocks
+ SearchForMatchingBlocks(file, foundBlocks, pindex,
+ blocksInIndex, sizesToScan, pDiffTimer);
+
+ // Is it completely different?
+ completelyDifferent = (foundBlocks.size() == 0);
+ }
+
+ // Create a recipe -- if the two files are completely different, don't put the from file ID in the recipe.
+ precipe = new BackupStoreFileEncodeStream::Recipe(pindex, blocksInIndex, completelyDifferent?(0):(DiffFromObjectID));
+ BlocksAvailableEntry *pindexKeptRef = pindex; // we need this later, but must set pindex == 0 now, because of exceptions
+ pindex = 0; // Recipe now has ownership
+
+ // Fill it in
+ GenerateRecipe(*precipe, pindexKeptRef, blocksInIndex, foundBlocks, sizeOfInputFile);
+ }
+ // foundBlocks no longer required
+
+ // Create the stream
+ std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
+
+ // Do the initial setup
+ ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime);
+ precipe = 0; // Stream has taken ownership of this
+
+ // Tell user about completely different status?
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = completelyDifferent;
+ }
+
+ // Return the stream for the caller
+ return stream;
+ }
+ catch(...)
+ {
+ // cleanup
+ if(pindex != 0)
+ {
+ ::free(pindex);
+ pindex = 0;
+ }
+ if(precipe != 0)
+ {
+ delete precipe;
+ precipe = 0;
+ }
+ throw;
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static LoadIndex(IOStream &, int64_t, BlocksAvailableEntry **, int64_t, bool &)
+// Purpose: Read in an index, and decrypt, and store in the in memory block format.
+// rCanDiffFromThis is set to false if the version of the from file is too old.
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis)
+{
+ // Reset
+ rNumBlocksOut = 0;
+ rCanDiffFromThis = false;
+
+ // Read header
+ file_BlockIndexHeader hdr;
+ if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ // Check against backwards comptaibility stuff
+ if(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0))
+ {
+ // Won't diff against old version
+
+ // Absorb rest of stream
+ char buffer[2048];
+ while(rBlockIndex.StreamDataLeft())
+ {
+ rBlockIndex.Read(buffer, sizeof(buffer), 1000 /* 1 sec timeout */);
+ }
+
+ // Tell caller
+ rCanDiffFromThis = false;
+ return;
+ }
+#endif
+
+ // Check magic
+ if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Check that we're not trying to diff against a file which references blocks from another file
+ if(((int64_t)box_ntoh64(hdr.mOtherFileID)) != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile)
+ }
+
+ // Mark as an acceptable diff.
+ rCanDiffFromThis = true;
+
+ // Get basic information
+ int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
+ uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase);
+
+ //TODO: Verify that these sizes look reasonable
+
+ // Allocate space for the index
+ BlocksAvailableEntry *pindex = (BlocksAvailableEntry*)::malloc(sizeof(BlocksAvailableEntry) * numBlocks);
+ if(pindex == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ try
+ {
+ for(int64_t b = 0; b < numBlocks; ++b)
+ {
+ // Read an entry from the stream
+ file_BlockIndexEntry entry;
+ if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ // Couldn't read entry
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Calculate IV for this entry
+ uint64_t iv = entryIVBase;
+ iv += b;
+ // Network byte order
+ iv = box_hton64(iv);
+ sBlowfishDecryptBlockEntry.SetIV(&iv);
+
+ // Decrypt the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+ entry.mEnEnc, sizeof(entry.mEnEnc));
+ if(sectionSize != sizeof(entryEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Check that we're not trying to diff against a file which references blocks from another file
+ if(((int64_t)box_ntoh64(entry.mEncodedSize)) <= 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile)
+ }
+
+ // Store all the required information
+ pindex[b].mpNextInHashList = 0; // hash list not set up yet
+ pindex[b].mSize = ntohl(entryEnc.mSize);
+ pindex[b].mWeakChecksum = ntohl(entryEnc.mWeakChecksum);
+ ::memcpy(pindex[b].mStrongChecksum, entryEnc.mStrongChecksum, sizeof(pindex[b].mStrongChecksum));
+ }
+
+ // Store index pointer for called
+ ASSERT(ppIndex != 0);
+ *ppIndex = pindex;
+
+ // Store number of blocks for caller
+ rNumBlocksOut = numBlocks;
+
+ }
+ catch(...)
+ {
+ // clean up and send the exception along its way
+ ::free(pindex);
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static FindMostUsedSizes(BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+// Purpose: Finds the most commonly used block sizes in the index
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+{
+ // Array for lengths
+ int64_t sizeCounts[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+
+ // Set arrays to lots of zeros (= unused entries)
+ for(int l = 0; l < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++l)
+ {
+ Sizes[l] = 0;
+ sizeCounts[l] = 0;
+ }
+
+ // Array for collecting sizes
+ std::map<int32_t, int64_t> foundSizes;
+
+ // Run through blocks and make a count of the entries
+ for(int64_t b = 0; b < NumBlocks; ++b)
+ {
+ // Only if the block size is bigger than the minimum size we'll scan for
+ if(pIndex[b].mSize > BACKUP_FILE_DIFF_MIN_BLOCK_SIZE)
+ {
+ // Find entry?
+ std::map<int32_t, int64_t>::const_iterator f(foundSizes.find(pIndex[b].mSize));
+ if(f != foundSizes.end())
+ {
+ // Increment existing entry
+ foundSizes[pIndex[b].mSize] = foundSizes[pIndex[b].mSize] + 1;
+ }
+ else
+ {
+ // New entry
+ foundSizes[pIndex[b].mSize] = 1;
+ }
+ }
+ }
+
+ // Make the block sizes
+ for(std::map<int32_t, int64_t>::const_iterator i(foundSizes.begin()); i != foundSizes.end(); ++i)
+ {
+ // Find the position of the size in the array
+ for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
+ {
+ // Instead of sorting on the raw count of blocks,
+ // take the file area covered by this block size.
+ if(i->second * i->first > sizeCounts[t] * Sizes[t])
+ {
+ // Then this size belong before this entry -- shuffle them up
+ for(int s = (BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1); s >= t; --s)
+ {
+ Sizes[s] = Sizes[s-1];
+ sizeCounts[s] = sizeCounts[s-1];
+ }
+
+ // Insert this size
+ Sizes[t] = i->first;
+ sizeCounts[t] = i->second;
+
+ // Shouldn't do any more searching
+ break;
+ }
+ }
+ }
+
+ // trace the size table in debug builds
+#ifndef NDEBUG
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
+ {
+ TRACE3("Diff block size %d: %d (count = %lld)\n", t, Sizes[t], sizeCounts[t]);
+ }
+ }
+#endif
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+// Purpose: Find the matching blocks within the file.
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks,
+ BlocksAvailableEntry *pIndex, int64_t NumBlocks,
+ int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer)
+{
+ time_t TimeMgmtEpoch = 0;
+ int MaximumDiffingTime = 0;
+ int KeepAliveTime = 0;
+
+ if (pDiffTimer)
+ {
+ TimeMgmtEpoch = pDiffTimer->GetTimeMgmtEpoch();
+ MaximumDiffingTime = pDiffTimer->GetMaximumDiffingTime();
+ KeepAliveTime = pDiffTimer->GetKeepaliveTime();
+ }
+
+ std::map<int64_t, int32_t> goodnessOfFit;
+
+ // Allocate the hash lookup table
+ BlocksAvailableEntry **phashTable = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024));
+
+ // Choose a size for the buffer, just a little bit more than the maximum block size
+ int32_t bufSize = Sizes[0];
+ for(int z = 1; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z)
+ {
+ if(Sizes[z] > bufSize) bufSize = Sizes[z];
+ }
+ bufSize += 4;
+ ASSERT(bufSize > Sizes[0]);
+ ASSERT(bufSize > 0);
+ if(bufSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // TODO: Use buffered file class
+ // Because we read in the file a scanned block size at a time, it is likely to be
+ // inefficient. Probably will be much better to use a buffering IOStream class which
+ // reads data in at the size of the filesystem block size.
+
+ // Allocate the buffers.
+ uint8_t *pbuffer0 = (uint8_t *)::malloc(bufSize);
+ uint8_t *pbuffer1 = (uint8_t *)::malloc(bufSize);
+ try
+ {
+ // Check buffer allocation
+ if(pbuffer0 == 0 || pbuffer1 == 0 || phashTable == 0)
+ {
+ // If a buffer got allocated, it will be cleaned up in the catch block
+ throw std::bad_alloc();
+ }
+
+ // Flag to abort the run, if too many blocks are found -- avoid using
+ // huge amounts of processor time when files contain many similar blocks.
+ bool abortSearch = false;
+
+ // Search for each block size in turn
+ // NOTE: Do the smallest size first, so that the scheme for adding
+ // entries in the found list works as expected and replaces smallers block
+ // with larger blocks when it finds matches at the same offset in the file.
+ for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s)
+ {
+ ASSERT(Sizes[s] <= bufSize);
+ //TRACE2("Diff pass %d, for block size %d\n", s, Sizes[s]);
+
+ // Check we haven't finished
+ if(Sizes[s] == 0)
+ {
+ // empty entry, try next size
+ continue;
+ }
+
+ // Set up the hash table entries
+ SetupHashTable(pIndex, NumBlocks, Sizes[s], phashTable);
+
+ // Shift file position to beginning
+ rFile.Seek(0, IOStream::SeekType_Absolute);
+
+ // Read first block
+ if(rFile.Read(pbuffer0, Sizes[s]) != Sizes[s])
+ {
+ // Size of file too short to match -- do next size
+ continue;
+ }
+
+ // Setup block pointers
+ uint8_t *beginnings = pbuffer0;
+ uint8_t *endings = pbuffer1;
+ int offset = 0;
+
+ // Calculate the first checksum, ready for rolling
+ RollingChecksum rolling(beginnings, Sizes[s]);
+
+ // Then roll, until the file is exhausted
+ int64_t fileBlockNumber = 0;
+ int64_t fileOffset = 0;
+ int rollOverInitialBytes = 0;
+ while(true)
+ {
+ if(sDiffTimerExpired)
+ {
+ ASSERT(TimeMgmtEpoch > 0);
+ ASSERT(pDiffTimer != NULL);
+
+ time_t tTotalRunIntvl = time(NULL) - TimeMgmtEpoch;
+
+ if(MaximumDiffingTime > 0 &&
+ tTotalRunIntvl >= MaximumDiffingTime)
+ {
+ TRACE0("MaximumDiffingTime reached - "
+ "suspending file diff\n");
+ abortSearch = true;
+ break;
+ }
+ else if(KeepAliveTime > 0)
+ {
+ TRACE0("KeepAliveTime reached - "
+ "initiating keep-alive\n");
+ pDiffTimer->DoKeepAlive();
+ }
+
+ sDiffTimerExpired = false;
+ }
+
+ // Load in another block of data, and record how big it is
+ int bytesInEndings = rFile.Read(endings, Sizes[s]);
+ int tmp;
+
+ // Skip any bytes from a previous matched block
+ if(rollOverInitialBytes > 0 && offset < bytesInEndings)
+ {
+ int spaceLeft = bytesInEndings - offset;
+ int thisRoll = (rollOverInitialBytes > spaceLeft) ? spaceLeft : rollOverInitialBytes;
+
+ rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll);
+
+ offset += thisRoll;
+ fileOffset += thisRoll;
+ rollOverInitialBytes -= thisRoll;
+
+ if(rollOverInitialBytes)
+ {
+ goto refresh;
+ }
+ }
+
+ if(goodnessOfFit.count(fileOffset))
+ {
+ tmp = goodnessOfFit[fileOffset];
+ }
+ else
+ {
+ tmp = 0;
+ }
+
+ if(tmp >= Sizes[s])
+ {
+ // Skip over bigger ready-matched blocks completely
+ rollOverInitialBytes = tmp;
+ int spaceLeft = bytesInEndings - offset;
+ int thisRoll = (rollOverInitialBytes > spaceLeft) ? spaceLeft : rollOverInitialBytes;
+
+ rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll);
+
+ offset += thisRoll;
+ fileOffset += thisRoll;
+ rollOverInitialBytes -= thisRoll;
+
+ if(rollOverInitialBytes)
+ {
+ goto refresh;
+ }
+ }
+
+ while(offset < bytesInEndings)
+ {
+ // Is current checksum in hash list?
+ uint16_t hash = rolling.GetComponentForHashing();
+ if(phashTable[hash] != 0 && (goodnessOfFit.count(fileOffset) == 0 || goodnessOfFit[fileOffset] < Sizes[s]))
+ {
+ if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks))
+ {
+ goodnessOfFit[fileOffset] = Sizes[s];
+
+ // Block matched, roll the checksum forward to the next block without doing
+ // any more comparisons, because these are pointless (as any more matches will be ignored when
+ // the receipe is generated) and just take up valuable processor time. Edge cases are
+ // especially nasty, using huge amounts of time and memory.
+ int skip = Sizes[s];
+ if(offset < bytesInEndings && skip > 0)
+ {
+ int spaceLeft = bytesInEndings - offset;
+ int thisRoll = (skip > spaceLeft) ? spaceLeft : skip;
+
+ rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll);
+
+ offset += thisRoll;
+ fileOffset += thisRoll;
+ skip -= thisRoll;
+ }
+ // Not all the bytes necessary will have been skipped, so get them
+ // skipped after the next block is loaded.
+ rollOverInitialBytes = skip;
+
+ // End this loop, so the final byte isn't used again
+ break;
+ }
+
+ int64_t NumBlocksFound = static_cast<int64_t>(
+ rFoundBlocks.size());
+ int64_t MaxBlocksFound = NumBlocks *
+ BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE;
+
+ if(NumBlocksFound > MaxBlocksFound)
+ {
+ abortSearch = true;
+ break;
+ }
+ }
+
+ // Roll checksum forward
+ rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
+
+ // Increment offsets
+ ++offset;
+ ++fileOffset;
+ }
+
+ if(abortSearch) break;
+
+ refresh:
+ // Finished?
+ if(bytesInEndings != Sizes[s])
+ {
+ // No more data in file -- check the final block
+ // (Do a copy and paste of 5 lines of code instead of introducing a comparison for
+ // each byte of the file)
+ uint16_t hash = rolling.GetComponentForHashing();
+ if(phashTable[hash] != 0 && (goodnessOfFit.count(fileOffset) == 0 || goodnessOfFit[fileOffset] < Sizes[s]))
+ {
+ if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks))
+ {
+ goodnessOfFit[fileOffset] = Sizes[s];
+ }
+ }
+
+ // finish
+ break;
+ }
+
+ // Switch buffers, reset offset
+ beginnings = endings;
+ endings = (beginnings == pbuffer0)?(pbuffer1):(pbuffer0); // ie the other buffer
+ offset = 0;
+
+ // And count the blocks which have been done
+ ++fileBlockNumber;
+ }
+
+ if(abortSearch) break;
+ }
+
+ // Free buffers and hash table
+ ::free(pbuffer1);
+ pbuffer1 = 0;
+ ::free(pbuffer0);
+ pbuffer0 = 0;
+ ::free(phashTable);
+ phashTable = 0;
+ }
+ catch(...)
+ {
+ // Cleanup and throw
+ if(pbuffer1 != 0) ::free(pbuffer1);
+ if(pbuffer0 != 0) ::free(pbuffer0);
+ if(phashTable != 0) ::free(phashTable);
+ throw;
+ }
+
+#ifndef NDEBUG
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ // Trace out the found blocks in debug mode
+ TRACE0("Diff: list of found blocks\n======== ======== ======== ========\n Offset BlkIdx Size Movement\n");
+ for(std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); i != rFoundBlocks.end(); ++i)
+ {
+ int64_t orgLoc = 0;
+ for(int64_t b = 0; b < i->second; ++b)
+ {
+ orgLoc += pIndex[b].mSize;
+ }
+ TRACE4("%8lld %8lld %8lld %8lld\n", i->first, i->second,
+ (int64_t)(pIndex[i->second].mSize), i->first - orgLoc);
+ }
+ TRACE0("======== ======== ======== ========\n");
+ }
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static SetupHashTable(BlocksAvailableEntry *, int64_t, in32_t, BlocksAvailableEntry **)
+// Purpose: Set up the hash table ready for a scan
+// Created: 14/1/04
+//
+// --------------------------------------------------------------------------
+static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable)
+{
+ // Set all entries in the hash table to zero
+ ::memset(pHashTable, 0, (sizeof(BlocksAvailableEntry *) * (64*1024)));
+
+ // Scan through the blocks, building the hash table
+ for(int64_t b = 0; b < NumBlocks; ++b)
+ {
+ // Only look at the required block size
+ if(pIndex[b].mSize == BlockSize)
+ {
+ // Get the value under which to hash this entry
+ uint16_t hash = RollingChecksum::ExtractHashingComponent(pIndex[b].mWeakChecksum);
+
+ // Already present in table?
+ if(pHashTable[hash] != 0)
+ {
+ //TRACE1("Another hash entry for %d found\n", hash);
+ // Yes -- need to set the pointer in this entry to the current entry to build the linked list
+ pIndex[b].mpNextInHashList = pHashTable[hash];
+ }
+
+ // Put a pointer to this entry in the hash table
+ pHashTable[hash] = pIndex + b;
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static bool SecondStageMatch(xxx)
+// Purpose: When a match in the hash table is found, scan for second stage match using strong checksum.
+// Created: 14/1/04
+//
+// --------------------------------------------------------------------------
+static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum &fastSum, uint8_t *pBeginnings, uint8_t *pEndings,
+ int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks)
+{
+ // Check parameters
+ ASSERT(pBeginnings != 0);
+ ASSERT(pEndings != 0);
+ ASSERT(Offset >= 0);
+ ASSERT(BlockSize > 0);
+ ASSERT(pFirstInHashList != 0);
+ ASSERT(pIndex != 0);
+
+#ifndef NDEBUG
+ uint16_t DEBUG_Hash = fastSum.GetComponentForHashing();
+#endif
+ uint32_t Checksum = fastSum.GetChecksum();
+
+ // Before we go to the expense of the MD5, make sure it's a darn good match on the checksum we already know.
+ BlocksAvailableEntry *scan = pFirstInHashList;
+ bool found=false;
+ while(scan != 0)
+ {
+ if(scan->mWeakChecksum == Checksum)
+ {
+ found = true;
+ break;
+ }
+ scan = scan->mpNextInHashList;
+ }
+ if(!found)
+ {
+ return false;
+ }
+
+ // Calculate the strong MD5 digest for this block
+ MD5Digest strong;
+ // Add the data from the beginnings
+ strong.Add(pBeginnings + Offset, BlockSize - Offset);
+ // Add any data from the endings
+ if(Offset > 0)
+ {
+ strong.Add(pEndings, Offset);
+ }
+ strong.Finish();
+
+ // Then go through the entries in the hash list, comparing with the strong digest calculated
+ scan = pFirstInHashList;
+ //TRACE0("second stage match\n");
+ while(scan != 0)
+ {
+ //TRACE3("scan size %d, block size %d, hash %d\n", scan->mSize, BlockSize, Hash);
+ ASSERT(scan->mSize == BlockSize);
+ ASSERT(RollingChecksum::ExtractHashingComponent(scan->mWeakChecksum) == DEBUG_Hash);
+
+ // Compare?
+ if(strong.DigestMatches(scan->mStrongChecksum))
+ {
+ //TRACE0("Match!\n");
+ // Found! Add to list of found blocks...
+ int64_t fileOffset = (FileBlockNumber * BlockSize) + Offset;
+ int64_t blockIndex = (scan - pIndex); // pointer arthmitic is frowned upon. But most efficient way of doing it here -- alternative is to use more memory
+
+ // We do NOT search for smallest blocks first, as this code originally assumed.
+ // To prevent this from potentially overwriting a better match, the caller must determine
+ // the relative "goodness" of any existing match and this one, and avoid the call if it
+ // could be detrimental.
+ rFoundBlocks[fileOffset] = blockIndex;
+
+ // No point in searching further, report success
+ return true;
+ }
+
+ // Next
+ scan = scan->mpNextInHashList;
+ }
+
+ // Not matched
+ return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static GenerateRecipe(BackupStoreFileEncodeStream::Recipe &, BlocksAvailableEntry *, int64_t, std::map<int64_t, int64_t> &)
+// Purpose: Fills in the recipe from the found block list
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex,
+ int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile)
+{
+ // NOTE: This function could be a lot more sophisiticated. For example, if
+ // a small block overlaps a big block like this
+ // ****
+ // *******************************
+ // then the small block will be used, not the big one. But it'd be better to
+ // just ignore the small block and keep the big one. However, some stats should
+ // be gathered about real world files before writing complex code which might
+ // go wrong.
+
+ // Initialise a blank instruction
+ BackupStoreFileEncodeStream::RecipeInstruction instruction;
+ #define RESET_INSTRUCTION \
+ instruction.mSpaceBefore = 0; \
+ instruction.mBlocks = 0; \
+ instruction.mpStartBlock = 0;
+ RESET_INSTRUCTION
+
+ // First, a special case for when there are no found blocks
+ if(rFoundBlocks.size() == 0)
+ {
+ // No blocks, just a load of space
+ instruction.mSpaceBefore = SizeOfInputFile;
+ rRecipe.push_back(instruction);
+
+ #ifndef NDEBUG
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ TRACE1("Diff: Default recipe generated, %lld bytes of file\n", SizeOfInputFile);
+ }
+ #endif
+
+ // Don't do anything
+ return;
+ }
+
+ // Current location
+ int64_t loc = 0;
+
+ // Then iterate through the list, generating the recipe
+ std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin());
+ ASSERT(i != rFoundBlocks.end()); // check logic
+
+ // Counting for debug tracing
+#ifndef NDEBUG
+ int64_t debug_NewBytesFound = 0;
+ int64_t debug_OldBlocksUsed = 0;
+#endif
+
+ for(; i != rFoundBlocks.end(); ++i)
+ {
+ // Remember... map is (position in file) -> (index of block in pIndex)
+
+ if(i->first < loc)
+ {
+ // This block overlaps the last one
+ continue;
+ }
+ else if(i->first > loc)
+ {
+ // There's a gap between the end of the last thing and this block.
+ // If there's an instruction waiting, push it onto the list
+ if(instruction.mSpaceBefore != 0 || instruction.mpStartBlock != 0)
+ {
+ rRecipe.push_back(instruction);
+ }
+ // Start a new instruction, with the gap ready
+ RESET_INSTRUCTION
+ instruction.mSpaceBefore = i->first - loc;
+ // Move location forward to match
+ loc += instruction.mSpaceBefore;
+#ifndef NDEBUG
+ debug_NewBytesFound += instruction.mSpaceBefore;
+#endif
+ }
+
+ // First, does the current instruction need pushing back, because this block is not
+ // sequential to the last one?
+ if(instruction.mpStartBlock != 0 && (pIndex + i->second) != (instruction.mpStartBlock + instruction.mBlocks))
+ {
+ rRecipe.push_back(instruction);
+ RESET_INSTRUCTION
+ }
+
+ // Add in this block
+ if(instruction.mpStartBlock == 0)
+ {
+ // This block starts a new instruction
+ instruction.mpStartBlock = pIndex + i->second;
+ instruction.mBlocks = 1;
+ }
+ else
+ {
+ // It continues the previous section of blocks
+ instruction.mBlocks += 1;
+ }
+
+#ifndef NDEBUG
+ debug_OldBlocksUsed++;
+#endif
+
+ // Move location forward
+ loc += pIndex[i->second].mSize;
+ }
+
+ // Push the last instruction generated
+ rRecipe.push_back(instruction);
+
+ // Is there any space left at the end which needs sending?
+ if(loc != SizeOfInputFile)
+ {
+ RESET_INSTRUCTION
+ instruction.mSpaceBefore = SizeOfInputFile - loc;
+#ifndef NDEBUG
+ debug_NewBytesFound += instruction.mSpaceBefore;
+#endif
+ rRecipe.push_back(instruction);
+ }
+
+ // dump out the recipe
+#ifndef NDEBUG
+ TRACE2("Diff: %lld new bytes found, %lld old blocks used\n", debug_NewBytesFound, debug_OldBlocksUsed);
+ if(BackupStoreFile::TraceDetailsOfDiffProcess)
+ {
+ TRACE1("Diff: Recipe generated (size %d)\n======== ========= ========\nSpace b4 FirstBlk NumBlks\n", rRecipe.size());
+ {
+ for(unsigned int e = 0; e < rRecipe.size(); ++e)
+ {
+ char b[64];
+#ifdef WIN32
+ sprintf(b, "%8I64d", (int64_t)(rRecipe[e].mpStartBlock - pIndex));
+#else
+ sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex));
+#endif
+ TRACE3("%8lld %s %8lld\n", rRecipe[e].mSpaceBefore, (rRecipe[e].mpStartBlock == 0)?" -":b, (int64_t)rRecipe[e].mBlocks);
+ }
+ }
+ TRACE0("======== ========= ========\n");
+ }
+#endif
+}
diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp
new file mode 100644
index 00000000..a9dc3b98
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileEncodeStream.cpp
@@ -0,0 +1,713 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileEncodeStream.cpp
+// Purpose: Implement stream-based file encoding for the backup store
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupStoreFileEncodeStream.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BoxTime.h"
+#include "BackupClientFileAttributes.h"
+#include "FileStream.h"
+#include "RollingChecksum.h"
+#include "Random.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream
+// Purpose: Constructor (opens file)
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::BackupStoreFileEncodeStream()
+ : mpRecipe(0),
+ mpFile(0),
+ mStatus(Status_Header),
+ mSendData(true),
+ mTotalBlocks(0),
+ mAbsoluteBlockNumber(-1),
+ mInstructionNumber(-1),
+ mNumBlocks(0),
+ mCurrentBlock(-1),
+ mCurrentBlockEncodedSize(0),
+ mPositionInCurrentBlock(0),
+ mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE),
+ mLastBlockSize(0),
+ mpRawBuffer(0),
+ mAllocatedBufferSize(0),
+ mEntryIVBase(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
+// Purpose: Destructor
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
+{
+ // Free buffers
+ if(mpRawBuffer)
+ {
+ ::free(mpRawBuffer);
+ mpRawBuffer = 0;
+ }
+
+ // Close the file, which we might have open
+ if(mpFile)
+ {
+ delete mpFile;
+ mpFile = 0;
+ }
+
+ // Free the recipe
+ if(mpRecipe != 0)
+ {
+ delete mpRecipe;
+ mpRecipe = 0;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *)
+// Purpose: Reads file information, and builds file header reading for sending.
+// Takes ownership of the Recipe.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEncodeStream::Recipe *pRecipe,
+ int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime)
+{
+ // Pointer to a blank recipe which we might create
+ BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0;
+
+ try
+ {
+ // Get file attributes
+ box_time_t modTime = 0;
+ int64_t fileSize = 0;
+ BackupClientFileAttributes attr;
+ attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime,
+ 0 /* not interested in attr mod time */, &fileSize);
+
+ // Might need to create a blank recipe...
+ if(pRecipe == 0)
+ {
+ pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0);
+
+ BackupStoreFileEncodeStream::RecipeInstruction instruction;
+ instruction.mSpaceBefore = fileSize; // whole file
+ instruction.mBlocks = 0; // no blocks
+ instruction.mpStartBlock = 0; // no block
+ pblankRecipe->push_back(instruction);
+
+ pRecipe = pblankRecipe;
+ }
+
+ // Tell caller?
+ if(pModificationTime != 0)
+ {
+ *pModificationTime = modTime;
+ }
+
+ // Go through each instruction in the recipe and work out how many blocks
+ // it will add, and the max clear size of these blocks
+ int maxBlockClearSize = 0;
+ for(uint64_t inst = 0; inst < pRecipe->size(); ++inst)
+ {
+ if((*pRecipe)[inst].mSpaceBefore > 0)
+ {
+ // Calculate the number of blocks the space before requires
+ int64_t numBlocks;
+ int32_t blockSize, lastBlockSize;
+ CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize);
+ // Add to accumlated total
+ mTotalBlocks += numBlocks;
+ // Update maximum clear size
+ if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize;
+ if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize;
+ }
+
+ // Add number of blocks copied from the previous file
+ mTotalBlocks += (*pRecipe)[inst].mBlocks;
+
+ // Check for bad things
+ if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Run through blocks to get the max clear size
+ for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b)
+ {
+ if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize;
+ }
+ }
+
+ // Send data? (symlinks don't have any data in them)
+ mSendData = !attr.IsSymLink();
+
+ // If not data is being sent, then the max clear block size is zero
+ if(!mSendData)
+ {
+ maxBlockClearSize = 0;
+ }
+
+ // Header
+ file_StreamFormat hdr;
+ hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1);
+ hdr.mNumBlocks = (mSendData)?(box_hton64(mTotalBlocks)):(0);
+ hdr.mContainerID = box_hton64(ContainerID);
+ hdr.mModificationTime = box_hton64(modTime);
+ // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size
+ hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128);
+ hdr.mOptions = 0; // no options defined yet
+
+ // Write header to stream
+ mData.Write(&hdr, sizeof(hdr));
+
+ // Write filename to stream
+ rStoreFilename.WriteToStream(mData);
+
+ // Write attributes to stream
+ attr.WriteToStream(mData);
+
+ // Allocate some buffers for writing data
+ if(mSendData)
+ {
+ // Open the file
+ mpFile = new FileStream(Filename);
+
+ // Work out the largest possible block required for the encoded data
+ mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize);
+
+ // Then allocate two blocks of this size
+ mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize);
+ if(mpRawBuffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+#ifndef NDEBUG
+ // In debug builds, make sure that the reallocation code is exercised.
+ mEncodedBuffer.Allocate(mAllocatedBufferSize / 4);
+#else
+ mEncodedBuffer.Allocate(mAllocatedBufferSize);
+#endif
+ }
+ else
+ {
+ // Write an empty block index for the symlink
+ file_BlockIndexHeader blkhdr;
+ blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
+ blkhdr.mOtherFileID = box_hton64(0); // not other file ID
+ blkhdr.mEntryIVBase = box_hton64(0);
+ blkhdr.mNumBlocks = box_hton64(0);
+ mData.Write(&blkhdr, sizeof(blkhdr));
+ }
+
+ // Ready for reading
+ mData.SetForReading();
+
+ // Update stats
+ BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize;
+
+ // Finally, store the pointer to the recipe, when we know exceptions won't occur
+ mpRecipe = pRecipe;
+ }
+ catch(...)
+ {
+ // Clean up any blank recipe
+ if(pblankRecipe != 0)
+ {
+ delete pblankRecipe;
+ pblankRecipe = 0;
+ }
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &)
+// Purpose: Calculates the sizes of blocks in a section of the file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut)
+{
+ // How many blocks, and how big?
+ rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2;
+ do
+ {
+ rBlockSizeOut *= 2;
+
+ rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut;
+
+ } while(rBlockSizeOut <= BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER);
+
+ // Last block size
+ rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut);
+
+ // Avoid small blocks?
+ if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN
+ && rNumBlocksOut > 1)
+ {
+ // Add the small bit of data to the last block
+ --rNumBlocksOut;
+ rLastBlockSizeOut += rBlockSizeOut;
+ }
+
+ // checks!
+ ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize);
+ //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut);
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Read(void *, int, int)
+// Purpose: As interface -- generates encoded file data on the fly from the raw file
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ // Check there's something to do.
+ if(mStatus == Status_Finished)
+ {
+ return 0;
+ }
+
+ int bytesToRead = NBytes;
+ uint8_t *buffer = (uint8_t*)pBuffer;
+
+ while(bytesToRead > 0 && mStatus != Status_Finished)
+ {
+ if(mStatus == Status_Header || mStatus == Status_BlockListing)
+ {
+ // Header or block listing phase -- send from the buffered stream
+
+ // Send bytes from the data buffer
+ int b = mData.Read(buffer, bytesToRead, Timeout);
+ bytesToRead -= b;
+ buffer += b;
+
+ // Check to see if all the data has been used from this stream
+ if(!mData.StreamDataLeft())
+ {
+ // Yes, move on to next phase (or finish, if there's no file data)
+ if(!mSendData)
+ {
+ mStatus = Status_Finished;
+ }
+ else
+ {
+ // Reset the buffer so it can be used for the next phase
+ mData.Reset();
+
+ // Get buffer ready for index?
+ if(mStatus == Status_Header)
+ {
+ // Just finished doing the stream header, create the block index header
+ file_BlockIndexHeader blkhdr;
+ blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
+ ASSERT(mpRecipe != 0);
+ blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID());
+ blkhdr.mNumBlocks = box_hton64(mTotalBlocks);
+
+ // Generate the IV base
+ Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase));
+ blkhdr.mEntryIVBase = box_hton64(mEntryIVBase);
+
+ mData.Write(&blkhdr, sizeof(blkhdr));
+ }
+
+ ++mStatus;
+ }
+ }
+ }
+ else if(mStatus == Status_Blocks)
+ {
+ // Block sending phase
+
+ if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize)
+ {
+ // Next block!
+ ++mCurrentBlock;
+ ++mAbsoluteBlockNumber;
+ if(mCurrentBlock >= mNumBlocks)
+ {
+ // Output extra blocks for this instruction and move forward in file
+ if(mInstructionNumber >= 0)
+ {
+ SkipPreviousBlocksInInstruction();
+ }
+
+ // Is there another instruction to go?
+ ++mInstructionNumber;
+
+ // Skip instructions which don't contain any data
+ while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size())
+ && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0)
+ {
+ SkipPreviousBlocksInInstruction();
+ ++mInstructionNumber;
+ }
+
+ if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size()))
+ {
+ // End of blocks, go to next phase
+ ++mStatus;
+
+ // Set the data to reading so the index can be written
+ mData.SetForReading();
+ }
+ else
+ {
+ // Get ready for this instruction
+ SetForInstruction();
+ }
+ }
+
+ // Can't use 'else' here as SetForInstruction() will change this
+ if(mCurrentBlock < mNumBlocks)
+ {
+ EncodeCurrentBlock();
+ }
+ }
+
+ // Send data from the current block (if there's data to send)
+ if(mPositionInCurrentBlock < mCurrentBlockEncodedSize)
+ {
+ // How much data to put in the buffer?
+ int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock;
+ if(s > bytesToRead) s = bytesToRead;
+
+ // Copy it in
+ ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s);
+
+ // Update variables
+ bytesToRead -= s;
+ buffer += s;
+ mPositionInCurrentBlock += s;
+ }
+ }
+ else
+ {
+ // Should never get here, as it'd be an invalid status
+ ASSERT(false);
+ }
+ }
+
+ // Add encoded size to stats
+ BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead);
+
+ // Return size of data to caller
+ return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction()
+// Purpose: Private. Stores the blocks of the old file referenced in the current
+// instruction into the index and skips over the data in the file
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction()
+{
+ // Check something is necessary
+ if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0)
+ {
+ return;
+ }
+
+ // Index of the first block in old file (being diffed from)
+ int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock);
+
+ int64_t sizeToSkip = 0;
+
+ for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b)
+ {
+ // Update stats
+ BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
+
+ // Store the entry
+ StoreBlockIndexEntry(0 - (firstIndex + b),
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize,
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum,
+ (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum);
+
+ // Increment the absolute block number -- kept encryption IV in sync
+ ++mAbsoluteBlockNumber;
+
+ // Add the size of this block to the size to skip
+ sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
+ }
+
+ // Move forward in the stream
+ mpFile->Seek(sizeToSkip, IOStream::SeekType_Relative);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::SetForInstruction()
+// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::SetForInstruction()
+{
+ // Calculate block sizes
+ CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize);
+
+ // Set variables
+ mCurrentBlock = 0;
+ mCurrentBlockEncodedSize = 0;
+ mPositionInCurrentBlock = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock()
+// Purpose: Private. Encodes the current block, and writes the block data to the index
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::EncodeCurrentBlock()
+{
+ // How big is the block, raw?
+ int blockRawSize = mBlockSize;
+ if(mCurrentBlock == (mNumBlocks - 1))
+ {
+ blockRawSize = mLastBlockSize;
+ }
+ ASSERT(blockRawSize < mAllocatedBufferSize);
+
+ // Check file open
+ if(mpFile == 0)
+ {
+ // File should be open, but isn't. So logical error.
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Read the data in
+ if(!mpFile->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */))
+ {
+ // TODO: Do something more intelligent, and abort this upload because the file
+ // has changed
+ THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer)
+ }
+
+ // Encode it
+ mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer);
+
+ //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize);
+
+ // Create block listing data -- generate checksums
+ RollingChecksum weakChecksum(mpRawBuffer, blockRawSize);
+ MD5Digest strongChecksum;
+ strongChecksum.Add(mpRawBuffer, blockRawSize);
+ strongChecksum.Finish();
+
+ // Add entry to the index
+ StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData());
+
+ // Set vars to reading this block
+ mPositionInCurrentBlock = 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *)
+// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream.
+// Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum)
+{
+ // First, the encrypted section
+ file_BlockIndexEntryEnc entryEnc;
+ entryEnc.mSize = htonl(ClearSize);
+ entryEnc.mWeakChecksum = htonl(WeakChecksum);
+ ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum));
+
+ // Then the clear section
+ file_BlockIndexEntry entry;
+ entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex));
+
+ // Then encrypt the encryted section
+ // Generate the IV from the block number
+ if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase))
+ {
+ THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements)
+ }
+ uint64_t iv = mEntryIVBase;
+ iv += mAbsoluteBlockNumber;
+ // Convert to network byte order before encrypting with it, so that restores work on
+ // platforms with different endiannesses.
+ iv = box_hton64(iv);
+ sBlowfishEncryptBlockEntry.SetIV(&iv);
+
+ // Encode the data
+ int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc));
+ if(encodedSize != sizeof(entry.mEnEnc))
+ {
+ THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+ }
+
+ // Save to data block for sending at the end of the stream
+ mData.Write(&entry, sizeof(entry));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Write(const void *, int)
+// Purpose: As interface. Exceptions.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes)
+{
+ THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StreamDataLeft()
+// Purpose: As interface -- end of stream reached?
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFileEncodeStream::StreamDataLeft()
+{
+ return (mStatus != Status_Finished);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::StreamClosed()
+// Purpose: As interface
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFileEncodeStream::StreamClosed()
+{
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t)
+// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex,
+ int64_t NumBlocksInIndex, int64_t OtherFileID)
+ : mpBlockIndex(pBlockIndex),
+ mNumBlocksInIndex(NumBlocksInIndex),
+ mOtherFileID(OtherFileID)
+{
+ ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0))
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFileEncodeStream::Recipe::~Recipe()
+// Purpose: Destructor
+// Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::Recipe::~Recipe()
+{
+ // Free the block index, if there is one
+ if(mpBlockIndex != 0)
+ {
+ ::free(mpBlockIndex);
+ }
+}
+
+
+
+
diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h
new file mode 100644
index 00000000..34c0d6b1
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileEncodeStream.h
@@ -0,0 +1,165 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileEncodeStream.h
+// Purpose: Implement stream-based file encoding for the backup store
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILEENCODESTREAM__H
+#define BACKUPSTOREFILEENCODESTREAM__H
+
+#include <vector>
+
+#include "IOStream.h"
+#include "BackupStoreFilename.h"
+#include "CollectInBufferStream.h"
+#include "MD5Digest.h"
+#include "BackupStoreFile.h"
+
+namespace BackupStoreFileCreation
+{
+ // Diffing and creation of files share some implementation details.
+ typedef struct _BlocksAvailableEntry
+ {
+ struct _BlocksAvailableEntry *mpNextInHashList;
+ int32_t mSize; // size in clear
+ uint32_t mWeakChecksum; // weak, rolling checksum
+ uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum
+ } BlocksAvailableEntry;
+
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFileEncodeStream
+// Purpose: Encode a file into a stream
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+class BackupStoreFileEncodeStream : public IOStream
+{
+public:
+ BackupStoreFileEncodeStream();
+ ~BackupStoreFileEncodeStream();
+
+ typedef struct
+ {
+ int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go
+ int32_t mBlocks; // number of block to reuse, starting at this one
+ BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null
+ } RecipeInstruction;
+
+ class Recipe : public std::vector<RecipeInstruction>
+ {
+ // NOTE: This class is rather tied in with the implementation of diffing.
+ public:
+ Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex,
+ int64_t OtherFileID = 0);
+ ~Recipe();
+
+ int64_t GetOtherFileID() {return mOtherFileID;}
+ int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock)
+ {
+ return pBlock - mpBlockIndex;
+ }
+
+ private:
+ BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex;
+ int64_t mNumBlocksInIndex;
+ int64_t mOtherFileID;
+ };
+
+ void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime);
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout);
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+private:
+ enum
+ {
+ Status_Header = 0,
+ Status_Blocks = 1,
+ Status_BlockListing = 2,
+ Status_Finished = 3
+ };
+
+private:
+ void EncodeCurrentBlock();
+ void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut);
+ void SkipPreviousBlocksInInstruction();
+ void SetForInstruction();
+ void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum);
+
+private:
+ Recipe *mpRecipe;
+ IOStream *mpFile; // source file
+ CollectInBufferStream mData; // buffer for header and index entries
+ int mStatus;
+ bool mSendData; // true if there's file data to send (ie not a symlink)
+ int64_t mTotalBlocks; // Total number of blocks in the file
+ int64_t mAbsoluteBlockNumber; // The absolute block number currently being output
+ // Instruction number
+ int64_t mInstructionNumber;
+ // All the below are within the current instruction
+ int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases
+ int64_t mCurrentBlock;
+ int32_t mCurrentBlockEncodedSize;
+ int32_t mPositionInCurrentBlock; // for reading out
+ int32_t mBlockSize; // Basic block size of most of the blocks in the file
+ int32_t mLastBlockSize; // the size (unencoded) of the last block in the file
+ // Buffers
+ uint8_t *mpRawBuffer; // buffer for raw data
+ BackupStoreFile::EncodingBuffer mEncodedBuffer;
+ // buffer for encoded data
+ int32_t mAllocatedBufferSize; // size of above two allocated blocks
+ uint64_t mEntryIVBase; // base for block entry IV
+};
+
+
+
+#endif // BACKUPSTOREFILEENCODESTREAM__H
+
diff --git a/lib/backupclient/BackupStoreFileRevDiff.cpp b/lib/backupclient/BackupStoreFileRevDiff.cpp
new file mode 100644
index 00000000..ac323532
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileRevDiff.cpp
@@ -0,0 +1,296 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileRevDiff.cpp
+// Purpose: Reverse a patch, to build a new patch from new to old files
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <stdlib.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t)
+// Purpose: Reverse a patch, to build a new patch from new to old files. Takes
+// two independent copies to the From file, for efficiency.
+// Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent)
+{
+ // Read and copy the header from the from file to the out file -- beginnings of the patch
+ file_StreamFormat hdr;
+ if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+ // Copy
+ rOut.Write(&hdr, sizeof(hdr));
+ // Copy over filename and attributes
+ // BLOCK
+ {
+ BackupStoreFilename filename;
+ filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ filename.WriteToStream(rOut);
+ StreamableMemBlock attr;
+ attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+ attr.WriteToStream(rOut);
+ }
+
+ // Build an index of common blocks.
+ // For each block in the from file, we want to know it's index in the
+ // diff file. Allocate memory for this information.
+ int64_t fromNumBlocks = box_ntoh64(hdr.mNumBlocks);
+ int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t));
+ if(pfromIndexInfo == 0)
+ {
+ throw std::bad_alloc();
+ }
+
+ // Buffer data
+ void *buffer = 0;
+ int bufferSize = 0;
+
+ // flag
+ bool isCompletelyDifferent = true;
+
+ try
+ {
+ // Initialise the index to be all 0, ie not filled in yet
+ for(int64_t i = 0; i < fromNumBlocks; ++i)
+ {
+ pfromIndexInfo[i] = 0;
+ }
+
+ // Within the from file, skip to the index
+ MoveStreamPositionToBlockIndex(rDiff);
+
+ // Read in header of index
+ file_BlockIndexHeader diffIdxHdr;
+ if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // And then read in each entry
+ int64_t diffNumBlocks = box_ntoh64(diffIdxHdr.mNumBlocks);
+ for(int64_t b = 0; b < diffNumBlocks; ++b)
+ {
+ file_BlockIndexEntry e;
+ if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Where's the block?
+ int64_t blockEn = box_ntoh64(e.mEncodedSize);
+ if(blockEn > 0)
+ {
+ // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file
+ }
+ else
+ {
+ // Block is in the original file, store which block it is in this file
+ int64_t fromIndex = 0 - blockEn;
+ if(fromIndex < 0 || fromIndex >= fromNumBlocks)
+ {
+ THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles)
+ }
+
+ // Store information about where it is in the new file
+ // NOTE: This is slight different to how it'll be stored in the final index.
+ pfromIndexInfo[fromIndex] = -1 - b;
+ }
+ }
+
+ // Open the index for the second copy of the from file
+ MoveStreamPositionToBlockIndex(rFrom2);
+
+ // Read in header of index
+ file_BlockIndexHeader fromIdxHdr;
+ if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+ || box_ntoh64(fromIdxHdr.mOtherFileID) != 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // So, we can now start building the data in the file
+ int64_t filePosition = rFrom.GetPosition();
+ for(int64_t b = 0; b < fromNumBlocks; ++b)
+ {
+ // Read entry from from index
+ file_BlockIndexEntry e;
+ if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Get size
+ int64_t blockSize = box_hton64(e.mEncodedSize);
+ if(blockSize < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+ }
+
+ // Copy this block?
+ if(pfromIndexInfo[b] == 0)
+ {
+ // Copy it, first move to file location
+ rFrom.Seek(filePosition, IOStream::SeekType_Absolute);
+
+ // Make sure there's memory available to copy this
+ if(bufferSize < blockSize || buffer == 0)
+ {
+ // Free old block
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ buffer = 0;
+ bufferSize = 0;
+ }
+ // Allocate new block
+ buffer = ::malloc(blockSize);
+ if(buffer == 0)
+ {
+ throw std::bad_alloc();
+ }
+ bufferSize = blockSize;
+ }
+ ASSERT(bufferSize >= blockSize);
+
+ // Copy the block
+ if(!rFrom.ReadFullBuffer(buffer, blockSize, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+ }
+ rOut.Write(buffer, blockSize);
+
+ // Store the size
+ pfromIndexInfo[b] = blockSize;
+ }
+ else
+ {
+ // Block isn't needed, so it's not completely different
+ isCompletelyDifferent = false;
+ }
+ filePosition += blockSize;
+ }
+
+ // Then write the index, modified header first
+ fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(box_hton64(ObjectIDOfFrom));
+ rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr));
+
+ // Move to start of index entries
+ rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute);
+
+ // Then copy modified entries
+ for(int64_t b = 0; b < fromNumBlocks; ++b)
+ {
+ // Read entry from from index
+ file_BlockIndexEntry e;
+ if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // Modify...
+ int64_t s = pfromIndexInfo[b];
+ // Adjust to reflect real block index (remember 0 has a different meaning here)
+ if(s < 0) ++s;
+ // Insert
+ e.mEncodedSize = box_hton64(s);
+ // Write
+ rOut.Write(&e, sizeof(e));
+ }
+ }
+ catch(...)
+ {
+ ::free(pfromIndexInfo);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+ throw;
+ }
+
+ // Free memory used (oh for finally {} blocks)
+ ::free(pfromIndexInfo);
+ if(buffer != 0)
+ {
+ ::free(buffer);
+ }
+
+ // return completely different flag
+ if(pIsCompletelyDifferent != 0)
+ {
+ *pIsCompletelyDifferent = isCompletelyDifferent;
+ }
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFileWire.h b/lib/backupclient/BackupStoreFileWire.h
new file mode 100644
index 00000000..18625ee0
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileWire.h
@@ -0,0 +1,112 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFileWire.h
+// Purpose: On the wire / disc formats for backup store files
+// Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILEWIRE__H
+#define BACKUPSTOREFILEWIRE__H
+
+#include "MD5Digest.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int64_t mNumBlocks; // number of blocks contained in the file
+ int64_t mContainerID;
+ int64_t mModificationTime;
+ int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block
+ int32_t mOptions; // bitmask of options used
+ // Then a BackupStoreFilename
+ // Then a BackupClientFileAttributes
+} file_StreamFormat;
+
+typedef struct
+{
+ int32_t mMagicValue; // different magic value
+ int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index
+ uint64_t mEntryIVBase; // base value for block IV
+ int64_t mNumBlocks; // repeat of value in file header
+} file_BlockIndexHeader;
+
+typedef struct
+{
+ int32_t mSize; // size in clear
+ uint32_t mWeakChecksum; // weak, rolling checksum
+ uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum
+} file_BlockIndexEntryEnc;
+
+typedef struct
+{
+ union
+ {
+ int64_t mEncodedSize; // size encoded, if > 0
+ int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0
+ };
+ uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section
+} file_BlockIndexEntry;
+
+// Use default packing
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// header for blocks of compressed data in files
+#define HEADER_CHUNK_IS_COMPRESSED 1 // bit
+#define HEADER_ENCODING_SHIFT 1 // shift value
+#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7
+#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7
+
+
+#endif // BACKUPSTOREFILEWIRE__H
+
diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp
new file mode 100644
index 00000000..f17fa1b1
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilename.cpp
@@ -0,0 +1,317 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilename.cpp
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreFilename.h"
+#include "Protocol.h"
+#include "BackupStoreException.h"
+#include "IOStream.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::BackupStoreFilename()
+// Purpose: Default constructor -- creates an invalid filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::BackupStoreFilename()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy)
+ : BackupStoreFilename_base(rToCopy)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::~BackupStoreFilename()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::~BackupStoreFilename()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::CheckValid(bool)
+// Purpose: Checks the encoded filename for validity
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const
+{
+ bool ok = true;
+
+ if(size() < 2)
+ {
+ // Isn't long enough to have a header
+ ok = false;
+ }
+ else
+ {
+ // Check size is consistent
+ unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(*this);
+ if(dsize != size())
+ {
+ ok = false;
+ }
+
+ // And encoding is an accepted value
+ unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this);
+ if(encoding < Encoding_Min || encoding > Encoding_Max)
+ {
+ ok = false;
+ }
+ }
+
+ // Exception?
+ if(!ok && ExceptionIfInvalid)
+ {
+ THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename)
+ }
+
+ return ok;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::ReadFromProtocol(Protocol &)
+// Purpose: Reads the filename from the protocol object
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol)
+{
+ // Read the header
+ char hdr[2];
+ rProtocol.Read(hdr, 2);
+
+ // How big is it?
+ int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
+
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ std::string data;
+ rProtocol.Read(data, dsize - 2);
+
+ // assign to this string, storing the header and the extra data
+ assign(hdr, 2);
+ append(data.c_str(), data.size());
+
+ // Check it
+ CheckValid();
+
+ // Alert derived classes
+ EncodedFilenameChanged();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::WriteToProtocol(Protocol &)
+// Purpose: Writes the filename to the protocol object
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const
+{
+ CheckValid();
+
+ rProtocol.Write(c_str(), size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::ReadFromStream(IOStream &)
+// Purpose: Reads the filename from a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout)
+{
+ // Read the header
+ char hdr[2];
+ if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+
+ // How big is it?
+ unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
+
+ // Assume most filenames are small
+ char buf[256];
+ if(dsize < sizeof(buf))
+ {
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ // Copy in header
+ buf[0] = hdr[0]; buf[1] = hdr[1];
+
+ // assign to this string, storing the header and the extra data
+ assign(buf, dsize);
+ }
+ else
+ {
+ // Block of memory to hold it
+ MemoryBlockGuard<char*> dataB(dsize+2);
+ char *data = dataB;
+
+ // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+ if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+ }
+ // Copy in header
+ data[0] = hdr[0]; data[1] = hdr[1];
+
+ // assign to this string, storing the header and the extra data
+ assign(data, dsize);
+ }
+
+ // Check it
+ CheckValid();
+
+ // Alert derived classes
+ EncodedFilenameChanged();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::WriteToStream(IOStream &)
+// Purpose: Writes the filename to a stream
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::WriteToStream(IOStream &rStream) const
+{
+ CheckValid();
+
+ rStream.Write(c_str(), size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::EncodedFilenameChanged()
+// Purpose: The encoded filename stored has changed
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::EncodedFilenameChanged()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::IsEncrypted()
+// Purpose: Returns true if the filename is stored using an encrypting encoding
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFilename::IsEncrypted() const
+{
+ return BACKUPSTOREFILENAME_GET_ENCODING(*this) != Encoding_Clear;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilename::SetAsClearFilename(const char *)
+// Purpose: Sets this object to be a valid filename, but with a filename in the clear.
+// Used on the server to create filenames when there's no way of encrypting it.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::SetAsClearFilename(const char *Clear)
+{
+ // Make std::string from the clear name
+ std::string toEncode(Clear);
+
+ // Make an encoded string
+ char hdr[2];
+ BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear);
+ std::string encoded(hdr, 2);
+ encoded += toEncode;
+ ASSERT(encoded.size() == toEncode.size() + 2);
+
+ // Store the encoded string
+ assign(encoded);
+
+ // Stuff which must be done
+ EncodedFilenameChanged();
+ CheckValid(false);
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h
new file mode 100644
index 00000000..e2007d7e
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilename.h
@@ -0,0 +1,123 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilename.h
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILENAME__H
+#define BACKUPSTOREFILENAME__H
+
+#include <string>
+
+class Protocol;
+class IOStream;
+
+// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+// don't define this -- the problem of memory usage still appears without this.
+// It's just that this class really showed up the problem. Instead, malloc allocation
+// is globally defined in BoxPlatform.h, for troublesome libraries.
+
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+ // Use a malloc_allocated string, because the STL default allocators really screw up with
+ // memory allocation, particularly with this class.
+ // Makes a few things a bit messy and inefficient with conversions.
+ // Given up using this, and use global malloc allocation instead, but thought it
+ // worth leaving this code in just in case it's useful for the future.
+ typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base;
+ // If this is changed, change GetClearFilename() back to returning a reference.
+#else
+ typedef std::string BackupStoreFilename_base;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFilename
+// Purpose: Filename for the backup store
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreFilename : public BackupStoreFilename_base
+{
+public:
+ BackupStoreFilename();
+ BackupStoreFilename(const BackupStoreFilename &rToCopy);
+ virtual ~BackupStoreFilename();
+
+ bool CheckValid(bool ExceptionIfInvalid = true) const;
+
+ void ReadFromProtocol(Protocol &rProtocol);
+ void WriteToProtocol(Protocol &rProtocol) const;
+
+ void ReadFromStream(IOStream &rStream, int Timeout);
+ void WriteToStream(IOStream &rStream) const;
+
+ void SetAsClearFilename(const char *Clear);
+
+ // Check that it's encrypted
+ bool IsEncrypted() const;
+
+ // These enumerated types belong in the base class so
+ // the CheckValid() function can make sure that the encoding
+ // is a valid encoding
+ enum
+ {
+ Encoding_Min = 1,
+ Encoding_Clear = 1,
+ Encoding_Blowfish = 2,
+ Encoding_Max = 2
+ };
+
+protected:
+ virtual void EncodedFilenameChanged();
+};
+
+// On the wire utilities for class and derived class
+#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2)
+#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3)
+
+#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;}
+
+#endif // BACKUPSTOREFILENAME__H
+
diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp
new file mode 100644
index 00000000..768fc5fe
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilenameClear.cpp
@@ -0,0 +1,367 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilenameClear.cpp
+// Purpose: BackupStoreFilenames in the clear
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupStoreException.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// Hide private variables from the rest of the world
+namespace
+{
+ int sEncodeMethod = BackupStoreFilename::Encoding_Clear;
+ CipherContext sBlowfishEncrypt;
+ CipherContext sBlowfishDecrypt;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear()
+// Purpose: Default constructor, creates an invalid filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &)
+// Purpose: Creates a filename, encoding from the given string
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode)
+{
+ SetClearFilename(rToEncode);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &)
+// Purpose: Copy constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy)
+ : BackupStoreFilename(rToCopy),
+ mClearFilename(rToCopy.mClearFilename)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
+// Purpose: Copy from base class
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
+ : BackupStoreFilename(rToCopy)
+{
+ // Will get a clear filename when it's required
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::~BackupStoreFilenameClear()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::GetClearFilename()
+// Purpose: Get the unencoded filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+const std::string BackupStoreFilenameClear::GetClearFilename() const
+{
+ MakeClearAvailable();
+ // When modifying, remember to change back to reference return if at all possible
+ // -- returns an object rather than a reference to allow easy use with other code.
+ return std::string(mClearFilename.c_str(), mClearFilename.size());
+}
+#else
+const std::string &BackupStoreFilenameClear::GetClearFilename() const
+{
+ MakeClearAvailable();
+ return mClearFilename;
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &)
+// Purpose: Encode and make available the clear filename
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode)
+{
+ // Only allow Blowfish encodings
+ if(sEncodeMethod != Encoding_Blowfish)
+ {
+ THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup)
+ }
+
+ // Make an encoded string with blowfish encryption
+ EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish);
+
+ // Store the clear filename
+ mClearFilename.assign(rToEncode.c_str(), rToEncode.size());
+
+ // Make sure we did the right thing
+ if(!CheckValid(false))
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::MakeClearAvailable()
+// Purpose: Private. Make sure the clear filename is available
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::MakeClearAvailable() const
+{
+ if(!mClearFilename.empty())
+ return; // nothing to do
+
+ // Check valid
+ CheckValid();
+
+ // Decode the header
+ int size = BACKUPSTOREFILENAME_GET_SIZE(*this);
+ int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this);
+
+ // Decode based on encoding given in the header
+ switch(encoding)
+ {
+ case Encoding_Clear:
+ TRACE0("**** BackupStoreFilename encoded with Clear encoding ****\n");
+ mClearFilename.assign(c_str() + 2, size - 2);
+ break;
+
+ case Encoding_Blowfish:
+ DecryptEncoded(sBlowfishDecrypt);
+ break;
+
+ default:
+ THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding)
+ break;
+ }
+}
+
+
+// Buffer for encoding and decoding -- do this all in one single buffer to
+// avoid lots of string allocation, which stuffs up memory usage.
+// These static memory vars are, of course, not thread safe, but we don't use threads.
+static int sEncDecBufferSize = 0;
+static MemoryBlockGuard<uint8_t *> *spEncDecBuffer = 0;
+
+static void EnsureEncDecBufferSize(int BufSize)
+{
+ if(spEncDecBuffer == 0)
+ {
+#ifndef WIN32
+ TRACE1("Allocating filename encoding/decoding buffer with size %d\n", BufSize);
+#endif
+ spEncDecBuffer = new MemoryBlockGuard<uint8_t *>(BufSize);
+ MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer);
+ MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer);
+ sEncDecBufferSize = BufSize;
+ }
+ else
+ {
+ if(sEncDecBufferSize < BufSize)
+ {
+#ifndef WIN32
+ TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, BufSize);
+#endif
+ spEncDecBuffer->Resize(BufSize);
+ sEncDecBufferSize = BufSize;
+ MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer);
+ }
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int)
+// Purpose: Private. Assigns the encoded filename string, encrypting.
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding)
+{
+ // Work out max size
+ int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4;
+
+ // Make sure encode/decode buffer has enough space
+ EnsureEncDecBufferSize(maxOutSize);
+
+ // Pointer to buffer
+ uint8_t *buffer = *spEncDecBuffer;
+
+ // Encode -- do entire block in one go
+ int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size());
+ // and add in header size
+ encSize += 2;
+
+ // Adjust header
+ BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding);
+
+ // Store the encoded string
+ assign((char*)buffer, encSize);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &)
+// Purpose: Decrypt the encoded filename using the cipher context
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const
+{
+ // Work out max size
+ int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(size()) + 4;
+
+ // Make sure encode/decode buffer has enough space
+ EnsureEncDecBufferSize(maxOutSize);
+
+ // Pointer to buffer
+ uint8_t *buffer = *spEncDecBuffer;
+
+ // Decrypt
+ const char *str = c_str() + 2;
+ int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, size() - 2);
+
+ // Assign to this
+ mClearFilename.assign((char*)buffer, sizeOut);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::EncodedFilenameChanged()
+// Purpose: The encoded filename stored has changed
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::EncodedFilenameChanged()
+{
+ BackupStoreFilename::EncodedFilenameChanged();
+
+ // Delete stored filename in clear
+ mClearFilename.erase();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int)
+// Purpose: Set the key used for Blowfish encryption of filenames
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength)
+{
+ // Initialisation vector not used. Can't use a different vector for each filename as
+ // that would stop comparisions on the server working.
+ sBlowfishEncrypt.Reset();
+ sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength);
+ sBlowfishEncrypt.SetIV(pIV);
+ sBlowfishDecrypt.Reset();
+ sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+ ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength);
+ sBlowfishDecrypt.SetIV(pIV);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFilenameClear::SetEncodingMethod(int)
+// Purpose: Set the encoding method used for filenames
+// Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetEncodingMethod(int Method)
+{
+ sEncodeMethod = Method;
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFilenameClear.h b/lib/backupclient/BackupStoreFilenameClear.h
new file mode 100644
index 00000000..39148f3d
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilenameClear.h
@@ -0,0 +1,98 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreFilenameClear.h
+// Purpose: BackupStoreFilenames in the clear
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILENAMECLEAR__H
+#define BACKUPSTOREFILENAMECLEAR__H
+
+#include "BackupStoreFilename.h"
+
+class CipherContext;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreFilenameClear
+// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreFilenameClear : public BackupStoreFilename
+{
+public:
+ BackupStoreFilenameClear();
+ BackupStoreFilenameClear(const std::string &rToEncode);
+ BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy);
+ BackupStoreFilenameClear(const BackupStoreFilename &rToCopy);
+ virtual ~BackupStoreFilenameClear();
+
+ // Because we need to use a different allocator for this class to avoid
+ // nasty things happening, can't return this as a reference. Which is a
+ // pity. But probably not too bad.
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+ const std::string GetClearFilename() const;
+#else
+ const std::string &GetClearFilename() const;
+#endif
+ void SetClearFilename(const std::string &rToEncode);
+
+ // Setup for encryption of filenames
+ static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength);
+ static void SetEncodingMethod(int Method);
+
+protected:
+ void MakeClearAvailable() const;
+ virtual void EncodedFilenameChanged();
+ void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding);
+ void DecryptEncoded(CipherContext &rCipherContext) const;
+
+private:
+ mutable BackupStoreFilename_base mClearFilename;
+};
+
+#endif // BACKUPSTOREFILENAMECLEAR__H
+
+
diff --git a/lib/backupclient/BackupStoreObjectDump.cpp b/lib/backupclient/BackupStoreObjectDump.cpp
new file mode 100644
index 00000000..14f9da39
--- /dev/null
+++ b/lib/backupclient/BackupStoreObjectDump.cpp
@@ -0,0 +1,256 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreObjectDump.cpp
+// Purpose: Implementations of dumping objects to stdout/TRACE
+// Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <map>
+
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreFilename.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreObjectMagic.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static void OutputLine(FILE *, bool, const char *, ...)
+// Purpose: Output a line for the object dumping, to file and/or trace...
+// Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+static void OutputLine(FILE *file, bool ToTrace, const char *format, ...)
+{
+ char text[512];
+ int r = 0;
+ va_list ap;
+ va_start(ap, format);
+ r = vsnprintf(text, sizeof(text), format, ap);
+ va_end(ap);
+
+ if(file != 0)
+ {
+ ::fprintf(file, "%s", text);
+ }
+ if(ToTrace)
+ {
+ TRACE1("%s", text);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace)
+// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere)
+// Dump the contents to a file, or trace.
+// Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace)
+{
+ FILE *file = (FILE*)clibFileHandle;
+
+ OutputLine(file, ToTrace, "Directory object.\nObject ID: %llx\nContainer ID: %llx\nNumber entries: %d\n"\
+ "Attributes mod time: %llx\nAttributes size: %d\n", mObjectID, mContainerID, mEntries.size(),
+ mAttributesModTime, mAttributes.GetSize());
+
+ // So repeated filenames can be illustrated, even though they can't be decoded
+ std::map<BackupStoreFilename, int> nameNum;
+ int nameNumI = 0;
+
+ // Dump items
+ OutputLine(file, ToTrace, "Items:\nID Size AttrHash AtSz NSz NIdx Flags\n");
+ for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+ {
+ // Choose file name index number for this file
+ std::map<BackupStoreFilename, int>::iterator nn(nameNum.find((*i)->GetName()));
+ int ni = nameNumI;
+ if(nn != nameNum.end())
+ {
+ ni = nn->second;
+ }
+ else
+ {
+ nameNum[(*i)->GetName()] = nameNumI;
+ ++nameNumI;
+ }
+
+ // Do dependencies
+ char depends[128];
+ depends[0] = '\0';
+ int depends_l = 0;
+ if((*i)->GetDependsNewer() != 0)
+ {
+#ifdef _MSC_VER
+ depends_l += ::sprintf(depends + depends_l, " depNew(%I64x)", (*i)->GetDependsNewer());
+#else
+ depends_l += ::sprintf(depends + depends_l, " depNew(%llx)", (long long)((*i)->GetDependsNewer()));
+#endif
+ }
+ if((*i)->GetDependsOlder() != 0)
+ {
+#ifdef _MSC_VER
+ depends_l += ::sprintf(depends + depends_l, " depOld(%I64x)", (*i)->GetDependsOlder());
+#else
+ depends_l += ::sprintf(depends + depends_l, " depOld(%llx)", (long long)((*i)->GetDependsOlder()));
+#endif
+ }
+
+ // Output item
+ int16_t f = (*i)->GetFlags();
+ OutputLine(file, ToTrace, "%06llx %4lld %016llx %4d %3d %4d%s%s%s%s%s%s\n",
+ (*i)->GetObjectID(),
+ (*i)->GetSizeInBlocks(),
+ (*i)->GetAttributesHash(),
+ (*i)->GetAttributes().GetSize(),
+ (*i)->GetName().size(),
+ ni,
+ ((f & BackupStoreDirectory::Entry::Flags_File)?" file":""),
+ ((f & BackupStoreDirectory::Entry::Flags_Dir)?" dir":""),
+ ((f & BackupStoreDirectory::Entry::Flags_Deleted)?" del":""),
+ ((f & BackupStoreDirectory::Entry::Flags_OldVersion)?" old":""),
+ ((f & BackupStoreDirectory::Entry::Flags_RemoveASAP)?" removeASAP":""),
+ depends);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreFile::DumpFile(void *, bool, IOStream &)
+// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere)
+// Dump the contents to a file, or trace.
+// Created: 4/5/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile)
+{
+ FILE *file = (FILE*)clibFileHandle;
+
+ // Read header
+ file_StreamFormat hdr;
+ if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr),
+ 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite))
+ {
+ // Couldn't read header
+ THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+ }
+
+ // Check and output header info
+ if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+ && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V0))
+ {
+ OutputLine(file, ToTrace, "File header doesn't have the correct magic, aborting dump\n");
+ return;
+ }
+
+ OutputLine(file, ToTrace, "File object.\nContainer ID: %llx\nModification time: %llx\n"\
+ "Max block clear size: %d\nOptions: %08x\nNum blocks: %d\n", box_ntoh64(hdr.mContainerID),
+ box_ntoh64(hdr.mModificationTime), ntohl(hdr.mMaxBlockClearSize), ntohl(hdr.mOptions),
+ box_ntoh64(hdr.mNumBlocks));
+
+ // Read the next two objects
+ BackupStoreFilename fn;
+ fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+ OutputLine(file, ToTrace, "Filename size: %d\n", fn.size());
+
+ BackupClientFileAttributes attr;
+ attr.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+ OutputLine(file, ToTrace, "Attributes size: %d\n", attr.GetSize());
+
+ // Dump the blocks
+ rFile.Seek(0, IOStream::SeekType_Absolute);
+ BackupStoreFile::MoveStreamPositionToBlockIndex(rFile);
+
+ // Read in header
+ file_BlockIndexHeader bhdr;
+ rFile.ReadFullBuffer(&bhdr, sizeof(bhdr), 0);
+ if(bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+ && bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0))
+ {
+ OutputLine(file, ToTrace, "WARNING: Block header doesn't have the correct magic\n");
+ }
+ // number of blocks
+ int64_t nblocks = box_ntoh64(bhdr.mNumBlocks);
+ OutputLine(file, ToTrace, "Other file ID (for block refs): %llx\nNum blocks (in blk hdr): %lld\n",
+ box_ntoh64(bhdr.mOtherFileID), nblocks);
+
+ // Dump info about each block
+ OutputLine(file, ToTrace, "======== ===== ==========\n Index Where EncSz/Idx\n");
+ int64_t nnew = 0, nold = 0;
+ for(int64_t b = 0; b < nblocks; ++b)
+ {
+ file_BlockIndexEntry en;
+ if(!rFile.ReadFullBuffer(&en, sizeof(en), 0))
+ {
+ OutputLine(file, ToTrace, "Didn't manage to read block %lld from file\n", b);
+ continue;
+ }
+ int64_t s = box_ntoh64(en.mEncodedSize);
+ if(s > 0)
+ {
+ nnew++;
+ TRACE2("%8lld this s=%8lld\n", b, s);
+ }
+ else
+ {
+ nold++;
+ TRACE2("%8lld other i=%8lld\n", b, 0 - s);
+ }
+ }
+ TRACE0("======== ===== ==========\n");
+}
+
diff --git a/lib/backupclient/BackupStoreObjectMagic.h b/lib/backupclient/BackupStoreObjectMagic.h
new file mode 100644
index 00000000..58b5d7ef
--- /dev/null
+++ b/lib/backupclient/BackupStoreObjectMagic.h
@@ -0,0 +1,69 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreObjectMagic.h
+// Purpose: Magic values for the start of objects in the backup store
+// Created: 19/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREOBJECTMAGIC__H
+#define BACKUPSTOREOBJECTMAGIC__H
+
+// Each of these values is the first 4 bytes of the object file.
+// Remember to swap from network to host byte order.
+
+// Magic value for file streams
+#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65
+// Do not use v0 in any new code!
+#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45
+
+// Magic for the block index at the file stream -- used to
+// ensure streams are reordered as expected
+#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478
+// Do not use v0 in any new code!
+#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B
+
+// Magic value for directory streams
+#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F
+
+#endif // BACKUPSTOREOBJECTMAGIC__H
+
diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra
new file mode 100644
index 00000000..66203e3c
--- /dev/null
+++ b/lib/backupclient/Makefile.extra
@@ -0,0 +1,16 @@
+
+MAKEPROTOCOL = ../../lib/server/makeprotocol.pl
+
+GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt
+
+# AUTOGEN SEEDING
+autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt
+ perl $(GEN_CMD_SRV)
+
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt
+ perl $(MAKEEXCEPTION) BackupStoreException.txt
+
diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp
new file mode 100644
index 00000000..8e276ba2
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccountDatabase.cpp
@@ -0,0 +1,408 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccountDatabase.cpp
+// Purpose: Database of accounts for the backup store
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <string>
+#include <map>
+#include <stdio.h>
+#include <sys/stat.h>
+
+#include "BackupStoreAccountDatabase.h"
+#include "Guards.h"
+#include "FdGetLine.h"
+#include "BackupStoreException.h"
+#include "CommonException.h"
+#include "FileModificationTime.h"
+
+#include "MemLeakFindOn.h"
+
+class _BackupStoreAccountDatabase
+{
+public:
+ std::string mFilename;
+ std::map<int32_t, BackupStoreAccountDatabase::Entry> mDatabase;
+ box_time_t mModificationTime;
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *)
+// Purpose: Constructor
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *Filename)
+ : pImpl(new _BackupStoreAccountDatabase)
+{
+ pImpl->mFilename = Filename;
+ pImpl->mModificationTime = 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::~BackupStoreAccountDatabase()
+// Purpose: Destructor
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::~BackupStoreAccountDatabase()
+{
+ delete pImpl;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::Entry()
+// Purpose: Default constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry()
+ : mID(-1),
+ mDiscSet(-1)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::Entry(int32_t, int)
+// Purpose: Constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry(int32_t ID, int DiscSet)
+ : mID(ID),
+ mDiscSet(DiscSet)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::Entry(const Entry &)
+// Purpose: Copy constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry(const Entry &rEntry)
+ : mID(rEntry.mID),
+ mDiscSet(rEntry.mDiscSet)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::~Entry()
+// Purpose: Destructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::~Entry()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Read(const char *)
+// Purpose: Read in a database from disc
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreAccountDatabase> BackupStoreAccountDatabase::Read(const char *Filename)
+{
+ // Database object to use
+ std::auto_ptr<BackupStoreAccountDatabase> db(new BackupStoreAccountDatabase(Filename));
+
+ // Read in the file
+ db->ReadFile();
+
+ // Return to called
+ return db;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::ReadFile()
+// Purpose: Read the file off disc
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::ReadFile() const
+{
+ // Open file
+ FileHandleGuard<> file(pImpl->mFilename.c_str());
+
+ // Clear existing entries
+ pImpl->mDatabase.clear();
+
+ // Read in lines
+ FdGetLine getLine(file);
+
+ while(!getLine.IsEOF())
+ {
+ // Read and split up line
+ std::string l(getLine.GetLine(true));
+
+ if(!l.empty())
+ {
+ // Check...
+ int32_t id;
+ int discSet;
+ if(::sscanf(l.c_str(), "%x:%d", &id, &discSet) != 2)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadAccountDatabaseFile)
+ }
+
+ // Make a new entry
+ pImpl->mDatabase[id] = Entry(id, discSet);
+ }
+ }
+
+ // Store the modification time of the file
+ pImpl->mModificationTime = GetDBFileModificationTime();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::CheckUpToDate()
+// Purpose: Private. Ensure that the in memory database matches the one on disc
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::CheckUpToDate() const
+{
+ if(pImpl->mModificationTime != GetDBFileModificationTime())
+ {
+ // File has changed -- load it in again
+ ReadFile();
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::GetDBFileModificationTime()
+// Purpose: Get the current modification time of the database
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+box_time_t BackupStoreAccountDatabase::GetDBFileModificationTime() const
+{
+ struct stat st;
+ if(::stat(pImpl->mFilename.c_str(), &st) == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ return FileModificationTime(st);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Write()
+// Purpose: Write the database back to disc after modifying it
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::Write()
+{
+ // Open file for writing
+ // Would use this...
+ // FileHandleGuard<O_WRONLY | O_TRUNC> file(pImpl->mFilename.c_str());
+ // but gcc fails randomly on it on some platforms. Weird.
+
+ int file = ::open(pImpl->mFilename.c_str(), O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+ if(file == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileOpenError)
+ }
+
+ try
+ {
+ // Then write each entry
+ for(std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.begin());
+ i != pImpl->mDatabase.end(); ++i)
+ {
+ // Write out the entry
+ char line[256]; // more than enough for a couple of integers in string form
+ int s = ::sprintf(line, "%x:%d\n", i->second.GetID(), i->second.GetDiscSet());
+ if(::write(file, line, s) != s)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+
+ ::close(file);
+ }
+ catch(...)
+ {
+ ::close(file);
+ throw;
+ }
+
+ // Done.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::EntryExists(int32_t)
+// Purpose: Does an entry exist in the database?
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool BackupStoreAccountDatabase::EntryExists(int32_t ID) const
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ return pImpl->mDatabase.find(ID) != pImpl->mDatabase.end();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::GetEntry(int32_t)
+// Purpose: Retrieve an entry
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+const BackupStoreAccountDatabase::Entry &BackupStoreAccountDatabase::GetEntry(int32_t ID) const
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.find(ID));
+ if(i == pImpl->mDatabase.end())
+ {
+ THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry)
+ }
+
+ return i->second;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::AddEntry(int32_t, int)
+// Purpose: Add a new entry to the database
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::AddEntry(int32_t ID, int DiscSet)
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ pImpl->mDatabase[ID] = Entry(ID, DiscSet);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::DeleteEntry(int32_t)
+// Purpose: Delete an entry from the database
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::DeleteEntry(int32_t ID)
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.find(ID));
+ if(i == pImpl->mDatabase.end())
+ {
+ THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry)
+ }
+
+ pImpl->mDatabase.erase(i);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t>)
+// Purpose:
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t> &rIDsOut)
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ // Delete everything in the output list
+ rIDsOut.clear();
+
+ std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.begin());
+ for(; i != pImpl->mDatabase.end(); ++i)
+ {
+ rIDsOut.push_back(i->first);
+ }
+}
+
+
+
+
diff --git a/lib/backupstore/BackupStoreAccountDatabase.h b/lib/backupstore/BackupStoreAccountDatabase.h
new file mode 100644
index 00000000..212a2838
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccountDatabase.h
@@ -0,0 +1,113 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccountDatabase.h
+// Purpose: Database of accounts for the backup store
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREACCOUNTDATABASE__H
+#define BACKUPSTOREACCOUNTDATABASE__H
+
+#include <memory>
+#include <vector>
+
+#include "BoxTime.h"
+
+class _BackupStoreAccountDatabase;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreAccountDatabase
+// Purpose: Database of accounts for the backup store
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+class BackupStoreAccountDatabase
+{
+public:
+ friend class _BackupStoreAccountDatabase; // to stop compiler warnings
+ ~BackupStoreAccountDatabase();
+private:
+ BackupStoreAccountDatabase(const char *Filename);
+ BackupStoreAccountDatabase(const BackupStoreAccountDatabase &);
+public:
+
+ static std::auto_ptr<BackupStoreAccountDatabase> Read(const char *Filename);
+ void Write();
+
+ class Entry
+ {
+ public:
+ Entry();
+ Entry(int32_t ID, int DiscSet);
+ Entry(const Entry &rEntry);
+ ~Entry();
+
+ int32_t GetID() const {return mID;}
+ int GetDiscSet() const {return mDiscSet;}
+
+ private:
+ int32_t mID;
+ int mDiscSet;
+ };
+
+ bool EntryExists(int32_t ID) const;
+ const Entry &GetEntry(int32_t ID) const;
+ void AddEntry(int32_t ID, int DiscSet);
+ void DeleteEntry(int32_t ID);
+
+ // This interface should change in the future. But for now it'll do.
+ void GetAllAccountIDs(std::vector<int32_t> &rIDsOut);
+
+private:
+ void ReadFile() const; // const in concept only
+ void CheckUpToDate() const; // const in concept only
+ box_time_t GetDBFileModificationTime() const;
+
+private:
+ mutable _BackupStoreAccountDatabase *pImpl;
+};
+
+#endif // BACKUPSTOREACCOUNTDATABASE__H
+
diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp
new file mode 100644
index 00000000..1b82a580
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccounts.cpp
@@ -0,0 +1,200 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccounts.cpp
+// Purpose: Account management for backup store server
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "BoxPortsAndFiles.h"
+#include "BackupStoreAccounts.h"
+#include "BackupStoreAccountDatabase.h"
+#include "RaidFileWrite.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+#include "UnixUser.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &)
+// Purpose: Constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase)
+ : mrDatabase(rDatabase)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::~BackupStoreAccounts()
+// Purpose: Destructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccounts::~BackupStoreAccounts()
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::Create(int32_t, int, int64_t, int64_t, const std::string &)
+// Purpose: Create a new account on the specified disc set.
+// If rAsUsername is not empty, then the account information will be written under the
+// username specified.
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername)
+{
+ {
+ // Become the user specified in the config file?
+ std::auto_ptr<UnixUser> user;
+ if(!rAsUsername.empty())
+ {
+ // Username specified, change...
+ user.reset(new UnixUser(rAsUsername.c_str()));
+ user->ChangeProcessUser(true /* temporary */);
+ // Change will be undone at the end of this function
+ }
+
+ // Get directory name
+ std::string dirName(MakeAccountRootDir(ID, DiscSet));
+
+ // Create a directory on disc
+ RaidFileWrite::CreateDirectory(DiscSet, dirName, true /* recursive */);
+
+ // Create an info file
+ BackupStoreInfo::CreateNew(ID, dirName, DiscSet, SizeSoftLimit, SizeHardLimit);
+
+ // And an empty directory
+ BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID);
+ int64_t rootDirSize = 0;
+ // Write it, knowing the directory scheme
+ {
+ RaidFileWrite rf(DiscSet, dirName + "o01");
+ rf.Open();
+ rootDir.WriteToStream(rf);
+ rootDirSize = rf.GetDiscUsageInBlocks();
+ rf.Commit(true);
+ }
+
+ // Update the store info to reflect the size of the root directory
+ std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, dirName, DiscSet, false /* ReadWrite */));
+ info->ChangeBlocksUsed(rootDirSize);
+ info->ChangeBlocksInDirectories(rootDirSize);
+
+ // Save it back
+ info->Save();
+ }
+
+ // As the original user...
+
+ // Create the entry in the database
+ mrDatabase.AddEntry(ID, DiscSet);
+
+ // Write the database back
+ mrDatabase.Write();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::GetAccountRoot(int32_t, std::string &, int &)
+// Purpose: Gets the root of an account, returning the info via references
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const
+{
+ // Find the account
+ const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID));
+
+ rRootDirOut = MakeAccountRootDir(ID, en.GetDiscSet());
+ rDiscSetOut = en.GetDiscSet();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::MakeAccountRootDir(int32_t, int)
+// Purpose: Private. Generates a root directory name for the account
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const
+{
+ char accid[64]; // big enough!
+ ::sprintf(accid, "%08x/", ID);
+ return std::string(std::string(BOX_RAIDFILE_ROOT_BBSTORED DIRECTORY_SEPARATOR) + accid);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::AccountExists(int32_t)
+// Purpose: Does an account exist?
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool BackupStoreAccounts::AccountExists(int32_t ID)
+{
+ return mrDatabase.EntryExists(ID);
+}
+
+
diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h
new file mode 100644
index 00000000..dee48148
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccounts.h
@@ -0,0 +1,85 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccounts.h
+// Purpose: Account management for backup store server
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREACCOUNTS__H
+#define BACKUPSTOREACCOUNTS__H
+
+#include <string>
+
+class BackupStoreAccountDatabase;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreAccounts
+// Purpose: Account management for backup store server
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+class BackupStoreAccounts
+{
+public:
+ BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase);
+ ~BackupStoreAccounts();
+private:
+ BackupStoreAccounts(const BackupStoreAccounts &rToCopy);
+
+public:
+ void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername);
+
+ bool AccountExists(int32_t ID);
+ void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const;
+
+private:
+ std::string MakeAccountRootDir(int32_t ID, int DiscSet) const;
+
+private:
+ BackupStoreAccountDatabase &mrDatabase;
+};
+
+#endif // BACKUPSTOREACCOUNTS__H
+
diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp
new file mode 100644
index 00000000..af849c40
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck.cpp
@@ -0,0 +1,783 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheck.cpp
+// Purpose: Check a store for consistency
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "BackupStoreCheck.h"
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::BackupStoreCheck(const std::string &, int, int32_t, bool, bool)
+// Purpose: Constructor
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet)
+ : mStoreRoot(rStoreRoot),
+ mDiscSetNumber(DiscSetNumber),
+ mAccountID(AccountID),
+ mFixErrors(FixErrors),
+ mQuiet(Quiet),
+ mNumberErrorsFound(0),
+ mLastIDInInfo(0),
+ mpInfoLastBlock(0),
+ mInfoLastBlockEntries(0),
+ mLostDirNameSerial(0),
+ mLostAndFoundDirectoryID(0),
+ mBlocksUsed(0),
+ mBlocksInOldFiles(0),
+ mBlocksInDeletedFiles(0),
+ mBlocksInDirectories(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::~BackupStoreCheck()
+// Purpose: Destructor
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::~BackupStoreCheck()
+{
+ // Clean up
+ FreeInfo();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::Check()
+// Purpose: Perform the check on the given account
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::Check()
+{
+ // Lock the account
+ {
+ std::string writeLockFilename;
+ StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename);
+
+ bool gotLock = false;
+ int triesLeft = 8;
+ do
+ {
+ gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */);
+
+ if(!gotLock)
+ {
+ --triesLeft;
+ ::sleep(1);
+ }
+ } while(!gotLock && triesLeft > 0);
+
+ if(!gotLock)
+ {
+ // Couldn't lock the account -- just stop now
+ if(!mQuiet)
+ {
+ ::printf("Couldn't lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server.\n");
+ }
+ THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount)
+ }
+ }
+
+ if(!mQuiet && mFixErrors)
+ {
+ ::printf("NOTE: Will fix errors encountered during checking.\n");
+ }
+
+ // Phase 1, check objects
+ if(!mQuiet)
+ {
+ ::printf("Check store account ID %08x\nPhase 1, check objects...\n", mAccountID);
+ }
+ CheckObjects();
+
+ // Phase 2, check directories
+ if(!mQuiet)
+ {
+ ::printf("Phase 2, check directories...\n");
+ }
+ CheckDirectories();
+
+ // Phase 3, check root
+ if(!mQuiet)
+ {
+ ::printf("Phase 3, check root...\n");
+ }
+ CheckRoot();
+
+ // Phase 4, check unattached objects
+ if(!mQuiet)
+ {
+ ::printf("Phase 4, fix unattached objects...\n");
+ }
+ CheckUnattachedObjects();
+
+ // Phase 5, fix bad info
+ if(!mQuiet)
+ {
+ ::printf("Phase 5, fix unrecovered inconsistencies...\n");
+ }
+ FixDirsWithWrongContainerID();
+ FixDirsWithLostDirs();
+
+ // Phase 6, regenerate store info
+ if(!mQuiet)
+ {
+ ::printf("Phase 6, regenerate store info...\n");
+ }
+ WriteNewStoreInfo();
+
+// DUMP_OBJECT_INFO
+
+ if(mNumberErrorsFound > 0)
+ {
+ ::printf("%lld errors found\n", mNumberErrorsFound);
+ if(!mFixErrors)
+ {
+ ::printf("NOTE: No changes to the store account have been made.\n");
+ }
+ if(!mFixErrors && mNumberErrorsFound > 0)
+ {
+ ::printf("Run again with fix option to fix these errors\n");
+ }
+ if(mNumberErrorsFound > 0)
+ {
+ ::printf("You should now use bbackupquery on the client machine to examine the store.\n");
+ if(mLostAndFoundDirectoryID != 0)
+ {
+ ::printf("A lost+found directory was created in the account root.\n"\
+ "This contains files and directories which could not be matched to existing directories.\n"\
+ "bbackupd will delete this directory in a few days time.\n");
+ }
+ }
+ }
+ else
+ {
+ ::printf("Store account checked, no errors found.\n");
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static TwoDigitHexToInt(const char *, int &)
+// Purpose: Convert a two digit hex string to an int, returning whether it's valid or not
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+static inline bool TwoDigitHexToInt(const char *String, int &rNumberOut)
+{
+ int n = 0;
+ // Char 0
+ if(String[0] >= '0' && String[0] <= '9')
+ {
+ n = (String[0] - '0') << 4;
+ }
+ else if(String[0] >= 'a' && String[0] <= 'f')
+ {
+ n = ((String[0] - 'a') + 0xa) << 4;
+ }
+ else
+ {
+ return false;
+ }
+ // Char 1
+ if(String[1] >= '0' && String[1] <= '9')
+ {
+ n |= String[1] - '0';
+ }
+ else if(String[1] >= 'a' && String[1] <= 'f')
+ {
+ n |= (String[1] - 'a') + 0xa;
+ }
+ else
+ {
+ return false;
+ }
+
+ // Return a valid number
+ rNumberOut = n;
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckObjects()
+// Purpose: Read in the contents of the directory, recurse to other levels,
+// checking objects for sanity and readability
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckObjects()
+{
+ // Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything
+ int64_t maxDir = 0;
+
+ // Find the maximum directory starting ID
+ {
+ // Make sure the starting root dir doesn't end with '/'.
+ std::string start(mStoreRoot);
+ if(start.size() > 0 && start[start.size() - 1] == '/')
+ {
+ start.resize(start.size() - 1);
+ }
+
+ maxDir = CheckObjectsScanDir(0, 1, mStoreRoot);
+ TRACE1("Max dir starting ID is %llx\n", maxDir);
+ }
+
+ // Then go through and scan all the objects within those directories
+ for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH))
+ {
+ CheckObjectsDir(d);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckObjectsScanDir(int64_t, int, int, const std::string &)
+// Purpose: Read in the contents of the directory, recurse to other levels,
+// return the maximum starting ID of any directory found.
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName)
+{
+ //TRACE2("Scan directory for max dir starting ID %s, StartID %lld\n", rDirName.c_str(), StartID);
+
+ int64_t maxID = StartID;
+
+ // Read in all the directories, and recurse downwards
+ {
+ std::vector<std::string> dirs;
+ RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName,
+ RaidFileRead::DirReadType_DirsOnly, dirs);
+
+ for(std::vector<std::string>::const_iterator i(dirs.begin()); i != dirs.end(); ++i)
+ {
+ // Check to see if it's the right name
+ int n = 0;
+ if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n)
+ && n < (1<<STORE_ID_SEGMENT_LENGTH))
+ {
+ // Next level down
+ int64_t mi = CheckObjectsScanDir(StartID | (n << (Level * STORE_ID_SEGMENT_LENGTH)), Level + 1,
+ rDirName + DIRECTORY_SEPARATOR + *i);
+ // Found a greater starting ID?
+ if(mi > maxID)
+ {
+ maxID = mi;
+ }
+ }
+ else
+ {
+ ::printf("Spurious or invalid directory %s/%s found%s -- delete manually\n", rDirName.c_str(), (*i).c_str(), mFixErrors?", deleting":"");
+ ++mNumberErrorsFound;
+ }
+ }
+ }
+
+ return maxID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckObjectsDir(int64_t)
+// Purpose: Check all the files within this directory which has the given starting ID.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
+{
+ // Make directory name -- first generate the filename of an entry in it
+ std::string dirName;
+ StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */);
+ // Check expectations
+ ASSERT(dirName.size() > 4 && dirName[dirName.size() - 4] == '/');
+ // Remove the filename from it
+ dirName.resize(dirName.size() - 4); // four chars for "/o00"
+
+ // Check directory exists
+ if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName))
+ {
+ TRACE1("RaidFile dir %s does not exist\n", dirName.c_str());
+ return;
+ }
+
+ // Read directory contents
+ std::vector<std::string> files;
+ RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName,
+ RaidFileRead::DirReadType_FilesOnly, files);
+
+ // Array of things present
+ bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)];
+ for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l)
+ {
+ idsPresent[l] = false;
+ }
+
+ // Parse each entry, building up a list of object IDs which are present in the dir.
+ // This is done so that whatever order is retured from the directory, objects are scanned
+ // in order.
+ // Filename must begin with a 'o' and be three characters long, otherwise it gets deleted.
+ for(std::vector<std::string>::const_iterator i(files.begin()); i != files.end(); ++i)
+ {
+ bool fileOK = true;
+ int n = 0;
+ if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n)
+ && n < (1<<STORE_ID_SEGMENT_LENGTH))
+ {
+ // Filename is valid, mark as existing
+ idsPresent[n] = true;
+ }
+ else
+ {
+ // info file in root dir is OK!
+ if(StartID != 0 || ::strcmp("info", (*i).c_str()) != 0)
+ {
+ fileOK = false;
+ }
+ }
+
+ if(!fileOK)
+ {
+ // Unexpected or bad file, delete it
+ ::printf("Spurious file %s/%s found%s\n", dirName.c_str(), (*i).c_str(), mFixErrors?", deleting":"");
+ ++mNumberErrorsFound;
+ if(mFixErrors)
+ {
+ RaidFileWrite del(mDiscSetNumber, dirName + DIRECTORY_SEPARATOR + *i);
+ del.Delete();
+ }
+ }
+ }
+
+ // Check all the objects found in this directory
+ for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i)
+ {
+ if(idsPresent[i])
+ {
+ // Check the object is OK, and add entry
+ char leaf[8];
+ ::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i);
+ if(!CheckAndAddObject(StartID | i, dirName + leaf))
+ {
+ // File was bad, delete it
+ ::printf("Corrupted file %s%s found%s\n", dirName.c_str(), leaf, mFixErrors?", deleting":"");
+ ++mNumberErrorsFound;
+ if(mFixErrors)
+ {
+ RaidFileWrite del(mDiscSetNumber, dirName + leaf);
+ del.Delete();
+ }
+ }
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckAndAddObject(int64_t, const std::string &)
+// Purpose: Check a specific object and add it to the list if it's OK -- if
+// there are any errors with the reading, return false and it'll be deleted.
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename)
+{
+ // Info on object...
+ bool isFile = true;
+ int64_t containerID = -1;
+ int64_t size = -1;
+
+ try
+ {
+ // Open file
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, rFilename));
+ size = file->GetDiscUsageInBlocks();
+
+ // Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile
+ uint32_t signature;
+ if(file->Read(&signature, sizeof(signature)) != sizeof(signature))
+ {
+ // Too short, can't read signature from it
+ return false;
+ }
+ // Seek back to beginning
+ file->Seek(0, IOStream::SeekType_Absolute);
+
+ // Then... check depending on the type
+ switch(ntohl(signature))
+ {
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
+#endif
+ // File... check
+ containerID = CheckFile(ObjectID, *file);
+ break;
+
+ case OBJECTMAGIC_DIR_MAGIC_VALUE:
+ isFile = false;
+ containerID = CheckDirInitial(ObjectID, *file);
+ break;
+
+ default:
+ // Unknown signature. Bad file. Very bad file.
+ return false;
+ break;
+ }
+
+ // Add to usage counts
+ int64_t s = file->GetDiscUsageInBlocks();
+ mBlocksUsed += s;
+ if(!isFile)
+ {
+ mBlocksInDirectories += s;
+ }
+ }
+ catch(...)
+ {
+ // Error caught, not a good file then, let it be deleted
+ return false;
+ }
+
+ // Got a container ID? (ie check was successful)
+ if(containerID == -1)
+ {
+ return false;
+ }
+
+ // Add to list of IDs known about
+ AddID(ObjectID, containerID, size, isFile);
+
+ // Report success
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckFile(int64_t, IOStream &)
+// Purpose: Do check on file, return original container ID if OK, or -1 on error
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream)
+{
+ // Check that it's not the root directory ID. Having a file as the root directory would be bad.
+ if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
+ {
+ // Get that dodgy thing deleted!
+ ::printf("Have file as root directory. This is bad.\n");
+ return -1;
+ }
+
+ // Check the format of the file, and obtain the container ID
+ int64_t originalContainerID = -1;
+ if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */,
+ &originalContainerID))
+ {
+ // Didn't verify
+ return -1;
+ }
+
+ return originalContainerID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckDirInitial(int64_t, IOStream &)
+// Purpose: Do initial check on directory, return container ID if OK, or -1 on error
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream)
+{
+ // Simply attempt to read in the directory
+ BackupStoreDirectory dir;
+ dir.ReadFromStream(rStream, IOStream::TimeOutInfinite);
+
+ // Check object ID
+ if(dir.GetObjectID() != ObjectID)
+ {
+ // Wrong object ID
+ return -1;
+ }
+
+ // Return container ID
+ return dir.GetContainerID();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckDirectories()
+// Purpose: Check the directories
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckDirectories()
+{
+ // Phase 1 did this:
+ // Checked that all the directories are readable
+ // Built a list of all directories and files which exist on the store
+ //
+ // This phase will check all the files in the directories, make
+ // a note of all directories which are missing, and do initial fixing.
+
+ // Scan all objects
+ for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ IDBlock *pblock = i->second;
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+
+ for(int e = 0; e < bentries; ++e)
+ {
+ uint8_t flags = GetFlags(pblock, e);
+ if(flags & Flags_IsDir)
+ {
+ // Found a directory. Read it in.
+ std::string filename;
+ StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */);
+ BackupStoreDirectory dir;
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Flag for modifications
+ bool isModified = false;
+
+ // Check for validity
+ if(dir.CheckAndFix())
+ {
+ // Wasn't quite right, and has been modified
+ ::printf("Directory ID %llx has bad structure\n", pblock->mID[e]);
+ ++mNumberErrorsFound;
+ isModified = true;
+ }
+
+ // Go through, and check that everything in that directory exists and is valid
+ std::vector<int64_t> toDelete;
+
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next()) != 0)
+ {
+ // Lookup the item
+ int32_t iIndex;
+ IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
+ bool badEntry = false;
+ if(piBlock != 0)
+ {
+ // Found. Get flags
+ uint8_t iflags = GetFlags(piBlock, iIndex);
+
+ // Is the type the same?
+ if(((iflags & Flags_IsDir) == Flags_IsDir)
+ != ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir))
+ {
+ // Entry is of wrong type
+ ::printf("Directory ID %llx references object %llx which has a different type than expected.\n", pblock->mID[e], en->GetObjectID());
+ badEntry = true;
+ }
+ else
+ {
+ // Check that the entry is not already contained.
+ if(iflags & Flags_IsContained)
+ {
+ badEntry = true;
+ ::printf("Directory ID %llx references object %llx which is already contained.\n", pblock->mID[e], en->GetObjectID());
+ }
+ else
+ {
+ // Not already contained -- mark as contained
+ SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
+
+ // Check that the container ID of the object is correct
+ if(piBlock->mContainer[iIndex] != pblock->mID[e])
+ {
+ // Needs fixing...
+ if(iflags & Flags_IsDir)
+ {
+ // Add to will fix later list
+ ::printf("Directory ID %llx has wrong container ID.\n", en->GetObjectID());
+ mDirsWithWrongContainerID.push_back(en->GetObjectID());
+ }
+ else
+ {
+ // This is OK for files, they might move
+ ::printf("File ID %llx has different container ID, probably moved\n", en->GetObjectID());
+ }
+
+ // Fix entry for now
+ piBlock->mContainer[iIndex] = pblock->mID[e];
+ }
+ }
+ }
+
+ // Check the object size, if it's OK and a file
+ if(!badEntry && !((iflags & Flags_IsDir) == Flags_IsDir))
+ {
+ if(en->GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[iIndex])
+ {
+ // Correct
+ en->SetSizeInBlocks(piBlock->mObjectSizeInBlocks[iIndex]);
+ // Mark as changed
+ isModified = true;
+ // Tell user
+ ::printf("Directory ID %llx has wrong size for object %llx\n", pblock->mID[e], en->GetObjectID());
+ }
+ }
+ }
+ else
+ {
+ // Item can't be found. Is it a directory?
+ if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir)
+ {
+ // Store the directory for later attention
+ mDirsWhichContainLostDirs[en->GetObjectID()] = pblock->mID[e];
+ }
+ else
+ {
+ // Just remove the entry
+ badEntry = true;
+ ::printf("Directory ID %llx references object %llx which does not exist.\n", pblock->mID[e], en->GetObjectID());
+ }
+ }
+
+ // Is this entry worth keeping?
+ if(badEntry)
+ {
+ toDelete.push_back(en->GetObjectID());
+ }
+ else
+ {
+ // Add to sizes?
+ if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion)
+ {
+ mBlocksInOldFiles += en->GetSizeInBlocks();
+ }
+ if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted)
+ {
+ mBlocksInDeletedFiles += en->GetSizeInBlocks();
+ }
+ }
+ }
+
+ if(toDelete.size() > 0)
+ {
+ // Delete entries from directory
+ for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d)
+ {
+ dir.DeleteEntry(*d);
+ }
+
+ // Mark as modified
+ isModified = true;
+
+ // Check the directory again, now that entries have been removed
+ dir.CheckAndFix();
+
+ // Errors found
+ ++mNumberErrorsFound;
+ }
+
+ if(isModified && mFixErrors)
+ {
+ ::printf("Fixing directory ID %llx\n", pblock->mID[e]);
+
+ // Save back to disc
+ RaidFileWrite fixed(mDiscSetNumber, filename);
+ fixed.Open(true /* allow overwriting */);
+ dir.WriteToStream(fixed);
+ // Commit it
+ fixed.Commit(true /* convert to raid representation now */);
+ }
+ }
+ }
+ }
+
+}
+
+
diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h
new file mode 100644
index 00000000..f176ebad
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck.h
@@ -0,0 +1,237 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheck.h
+// Purpose: Check a store for consistency
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECHECK__H
+#define BACKUPSTORECHECK__H
+
+#include <string>
+#include <map>
+#include <vector>
+#include <set>
+
+#include "NamedLock.h"
+class IOStream;
+class BackupStoreFilename;
+
+/*
+
+The following problems can be fixed:
+
+ * Spurious files deleted
+ * Corrupted files deleted
+ * Root ID as file, deleted
+ * Dirs with wrong object id inside, deleted
+ * Direcetory entries pointing to non-existant files, deleted
+ * Doubly references files have second reference deleted
+ * Wrong directory container IDs fixed
+ * Missing root recreated
+ * Reattach files which exist, but aren't referenced
+ - files go into directory original directory, if it still exists
+ - missing directories are inferred, and recreated
+ - or if all else fails, go into lost+found
+ - file dir entries take the original name and mod time
+ - directories go into lost+found
+ * Container IDs on directories corrected
+ * Inside directories,
+ - only one object per name has old version clear
+ - IDs aren't duplicated
+ * Bad store info files regenerated
+ * Bad sizes of files in directories fixed
+
+*/
+
+
+// Size of blocks in the list of IDs
+#ifdef NDEBUG
+ #define BACKUPSTORECHECK_BLOCK_SIZE (64*1024)
+#else
+ #define BACKUPSTORECHECK_BLOCK_SIZE 8
+#endif
+
+// The object ID type -- can redefine to uint32_t to produce a lower memory version for smaller stores
+typedef int64_t BackupStoreCheck_ID_t;
+// Can redefine the size type for lower memory usage too
+typedef int64_t BackupStoreCheck_Size_t;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreCheck
+// Purpose: Check a store for consistency
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+class BackupStoreCheck
+{
+public:
+ BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet);
+ ~BackupStoreCheck();
+private:
+ // no copying
+ BackupStoreCheck(const BackupStoreCheck &);
+ BackupStoreCheck &operator=(const BackupStoreCheck &);
+public:
+
+ // Do the exciting things
+ void Check();
+
+ bool ErrorsFound() {return mNumberErrorsFound > 0;}
+
+private:
+ enum
+ {
+ // Bit mask
+ Flags_IsDir = 1,
+ Flags_IsContained = 2,
+ // Mask
+ Flags__MASK = 3,
+ // Number of bits
+ Flags__NumFlags = 2,
+ // Items per uint8_t
+ Flags__NumItemsPerEntry = 4 // ie 8 / 2
+ };
+
+ typedef struct
+ {
+ // Note use arrays within the block, rather than the more obvious array of
+ // objects, to be more memory efficient -- think alignment of the byte values.
+ uint8_t mFlags[BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry];
+ BackupStoreCheck_ID_t mID[BACKUPSTORECHECK_BLOCK_SIZE];
+ BackupStoreCheck_ID_t mContainer[BACKUPSTORECHECK_BLOCK_SIZE];
+ BackupStoreCheck_Size_t mObjectSizeInBlocks[BACKUPSTORECHECK_BLOCK_SIZE];
+ } IDBlock;
+
+ // Phases of the check
+ void CheckObjects();
+ void CheckDirectories();
+ void CheckRoot();
+ void CheckUnattachedObjects();
+ void FixDirsWithWrongContainerID();
+ void FixDirsWithLostDirs();
+ void WriteNewStoreInfo();
+
+ // Checking functions
+ int64_t CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName);
+ void CheckObjectsDir(int64_t StartID);
+ bool CheckAndAddObject(int64_t ObjectID, const std::string &rFilename);
+ int64_t CheckFile(int64_t ObjectID, IOStream &rStream);
+ int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream);
+
+ // Fixing functions
+ bool TryToRecreateDirectory(int64_t MissingDirectoryID);
+ void InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory);
+ int64_t GetLostAndFoundDirID();
+ void CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID);
+
+ // Data handling
+ void FreeInfo();
+ void AddID(BackupStoreCheck_ID_t ID, BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile);
+ IDBlock *LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut);
+ inline void SetFlags(IDBlock *pBlock, int32_t Index, uint8_t Flags)
+ {
+ ASSERT(pBlock != 0);
+ ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE);
+ ASSERT(Flags < (1 << Flags__NumFlags));
+
+ pBlock->mFlags[Index / Flags__NumItemsPerEntry]
+ |= (Flags << ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags));
+ }
+ inline uint8_t GetFlags(IDBlock *pBlock, int32_t Index)
+ {
+ ASSERT(pBlock != 0);
+ ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE);
+
+ return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK;
+ }
+
+#ifndef NDEBUG
+ void DumpObjectInfo();
+ #define DUMP_OBJECT_INFO DumpObjectInfo();
+#else
+ #define DUMP_OBJECT_INFO
+#endif
+
+private:
+ std::string mStoreRoot;
+ int mDiscSetNumber;
+ int32_t mAccountID;
+ bool mFixErrors;
+ bool mQuiet;
+
+ int64_t mNumberErrorsFound;
+
+ // Lock for the store account
+ NamedLock mAccountLock;
+
+ // Storage for ID data
+ typedef std::map<BackupStoreCheck_ID_t, IDBlock*> Info_t;
+ Info_t mInfo;
+ BackupStoreCheck_ID_t mLastIDInInfo;
+ IDBlock *mpInfoLastBlock;
+ int32_t mInfoLastBlockEntries;
+
+ // List of stuff to fix
+ std::vector<BackupStoreCheck_ID_t> mDirsWithWrongContainerID;
+ // This is a map of lost dir ID -> existing dir ID
+ std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t> mDirsWhichContainLostDirs;
+
+ // Set of extra directories added
+ std::set<BackupStoreCheck_ID_t> mDirsAdded;
+
+ // Misc stuff
+ int32_t mLostDirNameSerial;
+ int64_t mLostAndFoundDirectoryID;
+
+ // Usage
+ int64_t mBlocksUsed;
+ int64_t mBlocksInOldFiles;
+ int64_t mBlocksInDeletedFiles;
+ int64_t mBlocksInDirectories;
+};
+
+#endif // BACKUPSTORECHECK__H
+
diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp
new file mode 100644
index 00000000..38e8bf92
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck2.cpp
@@ -0,0 +1,879 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheck2.cpp
+// Purpose: More backup store checking
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "BackupStoreCheck.h"
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreInfo.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckRoot()
+// Purpose: Check the root directory exists.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckRoot()
+{
+ int32_t index = 0;
+ IDBlock *pblock = LookupID(BACKUPSTORE_ROOT_DIRECTORY_ID, index);
+
+ if(pblock != 0)
+ {
+ // Found it. Which is lucky. Mark it as contained.
+ SetFlags(pblock, index, Flags_IsContained);
+ }
+ else
+ {
+ ::printf("Root directory doesn't exist\n");
+
+ ++mNumberErrorsFound;
+
+ if(mFixErrors)
+ {
+ // Create a new root directory
+ CreateBlankDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID);
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CreateBlankDirectory(int64_t, int64_t)
+// Purpose: Creates a blank directory
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID)
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ BackupStoreDirectory dir(DirectoryID, ContainingDirID);
+
+ // Serialise to disc
+ std::string filename;
+ StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */);
+ RaidFileWrite obj(mDiscSetNumber, filename);
+ obj.Open(false /* don't allow overwriting */);
+ dir.WriteToStream(obj);
+ int64_t size = obj.GetDiscUsageInBlocks();
+ obj.Commit(true /* convert to raid now */);
+
+ // Record the fact we've done this
+ mDirsAdded.insert(DirectoryID);
+
+ // Add to sizes
+ mBlocksUsed += size;
+ mBlocksInDirectories += size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckUnattachedObjects()
+// Purpose: Check for objects which aren't attached to anything
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckUnattachedObjects()
+{
+ // Scan all objects, finding ones which have no container
+ for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ IDBlock *pblock = i->second;
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+
+ for(int e = 0; e < bentries; ++e)
+ {
+ uint8_t flags = GetFlags(pblock, e);
+ if((flags & Flags_IsContained) == 0)
+ {
+ // Unattached object...
+ ::printf("Object %llx is unattached.\n", pblock->mID[e]);
+ ++mNumberErrorsFound;
+
+ // What's to be done?
+ int64_t putIntoDirectoryID = 0;
+
+ if((flags & Flags_IsDir) == Flags_IsDir)
+ {
+ // Directory. Just put into lost and found.
+ putIntoDirectoryID = GetLostAndFoundDirID();
+ }
+ else
+ {
+ // File. Only attempt to attach it somewhere if it isn't a patch
+ {
+ int64_t diffFromObjectID = 0;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */);
+ // The easiest way to do this is to verify it again. Not such a bad penalty, because
+ // this really shouldn't be done very often.
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ BackupStoreFile::VerifyEncodedFileFormat(*file, &diffFromObjectID);
+ }
+
+ // If not zero, then it depends on another file, which may or may not be available.
+ // Just delete it to be safe.
+ if(diffFromObjectID != 0)
+ {
+ ::printf("Object %llx is unattached, and is a patch. Deleting, cannot reliably recover.\n", pblock->mID[e]);
+
+ // Delete this object instead
+ if(mFixErrors)
+ {
+ RaidFileWrite del(mDiscSetNumber, filename);
+ del.Delete();
+ }
+
+ // Move on to next item
+ continue;
+ }
+ }
+
+ // Files contain their original filename, so perhaps the orginal directory still exists,
+ // or we can infer the existance of a directory?
+ // Look for a matching entry in the mDirsWhichContainLostDirs map.
+ // Can't do this with a directory, because the name just wouldn't be known, which is
+ // pretty useless as bbackupd would just delete it. So better to put it in lost+found
+ // where the admin can do something about it.
+ int32_t dirindex;
+ IDBlock *pdirblock = LookupID(pblock->mContainer[e], dirindex);
+ if(pdirblock != 0)
+ {
+ // Something with that ID has been found. Is it a directory?
+ if(GetFlags(pdirblock, dirindex) & Flags_IsDir)
+ {
+ // Directory exists, add to that one
+ putIntoDirectoryID = pblock->mContainer[e];
+ }
+ else
+ {
+ // Not a directory. Use lost and found dir
+ putIntoDirectoryID = GetLostAndFoundDirID();
+ }
+ }
+ else if(mDirsAdded.find(pblock->mContainer[e]) != mDirsAdded.end()
+ || TryToRecreateDirectory(pblock->mContainer[e]))
+ {
+ // The directory reappeared, or was created somehow elsewhere
+ putIntoDirectoryID = pblock->mContainer[e];
+ }
+ else
+ {
+ putIntoDirectoryID = GetLostAndFoundDirID();
+ }
+ }
+ ASSERT(putIntoDirectoryID != 0);
+
+ // Add it to the directory
+ InsertObjectIntoDirectory(pblock->mID[e], putIntoDirectoryID,
+ ((flags & Flags_IsDir) == Flags_IsDir));
+ }
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::TryToRecreateDirectory(int64_t)
+// Purpose: Recreate a missing directory
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID)
+{
+ // During the directory checking phase, a map of "missing directory" to
+ // containing directory was built. If we can find it here, then it's
+ // something which can be recreated!
+ std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator missing(
+ mDirsWhichContainLostDirs.find(MissingDirectoryID));
+ if(missing == mDirsWhichContainLostDirs.end())
+ {
+ // Not a missing directory, can't recreate.
+ return false;
+ }
+
+ // Can recreate this! Wooo!
+ if(!mFixErrors)
+ {
+ ::printf("Missing directory %llx could be recreated\n", MissingDirectoryID);
+ mDirsAdded.insert(MissingDirectoryID);
+ return true;
+ }
+ ::printf("Recreating missing directory %llx\n", MissingDirectoryID);
+
+ // Create a blank directory
+ BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */);
+ // Note that this directory already contains a directory entry pointing to
+ // this dir, so it doesn't have to be added.
+
+ // Serialise to disc
+ std::string filename;
+ StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */);
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(false /* don't allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+
+ // Record the fact we've done this
+ mDirsAdded.insert(MissingDirectoryID);
+
+ // Remove the entry from the map, so this doesn't happen again
+ mDirsWhichContainLostDirs.erase(missing);
+
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::GetLostAndFoundDirID()
+// Purpose: Returns the ID of the lost and found directory, creating it if necessary
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::GetLostAndFoundDirID()
+{
+ // Already allocated it?
+ if(mLostAndFoundDirectoryID != 0)
+ {
+ return mLostAndFoundDirectoryID;
+ }
+
+ if(!mFixErrors)
+ {
+ // The result will never be used anyway if errors aren't being fixed
+ return 1;
+ }
+
+ // Load up the root directory
+ BackupStoreDirectory dir;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(BACKUPSTORE_ROOT_DIRECTORY_ID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Find a suitable name
+ BackupStoreFilename lostAndFound;
+ int n = 0;
+ while(true)
+ {
+ char name[32];
+ ::sprintf(name, "lost+found%d", n++);
+ lostAndFound.SetAsClearFilename(name);
+ if(!dir.NameInUse(lostAndFound))
+ {
+ // Found a name which can be used
+ ::printf("Lost and found dir has name %s\n", name);
+ break;
+ }
+ }
+
+ // Allocate an ID
+ int64_t id = mLastIDInInfo + 1;
+
+ // Create a blank directory
+ CreateBlankDirectory(id, BACKUPSTORE_ROOT_DIRECTORY_ID);
+
+ // Add an entry for it
+ dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0);
+
+ // Write out root dir
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+
+ // Store
+ mLostAndFoundDirectoryID = id;
+
+ // Tell caller
+ return mLostAndFoundDirectoryID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::InsertObjectIntoDirectory(int64_t, int64_t, bool)
+// Purpose:
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory)
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ // Data for the object
+ BackupStoreFilename objectStoreFilename;
+ int64_t modTime = 100; // something which isn't zero or a special time
+ int32_t sizeInBlocks = 0; // suitable for directories
+
+ if(IsDirectory)
+ {
+ // Directory -- simply generate a name for it.
+ char name[32];
+ ::sprintf(name, "dir%08x", mLostDirNameSerial++);
+ objectStoreFilename.SetAsClearFilename(name);
+ }
+ else
+ {
+ // Files require a little more work...
+ // Open file
+ std::string fileFilename;
+ StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mDiscSetNumber, fileFilename, false /* don't make sure the dir exists */);
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, fileFilename));
+ // Fill in size information
+ sizeInBlocks = file->GetDiscUsageInBlocks();
+ // Read in header
+ file_StreamFormat hdr;
+ if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ ))
+ {
+ // This should never happen, everything has been checked before.
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+ // This tells us nice things
+ modTime = box_ntoh64(hdr.mModificationTime);
+ // And the filename comes next
+ objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Directory object
+ BackupStoreDirectory dir;
+
+ // Generate filename
+ std::string filename;
+ StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+
+ // Read it in
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Add a new entry in an appropraite place
+ dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks,
+ IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File));
+
+ // Fix any flags which have been broken, which there's a good change of going
+ dir.CheckAndFix();
+
+ // Write it out
+ if(mFixErrors)
+ {
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::FixDirsWithWrongContainerID()
+// Purpose: Rewrites container IDs where required
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FixDirsWithWrongContainerID()
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ // Run through things which need fixing
+ for(std::vector<BackupStoreCheck_ID_t>::iterator i(mDirsWithWrongContainerID.begin());
+ i != mDirsWithWrongContainerID.end(); ++i)
+ {
+ int32_t index = 0;
+ IDBlock *pblock = LookupID(*i, index);
+ if(pblock == 0) continue;
+
+ // Load in
+ BackupStoreDirectory dir;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(*i, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Adjust container ID
+ dir.SetContainerID(pblock->mContainer[index]);
+
+ // Write it out
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::FixDirsWithLostDirs()
+// Purpose: Fix directories
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FixDirsWithLostDirs()
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ // Run through things which need fixing
+ for(std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator i(mDirsWhichContainLostDirs.begin());
+ i != mDirsWhichContainLostDirs.end(); ++i)
+ {
+ int32_t index = 0;
+ IDBlock *pblock = LookupID(i->second, index);
+ if(pblock == 0) continue;
+
+ // Load in
+ BackupStoreDirectory dir;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(i->second, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Delete the dodgy entry
+ dir.DeleteEntry(i->first);
+
+ // Fix it up
+ dir.CheckAndFix();
+
+ // Write it out
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::WriteNewStoreInfo()
+// Purpose: Regenerate store info
+// Created: 23/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::WriteNewStoreInfo()
+{
+ // Attempt to load the existing store info file
+ std::auto_ptr<BackupStoreInfo> poldInfo;
+ try
+ {
+ poldInfo.reset(BackupStoreInfo::Load(mAccountID, mStoreRoot, mDiscSetNumber, true /* read only */).release());
+ }
+ catch(...)
+ {
+ ::printf("Load of existing store info failed, regenerating.\n");
+ ++mNumberErrorsFound;
+ }
+
+ // Minimum soft and hard limits
+ int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024;
+ int64_t minHard = ((minSoft * 11) / 10) + 1024;
+
+ // Need to do anything?
+ if(poldInfo.get() != 0 && mNumberErrorsFound == 0 && poldInfo->GetAccountID() == mAccountID)
+ {
+ // Leave the store info as it is, no need to alter it because nothing really changed,
+ // and the only essential thing was that the account ID was correct, which is was.
+ return;
+ }
+
+ // NOTE: We will always build a new store info, so the client store marker gets changed.
+
+ // Work out the new limits
+ int64_t softLimit = minSoft;
+ int64_t hardLimit = minHard;
+ if(poldInfo.get() != 0 && poldInfo->GetBlocksSoftLimit() > minSoft)
+ {
+ softLimit = poldInfo->GetBlocksSoftLimit();
+ }
+ else
+ {
+ ::printf("NOTE: Soft limit for account changed to ensure housekeeping doesn't delete files on next run\n");
+ }
+ if(poldInfo.get() != 0 && poldInfo->GetBlocksHardLimit() > minHard)
+ {
+ hardLimit = poldInfo->GetBlocksHardLimit();
+ }
+ else
+ {
+ ::printf("NOTE: Hard limit for account changed to ensure housekeeping doesn't delete files on next run\n");
+ }
+
+ // Object ID
+ int64_t lastObjID = mLastIDInInfo;
+ if(mLostAndFoundDirectoryID != 0)
+ {
+ mLastIDInInfo++;
+ }
+
+ // Build a new store info
+ std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::CreateForRegeneration(
+ mAccountID,
+ mStoreRoot,
+ mDiscSetNumber,
+ lastObjID,
+ mBlocksUsed,
+ mBlocksInOldFiles,
+ mBlocksInDeletedFiles,
+ mBlocksInDirectories,
+ softLimit,
+ hardLimit));
+
+ // Save to disc?
+ if(mFixErrors)
+ {
+ info->Save();
+ ::printf("New store info file written successfully.\n");
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::CheckAndFix()
+// Purpose: Check the directory for obvious logical problems, and fix them.
+// Return true if the directory was changed.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDirectory::CheckAndFix()
+{
+ bool changed = false;
+
+ // Check that if a file depends on a new version, that version is in this directory
+ {
+ std::vector<Entry*>::iterator i(mEntries.begin());
+ for(; i != mEntries.end(); ++i)
+ {
+ int64_t dependsNewer = (*i)->GetDependsNewer();
+ if(dependsNewer != 0)
+ {
+ BackupStoreDirectory::Entry *newerEn = FindEntryByID(dependsNewer);
+ if(newerEn == 0)
+ {
+ // Depends on something, but it isn't there.
+ TRACE2("Entry id %llx removed because depends on newer version %llx which doesn't exist\n", (*i)->GetObjectID(), dependsNewer);
+
+ // Remove
+ delete *i;
+ mEntries.erase(i);
+
+ // Start again at the beginning of the vector, the iterator is now invalid
+ i = mEntries.begin();
+
+ // Mark as changed
+ changed = true;
+ }
+ else
+ {
+ // Check that newerEn has it marked
+ if(newerEn->GetDependsOlder() != (*i)->GetObjectID())
+ {
+ // Wrong entry
+ TRACE3("Entry id %llx, correcting DependsOlder to %llx, was %llx\n", dependsNewer, (*i)->GetObjectID(), newerEn->GetDependsOlder());
+ newerEn->SetDependsOlder((*i)->GetObjectID());
+ // Mark as changed
+ changed = true;
+ }
+ }
+ }
+ }
+ }
+
+ // Check that if a file has a dependency marked, it exists, and remove it if it doesn't
+ {
+ std::vector<Entry*>::iterator i(mEntries.begin());
+ for(; i != mEntries.end(); ++i)
+ {
+ int64_t dependsOlder = (*i)->GetDependsOlder();
+ if(dependsOlder != 0 && FindEntryByID(dependsOlder) == 0)
+ {
+ // Has an older version marked, but this doesn't exist. Remove this mark
+ TRACE2("Entry id %llx was marked that %llx depended on it, which doesn't exist, dependency info cleaered\n", (*i)->GetObjectID(), dependsOlder);
+
+ (*i)->SetDependsOlder(0);
+
+ // Mark as changed
+ changed = true;
+ }
+ }
+ }
+
+ bool ch = false;
+ do
+ {
+ // Reset change marker
+ ch = false;
+
+ // Search backwards -- so see newer versions first
+ std::vector<Entry*>::iterator i(mEntries.end());
+ if(i == mEntries.begin())
+ {
+ // Directory is empty, stop now
+ return changed; // changed flag
+ }
+
+ // Records of things seen
+ std::set<int64_t> idsEncountered;
+ std::set<BackupStoreFilename> filenamesEncountered;
+
+ do
+ {
+ // Look at previous
+ --i;
+
+ bool removeEntry = false;
+ if((*i) == 0)
+ {
+ TRACE0("Remove because null pointer found\n");
+ removeEntry = true;
+ }
+ else
+ {
+ bool isDir = (((*i)->GetFlags() & Entry::Flags_Dir) == Entry::Flags_Dir);
+
+ // Check mutually exclusive flags
+ if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File))
+ {
+ // Bad! Unset the file flag
+ TRACE1("Entry %llx: File flag set when dir flag set\n", (*i)->GetObjectID());
+ (*i)->RemoveFlags(Entry::Flags_File);
+ changed = true;
+ }
+
+ // Check...
+ if(idsEncountered.find((*i)->GetObjectID()) != idsEncountered.end())
+ {
+ // ID already seen, or type doesn't match
+ TRACE1("Entry %llx: Remove because ID already seen\n", (*i)->GetObjectID());
+ removeEntry = true;
+ }
+ else
+ {
+ // Haven't already seen this ID, remember it
+ idsEncountered.insert((*i)->GetObjectID());
+
+ // Check to see if the name has already been encountered -- if not, then it
+ // needs to have the old version flag set
+ if(filenamesEncountered.find((*i)->GetName()) != filenamesEncountered.end())
+ {
+ // Seen before -- check old version flag set
+ if(((*i)->GetFlags() & Entry::Flags_OldVersion) != Entry::Flags_OldVersion
+ && ((*i)->GetFlags() & Entry::Flags_Deleted) == 0)
+ {
+ // Not set, set it
+ TRACE1("Entry %llx: Set old flag\n", (*i)->GetObjectID());
+ (*i)->AddFlags(Entry::Flags_OldVersion);
+ changed = true;
+ }
+ }
+ else
+ {
+ // Check old version flag NOT set
+ if(((*i)->GetFlags() & Entry::Flags_OldVersion) == Entry::Flags_OldVersion)
+ {
+ // Set, unset it
+ TRACE1("Entry %llx: Old flag unset\n", (*i)->GetObjectID());
+ (*i)->RemoveFlags(Entry::Flags_OldVersion);
+ changed = true;
+ }
+
+ // Remember filename
+ filenamesEncountered.insert((*i)->GetName());
+ }
+ }
+ }
+
+ if(removeEntry)
+ {
+ // Mark something as changed, in loop
+ ch = true;
+
+ // Mark something as globally changed
+ changed = true;
+
+ // erase the thing from the list
+ Entry *pentry = (*i);
+ mEntries.erase(i);
+
+ // And delete the entry object
+ delete pentry;
+
+ // Stop going around this loop, as the iterator is now invalid
+ break;
+ }
+ } while(i != mEntries.begin());
+
+ } while(ch != false);
+
+ return changed;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddUnattactedObject(...)
+// Purpose: Adds an object which is currently unattached. Assume that CheckAndFix() will be called afterwards.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::AddUnattactedObject(const BackupStoreFilename &rName,
+ box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags)
+{
+ Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags,
+ ModificationTime /* use as attr mod time too */);
+ try
+ {
+ // Want to order this just before the first object which has a higher ID,
+ // which is the place it's most likely to be correct.
+ std::vector<Entry*>::iterator i(mEntries.begin());
+ for(; i != mEntries.end(); ++i)
+ {
+ if((*i)->GetObjectID() > ObjectID)
+ {
+ // Found a good place to insert it
+ break;
+ }
+ }
+ if(i == mEntries.end())
+ {
+ mEntries.push_back(pnew);
+ }
+ else
+ {
+ mEntries.insert(i, 1 /* just the one copy */, pnew);
+ }
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::NameInUse(const BackupStoreFilename &)
+// Purpose: Returns true if the name is currently in use in the directory
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDirectory::NameInUse(const BackupStoreFilename &rName)
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+ {
+ if((*i)->GetName() == rName)
+ {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+
diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp
new file mode 100644
index 00000000..53a33d47
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheckData.cpp
@@ -0,0 +1,243 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheckData.cpp
+// Purpose: Data handling for store checking
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <memory>
+
+#include "BackupStoreCheck.h"
+#include "autogen_BackupStoreException.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::FreeInfo()
+// Purpose: Free all the data stored
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FreeInfo()
+{
+ // Free all the blocks
+ for(Info_t::iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ ::free(i->second);
+ }
+
+ // Clear the contents of the map
+ mInfo.clear();
+
+ // Reset the last ID, just in case
+ mpInfoLastBlock = 0;
+ mInfoLastBlockEntries = 0;
+ mLastIDInInfo = 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::AddID(BackupStoreCheck_ID_t, BackupStoreCheck_ID_t, bool)
+// Purpose: Add an ID to the list
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::AddID(BackupStoreCheck_ID_t ID,
+ BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile)
+{
+ // Check ID is OK.
+ if(ID <= mLastIDInInfo)
+ {
+ THROW_EXCEPTION(BackupStoreException, InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing)
+ }
+
+ // Can this go in the current block?
+ if(mpInfoLastBlock == 0 || mInfoLastBlockEntries >= BACKUPSTORECHECK_BLOCK_SIZE)
+ {
+ // No. Allocate a new one
+ IDBlock *pblk = (IDBlock*)::malloc(sizeof(IDBlock));
+ if(pblk == 0)
+ {
+ throw std::bad_alloc();
+ }
+ // Zero all the flags entries
+ for(int z = 0; z < (BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry); ++z)
+ {
+ pblk->mFlags[z] = 0;
+ }
+ // Store in map
+ mInfo[ID] = pblk;
+ // Allocated and stored OK, setup for use
+ mpInfoLastBlock = pblk;
+ mInfoLastBlockEntries = 0;
+ }
+ ASSERT(mpInfoLastBlock != 0 && mInfoLastBlockEntries < BACKUPSTORECHECK_BLOCK_SIZE);
+
+ // Add to block
+ mpInfoLastBlock->mID[mInfoLastBlockEntries] = ID;
+ mpInfoLastBlock->mContainer[mInfoLastBlockEntries] = Container;
+ mpInfoLastBlock->mObjectSizeInBlocks[mInfoLastBlockEntries] = ObjectSize;
+ SetFlags(mpInfoLastBlock, mInfoLastBlockEntries, IsFile?(0):(Flags_IsDir));
+
+ // Increment size
+ ++mInfoLastBlockEntries;
+
+ // Store last ID
+ mLastIDInInfo = ID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::LookupID(BackupStoreCheck_ID_t, int32_t
+// Purpose: Look up an ID. Return the block it's in, or zero if not found, and the
+// index within that block if the thing is found.
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut)
+{
+ IDBlock *pblock = 0;
+
+ // Find the lower matching block who's first entry is not less than ID
+ Info_t::const_iterator ib(mInfo.lower_bound(ID));
+
+ // Was there a block
+ if(ib == mInfo.end())
+ {
+ // Block wasn't found... could be in last block
+ pblock = mpInfoLastBlock;
+ }
+ else
+ {
+ // Found it as first entry?
+ if(ib->first == ID)
+ {
+ rIndexOut = 0;
+ return ib->second;
+ }
+
+ // Go back one block as it's not the first entry in this one
+ if(ib == mInfo.begin())
+ {
+ // Was first block, can't go back
+ return 0;
+ }
+ // Go back...
+ --ib;
+
+ // So, the ID will be in this block, if it's in anything
+ pblock = ib->second;
+ }
+
+ ASSERT(pblock != 0);
+ if(pblock == 0) return 0;
+
+ // How many entries are there in the block
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+
+ // Do binary search within block
+ int high = bentries;
+ int low = -1;
+ while(high - low > 1)
+ {
+ int i = (high + low) / 2;
+ if(ID <= pblock->mID[i])
+ {
+ high = i;
+ }
+ else
+ {
+ low = i;
+ }
+ }
+ if(ID == pblock->mID[high])
+ {
+ // Found
+ rIndexOut = high;
+ return pblock;
+ }
+
+ // Not found
+ return 0;
+}
+
+
+#ifndef NDEBUG
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::DumpObjectInfo()
+// Purpose: Debug only. Trace out all object info.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::DumpObjectInfo()
+{
+ for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ IDBlock *pblock = i->second;
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+ TRACE2("BLOCK @ 0x%08x, %d entries\n", pblock, bentries);
+
+ for(int e = 0; e < bentries; ++e)
+ {
+ uint8_t flags = GetFlags(pblock, e);
+ TRACE4("id %llx, c %llx, %s, %s\n",
+ pblock->mID[e], pblock->mContainer[e],
+ (flags & Flags_IsDir)?"dir":"file",
+ (flags & Flags_IsContained)?"contained":"unattached");
+ }
+ }
+}
+#endif
+
diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp
new file mode 100644
index 00000000..5ecdcbb5
--- /dev/null
+++ b/lib/backupstore/BackupStoreConfigVerify.cpp
@@ -0,0 +1,86 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreConfigVerify.h
+// Purpose: Configuration definition for the backup store server
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreConfigVerify.h"
+#include "ServerTLS.h"
+#include "BoxPortsAndFiles.h"
+
+#include "MemLeakFindOn.h"
+
+static const ConfigurationVerifyKey verifyserverkeys[] =
+{
+ SERVERTLS_VERIFY_SERVER_KEYS(0) // no default listen addresses
+};
+
+static const ConfigurationVerify verifyserver[] =
+{
+ {
+ "Server",
+ 0,
+ verifyserverkeys,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+ }
+};
+
+static const ConfigurationVerifyKey verifyrootkeys[] =
+{
+ {"AccountDatabase", 0, ConfigTest_Exists, 0},
+ {"TimeBetweenHousekeeping", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file
+ {"RaidFileConf", BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 0}
+};
+
+const ConfigurationVerify BackupConfigFileVerify =
+{
+ "root",
+ verifyserver,
+ verifyrootkeys,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+};
diff --git a/lib/backupstore/BackupStoreConfigVerify.h b/lib/backupstore/BackupStoreConfigVerify.h
new file mode 100644
index 00000000..fee8ae42
--- /dev/null
+++ b/lib/backupstore/BackupStoreConfigVerify.h
@@ -0,0 +1,56 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreConfigVerify.h
+// Purpose: Configuration definition for the backup store server
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECONFIGVERIFY__H
+#define BACKUPSTORECONFIGVERIFY__H
+
+#include "Configuration.h"
+
+extern const ConfigurationVerify BackupConfigFileVerify;
+
+#endif // BACKUPSTORECONFIGVERIFY__H
+
diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp
new file mode 100644
index 00000000..28508b60
--- /dev/null
+++ b/lib/backupstore/BackupStoreInfo.cpp
@@ -0,0 +1,630 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreInfo.cpp
+// Purpose: Main backup store information storage
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <algorithm>
+
+#include "BackupStoreInfo.h"
+#include "BackupStoreException.h"
+#include "RaidFileWrite.h"
+#include "RaidFileRead.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// ******************
+// make sure the defaults in CreateNew are modified!
+// ******************
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int32_t mAccountID;
+ int64_t mClientStoreMarker;
+ int64_t mLastObjectIDUsed;
+ int64_t mBlocksUsed;
+ int64_t mBlocksInOldFiles;
+ int64_t mBlocksInDeletedFiles;
+ int64_t mBlocksInDirectories;
+ int64_t mBlocksSoftLimit;
+ int64_t mBlocksHardLimit;
+ uint32_t mCurrentMarkNumber;
+ uint32_t mOptionsPresent; // bit mask of optional elements present
+ int64_t mNumberDeletedDirectories;
+ // Then loads of int64_t IDs for the deleted directories
+} info_StreamFormat;
+
+#define INFO_MAGIC_VALUE 0x34832476
+
+// Use default packing
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+#ifdef NDEBUG
+ #define NUM_DELETED_DIRS_BLOCK 256
+#else
+ #define NUM_DELETED_DIRS_BLOCK 2
+#endif
+
+#define INFO_FILENAME "info"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::BackupStoreInfo()
+// Purpose: Default constructor
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreInfo::BackupStoreInfo()
+ : mAccountID(-1),
+ mDiscSet(-1),
+ mReadOnly(true),
+ mIsModified(false),
+ mClientStoreMarker(0),
+ mLastObjectIDUsed(-1),
+ mBlocksUsed(0),
+ mBlocksInOldFiles(0),
+ mBlocksInDeletedFiles(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::~BackupStoreInfo
+// Purpose: Destructor
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreInfo::~BackupStoreInfo()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::CreateNew(int32_t, const std::string &, int)
+// Purpose: Create a new info file on disc
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+ // Initial header (is entire file)
+ info_StreamFormat hdr = {
+ htonl(INFO_MAGIC_VALUE), // mMagicValue
+ htonl(AccountID), // mAccountID
+ 0, // mClientStoreMarker
+ box_hton64(1), // mLastObjectIDUsed (which is the root directory)
+ 0, // mBlocksUsed
+ 0, // mBlocksInOldFiles
+ 0, // mBlocksInDeletedFiles
+ 0, // mBlocksInDirectories
+ box_hton64(BlockSoftLimit), // mBlocksSoftLimit
+ box_hton64(BlockHardLimit), // mBlocksHardLimit
+ 0, // mCurrentMarkNumber
+ 0, // mOptionsPresent
+ 0 // mNumberDeletedDirectories
+ };
+
+ // Generate the filename
+ ASSERT(rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR);
+ std::string fn(rRootDir + INFO_FILENAME);
+
+ // Open the file for writing
+ RaidFileWrite rf(DiscSet, fn);
+ rf.Open(false); // no overwriting, as this is a new file
+
+ // Write header
+ rf.Write(&hdr, sizeof(hdr));
+
+ // Commit it to disc, converting it to RAID now
+ rf.Commit(true);
+
+ // Done.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::Load(int32_t, const std::string &, int, bool)
+// Purpose: Loads the info from disc, given the root information. Can be marked as read only.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID)
+{
+ // Generate the filename
+ std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME);
+
+ // Open the file for reading (passing on optional request for revision ID)
+ std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID));
+
+ // Read in a header
+ info_StreamFormat hdr;
+ if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+ }
+
+ // Check it
+ if(ntohl(hdr.mMagicValue) != INFO_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != AccountID)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+ }
+
+ // Make new object
+ std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo);
+
+ // Put in basic location info
+ info->mAccountID = AccountID;
+ info->mDiscSet = DiscSet;
+ info->mFilename = fn;
+ info->mReadOnly = ReadOnly;
+
+ // Insert info from file
+ info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker);
+ info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed);
+ info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed);
+ info->mBlocksInOldFiles = box_ntoh64(hdr.mBlocksInOldFiles);
+ info->mBlocksInDeletedFiles = box_ntoh64(hdr.mBlocksInDeletedFiles);
+ info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories);
+ info->mBlocksSoftLimit = box_ntoh64(hdr.mBlocksSoftLimit);
+ info->mBlocksHardLimit = box_ntoh64(hdr.mBlocksHardLimit);
+
+ // Load up array of deleted objects
+ int64_t numDelObj = box_ntoh64(hdr.mNumberDeletedDirectories);
+
+ // Then load them in
+ if(numDelObj > 0)
+ {
+ int64_t objs[NUM_DELETED_DIRS_BLOCK];
+
+ int64_t toload = numDelObj;
+ while(toload > 0)
+ {
+ // How many in this one?
+ int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload));
+
+ if(!rf->ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+ }
+
+ // Add them
+ for(int t = 0; t < b; ++t)
+ {
+ info->mDeletedDirectories.push_back(box_ntoh64(objs[t]));
+ }
+
+ // Number loaded
+ toload -= b;
+ }
+ }
+
+ // Final check
+ if(static_cast<int64_t>(info->mDeletedDirectories.size()) != numDelObj)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+ }
+
+ // return it to caller
+ return info;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::CreateForRegeneration(...)
+// Purpose: Return an object which can be used to save for regeneration.
+// Created: 23/4/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration(int32_t AccountID, const std::string &rRootDir,
+ int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles,
+ int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+ // Generate the filename
+ std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME);
+
+ // Make new object
+ std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo);
+
+ // Put in basic info
+ info->mAccountID = AccountID;
+ info->mDiscSet = DiscSet;
+ info->mFilename = fn;
+ info->mReadOnly = false;
+
+ // Insert info starting info
+ info->mClientStoreMarker = 0;
+ info->mLastObjectIDUsed = LastObjectID;
+ info->mBlocksUsed = BlocksUsed;
+ info->mBlocksInOldFiles = BlocksInOldFiles;
+ info->mBlocksInDeletedFiles = BlocksInDeletedFiles;
+ info->mBlocksInDirectories = BlocksInDirectories;
+ info->mBlocksSoftLimit = BlockSoftLimit;
+ info->mBlocksHardLimit = BlockHardLimit;
+
+ // return it to caller
+ return info;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::Save()
+// Purpose: Save modified info back to disc
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::Save()
+{
+ // Make sure we're initialised (although should never come to this)
+ if(mFilename.empty() || mAccountID == -1 || mDiscSet == -1)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Can we do this?
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ // Then... open a write file
+ RaidFileWrite rf(mDiscSet, mFilename);
+ rf.Open(true); // allow overwriting
+
+ // Make header
+ info_StreamFormat hdr;
+ hdr.mMagicValue = htonl(INFO_MAGIC_VALUE);
+ hdr.mAccountID = htonl(mAccountID);
+ hdr.mClientStoreMarker = box_hton64(mClientStoreMarker);
+ hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed);
+ hdr.mBlocksUsed = box_hton64(mBlocksUsed);
+ hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles);
+ hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles);
+ hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories);
+ hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit);
+ hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit);
+ hdr.mCurrentMarkNumber = 0;
+ hdr.mOptionsPresent = 0;
+ hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size());
+
+ // Write header
+ rf.Write(&hdr, sizeof(hdr));
+
+ // Write the deleted object list
+ if(mDeletedDirectories.size() > 0)
+ {
+ int64_t objs[NUM_DELETED_DIRS_BLOCK];
+
+ int tosave = mDeletedDirectories.size();
+ std::vector<int64_t>::iterator i(mDeletedDirectories.begin());
+ while(tosave > 0)
+ {
+ // How many in this one?
+ int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave));
+
+ // Add them
+ for(int t = 0; t < b; ++t)
+ {
+ ASSERT(i != mDeletedDirectories.end());
+ objs[t] = box_hton64((*i));
+ i++;
+ }
+
+ // Write
+ rf.Write(objs, b * sizeof(int64_t));
+
+ // Number saved
+ tosave -= b;
+ }
+ }
+
+ // Commit it to disc, converting it to RAID now
+ rf.Commit(true);
+
+ // Mark is as not modified
+ mIsModified = false;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksUsed(int32_t)
+// Purpose: Change number of blocks used, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksUsed + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksUsed += Delta;
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksInOldFiles(int32_t)
+// Purpose: Change number of blocks in old files, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksInOldFiles + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksInOldFiles += Delta;
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksInDeletedFiles(int32_t)
+// Purpose: Change number of blocks in deleted files, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksInDeletedFiles + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksInDeletedFiles += Delta;
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksInDirectories(int32_t)
+// Purpose: Change number of blocks in directories, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksInDirectories + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksInDirectories += Delta;
+
+ mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::CorrectAllUsedValues(int64_t, int64_t, int64_t, int64_t)
+// Purpose: Set all the usage counts to specific values -- use for correcting in housekeeping
+// if something when wrong during the backup connection, and the store info wasn't
+// saved back to disc.
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ // Set the values
+ mBlocksUsed = Used;
+ mBlocksInOldFiles = InOldFiles;
+ mBlocksInDeletedFiles = InDeletedFiles;
+ mBlocksInDirectories = InDirectories;
+
+ mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::AddDeletedDirectory(int64_t)
+// Purpose: Add a directory ID to the deleted list
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::AddDeletedDirectory(int64_t DirID)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ mDeletedDirectories.push_back(DirID);
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::RemovedDeletedDirectory(int64_t)
+// Purpose: Remove a directory from the deleted list
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::RemovedDeletedDirectory(int64_t DirID)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ std::vector<int64_t>::iterator i(std::find(mDeletedDirectories.begin(), mDeletedDirectories.end(), DirID));
+ if(i == mDeletedDirectories.end())
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoDirNotInList)
+ }
+ mDeletedDirectories.erase(i);
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeLimits(int64_t, int64_t)
+// Purpose: Change the soft and hard limits
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ mBlocksSoftLimit = BlockSoftLimit;
+ mBlocksHardLimit = BlockHardLimit;
+
+ mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::AllocateObjectID()
+// Purpose: Allocate an ID for a new object in the store.
+// Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreInfo::AllocateObjectID()
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if(mLastObjectIDUsed < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotInitialised)
+ }
+
+ // Return the next object ID
+ return ++mLastObjectIDUsed;
+
+ mIsModified = true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::SetClientStoreMarker(int64_t)
+// Purpose: Sets the client store marker
+// Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ mClientStoreMarker = ClientStoreMarker;
+
+ mIsModified = true;
+}
+
+
+
diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h
new file mode 100644
index 00000000..0c4241d7
--- /dev/null
+++ b/lib/backupstore/BackupStoreInfo.h
@@ -0,0 +1,149 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreInfo.h
+// Purpose: Main backup store information storage
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREINFO__H
+#define BACKUPSTOREINFO__H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+class BackupStoreCheck;
+
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreInfo
+// Purpose: Main backup store information storage
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class BackupStoreInfo
+{
+ friend class BackupStoreCheck;
+public:
+ ~BackupStoreInfo();
+private:
+ // Creation through static functions only
+ BackupStoreInfo();
+ // No copying allowed
+ BackupStoreInfo(const BackupStoreInfo &);
+
+public:
+ // Create a New account, saving a blank info object to the disc
+ static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit);
+
+ // Load it from the store
+ static std::auto_ptr<BackupStoreInfo> Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0);
+
+ // Has info been modified?
+ bool IsModified() const {return mIsModified;}
+
+ // Save modified infomation back to store
+ void Save();
+
+ // Data access functions
+ int32_t GetAccountID() const {return mAccountID;}
+ int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;}
+ int64_t GetBlocksUsed() const {return mBlocksUsed;}
+ int64_t GetBlocksInOldFiles() const {return mBlocksInOldFiles;}
+ int64_t GetBlocksInDeletedFiles() const {return mBlocksInDeletedFiles;}
+ int64_t GetBlocksInDirectories() const {return mBlocksInDirectories;}
+ const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;}
+ int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;}
+ int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;}
+ bool IsReadOnly() const {return mReadOnly;}
+ int GetDiscSetNumber() const {return mDiscSet;}
+
+ // Data modification functions
+ void ChangeBlocksUsed(int64_t Delta);
+ void ChangeBlocksInOldFiles(int64_t Delta);
+ void ChangeBlocksInDeletedFiles(int64_t Delta);
+ void ChangeBlocksInDirectories(int64_t Delta);
+ void CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories);
+ void AddDeletedDirectory(int64_t DirID);
+ void RemovedDeletedDirectory(int64_t DirID);
+ void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit);
+
+ // Object IDs
+ int64_t AllocateObjectID();
+
+ // Client marker set and get
+ int64_t GetClientStoreMarker() {return mClientStoreMarker;}
+ void SetClientStoreMarker(int64_t ClientStoreMarker);
+
+private:
+ static std::auto_ptr<BackupStoreInfo> CreateForRegeneration(int32_t AccountID, const std::string &rRootDir,
+ int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles,
+ int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit);
+
+private:
+ // Location information
+ int32_t mAccountID;
+ int mDiscSet;
+ std::string mFilename;
+ bool mReadOnly;
+ bool mIsModified;
+
+ // Client infomation
+ int64_t mClientStoreMarker;
+
+ // Account information
+ int64_t mLastObjectIDUsed;
+ int64_t mBlocksUsed;
+ int64_t mBlocksInOldFiles;
+ int64_t mBlocksInDeletedFiles;
+ int64_t mBlocksInDirectories;
+ int64_t mBlocksSoftLimit;
+ int64_t mBlocksHardLimit;
+ std::vector<int64_t> mDeletedDirectories;
+};
+
+
+#endif // BACKUPSTOREINFO__H
+
+
diff --git a/lib/backupstore/StoreStructure.cpp b/lib/backupstore/StoreStructure.cpp
new file mode 100644
index 00000000..68876fd6
--- /dev/null
+++ b/lib/backupstore/StoreStructure.cpp
@@ -0,0 +1,133 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: StoreStructure.cpp
+// Purpose:
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "RaidFileController.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: StoreStructure::MakeObjectFilename(int64_t, const std::string &, int, std::string &, bool)
+// Purpose: Builds the object filename for a given object, given a root. Optionally ensure that the
+// directory exists.
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void StoreStructure::MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists)
+{
+ const static char *hex = "0123456789abcdef";
+
+ // Set output to root string
+ rFilenameOut = rStoreRoot;
+
+ // get the id value from the stored object ID so we can do
+ // bitwise operations on it.
+ uint64_t id = (uint64_t)ObjectID;
+
+ // get leafname, shift the bits which make up the leafname off
+ unsigned int leafname(id & STORE_ID_SEGMENT_MASK);
+ id >>= STORE_ID_SEGMENT_LENGTH;
+
+ // build pathname
+ while(id != 0)
+ {
+ // assumes that the segments are no bigger than 8 bits
+ int v = id & STORE_ID_SEGMENT_MASK;
+ rFilenameOut += hex[(v & 0xf0) >> 4];
+ rFilenameOut += hex[v & 0xf];
+ rFilenameOut += DIRECTORY_SEPARATOR_ASCHAR;
+
+ // shift the bits we used off the pathname
+ id >>= STORE_ID_SEGMENT_LENGTH;
+ }
+
+ // Want to make sure this exists?
+ if(EnsureDirectoryExists)
+ {
+ if(!RaidFileRead::DirectoryExists(DiscSet, rFilenameOut))
+ {
+ // Create it
+ RaidFileWrite::CreateDirectory(DiscSet, rFilenameOut, true /* recusive */);
+ }
+ }
+
+ // append the filename
+ rFilenameOut += 'o';
+ rFilenameOut += hex[(leafname & 0xf0) >> 4];
+ rFilenameOut += hex[leafname & 0xf];
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: StoreStructure::MakeWriteLockFilename(const std::string &, int, std::string &)
+// Purpose: Generate the on disc filename of the write lock file
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void StoreStructure::MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut)
+{
+ // Find the disc set
+ RaidFileController &rcontroller(RaidFileController::GetController());
+ RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(DiscSet));
+
+ // Make the filename
+ std::string writeLockFile(rdiscSet[0] + DIRECTORY_SEPARATOR + rStoreRoot + "write.lock");
+
+ // Return it to the caller
+ rFilenameOut = writeLockFile;
+}
+
+
diff --git a/lib/backupstore/StoreStructure.h b/lib/backupstore/StoreStructure.h
new file mode 100644
index 00000000..8c1e871c
--- /dev/null
+++ b/lib/backupstore/StoreStructure.h
@@ -0,0 +1,70 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: StoreStructure.h
+// Purpose: Functions for placing files in the store
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef STORESTRUCTURE__H
+#define STORESTRUCTURE__H
+
+#include <string>
+
+#ifdef NDEBUG
+ #define STORE_ID_SEGMENT_LENGTH 8
+ #define STORE_ID_SEGMENT_MASK 0xff
+#else
+ // Debug we'll use lots and lots of directories to stress things
+ #define STORE_ID_SEGMENT_LENGTH 2
+ #define STORE_ID_SEGMENT_MASK 0x03
+#endif
+
+
+namespace StoreStructure
+{
+ void MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists);
+ void MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut);
+};
+
+#endif // STORESTRUCTURE__H
+
diff --git a/lib/common/Archive.h b/lib/common/Archive.h
new file mode 100644
index 00000000..d8e93238
--- /dev/null
+++ b/lib/common/Archive.h
@@ -0,0 +1,199 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: Archive.h
+// Purpose: Backup daemon state archive
+// Created: 2005/04/11
+//
+// --------------------------------------------------------------------------
+
+#ifndef ARCHIVE__H
+#define ARCHIVE__H
+
+#include <vector>
+#include <string>
+#include <memory>
+
+#include "IOStream.h"
+#include "Guards.h"
+
+#define ARCHIVE_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2)
+
+#define ARCHIVE_MAGIC_VALUE_RECURSE 0x4449525F
+#define ARCHIVE_MAGIC_VALUE_NOOP 0x5449525F
+
+class Archive
+{
+public:
+ Archive(IOStream &Stream, int Timeout)
+ : mrStream(Stream)
+ {
+ mTimeout = Timeout;
+ }
+private:
+ // no copying
+ Archive(const Archive &);
+ Archive & operator=(const Archive &);
+public:
+ ~Archive()
+ {
+ }
+ //
+ //
+ //
+ void Write(bool Item)
+ {
+ Write((int) Item);
+ }
+ void Write(int Item)
+ {
+ int32_t privItem = htonl(Item);
+ mrStream.Write(&privItem, sizeof(privItem));
+ }
+ void Write(int64_t Item)
+ {
+ int64_t privItem = box_hton64(Item);
+ mrStream.Write(&privItem, sizeof(privItem));
+ }
+ void Write(uint64_t Item)
+ {
+ uint64_t privItem = box_hton64(Item);
+ mrStream.Write(&privItem, sizeof(privItem));
+ }
+ void Write(uint8_t Item)
+ {
+ int privItem = Item;
+ Write(privItem);
+ }
+ void Write(const std::string &Item)
+ {
+ int size = Item.size();
+ Write(size);
+ mrStream.Write(Item.c_str(), size);
+ }
+ //
+ //
+ //
+ void Read(bool &rItemOut)
+ {
+ int privItem;
+ Read(privItem);
+
+ if (privItem)
+ {
+ rItemOut = true;
+ }
+ else
+ {
+ rItemOut = false;
+ }
+ }
+ void Read(int &rItemOut)
+ {
+ int32_t privItem;
+ if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead)
+ }
+ rItemOut = ntohl(privItem);
+ }
+ void Read(int64_t &rItemOut)
+ {
+ int64_t privItem;
+ if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead)
+ }
+ rItemOut = box_ntoh64(privItem);
+ }
+ void Read(uint64_t &rItemOut)
+ {
+ uint64_t privItem;
+ if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead)
+ }
+ rItemOut = box_ntoh64(privItem);
+ }
+ void Read(uint8_t &rItemOut)
+ {
+ int privItem;
+ Read(privItem);
+ rItemOut = privItem;
+ }
+ void Read(std::string &rItemOut)
+ {
+ int size;
+ Read(size);
+
+ // Assume most strings are relatively small
+ char buf[256];
+ if(size < (int) sizeof(buf))
+ {
+ // Fetch rest of pPayload, relying on the Protocol to error on stupidly large sizes for us
+ if(!mrStream.ReadFullBuffer(buf, size, 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead)
+ }
+ // assign to this string, storing the header and the extra payload
+ rItemOut.assign(buf, size);
+ }
+ else
+ {
+ // Block of memory to hold it
+ MemoryBlockGuard<char*> dataB(size);
+ char *ppayload = dataB;
+
+ // Fetch rest of pPayload, relying on the Protocol to error on stupidly large sizes for us
+ if(!mrStream.ReadFullBuffer(ppayload, size, 0 /* not interested in bytes read if this fails */, mTimeout))
+ {
+ THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead)
+ }
+ // assign to this string, storing the header and the extra pPayload
+ rItemOut.assign(ppayload, size);
+ }
+ }
+private:
+ IOStream &mrStream;
+ int mTimeout;
+};
+
+#endif // ARCHIVE__H
diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h
new file mode 100644
index 00000000..61577e43
--- /dev/null
+++ b/lib/common/BannerText.h
@@ -0,0 +1,55 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BannerText.h
+// Purpose: Banner text for daemons and utilities
+// Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BANNERTEXT__H
+#define BANNERTEXT__H
+
+#define BANNER_TEXT(UtilityName) \
+ "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers and contributors 2003-2006\n"
+
+#endif // BANNERTEXT__H
+
diff --git a/lib/common/BeginStructPackForWire.h b/lib/common/BeginStructPackForWire.h
new file mode 100644
index 00000000..c7e3a069
--- /dev/null
+++ b/lib/common/BeginStructPackForWire.h
@@ -0,0 +1,61 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BeginStructPackForWire.h
+// Purpose: Begin structure packing for wire
+// Created: 25/11/03
+//
+// --------------------------------------------------------------------------
+
+// No header guard -- this is intentional
+
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+
+#pragma pack(1)
+
+#else
+
+ logical error -- check BoxPlatform.h and including file
+
+#endif
+
+
+
diff --git a/lib/common/Box.h b/lib/common/Box.h
new file mode 100644
index 00000000..14a4a6ac
--- /dev/null
+++ b/lib/common/Box.h
@@ -0,0 +1,191 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: Box.h
+// Purpose: Main header file for the Box project
+// Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOX__H
+#define BOX__H
+
+// Use the same changes as gcc3 for gcc4
+#ifdef PLATFORM_GCC4
+ #define PLATFORM_GCC3
+#endif
+
+#include "BoxPlatform.h"
+
+// uncomment this line to enable full memory leak finding on all malloc-ed blocks (at least, ones used by the STL)
+//#define MEMLEAKFINDER_FULL_MALLOC_MONITORING
+
+#ifndef NDEBUG
+ #ifdef HAVE_EXECINFO_H
+ #define SHOW_BACKTRACE_ON_EXCEPTION
+ #endif
+#endif
+
+#ifdef SHOW_BACKTRACE_ON_EXCEPTION
+ #include "Utils.h"
+ #define OPTIONAL_DO_BACKTRACE DumpStackBacktrace();
+#else
+ #define OPTIONAL_DO_BACKTRACE
+#endif
+
+#include "CommonException.h"
+
+#ifndef NDEBUG
+
+ extern bool AssertFailuresToSyslog;
+ #define ASSERT_FAILS_TO_SYSLOG_ON {AssertFailuresToSyslog = true;}
+ void BoxDebugAssertFailed(char *cond, char *file, int line);
+ #define ASSERT(cond) {if(!(cond)) {BoxDebugAssertFailed(#cond, __FILE__, __LINE__); THROW_EXCEPTION(CommonException, AssertFailed)}}
+
+ // Note that syslog tracing is independent of BoxDebugTraceOn, but stdout tracing is not
+ extern bool BoxDebugTraceToSyslog;
+ #define TRACE_TO_SYSLOG(x) {BoxDebugTraceToSyslog = x;}
+ extern bool BoxDebugTraceToStdout;
+ #define TRACE_TO_STDOUT(x) {BoxDebugTraceToStdout = x;}
+
+ extern bool BoxDebugTraceOn;
+ int BoxDebug_printf(const char *format, ...);
+ int BoxDebugTrace(const char *format, ...);
+ #define TRACE0(msg) {BoxDebugTrace("%s", msg);}
+ #define TRACE1(msg, a0) {BoxDebugTrace(msg, a0);}
+ #define TRACE2(msg, a0, a1) {BoxDebugTrace(msg, a0, a1);}
+ #define TRACE3(msg, a0, a1, a2) {BoxDebugTrace(msg, a0, a1, a2);}
+ #define TRACE4(msg, a0, a1, a2, a3) {BoxDebugTrace(msg, a0, a1, a2, a3);}
+ #define TRACE5(msg, a0, a1, a2, a3, a4) {BoxDebugTrace(msg, a0, a1, a2, a3, a4);}
+ #define TRACE6(msg, a0, a1, a2, a3, a4, a5) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5);}
+ #define TRACE7(msg, a0, a1, a2, a3, a4, a5, a6) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6);}
+ #define TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6, a7);}
+
+ #ifndef PLATFORM_DISABLE_MEM_LEAK_TESTING
+ #define BOX_MEMORY_LEAK_TESTING
+ #endif
+
+ // Exception names
+ #define EXCEPTION_CODENAMES_EXTENDED
+
+#else
+ #define ASSERT_FAILS_TO_SYSLOG_ON
+ #define ASSERT(cond)
+
+ #define TRACE_TO_SYSLOG(x) {}
+ #define TRACE_TO_STDOUT(x) {}
+
+ #define TRACE0(msg)
+ #define TRACE1(msg, a0)
+ #define TRACE2(msg, a0, a1)
+ #define TRACE3(msg, a0, a1, a2)
+ #define TRACE4(msg, a0, a1, a2, a3)
+ #define TRACE5(msg, a0, a1, a2, a3, a4)
+ #define TRACE6(msg, a0, a1, a2, a3, a4, a5)
+ #define TRACE7(msg, a0, a1, a2, a3, a4, a5, a6)
+ #define TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7)
+
+ // Box Backup builds release get extra information for exception logging
+ #define EXCEPTION_CODENAMES_EXTENDED
+ #define EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION
+
+#endif
+
+#ifdef BOX_MEMORY_LEAK_TESTING
+ // Memory leak testing
+ #include "MemLeakFinder.h"
+ #define MEMLEAKFINDER_NOT_A_LEAK(x) memleakfinder_notaleak(x);
+ #define MEMLEAKFINDER_START {memleakfinder_global_enable = true;}
+ #define MEMLEAKFINDER_STOP {memleakfinder_global_enable = false;}
+#else
+ #define DEBUG_NEW new
+ #define MEMLEAKFINDER_NOT_A_LEAK(x)
+ #define MEMLEAKFINDER_START
+ #define MEMLEAKFINDER_STOP
+#endif
+
+
+#define THROW_EXCEPTION(type, subtype) \
+ { \
+ OPTIONAL_DO_BACKTRACE \
+ TRACE1("Exception thrown: " #type "(" #subtype ") at " __FILE__ "(%d)\n", __LINE__) \
+ throw type(type::subtype); \
+ }
+
+// extra macros for converting to network byte order
+
+#ifdef HAVE_NETINET_IN_H
+ #include <netinet/in.h>
+#endif
+
+// Always define a swap64 function, as it's useful.
+inline uint64_t box_swap64(uint64_t x)
+{
+ return ((x & 0xff) << 56 |
+ (x & 0xff00LL) << 40 |
+ (x & 0xff0000LL) << 24 |
+ (x & 0xff000000LL) << 8 |
+ (x & 0xff00000000LL) >> 8 |
+ (x & 0xff0000000000LL) >> 24 |
+ (x & 0xff000000000000LL) >> 40 |
+ (x & 0xff00000000000000LL) >> 56);
+}
+
+#ifdef WORDS_BIGENDIAN
+ #define box_hton64(x) (x)
+ #define box_ntoh64(x) (x)
+#elif defined(HAVE_BSWAP64)
+ #ifdef HAVE_SYS_ENDIAN_H
+ #include <sys/endian.h>
+ #endif
+ #ifdef HAVE_ASM_BYTEORDER_H
+ #include <asm/byteorder.h>
+ #endif
+
+ #define box_hton64(x) BSWAP64(x)
+ #define box_ntoh64(x) BSWAP64(x)
+#else
+ #define box_hton64(x) box_swap64(x)
+ #define box_ntoh64(x) box_swap64(x)
+#endif
+
+#endif // BOX__H
+
diff --git a/lib/common/BoxConfig.h.in b/lib/common/BoxConfig.h.in
new file mode 100644
index 00000000..5e84c065
--- /dev/null
+++ b/lib/common/BoxConfig.h.in
@@ -0,0 +1,397 @@
+/* lib/common/BoxConfig.h.in. Generated from configure.ac by autoheader. */
+
+/* Define to major version for BDB_VERSION */
+#undef BDB_VERSION_MAJOR
+
+/* Define to minor version for BDB_VERSION */
+#undef BDB_VERSION_MINOR
+
+/* Define to point version for BDB_VERSION */
+#undef BDB_VERSION_POINT
+
+/* Name of the 64 bit endian swapping function */
+#undef BSWAP64
+
+/* Define to 1 if the `closedir' function returns void instead of `int'. */
+#undef CLOSEDIR_VOID
+
+/* Define to 1 if non-aligned int16 access will fail */
+#undef HAVE_ALIGNED_ONLY_INT16
+
+/* Define to 1 if non-aligned int32 access will fail */
+#undef HAVE_ALIGNED_ONLY_INT32
+
+/* Define to 1 if non-aligned int64 access will fail */
+#undef HAVE_ALIGNED_ONLY_INT64
+
+/* Define to 1 if you have the <asm/byteorder.h> header file. */
+#undef HAVE_ASM_BYTEORDER_H
+
+/* Define to 1 if BSWAP64 is defined to the name of a valid 64 bit endian
+ swapping function */
+#undef HAVE_BSWAP64
+
+/* Define to 1 if you have the <db.h> header file. */
+#undef HAVE_DB_H
+
+/* Define to 1 if you have the declaration of `F_SETLK', and to 0 if you
+ don't. */
+#undef HAVE_DECL_F_SETLK
+
+/* Define to 1 if you have the declaration of `INFTIM', and to 0 if you don't.
+ */
+#undef HAVE_DECL_INFTIM
+
+/* Define to 1 if you have the declaration of `O_EXLOCK', and to 0 if you
+ don't. */
+#undef HAVE_DECL_O_EXLOCK
+
+/* Define to 1 if you have the declaration of `SO_PEERCRED', and to 0 if you
+ don't. */
+#undef HAVE_DECL_SO_PEERCRED
+
+/* Define to 1 if you have the declaration of `XATTR_NOFOLLOW', and to 0 if
+ you don't. */
+#undef HAVE_DECL_XATTR_NOFOLLOW
+
+/* Define to 1 if #define of pragmas works */
+#undef HAVE_DEFINE_PRAGMA
+
+/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'.
+ */
+#undef HAVE_DIRENT_H
+
+/* Define to 1 if you have the <editline/readline.h> header file. */
+#undef HAVE_EDITLINE_READLINE_H
+
+/* define if the compiler supports exceptions */
+#undef HAVE_EXCEPTIONS
+
+/* Define to 1 if you have the <execinfo.h> header file. */
+#undef HAVE_EXECINFO_H
+
+/* Define to 1 if you have the `flock' function. */
+#undef HAVE_FLOCK
+
+/* Define to 1 if you have the `getmntent' function. */
+#undef HAVE_GETMNTENT
+
+/* Define to 1 if you have the `getpeereid' function. */
+#undef HAVE_GETPEEREID
+
+/* Define to 1 if you have the `getpid' function. */
+#undef HAVE_GETPID
+
+/* Define to 1 if you have the `getxattr' function. */
+#undef HAVE_GETXATTR
+
+/* Define to 1 if you have the <history.h> header file. */
+#undef HAVE_HISTORY_H
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define to 1 if you have the `kqueue' function. */
+#undef HAVE_KQUEUE
+
+/* Define to 1 if you have the `lchown' function. */
+#undef HAVE_LCHOWN
+
+/* Define to 1 if you have the `lgetxattr' function. */
+#undef HAVE_LGETXATTR
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#undef HAVE_LIBCRYPTO
+
+/* Define if you have a readline compatible library */
+#undef HAVE_LIBREADLINE
+
+/* Define to 1 if you have the `ssl' library (-lssl). */
+#undef HAVE_LIBSSL
+
+/* Define to 1 if you have the `z' library (-lz). */
+#undef HAVE_LIBZ
+
+/* Define to 1 if you have the `listxattr' function. */
+#undef HAVE_LISTXATTR
+
+/* Define to 1 if you have the `llistxattr' function. */
+#undef HAVE_LLISTXATTR
+
+/* Define to 1 if syscall lseek requires a dummy middle parameter */
+#undef HAVE_LSEEK_DUMMY_PARAM
+
+/* Define to 1 if you have the `lsetxattr' function. */
+#undef HAVE_LSETXATTR
+
+/* Define to 1 if you have the <memory.h> header file. */
+#undef HAVE_MEMORY_H
+
+/* Define to 1 if you have the <mntent.h> header file. */
+#undef HAVE_MNTENT_H
+
+/* Define to 1 if this platform supports mounts */
+#undef HAVE_MOUNTS
+
+/* define if the compiler implements namespaces */
+#undef HAVE_NAMESPACES
+
+/* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */
+#undef HAVE_NDIR_H
+
+/* Define to 1 if you have the <netinet/in.h> header file. */
+#undef HAVE_NETINET_IN_H
+
+/* Define to 1 if SSL is pre-0.9.7 */
+#undef HAVE_OLD_SSL
+
+/* Define to 1 if you have the <openssl/ssl.h> header file. */
+#undef HAVE_OPENSSL_SSL_H
+
+/* Define to 1 if you have the <process.h> header file. */
+#undef HAVE_PROCESS_H
+
+/* Define to 1 if you have the <pwd.h> header file. */
+#undef HAVE_PWD_H
+
+/* Define to 1 (and set RANDOM_DEVICE) if a random device is available */
+#undef HAVE_RANDOM_DEVICE
+
+/* Define to 1 if you have the <readline.h> header file. */
+#undef HAVE_READLINE_H
+
+/* Define if your readline library has add_history */
+#undef HAVE_READLINE_HISTORY
+
+/* Define to 1 if you have the <readline/history.h> header file. */
+#undef HAVE_READLINE_HISTORY_H
+
+/* Define to 1 if you have the <readline/readline.h> header file. */
+#undef HAVE_READLINE_READLINE_H
+
+/* Define to 1 if you have the <regex.h> header file. */
+#undef HAVE_REGEX_H
+
+/* Define to 1 if you have the `setproctitle' function. */
+#undef HAVE_SETPROCTITLE
+
+/* Define to 1 if you have the `setxattr' function. */
+#undef HAVE_SETXATTR
+
+/* Define to 1 if you have the <signal.h> header file. */
+#undef HAVE_SIGNAL_H
+
+/* Define to 1 if SSL is available */
+#undef HAVE_SSL
+
+/* Define to 1 if you have the `statfs' function. */
+#undef HAVE_STATFS
+
+/* Define to 1 if `stat' has the bug that it succeeds when given the
+ zero-length file name argument. */
+#undef HAVE_STAT_EMPTY_STRING_BUG
+
+/* Define to 1 if stdbool.h conforms to C99. */
+#undef HAVE_STDBOOL_H
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if `d_type' is member of `struct dirent'. */
+#undef HAVE_STRUCT_DIRENT_D_TYPE
+
+/* Define to 1 if `mnt_dir' is member of `struct mntent'. */
+#undef HAVE_STRUCT_MNTENT_MNT_DIR
+
+/* Define to 1 if `mnt_mountp' is member of `struct mnttab'. */
+#undef HAVE_STRUCT_MNTTAB_MNT_MOUNTP
+
+/* Define to 1 if `sin_len' is member of `struct sockaddr_in'. */
+#undef HAVE_STRUCT_SOCKADDR_IN_SIN_LEN
+
+/* Define to 1 if `f_mntonname' is member of `struct statfs'. */
+#undef HAVE_STRUCT_STATFS_F_MNTONNAME
+
+/* Define to 1 if `f_mntonname' is member of `struct statvfs'. */
+#undef HAVE_STRUCT_STATVFS_F_MNTONNAME
+
+/* Define to 1 if `st_flags' is member of `struct stat'. */
+#undef HAVE_STRUCT_STAT_ST_FLAGS
+
+/* Define to 1 if `st_mtimespec' is member of `struct stat'. */
+#undef HAVE_STRUCT_STAT_ST_MTIMESPEC
+
+/* Define to 1 if you have the `syscall' function. */
+#undef HAVE_SYSCALL
+
+/* Define to 1 if you have the <syslog.h> header file. */
+#undef HAVE_SYSLOG_H
+
+/* Define to 1 if you have the <sys/dir.h> header file, and it defines `DIR'.
+ */
+#undef HAVE_SYS_DIR_H
+
+/* Define to 1 if you have the <sys/endian.h> header file. */
+#undef HAVE_SYS_ENDIAN_H
+
+/* Define to 1 if you have the <sys/mnttab.h> header file. */
+#undef HAVE_SYS_MNTTAB_H
+
+/* Define to 1 if you have the <sys/mount.h> header file. */
+#undef HAVE_SYS_MOUNT_H
+
+/* Define to 1 if you have the <sys/ndir.h> header file, and it defines `DIR'.
+ */
+#undef HAVE_SYS_NDIR_H
+
+/* Define to 1 if you have the <sys/param.h> header file. */
+#undef HAVE_SYS_PARAM_H
+
+/* Define to 1 if you have the <sys/socket.h> header file. */
+#undef HAVE_SYS_SOCKET_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/syscall.h> header file. */
+#undef HAVE_SYS_SYSCALL_H
+
+/* Define to 1 if you have the <sys/time.h> header file. */
+#undef HAVE_SYS_TIME_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if you have the <sys/wait.h> header file. */
+#undef HAVE_SYS_WAIT_H
+
+/* Define to 1 if you have the <sys/xattr.h> header file. */
+#undef HAVE_SYS_XATTR_H
+
+/* Define to 1 if you have the <time.h> header file. */
+#undef HAVE_TIME_H
+
+/* Define to 1 if the system has the type `uint16_t'. */
+#undef HAVE_UINT16_T
+
+/* Define to 1 if the system has the type `uint32_t'. */
+#undef HAVE_UINT32_T
+
+/* Define to 1 if the system has the type `uint64_t'. */
+#undef HAVE_UINT64_T
+
+/* Define to 1 if the system has the type `uint8_t'. */
+#undef HAVE_UINT8_T
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* Define to 1 if the system has the type `u_int16_t'. */
+#undef HAVE_U_INT16_T
+
+/* Define to 1 if the system has the type `u_int32_t'. */
+#undef HAVE_U_INT32_T
+
+/* Define to 1 if the system has the type `u_int64_t'. */
+#undef HAVE_U_INT64_T
+
+/* Define to 1 if the system has the type `u_int8_t'. */
+#undef HAVE_U_INT8_T
+
+/* Define to 1 if struct dirent.d_type is valid */
+#undef HAVE_VALID_DIRENT_D_TYPE
+
+/* Define to 1 if the system has the type `_Bool'. */
+#undef HAVE__BOOL
+
+/* Define to 1 if you have the `__syscall' function. */
+#undef HAVE___SYSCALL
+
+/* Define to 1 if __syscall is available but needs a definition */
+#undef HAVE___SYSCALL_NEED_DEFN
+
+/* max value of long long calculated by configure */
+#undef LLONG_MAX
+
+/* min value of long long calculated by configure */
+#undef LLONG_MIN
+
+/* Define to 1 if `lstat' dereferences a symlink specified with a trailing
+ slash. */
+#undef LSTAT_FOLLOWS_SLASHED_SYMLINK
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Define to the filename of the random device (and set HAVE_RANDOM_DEVICE) */
+#undef RANDOM_DEVICE
+
+/* Define as the return type of signal handlers (`int' or `void'). */
+#undef RETSIGTYPE
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* TMP directory name */
+#undef TEMP_DIRECTORY_NAME
+
+/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
+#undef TIME_WITH_SYS_TIME
+
+/* Define to 1 if your <sys/time.h> declares `struct tm'. */
+#undef TM_IN_SYS_TIME
+
+/* Define to 1 if your processor stores words with the most significant byte
+ first (like Motorola and SPARC, unlike Intel and VAX). */
+#undef WORDS_BIGENDIAN
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+#undef _FILE_OFFSET_BITS
+
+/* Define for large files, on AIX-style hosts. */
+#undef _LARGE_FILES
+
+/* Define to 1 if __USE_MALLOC is required work around STL memory leaks */
+#undef __USE_MALLOC
+
+/* Define to empty if `const' does not conform to ANSI C. */
+#undef const
+
+/* Define to `int' if <sys/types.h> doesn't define. */
+#undef gid_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef mode_t
+
+/* Define to `long' if <sys/types.h> does not define. */
+#undef off_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef pid_t
+
+/* Define to `unsigned' if <sys/types.h> does not define. */
+#undef size_t
+
+/* Define to `int' if <sys/types.h> doesn't define. */
+#undef uid_t
diff --git a/lib/common/BoxException.cpp b/lib/common/BoxException.cpp
new file mode 100644
index 00000000..3e897f1b
--- /dev/null
+++ b/lib/common/BoxException.cpp
@@ -0,0 +1,59 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxException.cpp
+// Purpose: Exception
+// Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BoxException.h"
+
+#include "MemLeakFindOn.h"
+
+BoxException::BoxException()
+{
+}
+
+BoxException::~BoxException() throw ()
+{
+}
diff --git a/lib/common/BoxException.h b/lib/common/BoxException.h
new file mode 100644
index 00000000..00ba77d4
--- /dev/null
+++ b/lib/common/BoxException.h
@@ -0,0 +1,75 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxException.h
+// Purpose: Exception
+// Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXEXCEPTION__H
+#define BOXEXCEPTION__H
+
+#include <exception>
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BoxException
+// Purpose: Exception
+// Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+class BoxException : public std::exception
+{
+public:
+ BoxException();
+ ~BoxException() throw ();
+
+ virtual unsigned int GetType() const throw() = 0;
+ virtual unsigned int GetSubType() const throw() = 0;
+
+private:
+};
+
+
+#endif // BOXEXCEPTION__H
+
diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h
new file mode 100644
index 00000000..e3af29ff
--- /dev/null
+++ b/lib/common/BoxPlatform.h
@@ -0,0 +1,190 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxPlatform.h
+// Purpose: Specifies what each platform supports in more detail, and includes
+// extra files to get basic support for types.
+// Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXPLATFORM__H
+#define BOXPLATFORM__H
+
+#ifdef WIN32
+#define DIRECTORY_SEPARATOR "\\"
+#define DIRECTORY_SEPARATOR_ASCHAR '\\'
+#else
+#define DIRECTORY_SEPARATOR "/"
+#define DIRECTORY_SEPARATOR_ASCHAR '/'
+#endif
+
+#define PLATFORM_DEV_NULL "/dev/null"
+
+#include "BoxConfig.h"
+
+#ifdef WIN32
+ // need msvcrt version 6.1 or higher for _gmtime64()
+ // must define this before importing <sys/types.h>
+ #define __MSVCRT_VERSION__ 0x0601
+#endif
+
+#ifdef HAVE_SYS_TYPES_H
+ #include <sys/types.h>
+#endif
+#ifdef HAVE_INTTYPES_H
+ #include <inttypes.h>
+#else
+ #ifdef HAVE_STDINT_H
+ #include <stdint.h>
+ #endif
+#endif
+
+// Slight hack; disable interception on Darwin within raidfile test
+#ifdef __APPLE__
+ // TODO: Replace with autoconf test
+ #define PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE
+#endif
+
+// Disable memory testing under Darwin, it just doesn't like it very much.
+#ifdef __APPLE__
+ // TODO: We really should get some decent leak detection code.
+ #define PLATFORM_DISABLE_MEM_LEAK_TESTING
+#endif
+
+// Find out if credentials on UNIX sockets can be obtained
+#ifndef HAVE_GETPEEREID
+ #if !HAVE_DECL_SO_PEERCRED
+ #define PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET
+ #endif
+#endif
+
+#ifdef HAVE_DEFINE_PRAGMA
+ // set packing to one bytes (can't use push/pop on gcc)
+ #define BEGIN_STRUCTURE_PACKING_FOR_WIRE #pragma pack(1)
+
+ // Use default packing
+ #define END_STRUCTURE_PACKING_FOR_WIRE #pragma pack()
+#else
+ #define STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#endif
+
+// Handle differing xattr APIs
+#ifdef HAVE_SYS_XATTR_H
+ #if !defined(HAVE_LLISTXATTR) && defined(HAVE_LISTXATTR) && HAVE_DECL_XATTR_NOFOLLOW
+ #define llistxattr(a,b,c) listxattr(a,b,c,XATTR_NOFOLLOW)
+ #endif
+ #if !defined(HAVE_LGETXATTR) && defined(HAVE_GETXATTR) && HAVE_DECL_XATTR_NOFOLLOW
+ #define lgetxattr(a,b,c,d) getxattr(a,b,c,d,0,XATTR_NOFOLLOW)
+ #endif
+ #if !defined(HAVE_LSETXATTR) && defined(HAVE_SETXATTR) && HAVE_DECL_XATTR_NOFOLLOW
+ #define lsetxattr(a,b,c,d,e) setxattr(a,b,c,d,0,(e)|XATTR_NOFOLLOW)
+ #endif
+#endif
+
+#if defined WIN32 && !defined __MINGW32__
+ typedef __int8 int8_t;
+ typedef __int16 int16_t;
+ typedef __int32 int32_t;
+ typedef __int64 int64_t;
+
+ typedef unsigned __int8 u_int8_t;
+ typedef unsigned __int16 u_int16_t;
+ typedef unsigned __int32 u_int32_t;
+ typedef unsigned __int64 u_int64_t;
+
+ #define HAVE_U_INT8_T
+ #define HAVE_U_INT16_T
+ #define HAVE_U_INT32_T
+ #define HAVE_U_INT64_T
+
+ typedef int pid_t;
+#endif // WIN32 && !__MINGW32__
+
+// Define missing types
+#ifndef HAVE_UINT8_T
+ typedef u_int8_t uint8_t;
+#endif
+
+#ifndef HAVE_UINT16_T
+ typedef u_int16_t uint16_t;
+#endif
+
+#ifndef HAVE_UINT32_T
+ typedef u_int32_t uint32_t;
+#endif
+
+#ifndef HAVE_UINT64_T
+ typedef u_int64_t uint64_t;
+#endif
+
+#ifndef HAVE_U_INT8_T
+ typedef uint8_t u_int8_t;
+#endif
+
+#ifndef HAVE_U_INT16_T
+ typedef uint16_t u_int16_t;
+#endif
+
+#ifndef HAVE_U_INT32_T
+ typedef uint32_t u_int32_t;
+#endif
+
+#ifndef HAVE_U_INT64_T
+ typedef uint64_t u_int64_t;
+#endif
+
+#if !HAVE_DECL_INFTIM
+ #define INFTIM -1
+#endif
+
+#ifdef WIN32
+ typedef u_int64_t InodeRefType;
+#else
+ typedef ino_t InodeRefType;
+#endif
+
+#ifdef WIN32
+ #define WIN32_LEAN_AND_MEAN
+ #include "emu.h"
+#endif
+
+#endif // BOXPLATFORM__H
diff --git a/lib/common/BoxPortsAndFiles.h b/lib/common/BoxPortsAndFiles.h
new file mode 100644
index 00000000..b0500ee5
--- /dev/null
+++ b/lib/common/BoxPortsAndFiles.h
@@ -0,0 +1,74 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxPortsAndFiles.h
+// Purpose: Central list of which tcp/ip ports and hardcoded file locations
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXPORTSANDFILES__H
+#define BOXPORTSANDFILES__H
+
+#define BOX_PORT_BASE 2200
+
+
+// Backup store daemon
+#define BOX_PORT_BBSTORED (BOX_PORT_BASE+1)
+#define BOX_FILE_BBSTORED_DEFAULT_CONFIG "/etc/boxbackup/bbstored.conf"
+// directory within the RAIDFILE root for the backup store daemon
+#define BOX_RAIDFILE_ROOT_BBSTORED "backup"
+
+// Backup client daemon
+#ifdef WIN32
+#define BOX_FILE_BBACKUPD_DEFAULT_CONFIG "C:\\Program Files\\Box Backup\\bbackupd.conf"
+#else
+#define BOX_FILE_BBACKUPD_DEFAULT_CONFIG "/etc/boxbackup/bbackupd.conf"
+#endif
+
+// RaidFile conf location default
+#define BOX_FILE_RAIDFILE_DEFAULT_CONFIG "/etc/boxbackup/raidfile.conf"
+
+// Default name of the named pipe
+#define BOX_NAMED_PIPE_NAME L"\\\\.\\pipe\\boxbackup"
+
+#endif // BOXPORTSANDFILES__H
+
diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp
new file mode 100644
index 00000000..7c858f0d
--- /dev/null
+++ b/lib/common/BoxTime.cpp
@@ -0,0 +1,69 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxTime.cpp
+// Purpose: Time for the box
+// Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <time.h>
+
+#include "BoxTime.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: GetCurrentBoxTime()
+// Purpose: Returns the current time as a box time. (1 sec precision)
+// Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+box_time_t GetCurrentBoxTime()
+{
+ return SecondsToBoxTime(time(0));
+}
+
+
diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h
new file mode 100644
index 00000000..880985ca
--- /dev/null
+++ b/lib/common/BoxTime.h
@@ -0,0 +1,78 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxTime.h
+// Purpose: How time is represented
+// Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXTIME__H
+#define BOXTIME__H
+
+// Time is presented as an unsigned 64 bit integer, in microseconds
+typedef uint64_t box_time_t;
+
+#define NANO_SEC_IN_SEC (1000000000LL)
+#define NANO_SEC_IN_USEC (1000)
+#define NANO_SEC_IN_USEC_LL (1000LL)
+#define MICRO_SEC_IN_SEC (1000000)
+#define MICRO_SEC_IN_SEC_LL (1000000LL)
+#define MILLI_SEC_IN_NANO_SEC (1000)
+#define MILLI_SEC_IN_NANO_SEC_LL (1000LL)
+
+box_time_t GetCurrentBoxTime();
+
+inline box_time_t SecondsToBoxTime(time_t Seconds)
+{
+ return ((box_time_t)Seconds * MICRO_SEC_IN_SEC_LL);
+}
+inline time_t BoxTimeToSeconds(box_time_t Time)
+{
+ return Time / MICRO_SEC_IN_SEC_LL;
+}
+inline uint64_t BoxTimeToMilliSeconds(box_time_t Time)
+{
+ return Time / MILLI_SEC_IN_NANO_SEC_LL;
+}
+
+#endif // BOXTIME__H
+
diff --git a/lib/common/BoxTimeToText.cpp b/lib/common/BoxTimeToText.cpp
new file mode 100644
index 00000000..7997376b
--- /dev/null
+++ b/lib/common/BoxTimeToText.cpp
@@ -0,0 +1,100 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxTimeToText.cpp
+// Purpose: Convert box time to text
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <time.h>
+#include <stdio.h>
+
+#include "BoxTimeToText.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BoxTimeToISO8601String(box_time_t)
+// Purpose: Convert a 64 bit box time to a ISO 8601 complient string
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+std::string BoxTimeToISO8601String(box_time_t Time)
+{
+#ifdef WIN32
+ struct tm *time;
+ time_t bob = BoxTimeToSeconds(Time);
+
+ __time64_t winTime = bob;
+
+ time = _gmtime64(&winTime);
+ char str[128]; // more than enough space
+
+ if ( time == NULL )
+ {
+ // ::sprintf(str, "%016I64x ", bob);
+ return std::string("unable to convert time");
+ }
+
+ sprintf(str, "%04d-%02d-%02dT%02d:%02d:%02d", time->tm_year + 1900,
+ time->tm_mon + 1, time->tm_mday, time->tm_hour,
+ time->tm_min, time->tm_sec);
+#else // ! WIN32
+ time_t timeInSecs = BoxTimeToSeconds(Time);
+ struct tm time;
+ gmtime_r(&timeInSecs, &time);
+
+ char str[128]; // more than enough space
+ sprintf(str, "%04d-%02d-%02dT%02d:%02d:%02d", time.tm_year + 1900,
+ time.tm_mon + 1, time.tm_mday, time.tm_hour,
+ time.tm_min, time.tm_sec);
+#endif // WIN32
+
+ return std::string(str);
+}
+
+
diff --git a/lib/common/BoxTimeToText.h b/lib/common/BoxTimeToText.h
new file mode 100644
index 00000000..ce017c1d
--- /dev/null
+++ b/lib/common/BoxTimeToText.h
@@ -0,0 +1,57 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxTimeToText.h
+// Purpose: Convert box time to text
+// Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXTIMETOTEXT__H
+#define BOXTIMETOTEXT__H
+
+#include <string>
+#include "BoxTime.h"
+
+std::string BoxTimeToISO8601String(box_time_t Time);
+
+#endif // BOXTIMETOTEXT__H
+
diff --git a/lib/common/BoxTimeToUnix.h b/lib/common/BoxTimeToUnix.h
new file mode 100644
index 00000000..0a4bdcad
--- /dev/null
+++ b/lib/common/BoxTimeToUnix.h
@@ -0,0 +1,72 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BoxTimeToUnix.h
+// Purpose: Convert times in 64 bit values to UNIX structures
+// Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#ifndef FILEMODIFICATIONTIMETOTIMEVAL__H
+#define FILEMODIFICATIONTIMETOTIMEVAL__H
+
+#ifdef WIN32
+#include <time.h>
+#else
+#include <sys/time.h>
+#endif
+
+#include "BoxTime.h"
+
+inline void BoxTimeToTimeval(box_time_t Time, struct timeval &tv)
+{
+ tv.tv_sec = (long)(Time / MICRO_SEC_IN_SEC_LL);
+ tv.tv_usec = (long)(Time % MICRO_SEC_IN_SEC_LL);
+}
+
+inline void BoxTimeToTimespec(box_time_t Time, struct timespec &tv)
+{
+ tv.tv_sec = (time_t)(Time / MICRO_SEC_IN_SEC_LL);
+ tv.tv_nsec = ((long)(Time % MICRO_SEC_IN_SEC_LL)) * NANO_SEC_IN_USEC;
+}
+
+#endif // FILEMODIFICATIONTIMETOTIMEVAL__H
+
diff --git a/lib/common/CollectInBufferStream.cpp b/lib/common/CollectInBufferStream.cpp
new file mode 100644
index 00000000..93cba893
--- /dev/null
+++ b/lib/common/CollectInBufferStream.cpp
@@ -0,0 +1,312 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: CollectInBufferStream.cpp
+// Purpose: Collect data in a buffer, and then read it out.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "CollectInBufferStream.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+#define INITIAL_BUFFER_SIZE 1024
+#define MAX_BUFFER_ADDITION (1024*64)
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::CollectInBufferStream()
+// Purpose: Constructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+CollectInBufferStream::CollectInBufferStream()
+ : mBuffer(INITIAL_BUFFER_SIZE),
+ mBufferSize(INITIAL_BUFFER_SIZE),
+ mBytesInBuffer(0),
+ mReadPosition(0),
+ mInWritePhase(true)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::~CollectInBufferStream()
+// Purpose: Destructor
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+CollectInBufferStream::~CollectInBufferStream()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::Read(void *, int, int)
+// Purpose: As interface. But only works in read phase
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+int CollectInBufferStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+ if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+
+ // Adjust to number of bytes left
+ if(NBytes > (mBytesInBuffer - mReadPosition))
+ {
+ NBytes = (mBytesInBuffer - mReadPosition);
+ }
+ ASSERT(NBytes >= 0);
+ if(NBytes <= 0) return 0; // careful now
+
+ // Copy in the requested number of bytes and adjust the read pointer
+ ::memcpy(pBuffer, ((char*)mBuffer) + mReadPosition, NBytes);
+ mReadPosition += NBytes;
+
+ return NBytes;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::BytesLeftToRead()
+// Purpose: As interface. But only works in read phase
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type CollectInBufferStream::BytesLeftToRead()
+{
+ if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+
+ return (mBytesInBuffer - mReadPosition);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::Write(void *, int)
+// Purpose: As interface. But only works in write phase
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::Write(const void *pBuffer, int NBytes)
+{
+ if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+
+ // Enough space in the buffer
+ if((mBytesInBuffer + NBytes) > mBufferSize)
+ {
+ // Need to reallocate... what's the block size we'll use?
+ int allocateBlockSize = mBufferSize;
+ if(allocateBlockSize > MAX_BUFFER_ADDITION)
+ {
+ allocateBlockSize = MAX_BUFFER_ADDITION;
+ }
+
+ // Write it the easy way. Although it's not the most efficient...
+ int newSize = mBufferSize;
+ while(newSize < (mBytesInBuffer + NBytes))
+ {
+ newSize += allocateBlockSize;
+ }
+
+ // Reallocate buffer
+ mBuffer.Resize(newSize);
+
+ // Store new size
+ mBufferSize = newSize;
+ }
+
+ // Copy in data and adjust counter
+ ::memcpy(((char*)mBuffer) + mBytesInBuffer, pBuffer, NBytes);
+ mBytesInBuffer += NBytes;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::GetPosition()
+// Purpose: In write phase, returns the number of bytes written, in read
+// phase, the number of bytes to go
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type CollectInBufferStream::GetPosition() const
+{
+ return mInWritePhase?mBytesInBuffer:mReadPosition;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::Seek(pos_type, int)
+// Purpose: As interface. But read phase only.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::Seek(pos_type Offset, int SeekType)
+{
+ if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+
+ int newPos = 0;
+ switch(SeekType)
+ {
+ case IOStream::SeekType_Absolute:
+ newPos = Offset;
+ break;
+ case IOStream::SeekType_Relative:
+ newPos = mReadPosition + Offset;
+ break;
+ case IOStream::SeekType_End:
+ newPos = mBytesInBuffer + Offset;
+ break;
+ default:
+ THROW_EXCEPTION(CommonException, IOStreamBadSeekType)
+ break;
+ }
+
+ // Make sure it doesn't go over
+ if(newPos > mBytesInBuffer)
+ {
+ newPos = mBytesInBuffer;
+ }
+ // or under
+ if(newPos < 0)
+ {
+ newPos = 0;
+ }
+
+ // Set the new read position
+ mReadPosition = newPos;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::StreamDataLeft()
+// Purpose: As interface
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool CollectInBufferStream::StreamDataLeft()
+{
+ return mInWritePhase?(false):(mReadPosition < mBytesInBuffer);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::StreamClosed()
+// Purpose: As interface
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool CollectInBufferStream::StreamClosed()
+{
+ return !mInWritePhase;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::SetForReading()
+// Purpose: Switch to read phase, after all data written
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::SetForReading()
+{
+ if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+
+ // Move to read phase
+ mInWritePhase = false;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::GetBuffer()
+// Purpose: Returns the buffer
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void *CollectInBufferStream::GetBuffer() const
+{
+ return mBuffer.GetPtr();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::GetSize()
+// Purpose: Returns the buffer size
+// Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+int CollectInBufferStream::GetSize() const
+{
+ return mBytesInBuffer;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: CollectInBufferStream::Reset()
+// Purpose: Reset the stream, so it is empty and ready to be written to.
+// Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::Reset()
+{
+ mInWritePhase = true;
+ mBytesInBuffer = 0;
+ mReadPosition = 0;
+}
+
diff --git a/lib/common/CollectInBufferStream.h b/lib/common/CollectInBufferStream.h
new file mode 100644
index 00000000..90f6889b
--- /dev/null
+++ b/lib/common/CollectInBufferStream.h
@@ -0,0 +1,98 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: CollectInBufferStream.h
+// Purpose: Collect data in a buffer, and then read it out.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef COLLECTINBUFFERSTREAM__H
+#define COLLECTINBUFFERSTREAM__H
+
+#include "IOStream.h"
+#include "Guards.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: CollectInBufferStream
+// Purpose: Collect data in a buffer, and then read it out.
+// Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class CollectInBufferStream : public IOStream
+{
+public:
+ CollectInBufferStream();
+ ~CollectInBufferStream();
+private:
+ // No copying
+ CollectInBufferStream(const CollectInBufferStream &);
+ CollectInBufferStream(const IOStream &);
+public:
+
+ virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+ virtual pos_type BytesLeftToRead();
+ virtual void Write(const void *pBuffer, int NBytes);
+ virtual pos_type GetPosition() const;
+ virtual void Seek(pos_type Offset, int SeekType);
+ virtual bool StreamDataLeft();
+ virtual bool StreamClosed();
+
+ void SetForReading();
+
+ void Reset();
+
+ void *GetBuffer() const;
+ int GetSize() const;
+ bool IsSetForReading() const {return !mInWritePhase;}
+
+private:
+ MemoryBlockGuard<char*> mBuffer;
+ int mBufferSize;
+ int mBytesInBuffer;
+ int mReadPosition;
+ bool mInWritePhase;
+};
+
+#endif // COLLECTINBUFFERSTREAM__H
+
diff --git a/lib/common/CommonException.h b/lib/common/CommonException.h
new file mode 100644
index 00000000..61bd654a
--- /dev/null
+++ b/lib/common/CommonException.h
@@ -0,0 +1,55 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: CommonException.h
+// Purpose: Exception
+// Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef COMMONEXCEPTION__H
+#define COMMONEXCEPTION__H
+
+// Compatibility header with old non-autogen exception scheme
+#include "autogen_CommonException.h"
+
+#endif // COMMONEXCEPTION__H
+
diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt
new file mode 100644
index 00000000..5fa443d0
--- /dev/null
+++ b/lib/common/CommonException.txt
@@ -0,0 +1,46 @@
+
+# NOTE: Exception descriptions are for public distributions of Box Backup only -- do not rely for other applications.
+
+
+EXCEPTION Common 1
+
+Internal 0
+AssertFailed 1
+OSFileOpenError 2 Can't open a file -- attempted to load a non-existant config file or bad file referenced within?
+OSFileCloseError 3
+FileAlreadyClosed 4
+BadArguments 5
+ConfigNoKey 6
+ConfigNoSubConfig 7
+GetLineNoHandle 8
+OSFileError 9 Error accessing a file. Check permissions.
+GetLineEOF 10
+ConfigBadIntValue 11
+GetLineTooLarge 12 Protects against very large lines using up lots of memory.
+NotSupported 13
+OSFileReadError 14
+OSFileWriteError 15
+FileClosed 16
+IOStreamBadSeekType 17
+CantWriteToPartialReadStream 18
+CollectInBufferStreamNotInCorrectPhase 19
+NamedLockAlreadyLockingSomething 20
+NamedLockNotHeld 21
+StreamableMemBlockIncompleteRead 22
+MemBlockStreamNotSupported 23
+StreamDoesntHaveRequiredProperty 24
+CannotWriteToReadGatherStream 25
+ReadGatherStreamAddingBadBlock 26
+CouldNotLookUpUsername 27
+CouldNotRestoreProcessUser 28
+CouldNotChangeProcessUser 29
+RegexNotSupportedOnThisPlatform 30 Your platform does not have built in regular expression libraries.
+BadRegularExpression 31
+CouldNotCreateKQueue 32
+KEventErrorAdd 33
+KEventErrorWait 34
+KEventErrorRemove 35
+KQueueNotSupportedOnThisPlatform 36
+IOStreamGetLineNotEnoughDataToIgnore 37 Bad value passed to IOStreamGetLine::IgnoreBufferedData()
+TempDirPathTooLong 38 Your temporary directory path is too long. Check the TMP and TEMP environment variables.
+ArchiveBlockIncompleteRead 39 The Store Object Info File is too short or corrupted, and will be rewritten automatically when the next backup completes.
diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp
new file mode 100644
index 00000000..a766434d
--- /dev/null
+++ b/lib/common/Configuration.cpp
@@ -0,0 +1,779 @@
+// distribution boxbackup-0.10 (svn version: 494)
+//
+// Copyright (c) 2003 - 2006
+// Ben Summers and contributors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. All use of this software and associated advertising materials must
+// display the following acknowledgment:
+// This product includes software developed by Ben Summers.
+// 4. The names of the Authors may not be used to endorse or promote
+// products derived from this software without specific prior written
+// permission.
+//
+// [Where legally impermissible the Authors do not disclaim liability for
+// direct physical injury or death caused solely by defects in the software
+// unless it is modified by a third party.]
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//
+//
+// --------------------------------------------------------------------------
+//
+// File
+// Name: Configuration.cpp
+// Purpose: Reading configuration files
+// Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <limits.h>
+
+#