diff options
author | Ben Summers <ben@fluffy.co.uk> | 2005-10-14 08:50:54 +0000 |
---|---|---|
committer | Ben Summers <ben@fluffy.co.uk> | 2005-10-14 08:50:54 +0000 |
commit | 99f8ce096bc5569adbfea1911dbcda24c28d8d8b (patch) | |
tree | 049c302161fea1f2f6223e1e8f3c40d9e8aadc8b /lib |
Box Backup 0.09 with a few tweeks
Diffstat (limited to 'lib')
177 files changed, 31121 insertions, 0 deletions
diff --git a/lib/backupclient/BackupClientCryptoKeys.cpp b/lib/backupclient/BackupClientCryptoKeys.cpp new file mode 100755 index 00000000..015dadd2 --- /dev/null +++ b/lib/backupclient/BackupClientCryptoKeys.cpp @@ -0,0 +1,67 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientCryptoKeys.cpp +// Purpose: function for setting up all the backup client keys +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "BackupClientCryptoKeys.h" +#include "FileStream.h" +#include "BackupStoreFilenameClear.h" +#include "BackupStoreException.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreFile.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientCryptoKeys_Setup(const char *) +// Purpose: Read in the key material file, and set keys to all the backup elements required. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename) +{ + // Read in the key material + unsigned char KeyMaterial[BACKUPCRYPTOKEYS_FILE_SIZE]; + + // Open the file + FileStream file(KeyMaterialFilename); + // Read in data + if(!file.ReadFullBuffer(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE, 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntLoadClientKeyMaterial) + } + + // Tell the filename how to encrypt + BackupStoreFilenameClear::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH, + KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH); + BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish); + + // Tell the attributes how to encrypt + BackupClientFileAttributes::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH); + // and the secret for hashing + BackupClientFileAttributes::SetAttributeHashSecret(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH); + + // Tell the files how to encrypt + BackupStoreFile::SetBlowfishKeys(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH, + KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH); +#ifndef PLATFORM_OLD_OPENSSL + // Use AES where available + BackupStoreFile::SetAESKey(KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH); +#endif + + // Wipe the key material from memory + ::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE); +} + + + diff --git a/lib/backupclient/BackupClientCryptoKeys.h b/lib/backupclient/BackupClientCryptoKeys.h new file mode 100755 index 00000000..5e3a7df2 --- /dev/null +++ b/lib/backupclient/BackupClientCryptoKeys.h @@ -0,0 +1,55 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientCryptoKeys.h +// Purpose: Format of crypto keys file, and function for setting everything up +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTCRYTOKEYS__H +#define BACKUPCLIENTCRYTOKEYS__H + + +// All keys are the maximum size that Blowfish supports. Since only the +// setup time is affected by key length (encryption same speed whatever) +// there is no disadvantage to using long keys as they are never +// transmitted and are static over long periods of time. + + +// All sizes in bytes. Some gaps deliberately left in the used material. + +// How long the key material file is expected to be +#define BACKUPCRYPTOKEYS_FILE_SIZE 1024 + +// key for encrypting filenames (448 bits) +#define BACKUPCRYPTOKEYS_FILENAME_KEY_START 0 +#define BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH 56 +#define BACKUPCRYPTOKEYS_FILENAME_IV_START (0 + BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH) +#define BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH 8 + +// key for encrypting attributes (448 bits) +#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START (BACKUPCRYPTOKEYS_FILENAME_KEY_START+64) +#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH 56 + +// Blowfish key for encrypting file data (448 bits (max blowfish key length)) +#define BACKUPCRYPTOKEYS_FILE_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START+64) +#define BACKUPCRYPTOKEYS_FILE_KEY_LENGTH 56 + +// key for encrypting file block index entries +#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START (BACKUPCRYPTOKEYS_FILE_KEY_START+64) +#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH 56 + +// Secret for hashing attributes +#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START (BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START+64) +#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH 128 + +// AES key for encrypting file data (256 bits (max AES key length)) +#define BACKUPCRYPTOKEYS_FILE_AES_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START+128) +#define BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH 32 + + +void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename); + +#endif // BACKUPCLIENTCRYTOKEYS__H + diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp new file mode 100755 index 00000000..4aa4edba --- /dev/null +++ b/lib/backupclient/BackupClientFileAttributes.cpp @@ -0,0 +1,773 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientFileAttributes.cpp +// Purpose: Storage of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <string.h> +#include <unistd.h> +#include <limits.h> + +#include "BackupClientFileAttributes.h" +#include "CommonException.h" +#include "FileModificationTime.h" +#include "BoxTimeToUnix.h" +#include "BackupStoreException.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "MD5Digest.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +#define ATTRIBUTETYPE_GENERIC_UNIX 1 + +#define ATTRIBUTE_ENCODING_BLOWFISH 2 + +typedef struct +{ + int32_t AttributeType; + u_int32_t UID; + u_int32_t GID; + u_int64_t ModificationTime; + u_int64_t AttrModificationTime; + u_int32_t UserDefinedFlags; + u_int32_t FileGenerationNumber; + u_int16_t Mode; + // Symbolic link filename may follow +} attr_StreamFormat; + +// This has wire packing so it's compatible across platforms +// Use wider than necessary sizes, just to be careful. +typedef struct +{ + int32_t uid, gid, mode; +} attributeHashData; + +// Use default packing +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + + +#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256 + +// Hide private static variables from the rest of the world +// -- don't put them as static class variables to avoid openssl/evp.h being +// included all over the project. +namespace +{ + CipherContext sBlowfishEncrypt; + CipherContext sBlowfishDecrypt; + uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH]; + int sAttributeHashSecretLength = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::BackupClientFileAttributes() +// Purpose: Default constructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::BackupClientFileAttributes() + : mpClearAttributes(0) +{ + ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &) +// Purpose: Copy constructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy) + : StreamableMemBlock(rToCopy), // base class does the hard work + mpClearAttributes(0) +{ +} +BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy) + : StreamableMemBlock(rToCopy), // base class does the hard work + mpClearAttributes(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::~BackupClientFileAttributes() +// Purpose: Destructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::~BackupClientFileAttributes() +{ + if(mpClearAttributes) + { + delete mpClearAttributes; + mpClearAttributes = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &) +// Purpose: Assignment operator +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr) +{ + StreamableMemBlock::Set(rAttr); + RemoveClear(); // make sure no decrypted version held + return *this; +} +// Assume users play nice +BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr) +{ + StreamableMemBlock::Set(rAttr); + RemoveClear(); // make sure no decrypted version held + return *this; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &) +// Purpose: Comparison operator +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const +{ + EnsureClearAvailable(); + rAttr.EnsureClearAvailable(); + + return mpClearAttributes->operator==(*rAttr.mpClearAttributes); +} +// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data. +/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const +{ + StreamableMemBlock *pDecoded = 0; + + try + { + EnsureClearAvailable(); + StreamableMemBlock *pDecoded = MakeClear(rAttr); + + // Compare using clear version + bool compared = mpClearAttributes->operator==(rAttr); + + // Delete temporary + delete pDecoded; + + return compared; + } + catch(...) + { + delete pDecoded; + throw; + } +}*/ + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool) +// Purpose: Compare, optionally ignoring the attribute modification time and/or modification time, and some data which is +// irrelevant in practise (eg file generation number) +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime, bool IgnoreModTime) const +{ + EnsureClearAvailable(); + rAttr.EnsureClearAvailable(); + + // Check sizes are the same, as a first check + if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize()) + { + return false; + } + + // Then check the elements of the two things + // Bytes are checked in network order, but this doesn't matter as we're only checking for equality. + attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer(); + + if(a1->AttributeType != a2->AttributeType + || a1->UID != a2->UID + || a1->GID != a2->GID + || a1->UserDefinedFlags != a2->UserDefinedFlags + || a1->Mode != a2->Mode) + { + return false; + } + + if(!IgnoreModTime) + { + if(a1->ModificationTime != a2->ModificationTime) + { + return false; + } + } + + if(!IgnoreAttrModTime) + { + if(a1->AttrModificationTime != a2->AttrModificationTime) + { + return false; + } + } + + // Check symlink string? + unsigned int size = mpClearAttributes->GetSize(); + if(size > sizeof(attr_StreamFormat)) + { + // Symlink strings don't match + if(::memcmp(a1 + 1, a2 + 1, size - sizeof(attr_StreamFormat)) != 0) + { + return false; + } + } + + // Passes all test, must be OK + return true; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadAttributes(const char *) +// Purpose: Read the attributes of the file, and store them ready for streaming. +// Optionally retrieve the modification time and attribute modification time. +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroModificationTimes, box_time_t *pModTime, + box_time_t *pAttrModTime, int64_t *pFileSize, ino_t *pInodeNumber, bool *pHasMultipleLinks) +{ + StreamableMemBlock *pnewAttr = 0; + try + { + struct stat st; + if(::lstat(Filename, &st) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Modification times etc + if(pModTime) {*pModTime = FileModificationTime(st);} + if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);} + if(pFileSize) {*pFileSize = st.st_size;} + if(pInodeNumber) {*pInodeNumber = st.st_ino;} + if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);} + + // Is it a link? + if((st.st_mode & S_IFMT) == S_IFLNK) + { + ReadAttributesLink(Filename, &st, ZeroModificationTimes); + return; + } + ASSERT((st.st_mode & S_IFMT) != S_IFLNK); + + // Now, can allocate the block + pnewAttr = new StreamableMemBlock(sizeof(attr_StreamFormat)); + + // Fill in the entries + attr_StreamFormat *pattr = (attr_StreamFormat*)pnewAttr->GetBuffer(); + +#define FILL_IN_ATTRIBUTES \ + ::memset(pattr, 0, sizeof(attr_StreamFormat)); \ + ASSERT(pattr != 0); \ + pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX); \ + pattr->UID = htonl(st.st_uid); \ + pattr->GID = htonl(st.st_gid); \ + if(ZeroModificationTimes) \ + { \ + pattr->ModificationTime = 0; \ + pattr->AttrModificationTime = 0; \ + } \ + else \ + { \ + pattr->ModificationTime = hton64(FileModificationTime(st)); \ + pattr->AttrModificationTime = hton64(FileAttrModificationTime(st)); \ + } \ + pattr->Mode = htons(st.st_mode); +#ifdef PLATFORM_stat_NO_st_flags + #define FILL_IN_ATTRIBUTES_2 \ + pattr->UserDefinedFlags = 0; \ + pattr->FileGenerationNumber = 0; +#else + #define FILL_IN_ATTRIBUTES_2 \ + pattr->UserDefinedFlags = htonl(st.st_flags); \ + pattr->FileGenerationNumber = htonl(st.st_gen); +#endif + + FILL_IN_ATTRIBUTES + FILL_IN_ATTRIBUTES_2 + + // Attributes ready. Encrypt into this block + EncryptAttr(*pnewAttr); + + // Store the new attributes + RemoveClear(); + mpClearAttributes = pnewAttr; + pnewAttr = 0; + } + catch(...) + { + // clean up + delete pnewAttr; + pnewAttr = 0; + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadAttributesLink() +// Purpose: Private function, handles the case where a symbolic link is needed +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::ReadAttributesLink(const char *Filename, void *pst, bool ZeroModificationTimes) +{ + StreamableMemBlock *pnewAttr = 0; + try + { + // Avoid needing to have struct stat available when including header file + struct stat &st = *((struct stat *)pst); + // Make sure we're only called for symbolic links + ASSERT((st.st_mode & S_IFMT) == S_IFLNK); + + // Get the filename the link is linked to + char linkedTo[PATH_MAX+4]; + int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX); + if(linkedToSize == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Now, can allocate the block + pnewAttr = new StreamableMemBlock(sizeof(attr_StreamFormat) + linkedToSize + 1); + + // Fill in the entries + attr_StreamFormat *pattr = (attr_StreamFormat*)pnewAttr->GetBuffer(); + + FILL_IN_ATTRIBUTES + FILL_IN_ATTRIBUTES_2 + + // Add the path name for the symbolic link + ::memcpy(pattr + 1, linkedTo, linkedToSize); + // Make sure it's neatly terminated + ((char*)(pattr + 1))[linkedToSize] = '\0'; + + // Attributes ready. Encrypt into this block + EncryptAttr(*pnewAttr); + + // Store the new attributes + RemoveClear(); + mpClearAttributes = pnewAttr; + pnewAttr = 0; + } + catch(...) + { + // clean up + delete pnewAttr; + pnewAttr = 0; + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::WriteAttributes(const char *) +// Purpose: Apply the stored attributes to the file +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::WriteAttributes(const char *Filename) const +{ + // Got something loaded + if(GetSize() <= 0) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Make sure there are clear attributes to use + EnsureClearAvailable(); + ASSERT(mpClearAttributes != 0); + + // Check if the decrypted attributes are small enough, and the type of attributes stored + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) + { + // Don't know what to do with these + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + + // Check there is enough space for an attributes block + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get pointer to structure + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + + // is it a symlink? + int16_t mode = ntohs(pattr->Mode); + if((mode & S_IFMT) == S_IFLNK) + { + // Check things are sensible + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Make a symlink, first deleting anything in the way + ::unlink(Filename); + if(::symlink((char*)(pattr + 1), Filename) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + + // If working as root, set user IDs + if(::geteuid() == 0) + { + #ifdef PLATFORM_LCHOWN_NOT_SUPPORTED + // only if not a link, can't set their owner on this platform + if((mode & S_IFMT) != S_IFLNK) + { + // Not a link, use normal chown + if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + #else + if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) // use the version which sets things on symlinks + { + THROW_EXCEPTION(CommonException, OSFileError) + } + #endif + } + + // Stop now if symlink, because otherwise it'll just be applied to the target + if((mode & S_IFMT) == S_IFLNK) + { + return; + } + + // Set modification time? + box_time_t modtime = ntoh64(pattr->ModificationTime); + if(modtime != 0) + { + // Work out times as timevals + struct timeval times[2]; + BoxTimeToTimeval(modtime, times[1]); + // Copy access time as well, why not, got to set it to something + times[0] = times[1]; + // Attr modification time will be changed anyway, nothing that can be done about it + + // Try to apply + if(::utimes(Filename, times) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + + // Apply everything else... (allowable mode flags only) + if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID | S_ISGID | S_ISVTX)) != 0) // mode must be done last (think setuid) + { + THROW_EXCEPTION(CommonException, OSFileError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::IsSymLink() +// Purpose: Do these attributes represent a symbolic link? +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::IsSymLink() const +{ + EnsureClearAvailable(); + + // Got the right kind of thing? + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get the type of attributes stored + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat)) + { + // Check link + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK; + } + + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::RemoveClear() +// Purpose: Private. Deletes any clear version of the attributes that may be held +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::RemoveClear() const +{ + if(mpClearAttributes) + { + delete mpClearAttributes; + } + mpClearAttributes = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::EnsureClearAvailable() +// Purpose: Private. Makes sure the clear version is available +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::EnsureClearAvailable() const +{ + if(mpClearAttributes == 0) + { + mpClearAttributes = MakeClear(*this); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &) +// Purpose: Static. Decrypts stored attributes. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted) +{ + // New block + StreamableMemBlock *pdecrypted = 0; + + try + { + // Check the block is big enough for IV and header + int ivSize = sBlowfishEncrypt.GetIVLength(); + if(rEncrypted.GetSize() <= (ivSize + 1)) + { + THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes); + } + + // How much space is needed for the output? + int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize); + + // Allocate it + pdecrypted = new StreamableMemBlock(maxDecryptedSize); + + // ptr to block + uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer(); + + // Check that the header has right type + if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH) + { + THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding); + } + + // Set IV + sBlowfishDecrypt.SetIV(encBlock + 1); + + // Decrypt + int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1)); + + // Resize block to fit + pdecrypted->ResizeBlock(decryptedSize); + } + catch(...) + { + delete pdecrypted; + pdecrypted = 0; + } + + return pdecrypted; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int) +// Purpose: Static. Sets the key to use for encryption and decryption. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength) +{ + // IVs set later + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &) +// Purpose: Private. Encrypt the given attributes into this block. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt) +{ + // Free any existing block + FreeBlock(); + + // Work out the maximum amount of space we need + int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize()); + // And the size of the IV + int ivSize = sBlowfishEncrypt.GetIVLength(); + + // Allocate this space + AllocateBlock(maxEncryptedSize + ivSize + 1); + + // Store the encoding byte + uint8_t *block = (uint8_t*)GetBuffer(); + block[0] = ATTRIBUTE_ENCODING_BLOWFISH; + + // Generate and store an IV for this attribute block + int ivSize2 = 0; + const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2); + ASSERT(ivSize == ivSize2); + + // Copy into the encrypted block + ::memcpy(block + 1, iv, ivSize); + + // Do the transform + int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize()); + + // Resize this block + ResizeBlock(encrytedSize + ivSize + 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int) +// Purpose: Set the secret for the filename attribute hash +// Created: 25/4/04 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength) +{ + if(SecretLength > (int)sizeof(sAttributeHashSecret)) + { + SecretLength = sizeof(sAttributeHashSecret); + } + if(SecretLength < 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Copy + ::memcpy(sAttributeHashSecret, pSecret, SecretLength); + sAttributeHashSecretLength = SecretLength; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::GenerateAttributeHash(struct stat &, const std::string &) +// Purpose: Generate a 64 bit hash from the attributes, used to detect changes. +// Include filename in the hash, so that it changes from one file to another, +// so don't reveal identical attributes. +// Created: 25/4/04 +// +// -------------------------------------------------------------------------- +uint64_t BackupClientFileAttributes::GenerateAttributeHash(struct stat &st, const std::string &rFilename) +{ + if(sAttributeHashSecretLength == 0) + { + THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet) + } + + // Assemble stuff we're interested in + attributeHashData hashData; + memset(&hashData, 0, sizeof(hashData)); + // Use network byte order and large sizes to be cross platform + hashData.uid = htonl(st.st_uid); + hashData.gid = htonl(st.st_gid); + hashData.mode = htonl(st.st_mode); + + // Create a MD5 hash of the data, filename, and secret + MD5Digest digest; + digest.Add(&hashData, sizeof(hashData)); + digest.Add(rFilename.c_str(), rFilename.size()); + digest.Add(sAttributeHashSecret, sAttributeHashSecretLength); + digest.Finish(); + + // Return the first 64 bits of the hash + uint64_t result = *((uint64_t *)(digest.DigestAsData())); + return result; +} + + + diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h new file mode 100755 index 00000000..dd930684 --- /dev/null +++ b/lib/backupclient/BackupClientFileAttributes.h @@ -0,0 +1,70 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientFileAttributes.h +// Purpose: Storage of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTFILEATTRIBUTES__H +#define BACKUPCLIENTFILEATTRIBUTES__H + +#include <string> + +#include "StreamableMemBlock.h" +#include "BoxTime.h" + +struct stat; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientFileAttributes +// Purpose: Storage, streaming and application of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +class BackupClientFileAttributes : public StreamableMemBlock +{ +public: + BackupClientFileAttributes(); + BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy); + BackupClientFileAttributes(const StreamableMemBlock &rToCopy); + ~BackupClientFileAttributes(); + BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr); + BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr); + bool operator==(const BackupClientFileAttributes &rAttr) const; +// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous? + + bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const; + + // Prevent access to base class members accidently + void Set(); + + void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false, + box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0, + ino_t *pInodeNumber = 0, bool *pHasMultipleLinks = 0); + void WriteAttributes(const char *Filename) const; + + bool IsSymLink() const; + + static void SetBlowfishKey(const void *pKey, int KeyLength); + static void SetAttributeHashSecret(const void *pSecret, int SecretLength); + + static uint64_t GenerateAttributeHash(struct stat &st, const std::string &rFilename); + +private: + void ReadAttributesLink(const char *Filename, void *pst, bool ZeroModificationTimes); + + void RemoveClear() const; + void EnsureClearAvailable() const; + static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted); + void EncryptAttr(const StreamableMemBlock &rToEncrypt); + +private: + mutable StreamableMemBlock *mpClearAttributes; +}; + +#endif // BACKUPCLIENTFILEATTRIBUTES__H + diff --git a/lib/backupclient/BackupClientMakeExcludeList.cpp b/lib/backupclient/BackupClientMakeExcludeList.cpp new file mode 100755 index 00000000..0615faaa --- /dev/null +++ b/lib/backupclient/BackupClientMakeExcludeList.cpp @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientMakeExcludeList.cpp +// Purpose: Makes exclude lists from bbbackupd config location entries +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "BackupClientMakeExcludeList.h" +#include "Configuration.h" +#include "ExcludeList.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientMakeExcludeList(const Configuration &, const char *, const char *) +// Purpose: Given a Configuration object corresponding to a bbackupd Location, and the +// two names of the keys for definite and regex entries, return a ExcludeList. +// Or 0 if it isn't required. +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName, + const char *AlwaysIncludeDefiniteName, const char *AlwaysIncludeRegexName) +{ + // Check that at least one of the entries exists + if(!rConfig.KeyExists(DefiniteName) && !rConfig.KeyExists(RegexName)) + { + // Neither exists -- return 0 as an Exclude list isn't required. + return 0; + } + + // Create the exclude list + ExcludeList *pexclude = new ExcludeList; + + try + { + // Definite names to add? + if(rConfig.KeyExists(DefiniteName)) + { + pexclude->AddDefiniteEntries(rConfig.GetKeyValue(DefiniteName)); + } + // Regular expressions to add? + if(rConfig.KeyExists(RegexName)) + { + pexclude->AddRegexEntries(rConfig.GetKeyValue(RegexName)); + } + + // Add a "always include" list? + if(AlwaysIncludeDefiniteName != 0 && AlwaysIncludeRegexName != 0) + { + // This will accept NULL as a valid argument, so safe to do this. + pexclude->SetAlwaysIncludeList( + BackupClientMakeExcludeList(rConfig, AlwaysIncludeDefiniteName, AlwaysIncludeRegexName) + ); + } + } + catch(...) + { + // Clean up + delete pexclude; + throw; + } + + return pexclude; +} + + + diff --git a/lib/backupclient/BackupClientMakeExcludeList.h b/lib/backupclient/BackupClientMakeExcludeList.h new file mode 100755 index 00000000..d5dd8af4 --- /dev/null +++ b/lib/backupclient/BackupClientMakeExcludeList.h @@ -0,0 +1,48 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientMakeExcludeList.h +// Purpose: Makes exclude lists from bbbackupd config location entries +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTMAKEEXCLUDELIST__H +#define BACKUPCLIENTMAKEEXCLUDELIST__H + +class ExcludeList; +class Configuration; + +ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName, + const char *AlwaysIncludeDefiniteName = 0, const char *AlwaysIncludeRegexName = 0); + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientMakeExcludeList_Files(const Configuration &) +// Purpose: Create a exclude list from config file entries for files. May return 0. +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +inline ExcludeList *BackupClientMakeExcludeList_Files(const Configuration &rConfig) +{ + return BackupClientMakeExcludeList(rConfig, "ExcludeFile", "ExcludeFilesRegex", "AlwaysIncludeFile", "AlwaysIncludeFilesRegex"); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientMakeExcludeList_Dirs(const Configuration &) +// Purpose: Create a exclude list from config file entries for directories. May return 0. +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +inline ExcludeList *BackupClientMakeExcludeList_Dirs(const Configuration &rConfig) +{ + return BackupClientMakeExcludeList(rConfig, "ExcludeDir", "ExcludeDirsRegex", "AlwaysIncludeDir", "AlwaysIncludeDirsRegex"); +} + + +#endif // BACKUPCLIENTMAKEEXCLUDELIST__H + diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp new file mode 100755 index 00000000..ebe1d365 --- /dev/null +++ b/lib/backupclient/BackupClientRestore.cpp @@ -0,0 +1,468 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientRestore.cpp +// Purpose: +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <string> +#include <set> +#include <unistd.h> +#include <limits.h> +#include <stdio.h> + +#include "BackupClientRestore.h" +#include "autogen_BackupProtocolClient.h" +#include "CommonException.h" +#include "BackupClientFileAttributes.h" +#include "IOStream.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreFile.h" +#include "CollectInBufferStream.h" +#include "FileStream.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +#define MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES (128*1024) + +class RestoreResumeInfo +{ +public: + // constructor + RestoreResumeInfo() + : mNextLevelID(0), + mpNextLevel(0) + { + } + + // destructor + ~RestoreResumeInfo() + { + delete mpNextLevel; + mpNextLevel = 0; + } + + // Get a next level object + RestoreResumeInfo &AddLevel(int64_t ID, const std::string &rLocalName) + { + ASSERT(mpNextLevel == 0 && mNextLevelID == 0); + mpNextLevel = new RestoreResumeInfo; + mNextLevelID = ID; + mNextLevelLocalName = rLocalName; + return *mpNextLevel; + } + + // Remove the next level info + void RemoveLevel() + { + ASSERT(mpNextLevel != 0 && mNextLevelID != 0); + delete mpNextLevel; + mpNextLevel = 0; + mNextLevelID = 0; + mNextLevelLocalName.erase(); + } + + void Save(const std::string &rFilename) const + { + // TODO: use proper buffered streams when they're done + // Build info in memory buffer + CollectInBufferStream write; + + // Save this level + SaveLevel(write); + + // Store in file + write.SetForReading(); + FileStream file(rFilename.c_str(), O_WRONLY | O_CREAT); + write.CopyStreamTo(file, IOStream::TimeOutInfinite, 8*1024 /* large buffer */); + } + + void SaveLevel(IOStream &rWrite) const + { + // Write the restored objects + int64_t numObjects = mRestoredObjects.size(); + rWrite.Write(&numObjects, sizeof(numObjects)); + for(std::set<int64_t>::const_iterator i(mRestoredObjects.begin()); i != mRestoredObjects.end(); ++i) + { + int64_t id = *i; + rWrite.Write(&id, sizeof(id)); + } + + // Next level? + if(mpNextLevel != 0) + { + // ID + rWrite.Write(&mNextLevelID, sizeof(mNextLevelID)); + // Name string + int32_t nsize = mNextLevelLocalName.size(); + rWrite.Write(&nsize, sizeof(nsize)); + rWrite.Write(mNextLevelLocalName.c_str(), nsize); + // And then the level itself + mpNextLevel->SaveLevel(rWrite); + } + else + { + // Just write a zero + int64_t zero = 0; + rWrite.Write(&zero, sizeof(zero)); + } + } + + // Not written to be efficient -- shouldn't be called very often. + bool Load(const std::string &rFilename) + { + // Delete and reset if necessary + if(mpNextLevel != 0) + { + RemoveLevel(); + } + + // Open file + FileStream file(rFilename.c_str()); + + // Load this level + return LoadLevel(file); + } + + #define CHECKED_READ(x, s) if(!rRead.ReadFullBuffer(x, s, 0)) {return false;} + bool LoadLevel(IOStream &rRead) + { + // Load the restored objects list + mRestoredObjects.clear(); + int64_t numObjects = 0; + CHECKED_READ(&numObjects, sizeof(numObjects)); + for(int64_t o = 0; o < numObjects; ++o) + { + int64_t id; + CHECKED_READ(&id, sizeof(id)); + mRestoredObjects.insert(id); + } + + // ID of next level? + int64_t nextID = 0; + CHECKED_READ(&nextID, sizeof(nextID)); + if(nextID != 0) + { + // Load the next level! + std::string name; + int32_t nsize = 0; + CHECKED_READ(&nsize, sizeof(nsize)); + char n[PATH_MAX]; + if(nsize > PATH_MAX) return false; + CHECKED_READ(n, nsize); + name.assign(n, nsize); + + // Create a new level + mpNextLevel = new RestoreResumeInfo; + mNextLevelID = nextID; + mNextLevelLocalName = name; + + // And ask it to read itself in + if(!mpNextLevel->LoadLevel(rRead)) + { + return false; + } + } + + return true; + } + + // List of objects at this level which have been done already + std::set<int64_t> mRestoredObjects; + // Next level ID + int64_t mNextLevelID; + // Pointer to next level + RestoreResumeInfo *mpNextLevel; + // Local filename of next level + std::string mNextLevelLocalName; +}; + +// parameters structure +typedef struct +{ + bool PrintDots; + bool RestoreDeleted; + std::string mRestoreResumeInfoFilename; + RestoreResumeInfo mResumeInfo; +} RestoreParams; + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientRestoreDir(BackupProtocolClient &, int64_t, const char *, bool) +// Purpose: Restore a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t DirectoryID, std::string &rLocalDirectoryName, + RestoreParams &Params, RestoreResumeInfo &rLevel) +{ + // If we're resuming... check that we haven't got a next level to look at + if(rLevel.mpNextLevel != 0) + { + // Recurse immediately + std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + rLevel.mNextLevelLocalName); + BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, localDirname, Params, *rLevel.mpNextLevel); + + // Add it to the list of done itmes + rLevel.mRestoredObjects.insert(rLevel.mNextLevelID); + + // Remove the level for the recursed directory + rLevel.RemoveLevel(); + } + + // Save the resumption information + Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + + // Create the local directory (if not already done) -- path and owner set later, just use restrictive owner mode + switch(ObjectExists(rLocalDirectoryName.c_str())) + { + case ObjectExists_Dir: + // Do nothing + break; + case ObjectExists_File: + { + // File exists with this name, which is fun. Get rid of it. + ::printf("WARNING: File present with name '%s', removing out of the way of restored directory. Use specific restore with ID to restore this object.", rLocalDirectoryName.c_str()); + if(::unlink(rLocalDirectoryName.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + TRACE1("In restore, directory name collision with file %s", rLocalDirectoryName.c_str()); + } + // follow through to... (no break) + case ObjectExists_NoObject: + if(::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + break; + default: + ASSERT(false); + break; + } + + // Fetch the directory listing from the server -- getting a list of files which is approparite to the restore type + rConnection.QueryListDirectory( + DirectoryID, + Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING), + BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)), + true /* want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(rConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, rConnection.GetTimeout()); + + // Apply attributes to the directory + const StreamableMemBlock &dirAttrBlock(dir.GetAttributes()); + BackupClientFileAttributes dirAttr(dirAttrBlock); + dirAttr.WriteAttributes(rLocalDirectoryName.c_str()); + + int64_t bytesWrittenSinceLastRestoreInfoSave = 0; + + // Process files + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0) + { + // Check ID hasn't already been done + if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end()) + { + // Local name + BackupStoreFilenameClear nm(en->GetName()); + std::string localFilename(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename()); + + // Unlink anything which already exists -- for resuming restores, we can't overwrite files already there. + ::unlink(localFilename.c_str()); + + // Request it from the store + rConnection.QueryGetFile(DirectoryID, en->GetObjectID()); + + // Stream containing encoded file + std::auto_ptr<IOStream> objectStream(rConnection.ReceiveStream()); + + // Decode the file -- need to do different things depending on whether + // the directory entry has additional attributes + if(en->HasAttributes()) + { + // Use these attributes + const StreamableMemBlock &storeAttr(en->GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout(), &attr); + } + else + { + // Use attributes stored in file + BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout()); + } + + // Progress display? + if(Params.PrintDots) + { + printf("."); + fflush(stdout); + } + + // Add it to the list of done itmes + rLevel.mRestoredObjects.insert(en->GetObjectID()); + + // Save restore info? + int64_t fileSize; + if(FileExists(localFilename.c_str(), &fileSize, true /* treat links as not existing */)) + { + // File exists... + bytesWrittenSinceLastRestoreInfoSave += fileSize; + + if(bytesWrittenSinceLastRestoreInfoSave > MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES) + { + // Save the restore info, in case it's needed later + Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + bytesWrittenSinceLastRestoreInfoSave = 0; + } + } + } + } + } + + // Make sure the restore info has been saved + if(bytesWrittenSinceLastRestoreInfoSave != 0) + { + // Save the restore info, in case it's needed later + Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + bytesWrittenSinceLastRestoreInfoSave = 0; + } + + + // Recuse to directories + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0) + { + // Check ID hasn't already been done + if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end()) + { + // Local name + BackupStoreFilenameClear nm(en->GetName()); + std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename()); + + // Add the level for the next entry + RestoreResumeInfo &rnextLevel(rLevel.AddLevel(en->GetObjectID(), nm.GetClearFilename())); + + // Recurse + BackupClientRestoreDir(rConnection, en->GetObjectID(), localDirname, Params, rnextLevel); + + // Remove the level for the above call + rLevel.RemoveLevel(); + + // Add it to the list of done itmes + rLevel.mRestoredObjects.insert(en->GetObjectID()); + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientRestore(BackupProtocolClient &, int64_t, const char *, bool, bool) +// Purpose: Restore a directory on the server to a local directory on the disc. +// +// The local directory must not already exist. +// +// If a restore is aborted for any reason, then it may be resumed if +// Resume == true. If Resume == false and resumption is possible, then +// Restore_ResumePossible is returned. +// +// Set RestoreDeleted to restore a deleted directory. This may not give the +// directory structure when it was deleted, because files may have been deleted +// within it before it was deleted. +// +// Returns Restore_TargetExists if the target directory exists, but +// there is no restore possible. (Won't attempt to overwrite things.) +// +// Returns Restore_Complete on success. (Exceptions on error.) +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName, + bool PrintDots, bool RestoreDeleted, bool UndeleteAfterRestoreDeleted, bool Resume) +{ + // Parameter block + RestoreParams params; + params.PrintDots = PrintDots; + params.RestoreDeleted = RestoreDeleted; + params.mRestoreResumeInfoFilename = LocalDirectoryName; + params.mRestoreResumeInfoFilename += ".boxbackupresume"; + + // Target exists? + int targetExistance = ObjectExists(LocalDirectoryName); + + // Does any resumption information exist? + bool doingResume = false; + if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && targetExistance == ObjectExists_Dir) + { + if(!Resume) + { + // Caller didn't specify that resume should be done, so refuse to do it + // but say why. + return Restore_ResumePossible; + } + + // Attempt to load the resume info file + if(!params.mResumeInfo.Load(params.mRestoreResumeInfoFilename)) + { + // failed -- bad file, so things have gone a bit wrong + return Restore_TargetExists; + } + + // Flag as doing resume so next check isn't actually performed + doingResume = true; + } + + // Does the directory already exist? + if(targetExistance != ObjectExists_NoObject && !doingResume) + { + // Don't do anything in this case! + return Restore_TargetExists; + } + + // Restore the directory + std::string localName(LocalDirectoryName); + BackupClientRestoreDir(rConnection, DirectoryID, localName, params, params.mResumeInfo); + + // Undelete the directory on the server? + if(RestoreDeleted && UndeleteAfterRestoreDeleted) + { + // Send the command + rConnection.QueryUndeleteDirectory(DirectoryID); + } + + // Finish progress display? + if(PrintDots) + { + printf("\n"); + fflush(stdout); + } + + // Delete the resume information file + ::unlink(params.mRestoreResumeInfoFilename.c_str()); + + return Restore_Complete; +} + + + + diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h new file mode 100755 index 00000000..4b382771 --- /dev/null +++ b/lib/backupclient/BackupClientRestore.h @@ -0,0 +1,26 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientRestore.h +// Purpose: Functions to restore files from a backup store +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSCLIENTRESTORE_H +#define BACKUPSCLIENTRESTORE__H + +class BackupProtocolClient; + +enum +{ + Restore_Complete = 0, + Restore_ResumePossible = 1, + Restore_TargetExists = 2 +}; + +int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName, + bool PrintDots = false, bool RestoreDeleted = false, bool UndeleteAfterRestoreDeleted = false, bool Resume = false); + +#endif // BACKUPSCLIENTRESTORE__H + diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp new file mode 100755 index 00000000..eec2ceaf --- /dev/null +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -0,0 +1,105 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemonConfigVerify.cpp +// Purpose: Configuration file definition for bbackupd +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupDaemonConfigVerify.h" +#include "Daemon.h" +#include "BoxPortsAndFiles.h" + +#include "MemLeakFindOn.h" + + +static const ConfigurationVerifyKey backuplocationkeys[] = +{ + {"ExcludeFile", 0, ConfigTest_MultiValueAllowed, 0}, + {"ExcludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0}, + {"ExcludeDir", 0, ConfigTest_MultiValueAllowed, 0}, + {"ExcludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0}, + {"AlwaysIncludeFile", 0, ConfigTest_MultiValueAllowed, 0}, + {"AlwaysIncludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0}, + {"AlwaysIncludeDir", 0, ConfigTest_MultiValueAllowed, 0}, + {"AlwaysIncludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0}, + {"Path", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} +}; + +static const ConfigurationVerify backuplocations[] = +{ + { + "*", + 0, + backuplocationkeys, + ConfigTest_LastEntry, + 0 + } +}; + +static const ConfigurationVerifyKey verifyserverkeys[] = +{ + DAEMON_VERIFY_SERVER_KEYS +}; + +static const ConfigurationVerify verifyserver[] = +{ + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists, + 0 + }, + { + "BackupLocations", + backuplocations, + 0, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } +}; + +static const ConfigurationVerifyKey verifyrootkeys[] = +{ + {"AccountNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + + {"UpdateStoreInterval", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"MinimumFileAge", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"MaxUploadWait", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"MaxFileTimeInFuture", "172800", ConfigTest_IsInt, 0}, // file is uploaded if the file is this much in the future (2 days default) + + {"AutomaticBackup", "yes", ConfigTest_IsBool, 0}, + + {"SyncAllowScript", 0, 0, 0}, // optional script to run to see if the sync should be started now + // return "now" if it's allowed, or a number of seconds if it's not + + {"MaximumDiffingTime", 0, ConfigTest_IsInt, 0}, + + {"FileTrackingSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"DiffingUploadSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"StoreHostname", 0, ConfigTest_Exists, 0}, + {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file + + {"CommandSocket", 0, 0, 0}, // not compulsory to have this + + {"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full + + {"CertificateFile", 0, ConfigTest_Exists, 0}, + {"PrivateKeyFile", 0, ConfigTest_Exists, 0}, + {"TrustedCAsFile", 0, ConfigTest_Exists, 0}, + {"KeysFile", 0, ConfigTest_Exists, 0}, + {"DataDirectory", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} +}; + +const ConfigurationVerify BackupDaemonConfigVerify = +{ + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 +}; + diff --git a/lib/backupclient/BackupDaemonConfigVerify.h b/lib/backupclient/BackupDaemonConfigVerify.h new file mode 100755 index 00000000..fefa703c --- /dev/null +++ b/lib/backupclient/BackupDaemonConfigVerify.h @@ -0,0 +1,18 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemonConfigVerify.h +// Purpose: Configuration file definition for bbackupd +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPDAEMONCONFIGVERIFY__H +#define BACKUPDAEMONCONFIGVERIFY__H + +#include "Configuration.h" + +extern const ConfigurationVerify BackupDaemonConfigVerify; + +#endif // BACKUPDAEMONCONFIGVERIFY__H + diff --git a/lib/backupclient/BackupStoreConstants.h b/lib/backupclient/BackupStoreConstants.h new file mode 100755 index 00000000..8254e918 --- /dev/null +++ b/lib/backupclient/BackupStoreConstants.h @@ -0,0 +1,53 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreContants.h +// Purpose: constants for the backup system +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTORECONSTANTS__H +#define BACKUPSTORECONSTANTS__H + +#define BACKUPSTORE_ROOT_DIRECTORY_ID 1 + +#define BACKUP_STORE_SERVER_VERSION 1 + +// Minimum size for a chunk to be compressed +#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256 + +// min and max sizes for blocks +#define BACKUP_FILE_MIN_BLOCK_SIZE 4096 +#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024) + +// Increase the block size if there are more than this number of blocks +#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096 + +// Avoid creating blocks smaller than this +#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128 + +// Maximum number of sizes to do an rsync-like scan for +#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 8 + +// When doing rsync scans, do not scan for blocks smaller than +#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 256 + +// A limit to stop diffing running out of control: If more than this +// times the number of blocks in the original index are found, stop +// looking. This stops really bad cases of diffing files containing +// all the same byte using huge amounts of memory and processor time. +// This is a multiple of the number of blocks in the diff from file. +#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096 + +// How many seconds to wait before deleting unused root directory entries? +#ifndef NDEBUG + // Debug: 30 seconds (easier to test) + #define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER 30 +#else + // Release: 2 days (plenty of time for sysadmins to notice, or change their mind) + #define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER 172800 +#endif + +#endif // BACKUPSTORECONSTANTS__H + diff --git a/lib/backupclient/BackupStoreDirectory.cpp b/lib/backupclient/BackupStoreDirectory.cpp new file mode 100755 index 00000000..31fed78d --- /dev/null +++ b/lib/backupclient/BackupStoreDirectory.cpp @@ -0,0 +1,563 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory.h +// Purpose: Representation of a backup directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> + +#include "BackupStoreDirectory.h" +#include "IOStream.h" +#include "BackupStoreException.h" +#include "BackupStoreObjectMagic.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mNumEntries; + int64_t mObjectID; // this object ID + int64_t mContainerID; // ID of container + uint64_t mAttributesModTime; + int32_t mOptionsPresent; // bit mask of optional sections / features present + // Then a StreamableMemBlock for attributes +} dir_StreamFormat; + +enum +{ + Option_DependencyInfoPresent = 1 +} dir_StreamFormatOptions; + +typedef struct +{ + uint64_t mModificationTime; + int64_t mObjectID; + int64_t mSizeInBlocks; + uint64_t mAttributesHash; + int16_t mFlags; // order smaller items after bigger ones (for alignment) + // Then a BackupStoreFilename + // Then a StreamableMemBlock for attributes +} en_StreamFormat; + +typedef struct +{ + int64_t mDependsNewer; + int64_t mDependsOlder; +} en_StreamFormatDepends; + +// Use default packing +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::BackupStoreDirectory() +// Purpose: Constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::BackupStoreDirectory() + : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0) +{ + ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); +} + + +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t) +// Purpose: Constructor giving object and container IDs +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID) + : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::~BackupStoreDirectory() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::~BackupStoreDirectory() +{ + for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) + { + delete (*i); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int) +// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Get the header + dir_StreamFormat hdr; + if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic value... + if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue)) + { + THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat) + } + + // Get data + mObjectID = ntoh64(hdr.mObjectID); + mContainerID = ntoh64(hdr.mContainerID); + mAttributesModTime = ntoh64(hdr.mAttributesModTime); + + // Options + int32_t options = ntohl(hdr.mOptionsPresent); + + // Get attributes + mAttributes.ReadFromStream(rStream, Timeout); + + // Decode count + int count = ntohl(hdr.mNumEntries); + + // Clear existing list + mEntries.clear(); + + // Read them in! + for(int c = 0; c < count; ++c) + { + Entry *pen = new Entry; + try + { + // Read from stream + pen->ReadFromStream(rStream, Timeout); + + // Add to list + mEntries.push_back(pen); + } + catch(...) + { + delete pen; + throw; + } + } + + // Read in dependency info? + if(options & Option_DependencyInfoPresent) + { + // Read in extra dependency data + for(int c = 0; c < count; ++c) + { + mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool) +// Purpose: Writes a selection of entries to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const +{ + // Get count of entries + int32_t count = mEntries.size(); + if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING) + { + // Need to count the entries + count = 0; + Iterator i(*this); + while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0) + { + count++; + } + } + + // Check that sensible IDs have been set + ASSERT(mObjectID != 0); + ASSERT(mContainerID != 0); + + // Need dependency info? + bool dependencyInfoRequired = false; + if(StreamDependencyInfo) + { + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + if(pen->HasDependencies()) + { + dependencyInfoRequired = true; + } + } + } + + // Options + int32_t options = 0; + if(dependencyInfoRequired) options |= Option_DependencyInfoPresent; + + // Build header + dir_StreamFormat hdr; + hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE); + hdr.mNumEntries = htonl(count); + hdr.mObjectID = hton64(mObjectID); + hdr.mContainerID = hton64(mContainerID); + hdr.mAttributesModTime = hton64(mAttributesModTime); + hdr.mOptionsPresent = htonl(options); + + // Write header + rStream.Write(&hdr, sizeof(hdr)); + + // Write the attributes? + if(StreamAttributes) + { + mAttributes.WriteToStream(rStream); + } + else + { + // Write a blank header instead + StreamableMemBlock::WriteEmptyBlockToStream(rStream); + } + + // Then write all the entries + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + pen->WriteToStream(rStream); + } + + // Write dependency info? + if(dependencyInfoRequired) + { + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + pen->WriteToStreamDependencyInfo(rStream); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddEntry(const Entry &) +// Purpose: Adds entry to directory (no checking) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy) +{ + Entry *pnew = new Entry(rEntryToCopy); + try + { + mEntries.push_back(pnew); + } + catch(...) + { + delete pnew; + throw; + } + + return pnew; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t) +// Purpose: Adds entry to directory (no checking) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime) +{ + Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime); + try + { + mEntries.push_back(pnew); + } + catch(...) + { + delete pnew; + throw; + } + + return pnew; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::DeleteEntry(int64_t) +// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) +{ + for(std::vector<Entry*>::iterator i(mEntries.begin()); + i != mEntries.end(); ++i) + { + if((*i)->mObjectID == ObjectID) + { + // Delete + delete (*i); + // Remove from list + mEntries.erase(i); + // Done + return; + } + } + + // Not found + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::FindEntryByID(int64_t) +// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist. +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const +{ + for(std::vector<Entry*>::const_iterator i(mEntries.begin()); + i != mEntries.end(); ++i) + { + if((*i)->mObjectID == ObjectID) + { + // Found + return (*i); + } + } + + // Not found + return 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry() +// Purpose: Constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry() + : mModificationTime(0), + mObjectID(0), + mSizeInBlocks(0), + mFlags(0), + mAttributesHash(0), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::~Entry() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::~Entry() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry(const Entry &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) + : mName(rToCopy.mName), + mModificationTime(rToCopy.mModificationTime), + mObjectID(rToCopy.mObjectID), + mSizeInBlocks(rToCopy.mSizeInBlocks), + mFlags(rToCopy.mFlags), + mAttributesHash(rToCopy.mAttributesHash), + mAttributes(rToCopy.mAttributes), + mMinMarkNumber(rToCopy.mMinMarkNumber), + mMarkNumber(rToCopy.mMarkNumber), + mDependsNewer(rToCopy.mDependsNewer), + mDependsOlder(rToCopy.mDependsOlder) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t) +// Purpose: Constructor from values +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) + : mName(rName), + mModificationTime(ModificationTime), + mObjectID(ObjectID), + mSizeInBlocks(SizeInBlocks), + mFlags(Flags), + mAttributesHash(AttributesHash), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int) +// Purpose: Read an entry from a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Grab the raw bytes from the stream which compose the header + en_StreamFormat entry; + if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Do reading first before modifying the variables, to be more exception safe + + // Get the filename + BackupStoreFilename name; + name.ReadFromStream(rStream, Timeout); + + // Get the attributes + mAttributes.ReadFromStream(rStream, Timeout); + + // Store the rest of the bits + mModificationTime = ntoh64(entry.mModificationTime); + mObjectID = ntoh64(entry.mObjectID); + mSizeInBlocks = ntoh64(entry.mSizeInBlocks); + mAttributesHash = ntoh64(entry.mAttributesHash); + mFlags = ntohs(entry.mFlags); + mName = name; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &) +// Purpose: Writes the entry to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const +{ + // Build a structure + en_StreamFormat entry; + entry.mModificationTime = hton64(mModificationTime); + entry.mObjectID = hton64(mObjectID); + entry.mSizeInBlocks = hton64(mSizeInBlocks); + entry.mAttributesHash = hton64(mAttributesHash); + entry.mFlags = htons(mFlags); + + // Write it + rStream.Write(&entry, sizeof(entry)); + + // Write the filename + mName.WriteToStream(rStream); + + // Write any attributes + mAttributes.WriteToStream(rStream); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int) +// Purpose: Read the optional dependency info from a stream +// Created: 13/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout) +{ + // Grab the raw bytes from the stream which compose the header + en_StreamFormatDepends depends; + if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Store the data + mDependsNewer = ntoh64(depends.mDependsNewer); + mDependsOlder = ntoh64(depends.mDependsOlder); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &) +// Purpose: Write the optional dependency info to a stream +// Created: 13/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const +{ + // Build structure + en_StreamFormatDepends depends; + depends.mDependsNewer = hton64(mDependsNewer); + depends.mDependsOlder = hton64(mDependsOlder); + // Write + rStream.Write(&depends, sizeof(depends)); +} + + + diff --git a/lib/backupclient/BackupStoreDirectory.h b/lib/backupclient/BackupStoreDirectory.h new file mode 100755 index 00000000..958eee81 --- /dev/null +++ b/lib/backupclient/BackupStoreDirectory.h @@ -0,0 +1,268 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory.h +// Purpose: Representation of a backup directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREDIRECTORY__H +#define BACKUPSTOREDIRECTORY__H + +#include <string> +#include <vector> + +#include "BackupStoreFilenameClear.h" +#include "StreamableMemBlock.h" +#include "BoxTime.h" + +class IOStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreDirectory +// Purpose: In memory representation of a directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreDirectory +{ +public: + BackupStoreDirectory(); + BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID); +private: + // Copying not allowed + BackupStoreDirectory(const BackupStoreDirectory &rToCopy); +public: + ~BackupStoreDirectory(); + + class Entry + { + public: + friend class BackupStoreDirectory; + + Entry(); + ~Entry(); + Entry(const Entry &rToCopy); + Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash); + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream) const; + + const BackupStoreFilename &GetName() const {return mName;} + box_time_t GetModificationTime() const {return mModificationTime;} + int64_t GetObjectID() const {return mObjectID;} + int64_t GetSizeInBlocks() const {return mSizeInBlocks;} + int16_t GetFlags() const {return mFlags;} + void AddFlags(int16_t Flags) {mFlags |= Flags;} + void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;} + + // Some things can be changed + void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;} + void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;} + + // Attributes + bool HasAttributes() const {return !mAttributes.IsEmpty();} + void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;} + const StreamableMemBlock &GetAttributes() const {return mAttributes;} + uint64_t GetAttributesHash() const {return mAttributesHash;} + + // Marks + // The lowest mark number a version of a file of this name has ever had + uint32_t GetMinMarkNumber() const {return mMinMarkNumber;} + // The mark number on this file + uint32_t GetMarkNumber() const {return mMarkNumber;} + + // Make sure these flags are synced with those in backupprocotol.txt + // ListDirectory command + enum + { + Flags_INCLUDE_EVERYTHING = -1, + Flags_EXCLUDE_NOTHING = 0, + Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below! + Flags_File = 1, + Flags_Dir = 2, + Flags_Deleted = 4, + Flags_OldVersion = 8, + Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion + }; + // characters for textual listing of files -- see bbackupquery/BackupQueries + #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR" + + bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) + { + return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet)) + && ((mFlags & FlagsNotToBeSet) == 0); + }; + + // Get dependency info + // new version this depends on + int64_t GetDependsNewer() const {return mDependsNewer;} + void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;} + // older version which depends on this + int64_t GetDependsOlder() const {return mDependsOlder;} + void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;} + + // Dependency info saving + bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;} + void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout); + void WriteToStreamDependencyInfo(IOStream &rStream) const; + + private: + BackupStoreFilename mName; + box_time_t mModificationTime; + int64_t mObjectID; + int64_t mSizeInBlocks; + int16_t mFlags; + uint64_t mAttributesHash; + StreamableMemBlock mAttributes; + uint32_t mMinMarkNumber; + uint32_t mMarkNumber; + + uint64_t mDependsNewer; // new version this depends on + uint64_t mDependsOlder; // older version which depends on this + }; + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream, + int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, + int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING, + bool StreamAttributes = true, bool StreamDependencyInfo = true) const; + + Entry *AddEntry(const Entry &rEntryToCopy); + Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime); + void DeleteEntry(int64_t ObjectID); + Entry *FindEntryByID(int64_t ObjectID) const; + + int64_t GetObjectID() const {return mObjectID;} + int64_t GetContainerID() const {return mContainerID;} + + // Need to be able to update the container ID when moving objects + void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;} + + // Purely for use of server -- not serialised into streams + int64_t GetRevisionID() const {return mRevisionID;} + void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;} + + unsigned int GetNumberOfEntries() const {return mEntries.size();} + + // User info -- not serialised into streams + int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;} + void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;} + + // Attributes + bool HasAttributes() const {return !mAttributes.IsEmpty();} + void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;} + const StreamableMemBlock &GetAttributes() const {return mAttributes;} + box_time_t GetAttributesModTime() const {return mAttributesModTime;} + + class Iterator + { + public: + Iterator(const BackupStoreDirectory &rDir) + : mrDir(rDir), i(rDir.mEntries.begin()) + { + } + + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags + while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.end()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + // WARNING: This function is really very inefficient. + // Only use when you want to look up ONE filename, not in a loop looking up lots. + // In a looping situation, cache the decrypted filenames in another memory structure. + BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags or filename + while( (i != mrDir.mEntries.end()) + && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) ) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.end()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + private: + const BackupStoreDirectory &mrDir; + std::vector<Entry*>::const_iterator i; + }; + + friend class Iterator; + + class ReverseIterator + { + public: + ReverseIterator(const BackupStoreDirectory &rDir) + : mrDir(rDir), i(rDir.mEntries.rbegin()) + { + } + + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags + while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.rend()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + private: + const BackupStoreDirectory &mrDir; + std::vector<Entry*>::const_reverse_iterator i; + }; + + friend class ReverseIterator; + + // For recovery of the store + // Implemented in BackupStoreCheck2.cpp + bool CheckAndFix(); + void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags); + bool NameInUse(const BackupStoreFilename &rName); + // Don't use these functions in normal code! + + // For testing + void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;} + + // Debug and diagonistics + void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere + +private: + int64_t mRevisionID; + int64_t mObjectID; + int64_t mContainerID; + std::vector<Entry*> mEntries; + box_time_t mAttributesModTime; + StreamableMemBlock mAttributes; + int64_t mUserInfo1; +}; + +#endif // BACKUPSTOREDIRECTORY__H + diff --git a/lib/backupclient/BackupStoreException.h b/lib/backupclient/BackupStoreException.h new file mode 100755 index 00000000..981dfa60 --- /dev/null +++ b/lib/backupclient/BackupStoreException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREEXCEPTION__H +#define BACKUPSTOREEXCEPTION__H + +// Compatibility +#include "autogen_BackupStoreException.h" + +#endif // BACKUPSTOREEXCEPTION__H + diff --git a/lib/backupclient/BackupStoreException.txt b/lib/backupclient/BackupStoreException.txt new file mode 100644 index 00000000..50c615b2 --- /dev/null +++ b/lib/backupclient/BackupStoreException.txt @@ -0,0 +1,70 @@ +EXCEPTION BackupStore 4 + +Internal 0 +BadAccountDatabaseFile 1 +AccountDatabaseNoSuchEntry 2 +InvalidBackupStoreFilename 3 +UnknownFilenameEncoding 4 +CouldntReadEntireStructureFromStream 5 +BadDirectoryFormat 6 +CouldNotFindEntryInDirectory 7 +OutputFileAlreadyExists 8 +OSFileError 9 +StreamDoesntHaveRequiredFeatures 10 +BadBackupStoreFile 11 +CouldNotLoadStoreInfo 12 +BadStoreInfoOnLoad 13 +StoreInfoIsReadOnly 14 +StoreInfoDirNotInList 15 +StoreInfoBlockDeltaMakesValueNegative 16 +DirectoryHasBeenDeleted 17 +StoreInfoNotInitialised 18 +StoreInfoAlreadyLoaded 19 +StoreInfoNotLoaded 20 +ReadFileFromStreamTimedOut 21 +FileWrongSizeAfterBeingStored 22 +AddedFileDoesNotVerify 23 +StoreInfoForWrongAccount 24 +ContextIsReadOnly 25 +AttributesNotLoaded 26 +AttributesNotUnderstood 27 +WrongServerVersion 28 # client side +ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account. +NameAlreadyExistsInDirectory 30 +BerkelyDBFailure 31 # client side +InodeMapIsReadOnly 32 # client side +InodeMapNotOpen 33 # client side +FilenameEncryptionKeyNotKnown 34 +FilenameEncryptionNoKeyForSpecifiedMethod 35 +FilenameEncryptionNotSetup 36 +CouldntLoadClientKeyMaterial 37 +BadEncryptedAttributes 38 +EncryptedAttributesHaveUnknownEncoding 39 +OutputSizeTooSmallForChunk 40 +BadEncodedChunk 41 +NotEnoughSpaceToDecodeChunk 42 +ChunkHasUnknownEncoding 43 +ChunkContainsBadCompressedData 44 +CantWriteToEncodedFileStream 45 +Temp_FileEncodeStreamDidntReadBuffer 46 +CantWriteToDecodedFileStream 47 +WhenDecodingExpectedToReadButCouldnt 48 +BackupStoreFileFailedIntegrityCheck 49 +ThereIsNoDataInASymLink 50 +IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51 +BlockEntryEncodingDidntGiveExpectedLength 52 +CouldNotFindUnusedIDDuringAllocation 53 +AddedFileExceedsStorageLimit 54 +CannotDiffAnIncompleteStoreFile 55 +CannotDecodeDiffedFilesWithoutCombining 56 +FailedToReadBlockOnCombine 57 +OnCombineFromFileIsIncomplete 58 +BadNotifySysadminEventCode 59 +InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60 +CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server? +AttributeHashSecretNotSet 62 +AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES +SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection. +IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related +DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory +PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp new file mode 100755 index 00000000..2ea877b4 --- /dev/null +++ b/lib/backupclient/BackupStoreFile.cpp @@ -0,0 +1,1499 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFile.cpp +// Purpose: Utils for manipulating files +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <unistd.h> +#include <sys/stat.h> +#include <string.h> +#include <new> +#include <string.h> +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + #include <syslog.h> + #include <stdio.h> +#endif + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFilename.h" +#include "BackupStoreException.h" +#include "IOStream.h" +#include "Guards.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreObjectMagic.h" +#include "Compress.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "CipherAES.h" +#include "BackupStoreConstants.h" +#include "CollectInBufferStream.h" +#include "RollingChecksum.h" +#include "MD5Digest.h" +#include "ReadGatherStream.h" +#include "Random.h" +#include "BackupStoreFileEncodeStream.h" + +#include "MemLeakFindOn.h" + +using namespace BackupStoreFileCryptVar; + +// How big a buffer to use for copying files +#define COPY_BUFFER_SIZE (8*1024) + +// Statistics +BackupStoreFileStats BackupStoreFile::msStats = {0,0,0}; + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool sWarnedAboutBackwardsCompatiblity = false; +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &) +// Purpose: Encode a file into something for storing on file server. +// Requires a real filename so full info can be stored. +// +// Returns a stream. Most of the work is done by the stream +// when data is actually requested -- the file will be held +// open until the stream is deleted or the file finished. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime) +{ + // Create the stream + std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); + + // Do the initial setup + ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, 0 /* no recipe, just encode */, + ContainerID, rStoreFilename, pModificationTime); + + // Return the stream for the caller + return stream; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &) +// Purpose: Verify that an encoded file meets the format requirements. +// Doesn't verify that the data is intact and can be decoded. +// Optionally returns the ID of the file which it is diffed from, +// and the (original) container ID. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut) +{ + // Get the size of the file + int64_t fileSize = rFile.BytesLeftToRead(); + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Get the header... + file_StreamFormat hdr; + if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read header + return false; + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + return false; + } + + // Get a filename, see if it loads OK + try + { + BackupStoreFilename fn; + fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); + } + catch(...) + { + // an error occured while reading it, so that's not good + return false; + } + + // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not + try + { + int32_t size_s; + if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + int size = ntohl(size_s); + // Skip forward the size + rFile.Seek(size, IOStream::SeekType_Relative); + } + catch(...) + { + // an error occured while reading it, so that's not good + return false; + } + + // Get current position in file -- the end of the header + int64_t headerEnd = rFile.GetPosition(); + + // Get number of blocks + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + + // Calculate where the block index will be, check it's reasonable + int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + if(blockIndexLoc < headerEnd) + { + // Not enough space left for the block index, let alone the blocks themselves + return false; + } + + // Load the block index header + rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute); + file_BlockIndexHeader blkhdr; + if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read block index header -- assume bad file + return false; + } + + // Check header + if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + || (int64_t)ntoh64(blkhdr.mNumBlocks) != numBlocks) + { + // Bad header -- either magic value or number of blocks is wrong + return false; + } + + // Flag for recording whether a block is referenced from another file + bool blockFromOtherFileReferenced = false; + + // Read the index, checking that the length values all make sense + int64_t currentBlockStart = headerEnd; + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read block entry + file_BlockIndexEntry blk; + if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read block index entry -- assume bad file + return false; + } + + // Check size and location + int64_t blkSize = ntoh64(blk.mEncodedSize); + if(blkSize <= 0) + { + // Mark that this file references another file + blockFromOtherFileReferenced = true; + } + else + { + // This block is actually in this file + if((currentBlockStart + blkSize) > blockIndexLoc) + { + // Encoded size makes the block run over the index + return false; + } + + // Move the current block start ot the end of this block + currentBlockStart += blkSize; + } + } + + // Check that there's no empty space + if(currentBlockStart != blockIndexLoc) + { + return false; + } + + // Check that if another block is references, then the ID is there, and if one isn't there is no ID. + int64_t otherID = ntoh64(blkhdr.mOtherFileID); + if((otherID != 0 && blockFromOtherFileReferenced == false) + || (otherID == 0 && blockFromOtherFileReferenced == true)) + { + // Doesn't look good! + return false; + } + + // Does the caller want the other ID? + if(pDiffFromObjectIDOut) + { + *pDiffFromObjectIDOut = otherID; + } + + // Does the caller want the container ID? + if(pContainerIDOut) + { + *pContainerIDOut = ntoh64(hdr.mContainerID); + } + + // Passes all tests + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeFile(IOStream &, const char *) +// Purpose: Decode a file. Will set file attributes. File must not exist. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) +{ + // Does file exist? + struct stat st; + if(::stat(DecodedFilename, &st) == 0) + { + THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) + } + + // Try, delete output file if error + try + { + // Make a stream for outputting this file + FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL); + + // Get the decoding stream + std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr)); + + // Is it a symlink? + if(!stream->IsSymLink()) + { + // Copy it out to the file + stream->CopyStreamTo(out); + } + + // Write the attributes + stream->GetAttributes().WriteAttributes(DecodedFilename); + } + catch(...) + { + ::unlink(DecodedFilename); + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *) +// Purpose: Return a stream which will decode the encrypted file data on the fly. +// Accepts streams in block index first, or main header first, order. In the latter case, +// the stream must be Seek()able. +// +// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow +// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) +{ + // Create stream + std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout)); + + // Get it ready + stream->Setup(pAlterativeAttr); + + // Return to caller + return stream; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int) +// Purpose: Constructor +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout) + : mrEncodedFile(rEncodedFile), + mTimeout(Timeout), + mNumBlocks(0), + mpBlockIndex(0), + mpEncodedData(0), + mpClearData(0), + mClearDataSize(0), + mCurrentBlock(-1), + mCurrentBlockClearSize(0), + mPositionInCurrentBlock(0), + mEntryIVBase(42) // different to default value in the encoded stream! +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + , mIsOldVersion(false) +#endif +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::~DecodedStream() +// Purpose: Desctructor +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFile::DecodedStream::~DecodedStream() +{ + // Free any allocated memory + if(mpBlockIndex) + { + ::free(mpBlockIndex); + } + if(mpEncodedData) + { + BackupStoreFile::CodingChunkFree(mpEncodedData); + } + if(mpClearData) + { + ::free(mpClearData); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *) +// Purpose: Get the stream ready to decode -- reads in headers +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr) +{ + // Get the size of the file + int64_t fileSize = mrEncodedFile.BytesLeftToRead(); + + // Get the magic number to work out which order the stream is in + int32_t magic; + if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read magic value + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + bool inFileOrder = true; + switch(ntohl(magic)) + { +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_MAGIC_VALUE_V0: + mIsOldVersion = true; + // control flows on +#endif + case OBJECTMAGIC_FILE_MAGIC_VALUE_V1: + inFileOrder = true; + break; + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0: + mIsOldVersion = true; + // control flows on +#endif + case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1: + inFileOrder = false; + break; + + default: + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // If not in file order, then the index list must be read now + if(!inFileOrder) + { + ReadBlockIndex(true /* have already read and verified the magic number */); + } + + // Get header + file_StreamFormat hdr; + if(inFileOrder) + { + // Read the header, without the magic number + if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic), + 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + // Put in magic number + hdr.mMagicValue = magic; + } + else + { + // Not in file order, so need to read the full header + if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Get the filename + mFilename.ReadFromStream(mrEncodedFile, mTimeout); + + // Get the attributes (either from stream, or supplied attributes) + if(pAlterativeAttr != 0) + { + // Read dummy attributes + BackupClientFileAttributes attr; + attr.ReadFromStream(mrEncodedFile, mTimeout); + + // Set to supplied attributes + mAttributes = *pAlterativeAttr; + } + else + { + // Read the attributes from the stream + mAttributes.ReadFromStream(mrEncodedFile, mTimeout); + } + + // If it is in file order, go and read the file attributes + // Requires that the stream can seek + if(inFileOrder) + { + // Make sure the file size is known + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Store current location (beginning of encoded blocks) + int64_t endOfHeaderPos = mrEncodedFile.GetPosition(); + + // Work out where the index is + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + + // Seek to that position + mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute); + + // Read the block index + ReadBlockIndex(false /* magic number still to be read */); + + // Seek back to the end of header position, ready for reading the chunks + mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute); + } + + // Check view of blocks from block header and file header match + if(mNumBlocks != (int64_t)ntoh64(hdr.mNumBlocks)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Need to allocate some memory for the two blocks for reading encoded data, and clear data + if(mNumBlocks > 0) + { + // Find the maximum encoded data size + int32_t maxEncodedDataSize = 0; + const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; + ASSERT(entry != 0); + for(int64_t e = 0; e < mNumBlocks; e++) + { + // Get the clear and encoded size + int32_t encodedSize = ntoh64(entry[e].mEncodedSize); + ASSERT(encodedSize > 0); + + // Larger? + if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize; + } + + // Allocate those blocks! + mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32); + + // Allocate the block for the clear data, using the hint from the header. + // If this is wrong, things will exception neatly later on, so it can't be used + // to do anything more than cause an error on downloading. + mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32; + mpClearData = (uint8_t*)::malloc(mClearDataSize); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool) +// Purpose: Read the block index from the stream, and store in internal buffer (minus header) +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) +{ + // Header + file_BlockIndexHeader blkhdr; + + // Read it in -- way depends on how whether the magic number has already been read + if(MagicAlreadyRead) + { + // Read the header, without the magic number + if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue), + 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } + else + { + // Magic not already read, so need to read the full header + if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Check magic value + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + } + + // Get the number of blocks out of the header + mNumBlocks = ntoh64(blkhdr.mNumBlocks); + + // Read the IV base + mEntryIVBase = ntoh64(blkhdr.mEntryIVBase); + + // Load the block entries in? + if(mNumBlocks > 0) + { + // How big is the index? + int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks; + + // Allocate some memory + mpBlockIndex = ::malloc(indexSize); + if(mpBlockIndex == 0) + { + throw std::bad_alloc(); + } + + // Read it in + if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Read(void *, int, int) +// Purpose: As interface. Reads decrpyted data. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Symlinks don't have data. So can't read it. Not even zero bytes. + if(IsSymLink()) + { + // Don't allow reading in this case + THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink); + } + + // Already finished? + if(mCurrentBlock >= mNumBlocks) + { + // At end of stream, nothing to do + return 0; + } + + int bytesToRead = NBytes; + uint8_t *output = (uint8_t*)pBuffer; + + while(bytesToRead > 0 && mCurrentBlock < mNumBlocks) + { + // Anything left in the current block? + if(mPositionInCurrentBlock < mCurrentBlockClearSize) + { + // Copy data out of this buffer + int s = mCurrentBlockClearSize - mPositionInCurrentBlock; + if(s > bytesToRead) s = bytesToRead; // limit to requested data + + // Copy + ::memcpy(output, mpClearData + mPositionInCurrentBlock, s); + + // Update positions + output += s; + mPositionInCurrentBlock += s; + bytesToRead -= s; + } + + // Need to get some more data? + if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize) + { + // Number of next block + ++mCurrentBlock; + if(mCurrentBlock >= mNumBlocks) + { + // Stop now! + break; + } + + // Get the size from the block index + const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; + int32_t encodedSize = ntoh64(entry[mCurrentBlock].mEncodedSize); + if(encodedSize <= 0) + { + // The caller is attempting to decode a file which is the direct result of a diff + // operation, and so does not contain all the data. + // It needs to be combined with the previous version first. + THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining) + } + + // Load in next block + if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Decode the data + mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize); + + // Calculate IV for this entry + uint64_t iv = mEntryIVBase; + iv += mCurrentBlock; + // Convert to network byte order before encrypting with it, so that restores work on + // platforms with different endiannesses. + iv = hton64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Make sure this is the right size + if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) + { +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(!mIsOldVersion) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the + // IV for the encrypted section. Try again, with the thing the other way round + iv = box_swap64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + else + { + // Warn and log this issue + if(!sWarnedAboutBackwardsCompatiblity) + { + ::printf("WARNING: Decoded one or more files using backwards compatibility mode for block index.\n"); + ::syslog(LOG_ERR, "WARNING: Decoded one or more files using backwards compatibility mode for block index.\n"); + sWarnedAboutBackwardsCompatiblity = true; + } + } +#else + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) +#endif + } + + // Check the digest + MD5Digest md5; + md5.Add(mpClearData, mCurrentBlockClearSize); + md5.Finish(); + if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum)) + { + THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck) + } + + // Set vars to say what's happening + mPositionInCurrentBlock = 0; + } + } + + ASSERT(bytesToRead >= 0); + ASSERT(bytesToRead <= NBytes); + + return NBytes - bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::IsSymLink() +// Purpose: Is the unencoded file actually a symlink? +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::IsSymLink() +{ + // First, check in with the attributes + if(!mAttributes.IsSymLink()) + { + return false; + } + + // So the attributes think it is a symlink. + // Consistency check... + if(mNumBlocks != 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Write(const void *, int) +// Purpose: As interface. Throws exception, as you can't write to this stream. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::StreamDataLeft() +// Purpose: As interface. Any data left? +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::StreamDataLeft() +{ + return mCurrentBlock < mNumBlocks; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::StreamClosed() +// Purpose: As interface. Always returns true, no writing allowed. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::StreamClosed() +{ + // Can't write to this stream! + return true; +} + + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::SetBlowfishKey(const void *, int) +// Purpose: Static. Sets the key to use for encryption and decryption. +// Created: 7/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength) +{ + // IVs set later + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + + sBlowfishEncryptBlockEntry.Reset(); + sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); + sBlowfishEncryptBlockEntry.UsePadding(false); + sBlowfishDecryptBlockEntry.Reset(); + sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); + sBlowfishDecryptBlockEntry.UsePadding(false); +} + + +#ifndef PLATFORM_OLD_OPENSSL +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::SetAESKey(const void *, int) +// Purpose: Sets the AES key to use for file data encryption. Will select AES as +// the cipher to use when encrypting. +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength) +{ + // Setup context + sAESEncrypt.Reset(); + sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); + sAESDecrypt.Reset(); + sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); + + // Set encryption to use this key, instead of the "default" blowfish key + spEncrypt = &sAESEncrypt; + sEncryptCipherType = HEADER_AES_ENCODING; +} +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int) +// Purpose: The maximum output size of a block, given the chunk size +// Created: 7/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize) +{ + // Calculate... the maximum size of output by first the largest it could be after compression, + // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck + // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing + // an problem where a rather non-compressable file didn't fit in a block buffer.) + return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1 + + sBlowfishEncrypt.GetIVLength() + 128; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &) +// Purpose: Encodes a chunk (encryption, possible compressed beforehand) +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput) +{ + ASSERT(spEncrypt != 0); + + // Check there's some space in the output block + if(rOutput.mBufferSize < 256) + { + rOutput.Reallocate(256); + } + + // Check alignment of the block + ASSERT((((uint32_t)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + + // Want to compress it? + bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE); + + // Build header + uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT; + if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED; + + // Store header + rOutput.mpBuffer[0] = header; + int outOffset = 1; + + // Setup cipher, and store the IV + int ivLen = 0; + const void *iv = spEncrypt->SetRandomIV(ivLen); + ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen); + outOffset += ivLen; + + // Start encryption process + spEncrypt->Begin(); + + #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \ + { \ + if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \ + { \ + rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \ + } \ + } + + // Encode the chunk + if(compressChunk) + { + // buffer to compress into + uint8_t buffer[2048]; + + // Set compressor with all the chunk as an input + Compress<true> compress; + compress.Input(Chunk, ChunkSize); + compress.FinishInput(); + + // Get and encrypt output + while(!compress.OutputHasFinished()) + { + int s = compress.Output(buffer, sizeof(buffer)); + if(s > 0) + { + ENCODECHUNK_CHECK_SPACE(s) + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); + } + else + { + // Should never happen, as we put all the input in in one go. + // So if this happens, it means there's a logical problem somewhere + THROW_EXCEPTION(BackupStoreException, Internal) + } + } + ENCODECHUNK_CHECK_SPACE(16) + outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); + } + else + { + // Straight encryption + ENCODECHUNK_CHECK_SPACE(ChunkSize) + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize); + ENCODECHUNK_CHECK_SPACE(16) + outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); + } + + ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check + + return outOffset; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int) +// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find +// the extra output buffer size needed before calling. +// See notes in EncodeChunk() for notes re alignment of the +// encoded data. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize) +{ + // Check alignment of the encoded block + ASSERT((((uint32_t)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + + // First check + if(EncodedSize < 1) + { + THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) + } + + const uint8_t *input = (uint8_t*)Encoded; + + // Get header, make checks, etc + uint8_t header = input[0]; + bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED; + uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT); + if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING) + { + THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding) + } + +#ifndef PLATFORM_OLD_OPENSSL + // Choose cipher + CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt); +#else + // AES not supported with this version of OpenSSL + if(encodingType == HEADER_AES_ENCODING) + { + THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL) + } + CipherContext &cipher(sBlowfishDecrypt); +#endif + + // Check enough space for header, an IV and one byte of input + int ivLen = cipher.GetIVLength(); + if(EncodedSize < (1 + ivLen + 1)) + { + THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) + } + + // Set IV in decrypt context, and start + cipher.SetIV(input + 1); + cipher.Begin(); + + // Setup vars for code + int inOffset = 1 + ivLen; + uint8_t *output = (uint8_t*)Output; + int outOffset = 0; + + // Do action + if(chunkCompressed) + { + // Do things in chunks + uint8_t buffer[2048]; + int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer)); + + // Decompressor + Compress<false> decompress; + + while(inOffset < EncodedSize) + { + // Decrypt a block + int bl = inputBlockLen; + if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long + int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl); + inOffset += bl; + + // Decompress the decrypted data + if(s > 0) + { + decompress.Input(buffer, s); + int os = 0; + do + { + os = decompress.Output(output + outOffset, OutputSize - outOffset); + outOffset += os; + } while(os > 0); + + // Check that there's space left in the output buffer -- there always should be + if(outOffset >= OutputSize) + { + THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) + } + } + } + + // Get any compressed data remaining in the cipher context and compression + int s = cipher.Final(buffer, sizeof(buffer)); + decompress.Input(buffer, s); + decompress.FinishInput(); + while(!decompress.OutputHasFinished()) + { + int os = decompress.Output(output + outOffset, OutputSize - outOffset); + outOffset += os; + + // Check that there's space left in the output buffer -- there always should be + if(outOffset >= OutputSize) + { + THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) + } + } + } + else + { + // Easy decryption + outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset); + outOffset += cipher.Final(output + outOffset, OutputSize - outOffset); + } + + return outOffset; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool) +// Purpose: Returns a stream which gives a Stream order version of the encoded file. +// If TakeOwnership == true, then the input stream will be deleted when the +// returned stream is deleted. +// The input stream must be seekable. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership) +{ + ASSERT(pStream != 0); + + // Get the size of the file + int64_t fileSize = pStream->BytesLeftToRead(); + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Read the header + int bytesRead = 0; + file_StreamFormat hdr; + bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead); + + // Seek backwards to put the file pointer back where it was before we started this + pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative); + + // Check we got a block + if(!readBlock) + { + // Couldn't read header -- assume file bad + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Get number of blocks + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + + // Calculate where the block index will be, check it's reasonable + int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + int64_t blockIndexLoc = fileSize - blockIndexSize; + if(blockIndexLoc < 0) + { + // Doesn't look good! + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Build a reordered stream + std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership)); + + // Set it up... + ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get())); + int component = rreordered.AddComponent(pStream); + // Send out the block index + rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc); + // And then the rest of the file + rreordered.AddBlock(component, blockIndexLoc, true, 0); + + return reordered; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ResetStats() +// Purpose: Reset the gathered statistics +// Created: 20/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::ResetStats() +{ + msStats.mBytesInEncodedFiles = 0; + msStats.mBytesAlreadyOnServer = 0; + msStats.mTotalFileStreamSize = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &) +// Purpose: Compares the contents of a file against the checksums contained in the +// block index. Returns true if the checksums match, meaning the file is +// extremely likely to match the original. Will always consume the entire index. +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout) +{ + // is it a symlink? + bool sourceIsSymlink = false; + { + struct stat st; + if(::lstat(Filename, &st) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + if((st.st_mode & S_IFMT) == S_IFLNK) + { + sourceIsSymlink = true; + } + } + + // Open file, if it's not a symlink + std::auto_ptr<FileStream> in; + if(!sourceIsSymlink) + { + in.reset(new FileStream(Filename)); + } + + // Read header + file_BlockIndexHeader hdr; + if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic + if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0) +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0); +#endif + + // Get basic information + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + uint64_t entryIVBase = ntoh64(hdr.mEntryIVBase); + + //TODO: Verify that these sizes look reasonable + + // setup + void *data = 0; + int32_t dataSize = -1; + bool matches = true; + int64_t totalSizeInBlockIndex = 0; + + try + { + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read an entry from the stream + file_BlockIndexEntry entry; + if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read entry + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Calculate IV for this entry + uint64_t iv = entryIVBase; + iv += b; + iv = hton64(iv); +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(isOldVersion) + { + // Reverse the IV for compatibility + iv = box_swap64(iv); + } +#endif + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry.mEnEnc, sizeof(entry.mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Size of block + int32_t blockClearSize = ntohl(entryEnc.mSize); + if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + totalSizeInBlockIndex += blockClearSize; + + // Make sure there's enough memory allocated to load the block in + if(dataSize < blockClearSize) + { + // Too small, free the block if it's already allocated + if(data != 0) + { + ::free(data); + data = 0; + } + // Allocate a block + data = ::malloc(blockClearSize + 128); + if(data == 0) + { + throw std::bad_alloc(); + } + dataSize = blockClearSize + 128; + } + + // Load in the block from the file, if it's not a symlink + if(!sourceIsSymlink) + { + if(in->Read(data, blockClearSize) != blockClearSize) + { + // Not enough data left in the file, can't possibly match + matches = false; + } + else + { + // Check the checksum + MD5Digest md5; + md5.Add(data, blockClearSize); + md5.Finish(); + if(!md5.DigestMatches(entryEnc.mStrongChecksum)) + { + // Checksum didn't match + matches = false; + } + } + } + + // Keep on going regardless, to make sure the entire block index stream is read + // -- must always be consistent about what happens with the stream. + } + } + catch(...) + { + // clean up in case of errors + if(data != 0) + { + ::free(data); + data = 0; + } + throw; + } + + // free block + if(data != 0) + { + ::free(data); + data = 0; + } + + // Check for data left over if it's not a symlink + if(!sourceIsSymlink) + { + // Anything left to read in the file? + if(in->BytesLeftToRead() != 0) + { + // File has extra data at the end + matches = false; + } + } + + // Symlinks must have zero size on server + if(sourceIsSymlink) + { + matches = (totalSizeInBlockIndex == 0); + } + + return matches; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer() +// Purpose: Constructor +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +BackupStoreFile::EncodingBuffer::EncodingBuffer() + : mpBuffer(0), + mBufferSize(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer() +// Purpose: Destructor +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +BackupStoreFile::EncodingBuffer::~EncodingBuffer() +{ + if(mpBuffer != 0) + { + BackupStoreFile::CodingChunkFree(mpBuffer); + mpBuffer = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::Allocate(int) +// Purpose: Do initial allocation of block +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::EncodingBuffer::Allocate(int Size) +{ + ASSERT(mpBuffer == 0); + uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size); + if(buffer == 0) + { + throw std::bad_alloc(); + } + mpBuffer = buffer; + mBufferSize = Size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::Reallocate(int) +// Purpose: Reallocate the block. Try not to call this, it has to copy +// the entire contents as the block can't be reallocated straight. +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) +{ + TRACE2("Reallocating EncodingBuffer from %d to %d\n", mBufferSize, NewSize); + ASSERT(mpBuffer != 0); + uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + // Copy data + ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize); + + // Free old + BackupStoreFile::CodingChunkFree(mpBuffer); + + // Store new buffer + mpBuffer = buffer; + mBufferSize = NewSize; +} + + diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h new file mode 100755 index 00000000..0274107b --- /dev/null +++ b/lib/backupclient/BackupStoreFile.h @@ -0,0 +1,194 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFile.h +// Purpose: Utils for manipulating files +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILE__H +#define BACKUPSTOREFILE__H + +#include "IOStream.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreFilename.h" + +#include <memory> + +typedef struct +{ + int64_t mBytesInEncodedFiles; + int64_t mBytesAlreadyOnServer; + int64_t mTotalFileStreamSize; +} BackupStoreFileStats; + +// Uncomment to disable backwards compatibility +//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + + +// Output buffer to EncodeChunk and input data to DecodeChunk must +// have specific alignment, see function comments. +#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16 +#define BACKUPSTOREFILE_CODING_OFFSET 15 + +// Have some memory allocation commands, note closing "Off" at end of file. +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFile +// Purpose: Class to hold together utils for maniplating files. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +class BackupStoreFile +{ +public: + class DecodedStream : public IOStream + { + friend class BackupStoreFile; + private: + DecodedStream(IOStream &rEncodedFile, int Timeout); + DecodedStream(const DecodedStream &); // not allowed + DecodedStream &operator=(const DecodedStream &); // not allowed + public: + ~DecodedStream(); + + // Stream functions + virtual int Read(void *pBuffer, int NBytes, int Timeout); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + // Accessor functions + const BackupClientFileAttributes &GetAttributes() {return mAttributes;} + const BackupStoreFilename &GetFilename() {return mFilename;} + int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests + + bool IsSymLink(); + + private: + void Setup(const BackupClientFileAttributes *pAlterativeAttr); + void ReadBlockIndex(bool MagicAlreadyRead); + + private: + IOStream &mrEncodedFile; + int mTimeout; + BackupClientFileAttributes mAttributes; + BackupStoreFilename mFilename; + int64_t mNumBlocks; + void *mpBlockIndex; + uint8_t *mpEncodedData; + uint8_t *mpClearData; + int mClearDataSize; + int mCurrentBlock; + int mCurrentBlockClearSize; + int mPositionInCurrentBlock; + uint64_t mEntryIVBase; +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool mIsOldVersion; +#endif + }; + + + // Main interface + static std::auto_ptr<IOStream> EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0); + static std::auto_ptr<IOStream> EncodeFileDiff(const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, + int Timeout, int64_t *pModificationTime = 0, bool *pIsCompletelyDifferent = 0); + static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); + static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); + static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); + static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0); + static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); + static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); + static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout); + static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false); + + // Stream manipulation + static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership); + static void MoveStreamPositionToBlockIndex(IOStream &rStream); + + // Crypto setup + static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength); +#ifndef PLATFORM_OLD_OPENSSL + static void SetAESKey(const void *pKey, int KeyLength); +#endif + + // Allocation of properly aligning chunks for decoding and encoding chunks + inline static void *CodingChunkAlloc(int Size) + { + uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3)); + if(a == 0) return 0; + // Align to main block size + ASSERT(sizeof(uint32_t) == sizeof(void*)); // make sure casting the right pointer size, will need to fix on platforms with 64 bit pointers + uint32_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE - (((uint32_t)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE); + uint8_t *b = (a + adjustment); + // Store adjustment + *b = (uint8_t)adjustment; + // Return offset + return b + BACKUPSTOREFILE_CODING_OFFSET; + } + inline static void CodingChunkFree(void *Block) + { + // Check alignment is as expected + ASSERT(sizeof(uint32_t) == sizeof(void*)); // make sure casting the right pointer size, will need to fix on platforms with 64 bit pointers + ASSERT((((uint32_t)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + uint8_t *a = (uint8_t*)Block; + a -= BACKUPSTOREFILE_CODING_OFFSET; + // Adjust downwards... + a -= *a; + free(a); + } + + // Limits + static void SetMaximumDiffingTime(int Seconds); + + // Building blocks + class EncodingBuffer + { + public: + EncodingBuffer(); + ~EncodingBuffer(); + private: + // No copying + EncodingBuffer(const EncodingBuffer &); + EncodingBuffer &operator=(const EncodingBuffer &); + public: + void Allocate(int Size); + void Reallocate(int NewSize); + + uint8_t *mpBuffer; + int mBufferSize; + }; + static int MaxBlockSizeForChunkSize(int ChunkSize); + static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput); + + // Caller should know how big the output size is, but also allocate a bit more memory to cover various + // overheads allowed for in checks + static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize) + { + // Plenty big enough + return KnownChunkSize + 256; + } + static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize); + + // Statisitics, not designed to be completely reliable + static void ResetStats(); + static BackupStoreFileStats msStats; + + // For debug +#ifndef NDEBUG + static bool TraceDetailsOfDiffProcess; +#endif + + // For decoding encoded files + static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile); +}; + +#include "MemLeakFindOff.h" + +#endif // BACKUPSTOREFILE__H + diff --git a/lib/backupclient/BackupStoreFileCmbDiff.cpp b/lib/backupclient/BackupStoreFileCmbDiff.cpp new file mode 100644 index 00000000..039b00eb --- /dev/null +++ b/lib/backupclient/BackupStoreFileCmbDiff.cpp @@ -0,0 +1,326 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCmbDiff.cpp +// Purpose: Combine two diffs together +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <stdlib.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CombineDiffs(IOStream &, IOStream &, IOStream &rOut) +// Purpose: Given two diffs, combine them into a single diff, to produce a diff +// which, combined with the original file, creates the result of applying +// rDiff, then rDiff2. Two opens of rDiff2 are required +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut) +{ + // Skip header of first diff, record where the data starts, and skip to the index + int64_t diff1DataStarts = 0; + { + // Read the header for the From file + file_StreamFormat diff1Hdr; + if(!rDiff1.ReadFullBuffer(&diff1Hdr, sizeof(diff1Hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diff1Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Skip over the filename and attributes of the From file + // BLOCK + { + BackupStoreFilename filename2; + filename2.ReadFromStream(rDiff1, IOStream::TimeOutInfinite); + int32_t size_s; + if(!rDiff1.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + int size = ntohl(size_s); + // Skip forward the size + rDiff1.Seek(size, IOStream::SeekType_Relative); + } + // Record position + diff1DataStarts = rDiff1.GetPosition(); + // Skip to index + rDiff1.Seek(0 - (((ntoh64(diff1Hdr.mNumBlocks)) * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + } + + // Read the index of the first diff + // Header first + file_BlockIndexHeader diff1IdxHdr; + if(!rDiff1.ReadFullBuffer(&diff1IdxHdr, sizeof(diff1IdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(diff1IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + int64_t diff1NumBlocks = ntoh64(diff1IdxHdr.mNumBlocks); + // Allocate some memory + int64_t *diff1BlockStartPositions = (int64_t*)::malloc((diff1NumBlocks + 1) * sizeof(int64_t)); + if(diff1BlockStartPositions == 0) + { + throw std::bad_alloc(); + } + + // Buffer data + void *buffer = 0; + int bufferSize = 0; + + try + { + // Then the entries: + // For each entry, want to know if it's in the file, and if so, how big it is. + // We'll store this as an array of file positions in the file, with an additioal + // entry on the end so that we can work out the length of the last block. + // If an entry isn't in the file, then store 0 - (position in other file). + int64_t diff1Position = diff1DataStarts; + for(int64_t b = 0; b < diff1NumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff1.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Where's the block? + int64_t blockEn = ntoh64(e.mEncodedSize); + if(blockEn <= 0) + { + // Just store the negated block number + diff1BlockStartPositions[b] = blockEn; + } + else + { + // Block is present in this file + diff1BlockStartPositions[b] = diff1Position; + diff1Position += blockEn; + } + } + + // Finish off the list, so the last entry can have it's size calcuated. + diff1BlockStartPositions[diff1NumBlocks] = diff1Position; + + // Now read the second diff's header, copying it to the out file + file_StreamFormat diff2Hdr; + if(!rDiff2.ReadFullBuffer(&diff2Hdr, sizeof(diff2Hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diff2Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Copy + rOut.Write(&diff2Hdr, sizeof(diff2Hdr)); + // Copy over filename and attributes + // BLOCK + { + BackupStoreFilename filename; + filename.ReadFromStream(rDiff2, IOStream::TimeOutInfinite); + filename.WriteToStream(rOut); + StreamableMemBlock attr; + attr.ReadFromStream(rDiff2, IOStream::TimeOutInfinite); + attr.WriteToStream(rOut); + } + + // Get to the index of rDiff2b, and read the header + MoveStreamPositionToBlockIndex(rDiff2b); + file_BlockIndexHeader diff2IdxHdr; + if(!rDiff2b.ReadFullBuffer(&diff2IdxHdr, sizeof(diff2IdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(diff2IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + int64_t diff2NumBlocks = ntoh64(diff2IdxHdr.mNumBlocks); + int64_t diff2IndexEntriesStart = rDiff2b.GetPosition(); + + // Then read all the entries + int64_t diff2FilePosition = rDiff2.GetPosition(); + for(int64_t b = 0; b < diff2NumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // What do to next about copying data + bool copyBlock = false; + int copySize = 0; + int64_t copyFrom = 0; + bool fromFileDiff1 = false; + + // Where's the block? + int64_t blockEn = ntoh64(e.mEncodedSize); + if(blockEn > 0) + { + // Block is present in this file -- copy to out + copyBlock = true; + copyFrom = diff2FilePosition; + copySize = (int)blockEn; + + // Move pointer onwards + diff2FilePosition += blockEn; + } + else + { + // Block isn't present here -- is it present in the old one? + int64_t blockIndex = 0 - blockEn; + if(blockIndex < 0 || blockIndex > diff1NumBlocks) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + if(diff1BlockStartPositions[blockIndex] > 0) + { + // Block is in the old diff file, copy it across + copyBlock = true; + copyFrom = diff1BlockStartPositions[blockIndex]; + int nb = blockIndex + 1; + while(diff1BlockStartPositions[nb] <= 0) + { + // This is safe, because the last entry will terminate it properly! + ++nb; + ASSERT(nb <= diff1NumBlocks); + } + copySize = diff1BlockStartPositions[nb] - copyFrom; + fromFileDiff1 = true; + } + } + //TRACE4("%d %d %lld %d\n", copyBlock, copySize, copyFrom, fromFileDiff1); + + // Copy data to the output file? + if(copyBlock) + { + // Allocate enough space + if(bufferSize < copySize || buffer == 0) + { + // Free old block + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + bufferSize = 0; + } + // Allocate new block + buffer = ::malloc(copySize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + bufferSize = copySize; + } + ASSERT(bufferSize >= copySize); + + // Load in the data + if(fromFileDiff1) + { + rDiff1.Seek(copyFrom, IOStream::SeekType_Absolute); + if(!rDiff1.ReadFullBuffer(buffer, copySize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + } + else + { + rDiff2.Seek(copyFrom, IOStream::SeekType_Absolute); + if(!rDiff2.ReadFullBuffer(buffer, copySize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + } + // Write out data + rOut.Write(buffer, copySize); + } + } + + // Write the modified header + diff2IdxHdr.mOtherFileID = diff1IdxHdr.mOtherFileID; + rOut.Write(&diff2IdxHdr, sizeof(diff2IdxHdr)); + + // Then we'll write out the index, reading the data again + rDiff2b.Seek(diff2IndexEntriesStart, IOStream::SeekType_Absolute); + for(int64_t b = 0; b < diff2NumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Where's the block? + int64_t blockEn = ntoh64(e.mEncodedSize); + + // If it's not in this file, it needs modification... + if(blockEn <= 0) + { + int64_t blockIndex = 0 - blockEn; + // In another file. Need to translate this against the other diff + if(diff1BlockStartPositions[blockIndex] > 0) + { + // Block is in the first diff file, stick in size + int nb = blockIndex + 1; + while(diff1BlockStartPositions[nb] <= 0) + { + // This is safe, because the last entry will terminate it properly! + ++nb; + ASSERT(nb <= diff1NumBlocks); + } + int64_t size = diff1BlockStartPositions[nb] - diff1BlockStartPositions[blockIndex]; + e.mEncodedSize = hton64(size); + } + else + { + // Block in the original file, use translated value + e.mEncodedSize = hton64(diff1BlockStartPositions[blockIndex]); + } + } + + // Write entry + rOut.Write(&e, sizeof(e)); + } + } + catch(...) + { + // clean up + ::free(diff1BlockStartPositions); + if(buffer != 0) + { + ::free(buffer); + } + throw; + } + + // Clean up allocated memory + ::free(diff1BlockStartPositions); + if(buffer != 0) + { + ::free(buffer); + } +} + diff --git a/lib/backupclient/BackupStoreFileCmbIdx.cpp b/lib/backupclient/BackupStoreFileCmbIdx.cpp new file mode 100644 index 00000000..253001c2 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCmbIdx.cpp @@ -0,0 +1,324 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCmbIdx.cpp +// Purpose: Combine indicies of a delta file and the file it's a diff from. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <string.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" + +#include "MemLeakFindOn.h" + +// Hide from outside world +namespace +{ + +class BSFCombinedIndexStream : public IOStream +{ +public: + BSFCombinedIndexStream(IOStream *pDiff); + ~BSFCombinedIndexStream(); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + virtual void Initialise(IOStream &rFrom); + +private: + IOStream *mpDiff; + bool mIsInitialised; + bool mHeaderWritten; + file_BlockIndexHeader mHeader; + int64_t mNumEntriesToGo; + int64_t mNumEntriesInFromFile; + int64_t *mFromBlockSizes; // NOTE: Entries in network byte order +}; + +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CombineFileIndices(IOStream &, IOStream &, bool) +// Purpose: Given a diff file and the file it's a diff from, return a stream from which +// can be read the index of the combined file, without actually combining them. +// The stream of the diff must have a lifetime greater than or equal to the +// lifetime of the returned stream object. The full "from" file stream +// only needs to exist during the actual function call. +// If you pass in dodgy files which aren't related, then you will either +// get an error or bad results. So don't do that. +// If DiffIsIndexOnly is true, then rDiff is assumed to be a stream positioned +// at the beginning of the block index. Similarly for FromIsIndexOnly. +// WARNING: Reads of the returned streams with buffer sizes less than 64 bytes +// will not return any data. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly, bool FromIsIndexOnly) +{ + // Reposition file pointers? + if(!DiffIsIndexOnly) + { + MoveStreamPositionToBlockIndex(rDiff); + } + if(!FromIsIndexOnly) + { + MoveStreamPositionToBlockIndex(rFrom); + } + + // Create object + std::auto_ptr<IOStream> stream(new BSFCombinedIndexStream(&rDiff)); + + // Initialise it + ((BSFCombinedIndexStream *)stream.get())->Initialise(rFrom); + + // And return the stream + return stream; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::BSFCombinedIndexStream() +// Purpose: Private class. Constructor. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +BSFCombinedIndexStream::BSFCombinedIndexStream(IOStream *pDiff) + : mpDiff(pDiff), + mIsInitialised(false), + mHeaderWritten(false), + mNumEntriesToGo(0), + mNumEntriesInFromFile(0), + mFromBlockSizes(0) +{ + ASSERT(mpDiff != 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::~BSFCombinedIndexStream() +// Purpose: Private class. Destructor. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +BSFCombinedIndexStream::~BSFCombinedIndexStream() +{ + if(mFromBlockSizes != 0) + { + ::free(mFromBlockSizes); + mFromBlockSizes = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::Initialise(IOStream &) +// Purpose: Private class. Initalise from the streams (diff passed in constructor). +// Both streams must have file pointer positioned at the block index. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +void BSFCombinedIndexStream::Initialise(IOStream &rFrom) +{ + // Paranoia is good. + if(mIsInitialised) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Look at the diff file: Read in the header + if(!mpDiff->ReadFullBuffer(&mHeader, sizeof(mHeader), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(mHeader.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Read relevant data. + mNumEntriesToGo = ntoh64(mHeader.mNumBlocks); + + // Adjust a bit to reflect the fact it's no longer a diff + mHeader.mOtherFileID = hton64(0); + + // Now look at the from file: Read header + file_BlockIndexHeader fromHdr; + if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Then... allocate memory for the list of sizes + mNumEntriesInFromFile = ntoh64(fromHdr.mNumBlocks); + mFromBlockSizes = (int64_t*)::malloc(mNumEntriesInFromFile * sizeof(int64_t)); + if(mFromBlockSizes == 0) + { + throw std::bad_alloc(); + } + + // And read them all in! + for(int64_t b = 0; b < mNumEntriesInFromFile; ++b) + { + file_BlockIndexEntry e; + if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check that the from file isn't a delta in itself + if(ntoh64(e.mEncodedSize) <= 0) + { + THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) + } + + // Store size (in network byte order) + mFromBlockSizes[b] = e.mEncodedSize; + } + + // Flag as initialised + mIsInitialised = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::Read(void *, int, int) +// Purpose: Private class. As interface. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Paranoia is good. + if(!mIsInitialised || mFromBlockSizes == 0 || mpDiff == 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + int written = 0; + + // Header output yet? + if(!mHeaderWritten) + { + // Enough space? + if(NBytes < (int)sizeof(mHeader)) return 0; + + // Copy in + ::memcpy(pBuffer, &mHeader, sizeof(mHeader)); + NBytes -= sizeof(mHeader); + written += sizeof(mHeader); + + // Flag it's done + mHeaderWritten = true; + } + + // How many entries can be written? + int entriesToWrite = NBytes / sizeof(file_BlockIndexEntry); + if(entriesToWrite > mNumEntriesToGo) + { + entriesToWrite = mNumEntriesToGo; + } + + // Setup ready to go + file_BlockIndexEntry *poutput = (file_BlockIndexEntry*)(((uint8_t*)pBuffer) + written); + + // Write entries + for(int b = 0; b < entriesToWrite; ++b) + { + if(!mpDiff->ReadFullBuffer(&(poutput[b]), sizeof(file_BlockIndexEntry), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Does this need adjusting? + int s = ntoh64(poutput[b].mEncodedSize); + if(s <= 0) + { + // A reference to a block in the from file + int block = 0 - s; + ASSERT(block >= 0); + if(block >= mNumEntriesInFromFile) + { + // That's not good, the block doesn't exist + THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) + } + + // Adjust the entry in the buffer + poutput[b].mEncodedSize = mFromBlockSizes[block]; // stored in network byte order, no translation necessary + } + } + + // Update written count + written += entriesToWrite * sizeof(file_BlockIndexEntry); + mNumEntriesToGo -= entriesToWrite; + + return written; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::Write(const void *, int) +// Purpose: Private class. As interface. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::StreamDataLeft() +// Purpose: Private class. As interface +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +bool BSFCombinedIndexStream::StreamDataLeft() +{ + return (!mHeaderWritten) || (mNumEntriesToGo > 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::StreamClosed() +// Purpose: Private class. As interface. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +bool BSFCombinedIndexStream::StreamClosed() +{ + return true; // doesn't do writing +} + diff --git a/lib/backupclient/BackupStoreFileCombine.cpp b/lib/backupclient/BackupStoreFileCombine.cpp new file mode 100755 index 00000000..562a32d9 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCombine.cpp @@ -0,0 +1,410 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCombine.cpp +// Purpose: File combining for BackupStoreFile +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" +#include "FileStream.h" + +#include "MemLeakFindOn.h" + +typedef struct +{ + int64_t mFilePosition; +} FromIndexEntry; + +static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries); +static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut); +static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut); + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CombineFile(IOStream &, IOStream &, IOStream &) +// Purpose: Where rDiff is a store file which is incomplete as a result of a +// diffing operation, rFrom is the file it is diffed from, and +// rOut is the stream in which to place the result, the old file +// and new file are combined into a file containing all the data. +// rDiff2 is the same file as rDiff, opened again to get two +// independent streams to the same file. +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut) +{ + // Read and copy the header. + file_StreamFormat hdr; + if(!rDiff.ReadFullBuffer(&hdr, sizeof(hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Copy + rOut.Write(&hdr, sizeof(hdr)); + // Copy over filename and attributes + // BLOCK + { + BackupStoreFilename filename; + filename.ReadFromStream(rDiff, IOStream::TimeOutInfinite); + filename.WriteToStream(rOut); + StreamableMemBlock attr; + attr.ReadFromStream(rDiff, IOStream::TimeOutInfinite); + attr.WriteToStream(rOut); + } + + // Read the header for the From file + file_StreamFormat fromHdr; + if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Skip over the filename and attributes of the From file + // BLOCK + { + BackupStoreFilename filename2; + filename2.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + int32_t size_s; + if(!rFrom.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + int size = ntohl(size_s); + // Skip forward the size + rFrom.Seek(size, IOStream::SeekType_Relative); + } + + // Allocate memory for the block index of the From file + int64_t fromNumBlocks = ntoh64(fromHdr.mNumBlocks); + // NOTE: An extra entry is required so that the length of the last block can be calculated + FromIndexEntry *pFromIndex = (FromIndexEntry*)::malloc((fromNumBlocks+1) * sizeof(FromIndexEntry)); + if(pFromIndex == 0) + { + throw std::bad_alloc(); + } + + try + { + // Load the index from the From file, calculating the offsets in the + // file as we go along, and enforce that everything should be present. + LoadFromIndex(rFrom, pFromIndex, fromNumBlocks); + + // Read in the block index of the Diff file in small chunks, and output data + // for each block, either from this file, or the other file. + int64_t diffNumBlocks = ntoh64(hdr.mNumBlocks); + CopyData(rDiff /* positioned at start of data */, rDiff2, diffNumBlocks, rFrom, pFromIndex, fromNumBlocks, rOut); + + // Read in the block index again, and output the new block index, simply + // filling in the sizes of blocks from the old file. + WriteNewIndex(rDiff, diffNumBlocks, pFromIndex, fromNumBlocks, rOut); + + // Free buffers + ::free(pFromIndex); + pFromIndex = 0; + } + catch(...) + { + // Clean up + if(pFromIndex != 0) + { + ::free(pFromIndex); + pFromIndex = 0; + } + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static LoadFromIndex(IOStream &, FromIndexEntry *, int64_t) +// Purpose: Static. Load the index from the From file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries) +{ + ASSERT(pIndex != 0); + ASSERT(NumEntries >= 0); + + // Get the starting point in the file + int64_t filePos = rFrom.GetPosition(); + + // Jump to the end of the file to read the index + rFrom.Seek(0 - ((NumEntries * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + + // Read block index header + file_BlockIndexHeader blkhdr; + if(!rFrom.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || (int64_t)ntoh64(blkhdr.mNumBlocks) != NumEntries) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // And then the block entries + for(int64_t b = 0; b < NumEntries; ++b) + { + // Read + file_BlockIndexEntry en; + if(!rFrom.ReadFullBuffer(&en, sizeof(en), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // Add to list + pIndex[b].mFilePosition = filePos; + + // Encoded size? + int64_t encodedSize = ntoh64(en.mEncodedSize); + // Check that the block is actually there + if(encodedSize <= 0) + { + THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) + } + + // Move file pointer on + filePos += encodedSize; + } + + // Store the position in the very last entry, so the size of the last entry can be calculated + pIndex[NumEntries].mFilePosition = filePos; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static CopyData(IOStream &, IOStream &, int64_t, IOStream &, FromIndexEntry *, int64_t, IOStream &) +// Purpose: Static. Copy data from the Diff and From file to the out file. +// rDiffData is at beginning of data. +// rDiffIndex at any position. +// rFrom is at any position. +// rOut is after the header, ready for data +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, + IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut) +{ + // Jump to the end of the diff file to read the index + rDiffIndex.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + + // Read block index header + file_BlockIndexHeader diffBlkhdr; + if(!rDiffIndex.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || (int64_t)ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Record where the From file is + int64_t fromPos = rFrom.GetPosition(); + + // Buffer data + void *buffer = 0; + int bufferSize = 0; + + try + { + // Read the blocks in! + for(int64_t b = 0; b < DiffNumBlocks; ++b) + { + // Read + file_BlockIndexEntry en; + if(!rDiffIndex.ReadFullBuffer(&en, sizeof(en), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // What's the size value stored in the entry + int64_t encodedSize = ntoh64(en.mEncodedSize); + + // How much data will be read? + int32_t blockSize = 0; + if(encodedSize > 0) + { + // The block is actually in the diff file + blockSize = encodedSize; + } + else + { + // It's in the from file. First, check to see if it's valid + int64_t blockIdx = (0 - encodedSize); + if(blockIdx > FromNumBlocks) + { + // References a block which doesn't actually exist + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Calculate size. This operation is safe because of the extra entry at the end + blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition; + } + ASSERT(blockSize > 0); + + // Make sure there's memory available to copy this + if(bufferSize < blockSize || buffer == 0) + { + // Free old block + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + bufferSize = 0; + } + // Allocate new block + buffer = ::malloc(blockSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + bufferSize = blockSize; + } + ASSERT(bufferSize >= blockSize); + + // Load in data from one of the files + if(encodedSize > 0) + { + // Load from diff file + if(!rDiffData.ReadFullBuffer(buffer, blockSize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + } + else + { + // Locate and read the data from the from file + int64_t blockIdx = (0 - encodedSize); + // Seek if necessary + if(fromPos != pFromIndex[blockIdx].mFilePosition) + { + rFrom.Seek(pFromIndex[blockIdx].mFilePosition, IOStream::SeekType_Absolute); + fromPos = pFromIndex[blockIdx].mFilePosition; + } + // Read + if(!rFrom.ReadFullBuffer(buffer, blockSize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // Update fromPos to current position + fromPos += blockSize; + } + + // Write data to out file + rOut.Write(buffer, blockSize); + } + + // Free buffer, if allocated + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + } + } + catch(...) + { + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + } + throw; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static WriteNewIndex(IOStream &, int64_t, FromIndexEntry *, int64_t, IOStream &) +// Purpose: Write the index to the out file, just copying from the diff file and +// adjusting the entries. +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut) +{ + // Jump to the end of the diff file to read the index + rDiff.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + + // Read block index header + file_BlockIndexHeader diffBlkhdr; + if(!rDiff.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || (int64_t)ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Write it out with a blanked out other file ID + diffBlkhdr.mOtherFileID = hton64(0); + rOut.Write(&diffBlkhdr, sizeof(diffBlkhdr)); + + // Rewrite the index + for(int64_t b = 0; b < DiffNumBlocks; ++b) + { + file_BlockIndexEntry en; + if(!rDiff.ReadFullBuffer(&en, sizeof(en), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // What's the size value stored in the entry + int64_t encodedSize = ntoh64(en.mEncodedSize); + + // Need to adjust it? + if(encodedSize <= 0) + { + // This actually refers to a block in the from file. So rewrite this. + int64_t blockIdx = (0 - encodedSize); + if(blockIdx > FromNumBlocks) + { + // References a block which doesn't actually exist + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Calculate size. This operation is safe because of the extra entry at the end + int32_t blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition; + // Then replace entry + en.mEncodedSize = hton64(((uint64_t)blockSize)); + } + + // Write entry + rOut.Write(&en, sizeof(en)); + } +} + + + + + diff --git a/lib/backupclient/BackupStoreFileCryptVar.cpp b/lib/backupclient/BackupStoreFileCryptVar.cpp new file mode 100755 index 00000000..eeed64e4 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCryptVar.cpp @@ -0,0 +1,31 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCryptVar.cpp +// Purpose: Cryptographic keys for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileWire.h" + +#include "MemLeakFindOn.h" + +CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt; +CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt; + +#ifndef PLATFORM_OLD_OPENSSL + CipherContext BackupStoreFileCryptVar::sAESEncrypt; + CipherContext BackupStoreFileCryptVar::sAESDecrypt; +#endif + +// Default to blowfish +CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt; +uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING; + +CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry; +CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry; + diff --git a/lib/backupclient/BackupStoreFileCryptVar.h b/lib/backupclient/BackupStoreFileCryptVar.h new file mode 100755 index 00000000..00a34f71 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCryptVar.h @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCryptVar.h +// Purpose: Cryptographic keys for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILECRYPTVAR__H +#define BACKUPSTOREFILECRYPTVAR__H + +#include "CipherContext.h" + +// Hide private static variables from the rest of the world by putting them +// as static variables in a namespace. +// -- don't put them as static class variables to avoid openssl/evp.h being +// included all over the project. +namespace BackupStoreFileCryptVar +{ + // Keys for the main file data + extern CipherContext sBlowfishEncrypt; + extern CipherContext sBlowfishDecrypt; + // Use AES when available +#ifndef PLATFORM_OLD_OPENSSL + extern CipherContext sAESEncrypt; + extern CipherContext sAESDecrypt; +#endif + // How encoding will be done + extern CipherContext *spEncrypt; + extern uint8_t sEncryptCipherType; + + // Keys for the block indicies + extern CipherContext sBlowfishEncryptBlockEntry; + extern CipherContext sBlowfishDecryptBlockEntry; +} + +#endif // BACKUPSTOREFILECRYPTVAR__H + diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp new file mode 100755 index 00000000..b72ce328 --- /dev/null +++ b/lib/backupclient/BackupStoreFileDiff.cpp @@ -0,0 +1,973 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileDiff.cpp +// Purpose: Functions relating to diffing BackupStoreFiles +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <map> +#include <signal.h> +#include <sys/time.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreConstants.h" +#include "FileStream.h" +#include "RollingChecksum.h" +#include "MD5Digest.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +using namespace BackupStoreFileCryptVar; +using namespace BackupStoreFileCreation; + +// By default, don't trace out details of the diff as we go along -- would fill up logs significantly. +// But it's useful for the test. +#ifndef NDEBUG + bool BackupStoreFile::TraceDetailsOfDiffProcess = false; +#endif + +static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis); +static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]); +static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]); +static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable); +static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, uint16_t Hash, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks); +static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile); + +// Avoid running on too long with diffs +static int sMaximumDiffTime = 10; // maximum time to spend diffing +static bool sDiffTimedOut = false; +static bool sSetTimerSignelHandler = false; +static void TimerSignalHandler(int signal); +static void StartDiffTimer(); + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::SetMaximumDiffingTime(int) +// Purpose: Sets the maximum time to spend diffing, in seconds. Time is +// process virutal time. +// Created: 19/3/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::SetMaximumDiffingTime(int Seconds) +{ + sMaximumDiffTime = Seconds; + TRACE1("Set maximum diffing time to %d seconds\n", Seconds); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &) +// Purpose: Move the file pointer in this stream to just before the block index. +// Assumes that the stream is at the beginning, seekable, and +// reading from the stream is OK. +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream) +{ + // Size of file + int64_t fileSize = rStream.BytesLeftToRead(); + + // Get header + file_StreamFormat hdr; + + // Read the header + if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Work out where the index is + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + int64_t blockHeaderPosFromEnd = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + + // Sanity check + if(blockHeaderPosFromEnd > static_cast<int64_t>(fileSize - sizeof(file_StreamFormat))) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Seek to that position + rStream.Seek(0 - blockHeaderPosFromEnd, IOStream::SeekType_End); + + // Done. Stream now in right position (as long as the file is formatted correctly) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *) +// Purpose: Similar to EncodeFile, but takes the object ID of the file it's +// diffing from, and the index of the blocks in a stream. It'll then +// calculate which blocks can be reused from that old file. +// The timeout is the timeout value for reading the diff block index. +// If pIsCompletelyDifferent != 0, it will be set to true if the +// the two files are completely different (do not share any block), false otherwise. +// +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff(const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, + int Timeout, int64_t *pModificationTime, bool *pIsCompletelyDifferent) +{ + // Is it a symlink? + { + struct stat st; + if(::lstat(Filename, &st) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + if((st.st_mode & S_IFLNK) == S_IFLNK) + { + // Don't do diffs for symlinks + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = true; + } + return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + } + } + + // Load in the blocks + BlocksAvailableEntry *pindex = 0; + int64_t blocksInIndex = 0; + bool canDiffFromThis = false; + LoadIndex(rDiffFromBlockIndex, DiffFromObjectID, &pindex, blocksInIndex, Timeout, canDiffFromThis); + //TRACE1("Diff: Blocks in index: %lld\n", blocksInIndex); + + if(!canDiffFromThis) + { + // Don't do diffing... + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = true; + } + return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + } + + // Pointer to recipe we're going to create + BackupStoreFileEncodeStream::Recipe *precipe = 0; + + // Start the timeout timer, so that the operation doesn't continue for ever + StartDiffTimer(); + + try + { + // Find which sizes should be scanned + int32_t sizesToScan[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]; + FindMostUsedSizes(pindex, blocksInIndex, sizesToScan); + + // Flag for reporting to the user + bool completelyDifferent; + + // BLOCK + { + // Search the file to find matching blocks + std::map<int64_t, int64_t> foundBlocks; // map of offset in file to index in block index + int64_t sizeOfInputFile = 0; + // BLOCK + { + FileStream file(Filename); + // Get size of file + sizeOfInputFile = file.BytesLeftToRead(); + // Find all those lovely matching blocks + SearchForMatchingBlocks(file, foundBlocks, pindex, blocksInIndex, sizesToScan); + + // Is it completely different? + completelyDifferent = (foundBlocks.size() == 0); + } + + // Create a recipe -- if the two files are completely different, don't put the from file ID in the recipe. + precipe = new BackupStoreFileEncodeStream::Recipe(pindex, blocksInIndex, completelyDifferent?(0):(DiffFromObjectID)); + BlocksAvailableEntry *pindexKeptRef = pindex; // we need this later, but must set pindex == 0 now, because of exceptions + pindex = 0; // Recipe now has ownership + + // Fill it in + GenerateRecipe(*precipe, pindexKeptRef, blocksInIndex, foundBlocks, sizeOfInputFile); + } + // foundBlocks no longer required + + // Create the stream + std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); + + // Do the initial setup + ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime); + precipe = 0; // Stream has taken ownership of this + + // Tell user about completely different status? + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = completelyDifferent; + } + + // Return the stream for the caller + return stream; + } + catch(...) + { + // cleanup + if(pindex != 0) + { + ::free(pindex); + pindex = 0; + } + if(precipe != 0) + { + delete precipe; + precipe = 0; + } + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static LoadIndex(IOStream &, int64_t, BlocksAvailableEntry **, int64_t, bool &) +// Purpose: Read in an index, and decrypt, and store in the in memory block format. +// rCanDiffFromThis is set to false if the version of the from file is too old. +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis) +{ + // Reset + rNumBlocksOut = 0; + rCanDiffFromThis = false; + + // Read header + file_BlockIndexHeader hdr; + if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + // Check against backwards comptaibility stuff + if(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)) + { + // Won't diff against old version + + // Absorb rest of stream + char buffer[2048]; + while(rBlockIndex.StreamDataLeft()) + { + rBlockIndex.Read(buffer, sizeof(buffer), 1000 /* 1 sec timeout */); + } + + // Tell caller + rCanDiffFromThis = false; + return; + } +#endif + + // Check magic + if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Check that we're not trying to diff against a file which references blocks from another file + if(((int64_t)ntoh64(hdr.mOtherFileID)) != 0) + { + THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile) + } + + // Mark as an acceptable diff. + rCanDiffFromThis = true; + + // Get basic information + int64_t numBlocks = ntoh64(hdr.mNumBlocks); + uint64_t entryIVBase = ntoh64(hdr.mEntryIVBase); + + //TODO: Verify that these sizes look reasonable + + // Allocate space for the index + BlocksAvailableEntry *pindex = (BlocksAvailableEntry*)::malloc(sizeof(BlocksAvailableEntry) * numBlocks); + if(pindex == 0) + { + throw std::bad_alloc(); + } + + try + { + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read an entry from the stream + file_BlockIndexEntry entry; + if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read entry + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Calculate IV for this entry + uint64_t iv = entryIVBase; + iv += b; + // Network byte order + iv = hton64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry.mEnEnc, sizeof(entry.mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Check that we're not trying to diff against a file which references blocks from another file + if(((int64_t)ntoh64(entry.mEncodedSize)) <= 0) + { + THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile) + } + + // Store all the required information + pindex[b].mpNextInHashList = 0; // hash list not set up yet + pindex[b].mSize = ntohl(entryEnc.mSize); + pindex[b].mWeakChecksum = ntohl(entryEnc.mWeakChecksum); + ::memcpy(pindex[b].mStrongChecksum, entryEnc.mStrongChecksum, sizeof(pindex[b].mStrongChecksum)); + } + + // Store index pointer for called + ASSERT(ppIndex != 0); + *ppIndex = pindex; + + // Store number of blocks for caller + rNumBlocksOut = numBlocks; + + } + catch(...) + { + // clean up and send the exception along its way + ::free(pindex); + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static FindMostUsedSizes(BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +// Purpose: Finds the most commonly used block sizes in the index +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +{ + // Array for lengths + int64_t sizeCounts[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]; + + // Set arrays to lots of zeros (= unused entries) + for(int l = 0; l < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++l) + { + Sizes[l] = 0; + sizeCounts[l] = 0; + } + + // Array for collecting sizes + std::map<int32_t, int64_t> foundSizes; + + // Run through blocks and make a count of the entries + for(int64_t b = 0; b < NumBlocks; ++b) + { + // Only if the block size is bigger than the minimum size we'll scan for + if(pIndex[b].mSize > BACKUP_FILE_DIFF_MIN_BLOCK_SIZE) + { + // Find entry? + std::map<int32_t, int64_t>::const_iterator f(foundSizes.find(pIndex[b].mSize)); + if(f != foundSizes.end()) + { + // Increment existing entry + foundSizes[pIndex[b].mSize] = foundSizes[pIndex[b].mSize] + 1; + } + else + { + // New entry + foundSizes[pIndex[b].mSize] = 1; + } + } + } + + // Make the block sizes + for(std::map<int32_t, int64_t>::const_iterator i(foundSizes.begin()); i != foundSizes.end(); ++i) + { + // Find the position of the size in the array + for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t) + { + if(i->second > sizeCounts[t]) + { + // Then this size belong before this entry -- shuffle them up + for(int s = (BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1); s >= t; --s) + { + Sizes[s] = Sizes[s-1]; + sizeCounts[s] = sizeCounts[s-1]; + } + + // Insert this size + Sizes[t] = i->first; + sizeCounts[t] = i->second; + + // Shouldn't do any more searching + break; + } + } + } + + // trace the size table in debug builds +#ifndef NDEBUG + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t) + { + TRACE3("Diff block size %d: %d (count = %lld)\n", t, Sizes[t], sizeCounts[t]); + } + } +#endif +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +// Purpose: Find the matching blocks within the file. +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, + BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +{ + // Allocate the hash lookup table + BlocksAvailableEntry **phashTable = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024)); + + // Choose a size for the buffer, just a little bit more than the maximum block size + int32_t bufSize = Sizes[0]; + for(int z = 1; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z) + { + if(Sizes[z] > bufSize) bufSize = Sizes[z]; + } + bufSize += 4; + ASSERT(bufSize > Sizes[0]); + ASSERT(bufSize > 0); + if(bufSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // TODO: Use buffered file class + // Because we read in the file a scanned block size at a time, it is likely to be + // inefficient. Probably will be much better to use a buffering IOStream class which + // reads data in at the size of the filesystem block size. + + // Allocate the buffers. + uint8_t *pbuffer0 = (uint8_t *)::malloc(bufSize); + uint8_t *pbuffer1 = (uint8_t *)::malloc(bufSize); + try + { + // Check buffer allocation + if(pbuffer0 == 0 || pbuffer1 == 0 || phashTable == 0) + { + // If a buffer got alloocated, it will be cleaned up in the catch block + throw std::bad_alloc(); + } + + // Flag to abort the run, if too many blocks are found -- avoid using + // huge amounts of processor time when files contain many similar blocks. + bool abortSearch = false; + + // Search for each block size in turn + // NOTE: Do the smallest size first, so that the scheme for adding + // entries in the found list works as expected and replaces smallers block + // with larger blocks when it finds matches at the same offset in the file. + for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s) + { + ASSERT(Sizes[s] <= bufSize); + //TRACE2("Diff pass %d, for block size %d\n", s, Sizes[s]); + + // Check we haven't finished + if(Sizes[s] == 0) + { + // empty entry, try next size + continue; + } + + // Set up the hash table entries + SetupHashTable(pIndex, NumBlocks, Sizes[s], phashTable); + + // Shift file position to beginning + rFile.Seek(0, IOStream::SeekType_Absolute); + + // Read first block + if(rFile.Read(pbuffer0, Sizes[s]) != Sizes[s]) + { + // Size of file too short to match -- do next size + continue; + } + + // Setup block pointers + uint8_t *beginnings = pbuffer0; + uint8_t *endings = pbuffer1; + int offset = 0; + + // Calculate the first checksum, ready for rolling + RollingChecksum rolling(beginnings, Sizes[s]); + + // Then roll, until the file is exhausted + int64_t fileBlockNumber = 0; + int rollOverInitialBytes = 0; + while(true) + { + // Load in another block of data, and record how big it is + int bytesInEndings = rFile.Read(endings, Sizes[s]); + + // Skip any bytes from a previous matched block + while(rollOverInitialBytes > 0 && offset < bytesInEndings) + { + rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]); + ++offset; + --rollOverInitialBytes; + } + + while(offset < bytesInEndings) + { + // Put is current checksum in hash list? + uint16_t hash = rolling.GetComponentForHashing(); + if(phashTable[hash] != 0) + { + if(SecondStageMatch(phashTable[hash], hash, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks)) + { + // Block matched, roll the checksum forward to the next block without doing + // any more comparisons, because these are pointless (as any more matches will be ignored when + // the receipe is generated) and just take up valuable processor time. Edge cases are + // especially nasty, using huge amounts of time and memory. + int skip = Sizes[s]; + while(offset < bytesInEndings && skip > 0) + { + rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]); + ++offset; + --skip; + } + // Not all the bytes necessary will have been skipped, so get them + // skipped after the next block is loaded. + rollOverInitialBytes = skip; + + // End this loop, so the final byte isn't used again + break; + } + + if(static_cast<int64_t>(rFoundBlocks.size()) > (NumBlocks * BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE) + || sDiffTimedOut) + { + abortSearch = true; + break; + } + } + + // Roll checksum forward + rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]); + + // Increment offset + ++offset; + } + + if(abortSearch) break; + + // Finished? + if(bytesInEndings != Sizes[s]) + { + // No more data in file -- check the final block + // (Do a copy and paste of 5 lines of code instead of introducing a comparison for + // each byte of the file) + uint16_t hash = rolling.GetComponentForHashing(); + if(phashTable[hash] != 0) + { + SecondStageMatch(phashTable[hash], hash, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks); + } + + // finish + break; + } + + // Switch buffers, reset offset + beginnings = endings; + endings = (beginnings == pbuffer0)?(pbuffer1):(pbuffer0); // ie the other buffer + offset = 0; + + // And count the blocks which have been done + ++fileBlockNumber; + } + + if(abortSearch) break; + } + + // Free buffers and hash table + ::free(pbuffer1); + pbuffer1 = 0; + ::free(pbuffer0); + pbuffer0 = 0; + ::free(phashTable); + phashTable = 0; + } + catch(...) + { + // Cleanup and throw + if(pbuffer1 != 0) ::free(pbuffer1); + if(pbuffer0 != 0) ::free(pbuffer0); + if(phashTable != 0) ::free(phashTable); + throw; + } + +#ifndef NDEBUG + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + // Trace out the found blocks in debug mode + TRACE0("Diff: list of found blocks\n======== ======== ======== ========\n Offset BlkIdx Size Movement\n"); + for(std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); i != rFoundBlocks.end(); ++i) + { + int64_t orgLoc = 0; + for(int64_t b = 0; b < i->second; ++b) + { + orgLoc += pIndex[b].mSize; + } + TRACE4("%8lld %8lld %8lld %8lld\n", i->first, i->second, + (int64_t)(pIndex[i->second].mSize), i->first - orgLoc); + } + TRACE0("======== ======== ======== ========\n"); + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static SetupHashTable(BlocksAvailableEntry *, int64_t, in32_t, BlocksAvailableEntry **) +// Purpose: Set up the hash table ready for a scan +// Created: 14/1/04 +// +// -------------------------------------------------------------------------- +static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable) +{ + // Set all entries in the hash table to zero + ::memset(pHashTable, 0, (sizeof(BlocksAvailableEntry *) * (64*1024))); + + // Scan through the blocks, building the hash table + for(int64_t b = 0; b < NumBlocks; ++b) + { + // Only look at the required block size + if(pIndex[b].mSize == BlockSize) + { + // Get the value under which to hash this entry + uint16_t hash = RollingChecksum::ExtractHashingComponent(pIndex[b].mWeakChecksum); + + // Already present in table? + if(pHashTable[hash] != 0) + { + //TRACE1("Another hash entry for %d found\n", hash); + // Yes -- need to set the pointer in this entry to the current entry to build the linked list + pIndex[b].mpNextInHashList = pHashTable[hash]; + } + + // Put a pointer to this entry in the hash table + pHashTable[hash] = pIndex + b; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static bool SecondStageMatch(xxx) +// Purpose: When a match in the hash table is found, scan for second stage match using strong checksum. +// Created: 14/1/04 +// +// -------------------------------------------------------------------------- +static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, uint16_t Hash, uint8_t *pBeginnings, uint8_t *pEndings, + int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks) +{ + // Check parameters + ASSERT(pBeginnings != 0); + ASSERT(pEndings != 0); + ASSERT(Offset >= 0); + ASSERT(BlockSize > 0); + ASSERT(pFirstInHashList != 0); + ASSERT(pIndex != 0); + + // Calculate the strong MD5 digest for this block + MD5Digest strong; + // Add the data from the beginnings + strong.Add(pBeginnings + Offset, BlockSize - Offset); + // Add any data from the endings + if(Offset > 0) + { + strong.Add(pEndings, Offset); + } + strong.Finish(); + + // Then go through the entries in the hash list, comparing with the strong digest calculated + BlocksAvailableEntry *scan = pFirstInHashList; + //TRACE0("second stage match\n"); + while(scan != 0) + { + //TRACE3("scan size %d, block size %d, hash %d\n", scan->mSize, BlockSize, Hash); + ASSERT(scan->mSize == BlockSize); + ASSERT(RollingChecksum::ExtractHashingComponent(scan->mWeakChecksum) == Hash); + + // Compare? + if(strong.DigestMatches(scan->mStrongChecksum)) + { + //TRACE0("Match!\n"); + // Found! Add to list of found blocks... + int64_t fileOffset = (FileBlockNumber * BlockSize) + Offset; + int64_t blockIndex = (scan - pIndex); // pointer arthmitic is frowed apon. But most efficient way of doing it here -- alternative is to use more memory + + // Because we search for smallest blocks first, if there's an entry there + // we'll just be replacing it with a larger one, which is great news. + rFoundBlocks[fileOffset] = blockIndex; + + // No point in searching further, report success + return true; + } + + // Next + scan = scan->mpNextInHashList; + } + + // Not matched + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static GenerateRecipe(BackupStoreFileEncodeStream::Recipe &, BlocksAvailableEntry *, int64_t, std::map<int64_t, int64_t> &) +// Purpose: Fills in the recipe from the found block list +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, + int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile) +{ + // NOTE: This function could be a lot more sophisiticated. For example, if + // a small block overlaps a big block like this + // **** + // ******************************* + // then the small block will be used, not the big one. But it'd be better to + // just ignore the small block and keep the big one. However, some stats should + // be gathered about real world files before writing complex code which might + // go wrong. + + // Initialise a blank instruction + BackupStoreFileEncodeStream::RecipeInstruction instruction; + #define RESET_INSTRUCTION \ + instruction.mSpaceBefore = 0; \ + instruction.mBlocks = 0; \ + instruction.mpStartBlock = 0; + RESET_INSTRUCTION + + // First, a special case for when there are no found blocks + if(rFoundBlocks.size() == 0) + { + // No blocks, just a load of space + instruction.mSpaceBefore = SizeOfInputFile; + rRecipe.push_back(instruction); + + #ifndef NDEBUG + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + TRACE1("Diff: Default recipe generated, %lld bytes of file\n", SizeOfInputFile); + } + #endif + + // Don't do anything + return; + } + + // Current location + int64_t loc = 0; + + // Then iterate through the list, generating the recipe + std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); + ASSERT(i != rFoundBlocks.end()); // check logic + + // Counting for debug tracing +#ifndef NDEBUG + int64_t debug_NewBytesFound = 0; + int64_t debug_OldBlocksUsed = 0; +#endif + + for(; i != rFoundBlocks.end(); ++i) + { + // Remember... map is (position in file) -> (index of block in pIndex) + + if(i->first < loc) + { + // This block overlaps the last one + continue; + } + else if(i->first > loc) + { + // There's a gap between the end of the last thing and this block. + // If there's an instruction waiting, push it onto the list + if(instruction.mSpaceBefore != 0 || instruction.mpStartBlock != 0) + { + rRecipe.push_back(instruction); + } + // Start a new instruction, with the gap ready + RESET_INSTRUCTION + instruction.mSpaceBefore = i->first - loc; + // Move location forward to match + loc += instruction.mSpaceBefore; +#ifndef NDEBUG + debug_NewBytesFound += instruction.mSpaceBefore; +#endif + } + + // First, does the current instruction need pushing back, because this block is not + // sequential to the last one? + if(instruction.mpStartBlock != 0 && (pIndex + i->second) != (instruction.mpStartBlock + instruction.mBlocks)) + { + rRecipe.push_back(instruction); + RESET_INSTRUCTION + } + + // Add in this block + if(instruction.mpStartBlock == 0) + { + // This block starts a new instruction + instruction.mpStartBlock = pIndex + i->second; + instruction.mBlocks = 1; + } + else + { + // It continues the previous section of blocks + instruction.mBlocks += 1; + } + +#ifndef NDEBUG + debug_OldBlocksUsed++; +#endif + + // Move location forward + loc += pIndex[i->second].mSize; + } + + // Push the last instruction generated + rRecipe.push_back(instruction); + + // Is there any space left at the end which needs sending? + if(loc != SizeOfInputFile) + { + RESET_INSTRUCTION + instruction.mSpaceBefore = SizeOfInputFile - loc; +#ifndef NDEBUG + debug_NewBytesFound += instruction.mSpaceBefore; +#endif + rRecipe.push_back(instruction); + } + + // dump out the recipe +#ifndef NDEBUG + TRACE2("Diff: %lld new bytes found, %lld old blocks used\n", debug_NewBytesFound, debug_OldBlocksUsed); + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + TRACE1("Diff: Recipe generated (size %d)\n======== ========= ========\nSpace b4 FirstBlk NumBlks\n", rRecipe.size()); + { + for(unsigned int e = 0; e < rRecipe.size(); ++e) + { + char b[64]; + sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); + TRACE3("%8lld %s %8lld\n", rRecipe[e].mSpaceBefore, (rRecipe[e].mpStartBlock == 0)?" -":b, (int64_t)rRecipe[e].mBlocks); + } + } + TRACE0("======== ========= ========\n"); + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static TimerSignalHandler(int) +// Purpose: Signal handler +// Created: 19/3/04 +// +// -------------------------------------------------------------------------- +void TimerSignalHandler(int signal) +{ + sDiffTimedOut = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static StartDiffTimer() +// Purpose: Starts the diff timeout timer +// Created: 19/3/04 +// +// -------------------------------------------------------------------------- +void StartDiffTimer() +{ + // Set timer signal handler + if(!sSetTimerSignelHandler) + { + ::signal(SIGVTALRM, TimerSignalHandler); + sSetTimerSignelHandler = true; + } + + struct itimerval timeout; + // Don't want this to repeat + timeout.it_interval.tv_sec = 0; + timeout.it_interval.tv_usec = 0; + // Single timeout after the specified number of seconds + timeout.it_value.tv_sec = sMaximumDiffTime; + timeout.it_value.tv_usec = 0; + // Set timer + if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0) + { + TRACE0("WARNING: couldn't set diff timeout\n"); + } + + // Unset flag (last thing) + sDiffTimedOut = false; +} + + + diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp new file mode 100755 index 00000000..20e1fd80 --- /dev/null +++ b/lib/backupclient/BackupStoreFileEncodeStream.cpp @@ -0,0 +1,675 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileEncodeStream.cpp +// Purpose: Implement stream-based file encoding for the backup store +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BoxTime.h" +#include "BackupClientFileAttributes.h" +#include "FileStream.h" +#include "RollingChecksum.h" +#include "Random.h" + +#include "MemLeakFindOn.h" + +using namespace BackupStoreFileCryptVar; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream +// Purpose: Constructor (opens file) +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() + : mpRecipe(0), + mpFile(0), + mStatus(Status_Header), + mSendData(true), + mTotalBlocks(0), + mAbsoluteBlockNumber(-1), + mInstructionNumber(-1), + mNumBlocks(0), + mCurrentBlock(-1), + mCurrentBlockEncodedSize(0), + mPositionInCurrentBlock(0), + mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), + mLastBlockSize(0), + mpRawBuffer(0), + mAllocatedBufferSize(0), + mEntryIVBase(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() +// Purpose: Destructor +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() +{ + // Free buffers + if(mpRawBuffer) + { + ::free(mpRawBuffer); + mpRawBuffer = 0; + } + + // Close the file, which we might have open + if(mpFile) + { + delete mpFile; + mpFile = 0; + } + + // Free the recipe + if(mpRecipe != 0) + { + delete mpRecipe; + mpRecipe = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *) +// Purpose: Reads file information, and builds file header reading for sending. +// Takes ownership of the Recipe. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEncodeStream::Recipe *pRecipe, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime) +{ + // Pointer to a blank recipe which we might create + BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; + + try + { + // Get file attributes + box_time_t modTime = 0; + int64_t fileSize = 0; + BackupClientFileAttributes attr; + attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime, + 0 /* not interested in attr mod time */, &fileSize); + + // Might need to create a blank recipe... + if(pRecipe == 0) + { + pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); + + BackupStoreFileEncodeStream::RecipeInstruction instruction; + instruction.mSpaceBefore = fileSize; // whole file + instruction.mBlocks = 0; // no blocks + instruction.mpStartBlock = 0; // no block + pblankRecipe->push_back(instruction); + + pRecipe = pblankRecipe; + } + + // Tell caller? + if(pModificationTime != 0) + { + *pModificationTime = modTime; + } + + // Go through each instruction in the recipe and work out how many blocks + // it will add, and the max clear size of these blocks + int maxBlockClearSize = 0; + for(uint64_t inst = 0; inst < pRecipe->size(); ++inst) + { + if((*pRecipe)[inst].mSpaceBefore > 0) + { + // Calculate the number of blocks the space before requires + int64_t numBlocks; + int32_t blockSize, lastBlockSize; + CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize); + // Add to accumlated total + mTotalBlocks += numBlocks; + // Update maximum clear size + if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize; + if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize; + } + + // Add number of blocks copied from the previous file + mTotalBlocks += (*pRecipe)[inst].mBlocks; + + // Check for bad things + if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Run through blocks to get the max clear size + for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b) + { + if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize; + } + } + + // Send data? (symlinks don't have any data in them) + mSendData = !attr.IsSymLink(); + + // If not data is being sent, then the max clear block size is zero + if(!mSendData) + { + maxBlockClearSize = 0; + } + + // Header + file_StreamFormat hdr; + hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1); + hdr.mNumBlocks = (mSendData)?(hton64(mTotalBlocks)):(0); + hdr.mContainerID = hton64(ContainerID); + hdr.mModificationTime = hton64(modTime); + // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size + hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128); + hdr.mOptions = 0; // no options defined yet + + // Write header to stream + mData.Write(&hdr, sizeof(hdr)); + + // Write filename to stream + rStoreFilename.WriteToStream(mData); + + // Write attributes to stream + attr.WriteToStream(mData); + + // Allocate some buffers for writing data + if(mSendData) + { + // Open the file + mpFile = new FileStream(Filename); + + // Work out the largest possible block required for the encoded data + mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); + + // Then allocate two blocks of this size + mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize); + if(mpRawBuffer == 0) + { + throw std::bad_alloc(); + } +#ifndef NDEBUG + // In debug builds, make sure that the reallocation code is exercised. + mEncodedBuffer.Allocate(mAllocatedBufferSize / 4); +#else + mEncodedBuffer.Allocate(mAllocatedBufferSize); +#endif + } + else + { + // Write an empty block index for the symlink + file_BlockIndexHeader blkhdr; + blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); + blkhdr.mOtherFileID = hton64(0); // not other file ID + blkhdr.mEntryIVBase = hton64(0); + blkhdr.mNumBlocks = hton64(0); + mData.Write(&blkhdr, sizeof(blkhdr)); + } + + // Ready for reading + mData.SetForReading(); + + // Update stats + BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize; + + // Finally, store the pointer to the recipe, when we know exceptions won't occur + mpRecipe = pRecipe; + } + catch(...) + { + // Clean up any blank recipe + if(pblankRecipe != 0) + { + delete pblankRecipe; + pblankRecipe = 0; + } + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &) +// Purpose: Calculates the sizes of blocks in a section of the file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut) +{ + // How many blocks, and how big? + rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2; + do + { + rBlockSizeOut *= 2; + + rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut; + + } while(rBlockSizeOut <= BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); + + // Last block size + rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut); + + // Avoid small blocks? + if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN + && rNumBlocksOut > 1) + { + // Add the small bit of data to the last block + --rNumBlocksOut; + rLastBlockSizeOut += rBlockSizeOut; + } + + // checks! + ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize); + //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Read(void *, int, int) +// Purpose: As interface -- generates encoded file data on the fly from the raw file +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Check there's something to do. + if(mStatus == Status_Finished) + { + return 0; + } + + int bytesToRead = NBytes; + uint8_t *buffer = (uint8_t*)pBuffer; + + while(bytesToRead > 0 && mStatus != Status_Finished) + { + if(mStatus == Status_Header || mStatus == Status_BlockListing) + { + // Header or block listing phase -- send from the buffered stream + + // Send bytes from the data buffer + int b = mData.Read(buffer, bytesToRead, Timeout); + bytesToRead -= b; + buffer += b; + + // Check to see if all the data has been used from this stream + if(!mData.StreamDataLeft()) + { + // Yes, move on to next phase (or finish, if there's no file data) + if(!mSendData) + { + mStatus = Status_Finished; + } + else + { + // Reset the buffer so it can be used for the next phase + mData.Reset(); + + // Get buffer ready for index? + if(mStatus == Status_Header) + { + // Just finished doing the stream header, create the block index header + file_BlockIndexHeader blkhdr; + blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); + ASSERT(mpRecipe != 0); + blkhdr.mOtherFileID = hton64(mpRecipe->GetOtherFileID()); + blkhdr.mNumBlocks = hton64(mTotalBlocks); + + // Generate the IV base + Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase)); + blkhdr.mEntryIVBase = hton64(mEntryIVBase); + + mData.Write(&blkhdr, sizeof(blkhdr)); + } + + ++mStatus; + } + } + } + else if(mStatus == Status_Blocks) + { + // Block sending phase + + if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize) + { + // Next block! + ++mCurrentBlock; + ++mAbsoluteBlockNumber; + if(mCurrentBlock >= mNumBlocks) + { + // Output extra blocks for this instruction and move forward in file + if(mInstructionNumber >= 0) + { + SkipPreviousBlocksInInstruction(); + } + + // Is there another instruction to go? + ++mInstructionNumber; + + // Skip instructions which don't contain any data + while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size()) + && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0) + { + SkipPreviousBlocksInInstruction(); + ++mInstructionNumber; + } + + if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size())) + { + // End of blocks, go to next phase + ++mStatus; + + // Set the data to reading so the index can be written + mData.SetForReading(); + } + else + { + // Get ready for this instruction + SetForInstruction(); + } + } + + // Can't use 'else' here as SetForInstruction() will change this + if(mCurrentBlock < mNumBlocks) + { + EncodeCurrentBlock(); + } + } + + // Send data from the current block (if there's data to send) + if(mPositionInCurrentBlock < mCurrentBlockEncodedSize) + { + // How much data to put in the buffer? + int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock; + if(s > bytesToRead) s = bytesToRead; + + // Copy it in + ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s); + + // Update variables + bytesToRead -= s; + buffer += s; + mPositionInCurrentBlock += s; + } + } + else + { + // Should never get here, as it'd be an invalid status + ASSERT(false); + } + } + + // Add encoded size to stats + BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead); + + // Return size of data to caller + return NBytes - bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction() +// Purpose: Private. Stores the blocks of the old file referenced in the current +// instruction into the index and skips over the data in the file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction() +{ + // Check something is necessary + if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0) + { + return; + } + + // Index of the first block in old file (being diffed from) + int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock); + + int64_t sizeToSkip = 0; + + for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b) + { + // Update stats + BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; + + // Store the entry + StoreBlockIndexEntry(0 - (firstIndex + b), + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize, + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum, + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); + + // Increment the absolute block number -- kept encryption IV in sync + ++mAbsoluteBlockNumber; + + // Add the size of this block to the size to skip + sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; + } + + // Move forward in the stream + mpFile->Seek(sizeToSkip, IOStream::SeekType_Relative); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::SetForInstruction() +// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::SetForInstruction() +{ + // Calculate block sizes + CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize); + + // Set variables + mCurrentBlock = 0; + mCurrentBlockEncodedSize = 0; + mPositionInCurrentBlock = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock() +// Purpose: Private. Encodes the current block, and writes the block data to the index +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::EncodeCurrentBlock() +{ + // How big is the block, raw? + int blockRawSize = mBlockSize; + if(mCurrentBlock == (mNumBlocks - 1)) + { + blockRawSize = mLastBlockSize; + } + ASSERT(blockRawSize < mAllocatedBufferSize); + + // Check file open + if(mpFile == 0) + { + // File should be open, but isn't. So logical error. + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Read the data in + if(!mpFile->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */)) + { + // TODO: Do something more intelligent, and abort this upload because the file + // has changed + THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer) + } + + // Encode it + mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer); + + //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); + + // Create block listing data -- generate checksums + RollingChecksum weakChecksum(mpRawBuffer, blockRawSize); + MD5Digest strongChecksum; + strongChecksum.Add(mpRawBuffer, blockRawSize); + strongChecksum.Finish(); + + // Add entry to the index + StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); + + // Set vars to reading this block + mPositionInCurrentBlock = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *) +// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream. +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum) +{ + // First, the encrypted section + file_BlockIndexEntryEnc entryEnc; + entryEnc.mSize = htonl(ClearSize); + entryEnc.mWeakChecksum = htonl(WeakChecksum); + ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum)); + + // Then the clear section + file_BlockIndexEntry entry; + entry.mEncodedSize = hton64(((uint64_t)EncSizeOrBlkIndex)); + + // Then encrypt the encryted section + // Generate the IV from the block number + if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase)) + { + THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements) + } + uint64_t iv = mEntryIVBase; + iv += mAbsoluteBlockNumber; + // Convert to network byte order before encrypting with it, so that restores work on + // platforms with different endiannesses. + iv = hton64(iv); + sBlowfishEncryptBlockEntry.SetIV(&iv); + + // Encode the data + int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc)); + if(encodedSize != sizeof(entry.mEnEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Save to data block for sending at the end of the stream + mData.Write(&entry, sizeof(entry)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Write(const void *, int) +// Purpose: As interface. Exceptions. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StreamDataLeft() +// Purpose: As interface -- end of stream reached? +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFileEncodeStream::StreamDataLeft() +{ + return (mStatus != Status_Finished); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StreamClosed() +// Purpose: As interface +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFileEncodeStream::StreamClosed() +{ + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t) +// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, + int64_t NumBlocksInIndex, int64_t OtherFileID) + : mpBlockIndex(pBlockIndex), + mNumBlocksInIndex(NumBlocksInIndex), + mOtherFileID(OtherFileID) +{ + ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Recipe::~Recipe() +// Purpose: Destructor +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::Recipe::~Recipe() +{ + // Free the block index, if there is one + if(mpBlockIndex != 0) + { + ::free(mpBlockIndex); + } +} + + + + diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h new file mode 100755 index 00000000..1c748798 --- /dev/null +++ b/lib/backupclient/BackupStoreFileEncodeStream.h @@ -0,0 +1,127 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileEncodeStream.h +// Purpose: Implement stream-based file encoding for the backup store +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILEENCODESTREAM__H +#define BACKUPSTOREFILEENCODESTREAM__H + +#include <vector> + +#include "IOStream.h" +#include "BackupStoreFilename.h" +#include "CollectInBufferStream.h" +#include "MD5Digest.h" +#include "BackupStoreFile.h" + +namespace BackupStoreFileCreation +{ + // Diffing and creation of files share some implementation details. + typedef struct _BlocksAvailableEntry + { + struct _BlocksAvailableEntry *mpNextInHashList; + int32_t mSize; // size in clear + uint32_t mWeakChecksum; // weak, rolling checksum + uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum + } BlocksAvailableEntry; + +} + + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFileEncodeStream +// Purpose: Encode a file into a stream +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +class BackupStoreFileEncodeStream : public IOStream +{ +public: + BackupStoreFileEncodeStream(); + ~BackupStoreFileEncodeStream(); + + typedef struct + { + int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go + int32_t mBlocks; // number of block to reuse, starting at this one + BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null + } RecipeInstruction; + + class Recipe : public std::vector<RecipeInstruction> + { + // NOTE: This class is rather tied in with the implementation of diffing. + public: + Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex, + int64_t OtherFileID = 0); + ~Recipe(); + + int64_t GetOtherFileID() {return mOtherFileID;} + int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock) + { + return pBlock - mpBlockIndex; + } + + private: + BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex; + int64_t mNumBlocksInIndex; + int64_t mOtherFileID; + }; + + void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime); + + virtual int Read(void *pBuffer, int NBytes, int Timeout); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + enum + { + Status_Header = 0, + Status_Blocks = 1, + Status_BlockListing = 2, + Status_Finished = 3 + }; + +private: + void EncodeCurrentBlock(); + void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); + void SkipPreviousBlocksInInstruction(); + void SetForInstruction(); + void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum); + +private: + Recipe *mpRecipe; + IOStream *mpFile; // source file + CollectInBufferStream mData; // buffer for header and index entries + int mStatus; + bool mSendData; // true if there's file data to send (ie not a symlink) + int64_t mTotalBlocks; // Total number of blocks in the file + int64_t mAbsoluteBlockNumber; // The absolute block number currently being output + // Instruction number + int64_t mInstructionNumber; + // All the below are within the current instruction + int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases + int64_t mCurrentBlock; + int32_t mCurrentBlockEncodedSize; + int32_t mPositionInCurrentBlock; // for reading out + int32_t mBlockSize; // Basic block size of most of the blocks in the file + int32_t mLastBlockSize; // the size (unencoded) of the last block in the file + // Buffers + uint8_t *mpRawBuffer; // buffer for raw data + BackupStoreFile::EncodingBuffer mEncodedBuffer; + // buffer for encoded data + int32_t mAllocatedBufferSize; // size of above two allocated blocks + uint64_t mEntryIVBase; // base for block entry IV +}; + + + +#endif // BACKUPSTOREFILEENCODESTREAM__H + diff --git a/lib/backupclient/BackupStoreFileRevDiff.cpp b/lib/backupclient/BackupStoreFileRevDiff.cpp new file mode 100644 index 00000000..f1dc52d8 --- /dev/null +++ b/lib/backupclient/BackupStoreFileRevDiff.cpp @@ -0,0 +1,258 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileRevDiff.cpp +// Purpose: Reverse a patch, to build a new patch from new to old files +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <stdlib.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t) +// Purpose: Reverse a patch, to build a new patch from new to old files. Takes +// two independent copies to the From file, for efficiency. +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent) +{ + // Read and copy the header from the from file to the out file -- beginnings of the patch + file_StreamFormat hdr; + if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Copy + rOut.Write(&hdr, sizeof(hdr)); + // Copy over filename and attributes + // BLOCK + { + BackupStoreFilename filename; + filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + filename.WriteToStream(rOut); + StreamableMemBlock attr; + attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + attr.WriteToStream(rOut); + } + + // Build an index of common blocks. + // For each block in the from file, we want to know it's index in the + // diff file. Allocate memory for this information. + int64_t fromNumBlocks = ntoh64(hdr.mNumBlocks); + int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t)); + if(pfromIndexInfo == 0) + { + throw std::bad_alloc(); + } + + // Buffer data + void *buffer = 0; + int bufferSize = 0; + + // flag + bool isCompletelyDifferent = true; + + try + { + // Initialise the index to be all 0, ie not filled in yet + for(int64_t i = 0; i < fromNumBlocks; ++i) + { + pfromIndexInfo[i] = 0; + } + + // Within the from file, skip to the index + MoveStreamPositionToBlockIndex(rDiff); + + // Read in header of index + file_BlockIndexHeader diffIdxHdr; + if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // And then read in each entry + int64_t diffNumBlocks = ntoh64(diffIdxHdr.mNumBlocks); + for(int64_t b = 0; b < diffNumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Where's the block? + int64_t blockEn = ntoh64(e.mEncodedSize); + if(blockEn > 0) + { + // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file + } + else + { + // Block is in the original file, store which block it is in this file + int64_t fromIndex = 0 - blockEn; + if(fromIndex < 0 || fromIndex >= fromNumBlocks) + { + THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles) + } + + // Store information about where it is in the new file + // NOTE: This is slight different to how it'll be stored in the final index. + pfromIndexInfo[fromIndex] = -1 - b; + } + } + + // Open the index for the second copy of the from file + MoveStreamPositionToBlockIndex(rFrom2); + + // Read in header of index + file_BlockIndexHeader fromIdxHdr; + if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || ntoh64(fromIdxHdr.mOtherFileID) != 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // So, we can now start building the data in the file + int64_t filePosition = rFrom.GetPosition(); + for(int64_t b = 0; b < fromNumBlocks; ++b) + { + // Read entry from from index + file_BlockIndexEntry e; + if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Get size + int64_t blockSize = hton64(e.mEncodedSize); + if(blockSize < 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Copy this block? + if(pfromIndexInfo[b] == 0) + { + // Copy it, first move to file location + rFrom.Seek(filePosition, IOStream::SeekType_Absolute); + + // Make sure there's memory available to copy this + if(bufferSize < blockSize || buffer == 0) + { + // Free old block + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + bufferSize = 0; + } + // Allocate new block + buffer = ::malloc(blockSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + bufferSize = blockSize; + } + ASSERT(bufferSize >= blockSize); + + // Copy the block + if(!rFrom.ReadFullBuffer(buffer, blockSize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + rOut.Write(buffer, blockSize); + + // Store the size + pfromIndexInfo[b] = blockSize; + } + else + { + // Block isn't needed, so it's not completely different + isCompletelyDifferent = false; + } + filePosition += blockSize; + } + + // Then write the index, modified header first + fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(hton64(ObjectIDOfFrom)); + rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr)); + + // Move to start of index entries + rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute); + + // Then copy modified entries + for(int64_t b = 0; b < fromNumBlocks; ++b) + { + // Read entry from from index + file_BlockIndexEntry e; + if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Modify... + int64_t s = pfromIndexInfo[b]; + // Adjust to reflect real block index (remember 0 has a different meaning here) + if(s < 0) ++s; + // Insert + e.mEncodedSize = hton64(s); + // Write + rOut.Write(&e, sizeof(e)); + } + } + catch(...) + { + ::free(pfromIndexInfo); + if(buffer != 0) + { + ::free(buffer); + } + throw; + } + + // Free memory used (oh for finally {} blocks) + ::free(pfromIndexInfo); + if(buffer != 0) + { + ::free(buffer); + } + + // return completely different flag + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = isCompletelyDifferent; + } +} + + + diff --git a/lib/backupclient/BackupStoreFileWire.h b/lib/backupclient/BackupStoreFileWire.h new file mode 100755 index 00000000..5b1bc819 --- /dev/null +++ b/lib/backupclient/BackupStoreFileWire.h @@ -0,0 +1,74 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileWire.h +// Purpose: On the wire / disc formats for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILEWIRE__H +#define BACKUPSTOREFILEWIRE__H + +#include "MD5Digest.h" + +// set packing to one byte +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + int32_t mMagicValue; // also the version number + int64_t mNumBlocks; // number of blocks contained in the file + int64_t mContainerID; + int64_t mModificationTime; + int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block + int32_t mOptions; // bitmask of options used + // Then a BackupStoreFilename + // Then a BackupClientFileAttributes +} file_StreamFormat; + +typedef struct +{ + int32_t mMagicValue; // different magic value + int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index + uint64_t mEntryIVBase; // base value for block IV + int64_t mNumBlocks; // repeat of value in file header +} file_BlockIndexHeader; + +typedef struct +{ + int32_t mSize; // size in clear + uint32_t mWeakChecksum; // weak, rolling checksum + uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum +} file_BlockIndexEntryEnc; + +typedef struct +{ + union + { + int64_t mEncodedSize; // size encoded, if > 0 + int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0 + }; + uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section +} file_BlockIndexEntry; + +// Use default packing +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +// header for blocks of compressed data in files +#define HEADER_CHUNK_IS_COMPRESSED 1 // bit +#define HEADER_ENCODING_SHIFT 1 // shift value +#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7 +#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7 + + +#endif // BACKUPSTOREFILEWIRE__H + diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp new file mode 100755 index 00000000..fbfe3313 --- /dev/null +++ b/lib/backupclient/BackupStoreFilename.cpp @@ -0,0 +1,279 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilename.cpp +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreFilename.h" +#include "Protocol.h" +#include "BackupStoreException.h" +#include "IOStream.h" +#include "Guards.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::BackupStoreFilename() +// Purpose: Default constructor -- creates an invalid filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::BackupStoreFilename() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy) + : BackupStoreFilename_base(rToCopy) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::~BackupStoreFilename() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::~BackupStoreFilename() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::CheckValid(bool) +// Purpose: Checks the encoded filename for validity +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const +{ + bool ok = true; + + if(size() < 2) + { + // Isn't long enough to have a header + ok = false; + } + else + { + // Check size is consistent + unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(*this); + if(dsize != size()) + { + ok = false; + } + + // And encoding is an accepted value + unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this); + if(encoding < Encoding_Min || encoding > Encoding_Max) + { + ok = false; + } + } + + // Exception? + if(!ok && ExceptionIfInvalid) + { + THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename) + } + + return ok; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::ReadFromProtocol(Protocol &) +// Purpose: Reads the filename from the protocol object +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol) +{ + // Read the header + char hdr[2]; + rProtocol.Read(hdr, 2); + + // How big is it? + int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); + + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + std::string data; + rProtocol.Read(data, dsize - 2); + + // assign to this string, storing the header and the extra data + assign(hdr, 2); + append(data.c_str(), data.size()); + + // Check it + CheckValid(); + + // Alert derived classes + EncodedFilenameChanged(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::WriteToProtocol(Protocol &) +// Purpose: Writes the filename to the protocol object +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const +{ + CheckValid(); + + rProtocol.Write(c_str(), size()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::ReadFromStream(IOStream &) +// Purpose: Reads the filename from a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Read the header + char hdr[2]; + if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // How big is it? + unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); + + // Assume most filenames are small + char buf[256]; + if(dsize < sizeof(buf)) + { + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + // Copy in header + buf[0] = hdr[0]; buf[1] = hdr[1]; + + // assign to this string, storing the header and the extra data + assign(buf, dsize); + } + else + { + // Block of memory to hold it + MemoryBlockGuard<char*> dataB(dsize+2); + char *data = dataB; + + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + // Copy in header + data[0] = hdr[0]; data[1] = hdr[1]; + + // assign to this string, storing the header and the extra data + assign(data, dsize); + } + + // Check it + CheckValid(); + + // Alert derived classes + EncodedFilenameChanged(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::WriteToStream(IOStream &) +// Purpose: Writes the filename to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::WriteToStream(IOStream &rStream) const +{ + CheckValid(); + + rStream.Write(c_str(), size()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::EncodedFilenameChanged() +// Purpose: The encoded filename stored has changed +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::EncodedFilenameChanged() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::IsEncrypted() +// Purpose: Returns true if the filename is stored using an encrypting encoding +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFilename::IsEncrypted() const +{ + return BACKUPSTOREFILENAME_GET_ENCODING(*this) != Encoding_Clear; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::SetAsClearFilename(const char *) +// Purpose: Sets this object to be a valid filename, but with a filename in the clear. +// Used on the server to create filenames when there's no way of encrypting it. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::SetAsClearFilename(const char *Clear) +{ + // Make std::string from the clear name + std::string toEncode(Clear); + + // Make an encoded string + char hdr[2]; + BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear); + std::string encoded(hdr, 2); + encoded += toEncode; + ASSERT(encoded.size() == toEncode.size() + 2); + + // Store the encoded string + assign(encoded); + + // Stuff which must be done + EncodedFilenameChanged(); + CheckValid(false); +} + + + diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h new file mode 100755 index 00000000..4c951b6f --- /dev/null +++ b/lib/backupclient/BackupStoreFilename.h @@ -0,0 +1,85 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilename.h +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILENAME__H +#define BACKUPSTOREFILENAME__H + +#include <string> + +class Protocol; +class IOStream; + +// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE +// don't define this -- the problem of memory usage still appears without this. +// It's just that this class really showed up the problem. Instead, malloc allocation +// is globally defined in BoxPlatform.h, for troublesome libraries. + +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE + // Use a malloc_allocated string, because the STL default allocators really screw up with + // memory allocation, particularly with this class. + // Makes a few things a bit messy and inefficient with conversions. + // Given up using this, and use global malloc allocation instead, but thought it + // worth leaving this code in just in case it's useful for the future. + typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base; + // If this is changed, change GetClearFilename() back to returning a reference. +#else + typedef std::string BackupStoreFilename_base; +#endif // PLATFORM_HAVE_STL_MALLOC_ALLOC + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFilename +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreFilename : public BackupStoreFilename_base +{ +public: + BackupStoreFilename(); + BackupStoreFilename(const BackupStoreFilename &rToCopy); + virtual ~BackupStoreFilename(); + + bool CheckValid(bool ExceptionIfInvalid = true) const; + + void ReadFromProtocol(Protocol &rProtocol); + void WriteToProtocol(Protocol &rProtocol) const; + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream) const; + + void SetAsClearFilename(const char *Clear); + + // Check that it's encrypted + bool IsEncrypted() const; + + // These enumerated types belong in the base class so + // the CheckValid() function can make sure that the encoding + // is a valid encoding + enum + { + Encoding_Min = 1, + Encoding_Clear = 1, + Encoding_Blowfish = 2, + Encoding_Max = 2 + }; + +protected: + virtual void EncodedFilenameChanged(); +}; + +// On the wire utilities for class and derived class +#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2) +#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3) + +#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;} + +#endif // BACKUPSTOREFILENAME__H + diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp new file mode 100755 index 00000000..3069f612 --- /dev/null +++ b/lib/backupclient/BackupStoreFilenameClear.cpp @@ -0,0 +1,335 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilenameClear.cpp +// Purpose: BackupStoreFilenames in the clear +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreFilenameClear.h" +#include "BackupStoreException.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "Guards.h" + +#include "MemLeakFindOn.h" + +// Hide private variables from the rest of the world +namespace +{ + int sEncodeMethod = BackupStoreFilename::Encoding_Clear; + CipherContext sBlowfishEncrypt; + CipherContext sBlowfishDecrypt; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear() +// Purpose: Default constructor, creates an invalid filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &) +// Purpose: Creates a filename, encoding from the given string +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode) +{ + SetClearFilename(rToEncode); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy) + : BackupStoreFilename(rToCopy), + mClearFilename(rToCopy.mClearFilename) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) +// Purpose: Copy from base class +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) + : BackupStoreFilename(rToCopy) +{ + // Will get a clear filename when it's required +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::~BackupStoreFilenameClear() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::GetClearFilename() +// Purpose: Get the unencoded filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE +const std::string BackupStoreFilenameClear::GetClearFilename() const +{ + MakeClearAvailable(); + // When modifying, remember to change back to reference return if at all possible + // -- returns an object rather than a reference to allow easy use with other code. + return std::string(mClearFilename.c_str(), mClearFilename.size()); +} +#else +const std::string &BackupStoreFilenameClear::GetClearFilename() const +{ + MakeClearAvailable(); + return mClearFilename; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &) +// Purpose: Encode and make available the clear filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode) +{ + // Only allow Blowfish encodings + if(sEncodeMethod != Encoding_Blowfish) + { + THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup) + } + + // Make an encoded string with blowfish encryption + EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish); + + // Store the clear filename + mClearFilename.assign(rToEncode.c_str(), rToEncode.size()); + + // Make sure we did the right thing + if(!CheckValid(false)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::MakeClearAvailable() +// Purpose: Private. Make sure the clear filename is available +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::MakeClearAvailable() const +{ + if(!mClearFilename.empty()) + return; // nothing to do + + // Check valid + CheckValid(); + + // Decode the header + int size = BACKUPSTOREFILENAME_GET_SIZE(*this); + int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this); + + // Decode based on encoding given in the header + switch(encoding) + { + case Encoding_Clear: + TRACE0("**** BackupStoreFilename encoded with Clear encoding ****\n"); + mClearFilename.assign(c_str() + 2, size - 2); + break; + + case Encoding_Blowfish: + DecryptEncoded(sBlowfishDecrypt); + break; + + default: + THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding) + break; + } +} + + +// Buffer for encoding and decoding -- do this all in one single buffer to +// avoid lots of string allocation, which stuffs up memory usage. +// These static memory vars are, of course, not thread safe, but we don't use threads. +#ifndef NDEBUG +static int sEncDecBufferSize = 2; // small size for debug builds +#else +static int sEncDecBufferSize = 256; +#endif +static MemoryBlockGuard<uint8_t *> spEncDecBuffer(sEncDecBufferSize); + +// fudge to stop leak reporting +#ifdef BOX_MEMORY_LEAK_TESTING +namespace +{ + class leak_off + { + public: + leak_off() + { + MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer); + } + }; + leak_off dont_report_as_leak; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int) +// Purpose: Private. Assigns the encoded filename string, encrypting. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding) +{ + // Work out max size + int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4; + + // Make sure encode/decode buffer has enough space + if(sEncDecBufferSize < maxOutSize) + { + TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, maxOutSize); + spEncDecBuffer.Resize(maxOutSize); + sEncDecBufferSize = maxOutSize; + } + + // Pointer to buffer + uint8_t *buffer = spEncDecBuffer; + MEMLEAKFINDER_NOT_A_LEAK(buffer); + + // Encode -- do entire block in one go + int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size()); + // and add in header size + encSize += 2; + + // Adjust header + BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding); + + // Store the encoded string + assign((char*)buffer, encSize); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &) +// Purpose: Decrypt the encoded filename using the cipher context +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const +{ + // Work out max size + int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(size()) + 4; + + // Make sure encode/decode buffer has enough space + if(sEncDecBufferSize < maxOutSize) + { + TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, maxOutSize); + spEncDecBuffer.Resize(maxOutSize); + sEncDecBufferSize = maxOutSize; + } + + // Pointer to buffer + uint8_t *buffer = spEncDecBuffer; + MEMLEAKFINDER_NOT_A_LEAK(buffer); + + // Decrypt + const char *str = c_str() + 2; + int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, size() - 2); + + // Assign to this + mClearFilename.assign((char*)buffer, sizeOut); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::EncodedFilenameChanged() +// Purpose: The encoded filename stored has changed +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::EncodedFilenameChanged() +{ + BackupStoreFilename::EncodedFilenameChanged(); + + // Delete stored filename in clear + mClearFilename.erase(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int) +// Purpose: Set the key used for Blowfish encryption of filenames +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength) +{ + // Initialisation vector not used. Can't use a different vector for each filename as + // that would stop comparisions on the server working. + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength); + sBlowfishEncrypt.SetIV(pIV); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength); + sBlowfishDecrypt.SetIV(pIV); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetEncodingMethod(int) +// Purpose: Set the encoding method used for filenames +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetEncodingMethod(int Method) +{ + sEncodeMethod = Method; +} + + + diff --git a/lib/backupclient/BackupStoreFilenameClear.h b/lib/backupclient/BackupStoreFilenameClear.h new file mode 100755 index 00000000..d4c45701 --- /dev/null +++ b/lib/backupclient/BackupStoreFilenameClear.h @@ -0,0 +1,60 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilenameClear.h +// Purpose: BackupStoreFilenames in the clear +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILENAMECLEAR__H +#define BACKUPSTOREFILENAMECLEAR__H + +#include "BackupStoreFilename.h" + +class CipherContext; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFilenameClear +// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreFilenameClear : public BackupStoreFilename +{ +public: + BackupStoreFilenameClear(); + BackupStoreFilenameClear(const std::string &rToEncode); + BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy); + BackupStoreFilenameClear(const BackupStoreFilename &rToCopy); + virtual ~BackupStoreFilenameClear(); + + // Because we need to use a different allocator for this class to avoid + // nasty things happening, can't return this as a reference. Which is a + // pity. But probably not too bad. +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE + const std::string GetClearFilename() const; +#else + const std::string &GetClearFilename() const; +#endif + void SetClearFilename(const std::string &rToEncode); + + // Setup for encryption of filenames + static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength); + static void SetEncodingMethod(int Method); + +protected: + void MakeClearAvailable() const; + virtual void EncodedFilenameChanged(); + void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding); + void DecryptEncoded(CipherContext &rCipherContext) const; + +private: + mutable BackupStoreFilename_base mClearFilename; +}; + +#endif // BACKUPSTOREFILENAMECLEAR__H + + diff --git a/lib/backupclient/BackupStoreObjectDump.cpp b/lib/backupclient/BackupStoreObjectDump.cpp new file mode 100644 index 00000000..caa0e82f --- /dev/null +++ b/lib/backupclient/BackupStoreObjectDump.cpp @@ -0,0 +1,210 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreObjectDump.cpp +// Purpose: Implementations of dumping objects to stdout/TRACE +// Created: 3/5/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <stdarg.h> +#include <map> + +#include "BackupStoreDirectory.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreFilename.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreObjectMagic.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void OutputLine(FILE *, bool, const char *, ...) +// Purpose: Output a line for the object dumping, to file and/or trace... +// Created: 3/5/04 +// +// -------------------------------------------------------------------------- +static void OutputLine(FILE *file, bool ToTrace, const char *format, ...) +{ + char text[512]; + int r = 0; + va_list ap; + va_start(ap, format); + r = vsnprintf(text, sizeof(text), format, ap); + va_end(ap); + + if(file != 0) + { + ::fprintf(file, "%s", text); + } + if(ToTrace) + { + TRACE1("%s", text); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) +// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere) +// Dump the contents to a file, or trace. +// Created: 3/5/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) +{ + FILE *file = (FILE*)clibFileHandle; + + OutputLine(file, ToTrace, "Directory object.\nObject ID: %llx\nContainer ID: %llx\nNumber entries: %d\n"\ + "Attributes mod time: %llx\nAttributes size: %d\n", mObjectID, mContainerID, mEntries.size(), + mAttributesModTime, mAttributes.GetSize()); + + // So repeated filenames can be illustrated, even though they can't be decoded + std::map<BackupStoreFilename, int> nameNum; + int nameNumI = 0; + + // Dump items + OutputLine(file, ToTrace, "Items:\nID Size AttrHash AtSz NSz NIdx Flags\n"); + for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i) + { + // Choose file name index number for this file + std::map<BackupStoreFilename, int>::iterator nn(nameNum.find((*i)->GetName())); + int ni = nameNumI; + if(nn != nameNum.end()) + { + ni = nn->second; + } + else + { + nameNum[(*i)->GetName()] = nameNumI; + ++nameNumI; + } + + // Do dependencies + char depends[128]; + depends[0] = '\0'; + int depends_l = 0; + if((*i)->GetDependsNewer() != 0) + { + depends_l += ::sprintf(depends + depends_l, " depNew(%llx)", (*i)->GetDependsNewer()); + } + if((*i)->GetDependsOlder() != 0) + { + depends_l += ::sprintf(depends + depends_l, " depOld(%llx)", (*i)->GetDependsOlder()); + } + + // Output item + int16_t f = (*i)->GetFlags(); + OutputLine(file, ToTrace, "%06llx %4lld %016llx %4d %3d %4d%s%s%s%s%s%s\n", + (*i)->GetObjectID(), + (*i)->GetSizeInBlocks(), + (*i)->GetAttributesHash(), + (*i)->GetAttributes().GetSize(), + (*i)->GetName().size(), + ni, + ((f & BackupStoreDirectory::Entry::Flags_File)?" file":""), + ((f & BackupStoreDirectory::Entry::Flags_Dir)?" dir":""), + ((f & BackupStoreDirectory::Entry::Flags_Deleted)?" del":""), + ((f & BackupStoreDirectory::Entry::Flags_OldVersion)?" old":""), + ((f & BackupStoreDirectory::Entry::Flags_RemoveASAP)?" removeASAP":""), + depends); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DumpFile(void *, bool, IOStream &) +// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere) +// Dump the contents to a file, or trace. +// Created: 4/5/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile) +{ + FILE *file = (FILE*)clibFileHandle; + + // Read header + file_StreamFormat hdr; + if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), + 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Check and output header info + if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V0)) + { + OutputLine(file, ToTrace, "File header doesn't have the correct magic, aborting dump\n"); + return; + } + + OutputLine(file, ToTrace, "File object.\nContainer ID: %llx\nModification time: %llx\n"\ + "Max block clear size: %d\nOptions: %08x\nNum blocks: %d\n", ntoh64(hdr.mContainerID), + ntoh64(hdr.mModificationTime), ntohl(hdr.mMaxBlockClearSize), ntohl(hdr.mOptions), + ntoh64(hdr.mNumBlocks)); + + // Read the next two objects + BackupStoreFilename fn; + fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); + OutputLine(file, ToTrace, "Filename size: %d\n", fn.size()); + + BackupClientFileAttributes attr; + attr.ReadFromStream(rFile, IOStream::TimeOutInfinite); + OutputLine(file, ToTrace, "Attributes size: %d\n", attr.GetSize()); + + // Dump the blocks + rFile.Seek(0, IOStream::SeekType_Absolute); + BackupStoreFile::MoveStreamPositionToBlockIndex(rFile); + + // Read in header + file_BlockIndexHeader bhdr; + rFile.ReadFullBuffer(&bhdr, sizeof(bhdr), 0); + if(bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + && bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)) + { + OutputLine(file, ToTrace, "WARNING: Block header doesn't have the correct magic\n"); + } + // number of blocks + int64_t nblocks = ntoh64(bhdr.mNumBlocks); + OutputLine(file, ToTrace, "Other file ID (for block refs): %llx\nNum blocks (in blk hdr): %lld\n", + ntoh64(bhdr.mOtherFileID), nblocks); + + // Dump info about each block + OutputLine(file, ToTrace, "======== ===== ==========\n Index Where EncSz/Idx\n"); + int64_t nnew = 0, nold = 0; + for(int64_t b = 0; b < nblocks; ++b) + { + file_BlockIndexEntry en; + if(!rFile.ReadFullBuffer(&en, sizeof(en), 0)) + { + OutputLine(file, ToTrace, "Didn't manage to read block %lld from file\n", b); + continue; + } + int64_t s = ntoh64(en.mEncodedSize); + if(s > 0) + { + nnew++; + TRACE2("%8lld this s=%8lld\n", b, s); + } + else + { + nold++; + TRACE2("%8lld other i=%8lld\n", b, 0 - s); + } + } + TRACE0("======== ===== ==========\n"); +} + diff --git a/lib/backupclient/BackupStoreObjectMagic.h b/lib/backupclient/BackupStoreObjectMagic.h new file mode 100755 index 00000000..7ee600a2 --- /dev/null +++ b/lib/backupclient/BackupStoreObjectMagic.h @@ -0,0 +1,31 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreObjectMagic.h +// Purpose: Magic values for the start of objects in the backup store +// Created: 19/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREOBJECTMAGIC__H +#define BACKUPSTOREOBJECTMAGIC__H + +// Each of these values is the first 4 bytes of the object file. +// Remember to swap from network to host byte order. + +// Magic value for file streams +#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65 +// Do not use v0 in any new code! +#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45 + +// Magic for the block index at the file stream -- used to +// ensure streams are reordered as expected +#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478 +// Do not use v0 in any new code! +#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B + +// Magic value for directory streams +#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F + +#endif // BACKUPSTOREOBJECTMAGIC__H + diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra new file mode 100755 index 00000000..66203e3c --- /dev/null +++ b/lib/backupclient/Makefile.extra @@ -0,0 +1,16 @@ + +MAKEPROTOCOL = ../../lib/server/makeprotocol.pl + +GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt + +# AUTOGEN SEEDING +autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt + perl $(GEN_CMD_SRV) + + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt + perl $(MAKEEXCEPTION) BackupStoreException.txt + diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp new file mode 100755 index 00000000..91a6b758 --- /dev/null +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -0,0 +1,370 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccountDatabase.cpp +// Purpose: Database of accounts for the backup store +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> +#include <string> +#include <map> +#include <stdio.h> +#include <sys/stat.h> + +#include "BackupStoreAccountDatabase.h" +#include "Guards.h" +#include "FdGetLine.h" +#include "BackupStoreException.h" +#include "CommonException.h" +#include "FileModificationTime.h" + +#include "MemLeakFindOn.h" + +class _BackupStoreAccountDatabase +{ +public: + std::string mFilename; + std::map<int32_t, BackupStoreAccountDatabase::Entry> mDatabase; + box_time_t mModificationTime; +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *) +// Purpose: Constructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *Filename) + : pImpl(new _BackupStoreAccountDatabase) +{ + pImpl->mFilename = Filename; + pImpl->mModificationTime = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::~BackupStoreAccountDatabase() +// Purpose: Destructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::~BackupStoreAccountDatabase() +{ + delete pImpl; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::Entry() +// Purpose: Default constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::Entry() + : mID(-1), + mDiscSet(-1) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::Entry(int32_t, int) +// Purpose: Constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::Entry(int32_t ID, int DiscSet) + : mID(ID), + mDiscSet(DiscSet) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::Entry(const Entry &) +// Purpose: Copy constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::Entry(const Entry &rEntry) + : mID(rEntry.mID), + mDiscSet(rEntry.mDiscSet) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::~Entry() +// Purpose: Destructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::~Entry() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Read(const char *) +// Purpose: Read in a database from disc +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreAccountDatabase> BackupStoreAccountDatabase::Read(const char *Filename) +{ + // Database object to use + std::auto_ptr<BackupStoreAccountDatabase> db(new BackupStoreAccountDatabase(Filename)); + + // Read in the file + db->ReadFile(); + + // Return to called + return db; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::ReadFile() +// Purpose: Read the file off disc +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::ReadFile() const +{ + // Open file + FileHandleGuard<> file(pImpl->mFilename.c_str()); + + // Clear existing entries + pImpl->mDatabase.clear(); + + // Read in lines + FdGetLine getLine(file); + + while(!getLine.IsEOF()) + { + // Read and split up line + std::string l(getLine.GetLine(true)); + + if(!l.empty()) + { + // Check... + int32_t id; + int discSet; + if(::sscanf(l.c_str(), "%x:%d", &id, &discSet) != 2) + { + THROW_EXCEPTION(BackupStoreException, BadAccountDatabaseFile) + } + + // Make a new entry + pImpl->mDatabase[id] = Entry(id, discSet); + } + } + + // Store the modification time of the file + pImpl->mModificationTime = GetDBFileModificationTime(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::CheckUpToDate() +// Purpose: Private. Ensure that the in memory database matches the one on disc +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::CheckUpToDate() const +{ + if(pImpl->mModificationTime != GetDBFileModificationTime()) + { + // File has changed -- load it in again + ReadFile(); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::GetDBFileModificationTime() +// Purpose: Get the current modification time of the database +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +box_time_t BackupStoreAccountDatabase::GetDBFileModificationTime() const +{ + struct stat st; + if(::stat(pImpl->mFilename.c_str(), &st) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + return FileModificationTime(st); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Write() +// Purpose: Write the database back to disc after modifying it +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::Write() +{ + // Open file for writing + // Would use this... + // FileHandleGuard<O_WRONLY | O_TRUNC> file(pImpl->mFilename.c_str()); + // but gcc fails randomly on it on some platforms. Weird. + + int file = ::open(pImpl->mFilename.c_str(), O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + if(file == -1) + { + THROW_EXCEPTION(CommonException, OSFileOpenError) + } + + try + { + // Then write each entry + for(std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.begin()); + i != pImpl->mDatabase.end(); ++i) + { + // Write out the entry + char line[256]; // more than enough for a couple of integers in string form + int s = ::sprintf(line, "%x:%d\n", i->second.GetID(), i->second.GetDiscSet()); + if(::write(file, line, s) != s) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + + ::close(file); + } + catch(...) + { + ::close(file); + throw; + } + + // Done. +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::EntryExists(int32_t) +// Purpose: Does an entry exist in the database? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreAccountDatabase::EntryExists(int32_t ID) const +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + return pImpl->mDatabase.find(ID) != pImpl->mDatabase.end(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::GetEntry(int32_t) +// Purpose: Retrieve an entry +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +const BackupStoreAccountDatabase::Entry &BackupStoreAccountDatabase::GetEntry(int32_t ID) const +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.find(ID)); + if(i == pImpl->mDatabase.end()) + { + THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry) + } + + return i->second; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::AddEntry(int32_t, int) +// Purpose: Add a new entry to the database +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::AddEntry(int32_t ID, int DiscSet) +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + pImpl->mDatabase[ID] = Entry(ID, DiscSet); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::DeleteEntry(int32_t) +// Purpose: Delete an entry from the database +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::DeleteEntry(int32_t ID) +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.find(ID)); + if(i == pImpl->mDatabase.end()) + { + THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry) + } + + pImpl->mDatabase.erase(i); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t>) +// Purpose: +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t> &rIDsOut) +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + // Delete everything in the output list + rIDsOut.clear(); + + std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.begin()); + for(; i != pImpl->mDatabase.end(); ++i) + { + rIDsOut.push_back(i->first); + } +} + + + + diff --git a/lib/backupstore/BackupStoreAccountDatabase.h b/lib/backupstore/BackupStoreAccountDatabase.h new file mode 100755 index 00000000..8d6e7ad8 --- /dev/null +++ b/lib/backupstore/BackupStoreAccountDatabase.h @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccountDatabase.h +// Purpose: Database of accounts for the backup store +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREACCOUNTDATABASE__H +#define BACKUPSTOREACCOUNTDATABASE__H + +#include <memory> +#include <vector> + +#include "BoxTime.h" + +class _BackupStoreAccountDatabase; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreAccountDatabase +// Purpose: Database of accounts for the backup store +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +class BackupStoreAccountDatabase +{ +public: + friend class _BackupStoreAccountDatabase; // to stop compiler warnings + ~BackupStoreAccountDatabase(); +private: + BackupStoreAccountDatabase(const char *Filename); + BackupStoreAccountDatabase(const BackupStoreAccountDatabase &); +public: + + static std::auto_ptr<BackupStoreAccountDatabase> Read(const char *Filename); + void Write(); + + class Entry + { + public: + Entry(); + Entry(int32_t ID, int DiscSet); + Entry(const Entry &rEntry); + ~Entry(); + + int32_t GetID() const {return mID;} + int GetDiscSet() const {return mDiscSet;} + + private: + int32_t mID; + int mDiscSet; + }; + + bool EntryExists(int32_t ID) const; + const Entry &GetEntry(int32_t ID) const; + void AddEntry(int32_t ID, int DiscSet); + void DeleteEntry(int32_t ID); + + // This interface should change in the future. But for now it'll do. + void GetAllAccountIDs(std::vector<int32_t> &rIDsOut); + +private: + void ReadFile() const; // const in concept only + void CheckUpToDate() const; // const in concept only + box_time_t GetDBFileModificationTime() const; + +private: + mutable _BackupStoreAccountDatabase *pImpl; +}; + +#endif // BACKUPSTOREACCOUNTDATABASE__H + diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp new file mode 100755 index 00000000..36d9cad3 --- /dev/null +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -0,0 +1,162 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccounts.cpp +// Purpose: Account management for backup store server +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> + +#include "BoxPortsAndFiles.h" +#include "BackupStoreAccounts.h" +#include "BackupStoreAccountDatabase.h" +#include "RaidFileWrite.h" +#include "BackupStoreInfo.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" +#include "UnixUser.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &) +// Purpose: Constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase) + : mrDatabase(rDatabase) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::~BackupStoreAccounts() +// Purpose: Destructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccounts::~BackupStoreAccounts() +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::Create(int32_t, int, int64_t, int64_t, const std::string &) +// Purpose: Create a new account on the specified disc set. +// If rAsUsername is not empty, then the account information will be written under the +// username specified. +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername) +{ + { + // Become the user specified in the config file? + std::auto_ptr<UnixUser> user; + if(!rAsUsername.empty()) + { + // Username specified, change... + user.reset(new UnixUser(rAsUsername.c_str())); + user->ChangeProcessUser(true /* temporary */); + // Change will be undone at the end of this function + } + + // Get directory name + std::string dirName(MakeAccountRootDir(ID, DiscSet)); + + // Create a directory on disc + RaidFileWrite::CreateDirectory(DiscSet, dirName, true /* recursive */); + + // Create an info file + BackupStoreInfo::CreateNew(ID, dirName, DiscSet, SizeSoftLimit, SizeHardLimit); + + // And an empty directory + BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); + int64_t rootDirSize = 0; + // Write it, knowing the directory scheme + { + RaidFileWrite rf(DiscSet, dirName + "o01"); + rf.Open(); + rootDir.WriteToStream(rf); + rootDirSize = rf.GetDiscUsageInBlocks(); + rf.Commit(true); + } + + // Update the store info to reflect the size of the root directory + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, dirName, DiscSet, false /* ReadWrite */)); + info->ChangeBlocksUsed(rootDirSize); + info->ChangeBlocksInDirectories(rootDirSize); + + // Save it back + info->Save(); + } + + // As the original user... + + // Create the entry in the database + mrDatabase.AddEntry(ID, DiscSet); + + // Write the database back + mrDatabase.Write(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::GetAccountRoot(int32_t, std::string &, int &) +// Purpose: Gets the root of an account, returning the info via references +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const +{ + // Find the account + const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID)); + + rRootDirOut = MakeAccountRootDir(ID, en.GetDiscSet()); + rDiscSetOut = en.GetDiscSet(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::MakeAccountRootDir(int32_t, int) +// Purpose: Private. Generates a root directory name for the account +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const +{ + char accid[64]; // big enough! + ::sprintf(accid, "%08x/", ID); + return std::string(std::string(BOX_RAIDFILE_ROOT_BBSTORED DIRECTORY_SEPARATOR) + accid); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::AccountExists(int32_t) +// Purpose: Does an account exist? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreAccounts::AccountExists(int32_t ID) +{ + return mrDatabase.EntryExists(ID); +} + + diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h new file mode 100755 index 00000000..0c3dd103 --- /dev/null +++ b/lib/backupstore/BackupStoreAccounts.h @@ -0,0 +1,47 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccounts.h +// Purpose: Account management for backup store server +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREACCOUNTS__H +#define BACKUPSTOREACCOUNTS__H + +#include <string> + +class BackupStoreAccountDatabase; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreAccounts +// Purpose: Account management for backup store server +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +class BackupStoreAccounts +{ +public: + BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase); + ~BackupStoreAccounts(); +private: + BackupStoreAccounts(const BackupStoreAccounts &rToCopy); + +public: + void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername); + + bool AccountExists(int32_t ID); + void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const; + +private: + std::string MakeAccountRootDir(int32_t ID, int DiscSet) const; + +private: + BackupStoreAccountDatabase &mrDatabase; +}; + +#endif // BACKUPSTOREACCOUNTS__H + diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp new file mode 100644 index 00000000..fb48c3da --- /dev/null +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -0,0 +1,745 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheck.cpp +// Purpose: Check a store for consistency +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <string.h> +#include <unistd.h> + +#include "BackupStoreCheck.h" +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreFile.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::BackupStoreCheck(const std::string &, int, int32_t, bool, bool) +// Purpose: Constructor +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet) + : mStoreRoot(rStoreRoot), + mDiscSetNumber(DiscSetNumber), + mAccountID(AccountID), + mFixErrors(FixErrors), + mQuiet(Quiet), + mNumberErrorsFound(0), + mLastIDInInfo(0), + mpInfoLastBlock(0), + mInfoLastBlockEntries(0), + mLostDirNameSerial(0), + mLostAndFoundDirectoryID(0), + mBlocksUsed(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::~BackupStoreCheck() +// Purpose: Destructor +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +BackupStoreCheck::~BackupStoreCheck() +{ + // Clean up + FreeInfo(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::Check() +// Purpose: Perform the check on the given account +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::Check() +{ + // Lock the account + { + std::string writeLockFilename; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); + + bool gotLock = false; + int triesLeft = 8; + do + { + gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */); + + if(!gotLock) + { + --triesLeft; + ::sleep(1); + } + } while(!gotLock && triesLeft > 0); + + if(!gotLock) + { + // Couldn't lock the account -- just stop now + if(!mQuiet) + { + ::printf("Couldn't lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server.\n"); + } + THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount) + } + } + + if(!mQuiet && mFixErrors) + { + ::printf("NOTE: Will fix errors encountered during checking.\n"); + } + + // Phase 1, check objects + if(!mQuiet) + { + ::printf("Check store account ID %08x\nPhase 1, check objects...\n", mAccountID); + } + CheckObjects(); + + // Phase 2, check directories + if(!mQuiet) + { + ::printf("Phase 2, check directories...\n"); + } + CheckDirectories(); + + // Phase 3, check root + if(!mQuiet) + { + ::printf("Phase 3, check root...\n"); + } + CheckRoot(); + + // Phase 4, check unattached objects + if(!mQuiet) + { + ::printf("Phase 4, fix unattached objects...\n"); + } + CheckUnattachedObjects(); + + // Phase 5, fix bad info + if(!mQuiet) + { + ::printf("Phase 5, fix unrecovered inconsistencies...\n"); + } + FixDirsWithWrongContainerID(); + FixDirsWithLostDirs(); + + // Phase 6, regenerate store info + if(!mQuiet) + { + ::printf("Phase 6, regenerate store info...\n"); + } + WriteNewStoreInfo(); + +// DUMP_OBJECT_INFO + + if(mNumberErrorsFound > 0) + { + ::printf("%lld errors found\n", mNumberErrorsFound); + if(!mFixErrors) + { + ::printf("NOTE: No changes to the store account have been made.\n"); + } + if(!mFixErrors && mNumberErrorsFound > 0) + { + ::printf("Run again with fix option to fix these errors\n"); + } + if(mNumberErrorsFound > 0) + { + ::printf("You should now use bbackupquery on the client machine to examine the store.\n"); + if(mLostAndFoundDirectoryID != 0) + { + ::printf("A lost+found directory was created in the account root.\n"\ + "This contains files and directories which could not be matched to existing directories.\n"\ + "bbackupd will delete this directory in a few days time.\n"); + } + } + } + else + { + ::printf("Store account checked, no errors found.\n"); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static TwoDigitHexToInt(const char *, int &) +// Purpose: Convert a two digit hex string to an int, returning whether it's valid or not +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +static inline bool TwoDigitHexToInt(const char *String, int &rNumberOut) +{ + int n = 0; + // Char 0 + if(String[0] >= '0' && String[0] <= '9') + { + n = (String[0] - '0') << 4; + } + else if(String[0] >= 'a' && String[0] <= 'f') + { + n = ((String[0] - 'a') + 0xa) << 4; + } + else + { + return false; + } + // Char 1 + if(String[1] >= '0' && String[1] <= '9') + { + n |= String[1] - '0'; + } + else if(String[1] >= 'a' && String[1] <= 'f') + { + n |= (String[1] - 'a') + 0xa; + } + else + { + return false; + } + + // Return a valid number + rNumberOut = n; + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckObjects() +// Purpose: Read in the contents of the directory, recurse to other levels, +// checking objects for sanity and readability +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckObjects() +{ + // Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything + int64_t maxDir = 0; + + // Find the maximum directory starting ID + { + // Make sure the starting root dir doesn't end with '/'. + std::string start(mStoreRoot); + if(start.size() > 0 && start[start.size() - 1] == '/') + { + start.resize(start.size() - 1); + } + + maxDir = CheckObjectsScanDir(0, 1, mStoreRoot); + TRACE1("Max dir starting ID is %llx\n", maxDir); + } + + // Then go through and scan all the objects within those directories + for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH)) + { + CheckObjectsDir(d); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckObjectsScanDir(int64_t, int, int, const std::string &) +// Purpose: Read in the contents of the directory, recurse to other levels, +// return the maximum starting ID of any directory found. +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName) +{ + //TRACE2("Scan directory for max dir starting ID %s, StartID %lld\n", rDirName.c_str(), StartID); + + int64_t maxID = StartID; + + // Read in all the directories, and recurse downwards + { + std::vector<std::string> dirs; + RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName, + RaidFileRead::DirReadType_DirsOnly, dirs); + + for(std::vector<std::string>::const_iterator i(dirs.begin()); i != dirs.end(); ++i) + { + // Check to see if it's the right name + int n = 0; + if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n) + && n < (1<<STORE_ID_SEGMENT_LENGTH)) + { + // Next level down + int64_t mi = CheckObjectsScanDir(StartID | (n << (Level * STORE_ID_SEGMENT_LENGTH)), Level + 1, + rDirName + DIRECTORY_SEPARATOR + *i); + // Found a greater starting ID? + if(mi > maxID) + { + maxID = mi; + } + } + else + { + ::printf("Spurious or invalid directory %s/%s found%s -- delete manually\n", rDirName.c_str(), (*i).c_str(), mFixErrors?", deleting":""); + ++mNumberErrorsFound; + } + } + } + + return maxID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckObjectsDir(int64_t) +// Purpose: Check all the files within this directory which has the given starting ID. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckObjectsDir(int64_t StartID) +{ + // Make directory name -- first generate the filename of an entry in it + std::string dirName; + StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */); + // Check expectations + ASSERT(dirName.size() > 4 && dirName[dirName.size() - 4] == '/'); + // Remove the filename from it + dirName.resize(dirName.size() - 4); // four chars for "/o00" + + // Check directory exists + if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName)) + { + TRACE1("RaidFile dir %s does not exist\n", dirName.c_str()); + return; + } + + // Read directory contents + std::vector<std::string> files; + RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName, + RaidFileRead::DirReadType_FilesOnly, files); + + // Array of things present + bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)]; + for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l) + { + idsPresent[l] = false; + } + + // Parse each entry, building up a list of object IDs which are present in the dir. + // This is done so that whatever order is retured from the directory, objects are scanned + // in order. + // Filename must begin with a 'o' and be three characters long, otherwise it gets deleted. + for(std::vector<std::string>::const_iterator i(files.begin()); i != files.end(); ++i) + { + bool fileOK = true; + int n = 0; + if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n) + && n < (1<<STORE_ID_SEGMENT_LENGTH)) + { + // Filename is valid, mark as existing + idsPresent[n] = true; + } + else + { + // info file in root dir is OK! + if(StartID != 0 || ::strcmp("info", (*i).c_str()) != 0) + { + fileOK = false; + } + } + + if(!fileOK) + { + // Unexpected or bad file, delete it + ::printf("Spurious file %s/%s found%s\n", dirName.c_str(), (*i).c_str(), mFixErrors?", deleting":""); + ++mNumberErrorsFound; + if(mFixErrors) + { + RaidFileWrite del(mDiscSetNumber, dirName + DIRECTORY_SEPARATOR + *i); + del.Delete(); + } + } + } + + // Check all the objects found in this directory + for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i) + { + if(idsPresent[i]) + { + // Check the object is OK, and add entry + char leaf[8]; + ::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i); + if(!CheckAndAddObject(StartID | i, dirName + leaf)) + { + // File was bad, delete it + ::printf("Corrupted file %s%s found%s\n", dirName.c_str(), leaf, mFixErrors?", deleting":""); + ++mNumberErrorsFound; + if(mFixErrors) + { + RaidFileWrite del(mDiscSetNumber, dirName + leaf); + del.Delete(); + } + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckAndAddObject(int64_t, const std::string &) +// Purpose: Check a specific object and add it to the list if it's OK -- if +// there are any errors with the reading, return false and it'll be deleted. +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename) +{ + // Info on object... + bool isFile = true; + int64_t containerID = -1; + int64_t size = -1; + + try + { + // Open file + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, rFilename)); + size = file->GetDiscUsageInBlocks(); + + // Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile + uint32_t signature; + if(file->Read(&signature, sizeof(signature)) != sizeof(signature)) + { + // Too short, can't read signature from it + return false; + } + // Seek back to beginning + file->Seek(0, IOStream::SeekType_Absolute); + + // Then... check depending on the type + switch(ntohl(signature)) + { + case OBJECTMAGIC_FILE_MAGIC_VALUE_V1: +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_MAGIC_VALUE_V0: +#endif + // File... check + containerID = CheckFile(ObjectID, *file); + break; + + case OBJECTMAGIC_DIR_MAGIC_VALUE: + isFile = false; + containerID = CheckDirInitial(ObjectID, *file); + break; + + default: + // Unknown signature. Bad file. Very bad file. + return false; + break; + } + + // Add to usage counts + int64_t s = file->GetDiscUsageInBlocks(); + mBlocksUsed += s; + if(!isFile) + { + mBlocksInDirectories += s; + } + } + catch(...) + { + // Error caught, not a good file then, let it be deleted + return false; + } + + // Got a container ID? (ie check was successful) + if(containerID == -1) + { + return false; + } + + // Add to list of IDs known about + AddID(ObjectID, containerID, size, isFile); + + // Report success + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckFile(int64_t, IOStream &) +// Purpose: Do check on file, return original container ID if OK, or -1 on error +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) +{ + // Check that it's not the root directory ID. Having a file as the root directory would be bad. + if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID) + { + // Get that dodgy thing deleted! + ::printf("Have file as root directory. This is bad.\n"); + return -1; + } + + // Check the format of the file, and obtain the container ID + int64_t originalContainerID = -1; + if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */, + &originalContainerID)) + { + // Didn't verify + return -1; + } + + return originalContainerID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckDirInitial(int64_t, IOStream &) +// Purpose: Do initial check on directory, return container ID if OK, or -1 on error +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream) +{ + // Simply attempt to read in the directory + BackupStoreDirectory dir; + dir.ReadFromStream(rStream, IOStream::TimeOutInfinite); + + // Check object ID + if(dir.GetObjectID() != ObjectID) + { + // Wrong object ID + return -1; + } + + // Return container ID + return dir.GetContainerID(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckDirectories() +// Purpose: Check the directories +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckDirectories() +{ + // Phase 1 did this: + // Checked that all the directories are readable + // Built a list of all directories and files which exist on the store + // + // This phase will check all the files in the directories, make + // a note of all directories which are missing, and do initial fixing. + + // Scan all objects + for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + IDBlock *pblock = i->second; + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + + for(int e = 0; e < bentries; ++e) + { + uint8_t flags = GetFlags(pblock, e); + if(flags & Flags_IsDir) + { + // Found a directory. Read it in. + std::string filename; + StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */); + BackupStoreDirectory dir; + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Flag for modifications + bool isModified = false; + + // Check for validity + if(dir.CheckAndFix()) + { + // Wasn't quite right, and has been modified + ::printf("Directory ID %llx has bad structure\n", pblock->mID[e]); + ++mNumberErrorsFound; + isModified = true; + } + + // Go through, and check that everything in that directory exists and is valid + std::vector<int64_t> toDelete; + + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + // Lookup the item + int32_t iIndex; + IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); + bool badEntry = false; + if(piBlock != 0) + { + // Found. Get flags + uint8_t iflags = GetFlags(piBlock, iIndex); + + // Is the type the same? + if(((iflags & Flags_IsDir) == Flags_IsDir) + != ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir)) + { + // Entry is of wrong type + ::printf("Directory ID %llx references object %llx which has a different type than expected.\n", pblock->mID[e], en->GetObjectID()); + badEntry = true; + } + else + { + // Check that the entry is not already contained. + if(iflags & Flags_IsContained) + { + badEntry = true; + ::printf("Directory ID %llx references object %llx which is already contained.\n", pblock->mID[e], en->GetObjectID()); + } + else + { + // Not already contained -- mark as contained + SetFlags(piBlock, iIndex, iflags | Flags_IsContained); + + // Check that the container ID of the object is correct + if(piBlock->mContainer[iIndex] != pblock->mID[e]) + { + // Needs fixing... + if(iflags & Flags_IsDir) + { + // Add to will fix later list + ::printf("Directory ID %llx has wrong container ID.\n", en->GetObjectID()); + mDirsWithWrongContainerID.push_back(en->GetObjectID()); + } + else + { + // This is OK for files, they might move + ::printf("File ID %llx has different container ID, probably moved\n", en->GetObjectID()); + } + + // Fix entry for now + piBlock->mContainer[iIndex] = pblock->mID[e]; + } + } + } + + // Check the object size, if it's OK and a file + if(!badEntry && !((iflags & Flags_IsDir) == Flags_IsDir)) + { + if(en->GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[iIndex]) + { + // Correct + en->SetSizeInBlocks(piBlock->mObjectSizeInBlocks[iIndex]); + // Mark as changed + isModified = true; + // Tell user + ::printf("Directory ID %llx has wrong size for object %llx\n", pblock->mID[e], en->GetObjectID()); + } + } + } + else + { + // Item can't be found. Is it a directory? + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) + { + // Store the directory for later attention + mDirsWhichContainLostDirs[en->GetObjectID()] = pblock->mID[e]; + } + else + { + // Just remove the entry + badEntry = true; + ::printf("Directory ID %llx references object %llx which does not exist.\n", pblock->mID[e], en->GetObjectID()); + } + } + + // Is this entry worth keeping? + if(badEntry) + { + toDelete.push_back(en->GetObjectID()); + } + else + { + // Add to sizes? + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) + { + mBlocksInOldFiles += en->GetSizeInBlocks(); + } + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) + { + mBlocksInDeletedFiles += en->GetSizeInBlocks(); + } + } + } + + if(toDelete.size() > 0) + { + // Delete entries from directory + for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) + { + dir.DeleteEntry(*d); + } + + // Mark as modified + isModified = true; + + // Check the directory again, now that entries have been removed + dir.CheckAndFix(); + + // Errors found + ++mNumberErrorsFound; + } + + if(isModified && mFixErrors) + { + ::printf("Fixing directory ID %llx\n", pblock->mID[e]); + + // Save back to disc + RaidFileWrite fixed(mDiscSetNumber, filename); + fixed.Open(true /* allow overwriting */); + dir.WriteToStream(fixed); + // Commit it + fixed.Commit(true /* convert to raid representation now */); + } + } + } + } + +} + + diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h new file mode 100644 index 00000000..3f48312a --- /dev/null +++ b/lib/backupstore/BackupStoreCheck.h @@ -0,0 +1,199 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheck.h +// Purpose: Check a store for consistency +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTORECHECK__H +#define BACKUPSTORECHECK__H + +#include <string> +#include <map> +#include <vector> +#include <set> + +#include "NamedLock.h" +class IOStream; +class BackupStoreFilename; + +/* + +The following problems can be fixed: + + * Spurious files deleted + * Corrupted files deleted + * Root ID as file, deleted + * Dirs with wrong object id inside, deleted + * Direcetory entries pointing to non-existant files, deleted + * Doubly references files have second reference deleted + * Wrong directory container IDs fixed + * Missing root recreated + * Reattach files which exist, but aren't referenced + - files go into directory original directory, if it still exists + - missing directories are inferred, and recreated + - or if all else fails, go into lost+found + - file dir entries take the original name and mod time + - directories go into lost+found + * Container IDs on directories corrected + * Inside directories, + - only one object per name has old version clear + - IDs aren't duplicated + * Bad store info files regenerated + * Bad sizes of files in directories fixed + +*/ + + +// Size of blocks in the list of IDs +#ifdef NDEBUG + #define BACKUPSTORECHECK_BLOCK_SIZE (64*1024) +#else + #define BACKUPSTORECHECK_BLOCK_SIZE 8 +#endif + +// The object ID type -- can redefine to uint32_t to produce a lower memory version for smaller stores +typedef int64_t BackupStoreCheck_ID_t; +// Can redefine the size type for lower memory usage too +typedef int64_t BackupStoreCheck_Size_t; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreCheck +// Purpose: Check a store for consistency +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +class BackupStoreCheck +{ +public: + BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet); + ~BackupStoreCheck(); +private: + // no copying + BackupStoreCheck(const BackupStoreCheck &); + BackupStoreCheck &operator=(const BackupStoreCheck &); +public: + + // Do the exciting things + void Check(); + + bool ErrorsFound() {return mNumberErrorsFound > 0;} + +private: + enum + { + // Bit mask + Flags_IsDir = 1, + Flags_IsContained = 2, + // Mask + Flags__MASK = 3, + // Number of bits + Flags__NumFlags = 2, + // Items per uint8_t + Flags__NumItemsPerEntry = 4 // ie 8 / 2 + }; + + typedef struct + { + // Note use arrays within the block, rather than the more obvious array of + // objects, to be more memory efficient -- think alignment of the byte values. + uint8_t mFlags[BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry]; + BackupStoreCheck_ID_t mID[BACKUPSTORECHECK_BLOCK_SIZE]; + BackupStoreCheck_ID_t mContainer[BACKUPSTORECHECK_BLOCK_SIZE]; + BackupStoreCheck_Size_t mObjectSizeInBlocks[BACKUPSTORECHECK_BLOCK_SIZE]; + } IDBlock; + + // Phases of the check + void CheckObjects(); + void CheckDirectories(); + void CheckRoot(); + void CheckUnattachedObjects(); + void FixDirsWithWrongContainerID(); + void FixDirsWithLostDirs(); + void WriteNewStoreInfo(); + + // Checking functions + int64_t CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName); + void CheckObjectsDir(int64_t StartID); + bool CheckAndAddObject(int64_t ObjectID, const std::string &rFilename); + int64_t CheckFile(int64_t ObjectID, IOStream &rStream); + int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); + + // Fixing functions + bool TryToRecreateDirectory(int64_t MissingDirectoryID); + void InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory); + int64_t GetLostAndFoundDirID(); + void CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID); + + // Data handling + void FreeInfo(); + void AddID(BackupStoreCheck_ID_t ID, BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile); + IDBlock *LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut); + inline void SetFlags(IDBlock *pBlock, int32_t Index, uint8_t Flags) + { + ASSERT(pBlock != 0); + ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE); + ASSERT(Flags < (1 << Flags__NumFlags)); + + pBlock->mFlags[Index / Flags__NumItemsPerEntry] + |= (Flags << ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)); + } + inline uint8_t GetFlags(IDBlock *pBlock, int32_t Index) + { + ASSERT(pBlock != 0); + ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE); + + return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK; + } + +#ifndef NDEBUG + void DumpObjectInfo(); + #define DUMP_OBJECT_INFO DumpObjectInfo(); +#else + #define DUMP_OBJECT_INFO +#endif + +private: + std::string mStoreRoot; + int mDiscSetNumber; + int32_t mAccountID; + bool mFixErrors; + bool mQuiet; + + int64_t mNumberErrorsFound; + + // Lock for the store account + NamedLock mAccountLock; + + // Storage for ID data + typedef std::map<BackupStoreCheck_ID_t, IDBlock*> Info_t; + Info_t mInfo; + BackupStoreCheck_ID_t mLastIDInInfo; + IDBlock *mpInfoLastBlock; + int32_t mInfoLastBlockEntries; + + // List of stuff to fix + std::vector<BackupStoreCheck_ID_t> mDirsWithWrongContainerID; + // This is a map of lost dir ID -> existing dir ID + std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t> mDirsWhichContainLostDirs; + + // Set of extra directories added + std::set<BackupStoreCheck_ID_t> mDirsAdded; + + // Misc stuff + int32_t mLostDirNameSerial; + int64_t mLostAndFoundDirectoryID; + + // Usage + int64_t mBlocksUsed; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; +}; + +#endif // BACKUPSTORECHECK__H + diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp new file mode 100644 index 00000000..fe91d00d --- /dev/null +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -0,0 +1,841 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheck2.cpp +// Purpose: More backup store checking +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <string.h> + +#include "BackupStoreCheck.h" +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" +#include "BackupStoreInfo.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckRoot() +// Purpose: Check the root directory exists. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckRoot() +{ + int32_t index = 0; + IDBlock *pblock = LookupID(BACKUPSTORE_ROOT_DIRECTORY_ID, index); + + if(pblock != 0) + { + // Found it. Which is lucky. Mark it as contained. + SetFlags(pblock, index, Flags_IsContained); + } + else + { + ::printf("Root directory doesn't exist\n"); + + ++mNumberErrorsFound; + + if(mFixErrors) + { + // Create a new root directory + CreateBlankDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CreateBlankDirectory(int64_t, int64_t) +// Purpose: Creates a blank directory +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID) +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + BackupStoreDirectory dir(DirectoryID, ContainingDirID); + + // Serialise to disc + std::string filename; + StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); + RaidFileWrite obj(mDiscSetNumber, filename); + obj.Open(false /* don't allow overwriting */); + dir.WriteToStream(obj); + int64_t size = obj.GetDiscUsageInBlocks(); + obj.Commit(true /* convert to raid now */); + + // Record the fact we've done this + mDirsAdded.insert(DirectoryID); + + // Add to sizes + mBlocksUsed += size; + mBlocksInDirectories += size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckUnattachedObjects() +// Purpose: Check for objects which aren't attached to anything +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckUnattachedObjects() +{ + // Scan all objects, finding ones which have no container + for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + IDBlock *pblock = i->second; + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + + for(int e = 0; e < bentries; ++e) + { + uint8_t flags = GetFlags(pblock, e); + if((flags & Flags_IsContained) == 0) + { + // Unattached object... + ::printf("Object %llx is unattached.\n", pblock->mID[e]); + ++mNumberErrorsFound; + + // What's to be done? + int64_t putIntoDirectoryID = 0; + + if((flags & Flags_IsDir) == Flags_IsDir) + { + // Directory. Just put into lost and found. + putIntoDirectoryID = GetLostAndFoundDirID(); + } + else + { + // File. Only attempt to attach it somewhere if it isn't a patch + { + int64_t diffFromObjectID = 0; + std::string filename; + StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */); + // The easiest way to do this is to verify it again. Not such a bad penalty, because + // this really shouldn't be done very often. + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + BackupStoreFile::VerifyEncodedFileFormat(*file, &diffFromObjectID); + } + + // If not zero, then it depends on another file, which may or may not be available. + // Just delete it to be safe. + if(diffFromObjectID != 0) + { + ::printf("Object %llx is unattached, and is a patch. Deleting, cannot reliably recover.\n", pblock->mID[e]); + + // Delete this object instead + if(mFixErrors) + { + RaidFileWrite del(mDiscSetNumber, filename); + del.Delete(); + } + + // Move on to next item + continue; + } + } + + // Files contain their original filename, so perhaps the orginal directory still exists, + // or we can infer the existance of a directory? + // Look for a matching entry in the mDirsWhichContainLostDirs map. + // Can't do this with a directory, because the name just wouldn't be known, which is + // pretty useless as bbackupd would just delete it. So better to put it in lost+found + // where the admin can do something about it. + int32_t dirindex; + IDBlock *pdirblock = LookupID(pblock->mContainer[e], dirindex); + if(pdirblock != 0) + { + // Something with that ID has been found. Is it a directory? + if(GetFlags(pdirblock, dirindex) & Flags_IsDir) + { + // Directory exists, add to that one + putIntoDirectoryID = pblock->mContainer[e]; + } + else + { + // Not a directory. Use lost and found dir + putIntoDirectoryID = GetLostAndFoundDirID(); + } + } + else if(mDirsAdded.find(pblock->mContainer[e]) != mDirsAdded.end() + || TryToRecreateDirectory(pblock->mContainer[e])) + { + // The directory reappeared, or was created somehow elsewhere + putIntoDirectoryID = pblock->mContainer[e]; + } + else + { + putIntoDirectoryID = GetLostAndFoundDirID(); + } + } + ASSERT(putIntoDirectoryID != 0); + + // Add it to the directory + InsertObjectIntoDirectory(pblock->mID[e], putIntoDirectoryID, + ((flags & Flags_IsDir) == Flags_IsDir)); + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::TryToRecreateDirectory(int64_t) +// Purpose: Recreate a missing directory +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) +{ + // During the directory checking phase, a map of "missing directory" to + // containing directory was built. If we can find it here, then it's + // something which can be recreated! + std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator missing( + mDirsWhichContainLostDirs.find(MissingDirectoryID)); + if(missing == mDirsWhichContainLostDirs.end()) + { + // Not a missing directory, can't recreate. + return false; + } + + // Can recreate this! Wooo! + if(!mFixErrors) + { + ::printf("Missing directory %llx could be recreated\n", MissingDirectoryID); + mDirsAdded.insert(MissingDirectoryID); + return true; + } + ::printf("Recreating missing directory %llx\n", MissingDirectoryID); + + // Create a blank directory + BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */); + // Note that this directory already contains a directory entry pointing to + // this dir, so it doesn't have to be added. + + // Serialise to disc + std::string filename; + StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(false /* don't allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + + // Record the fact we've done this + mDirsAdded.insert(MissingDirectoryID); + + // Remove the entry from the map, so this doesn't happen again + mDirsWhichContainLostDirs.erase(missing); + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::GetLostAndFoundDirID() +// Purpose: Returns the ID of the lost and found directory, creating it if necessary +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::GetLostAndFoundDirID() +{ + // Already allocated it? + if(mLostAndFoundDirectoryID != 0) + { + return mLostAndFoundDirectoryID; + } + + if(!mFixErrors) + { + // The result will never be used anyway if errors aren't being fixed + return 1; + } + + // Load up the root directory + BackupStoreDirectory dir; + std::string filename; + StoreStructure::MakeObjectFilename(BACKUPSTORE_ROOT_DIRECTORY_ID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Find a suitable name + BackupStoreFilename lostAndFound; + int n = 0; + while(true) + { + char name[32]; + ::sprintf(name, "lost+found%d", n++); + lostAndFound.SetAsClearFilename(name); + if(!dir.NameInUse(lostAndFound)) + { + // Found a name which can be used + ::printf("Lost and found dir has name %s\n", name); + break; + } + } + + // Allocate an ID + int64_t id = mLastIDInInfo + 1; + + // Create a blank directory + CreateBlankDirectory(id, BACKUPSTORE_ROOT_DIRECTORY_ID); + + // Add an entry for it + dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0); + + // Write out root dir + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + + // Store + mLostAndFoundDirectoryID = id; + + // Tell caller + return mLostAndFoundDirectoryID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::InsertObjectIntoDirectory(int64_t, int64_t, bool) +// Purpose: +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory) +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + // Data for the object + BackupStoreFilename objectStoreFilename; + int64_t modTime = 100; // something which isn't zero or a special time + int32_t sizeInBlocks = 0; // suitable for directories + + if(IsDirectory) + { + // Directory -- simply generate a name for it. + char name[32]; + ::sprintf(name, "dir%08x", mLostDirNameSerial++); + objectStoreFilename.SetAsClearFilename(name); + } + else + { + // Files require a little more work... + // Open file + std::string fileFilename; + StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mDiscSetNumber, fileFilename, false /* don't make sure the dir exists */); + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, fileFilename)); + // Fill in size information + sizeInBlocks = file->GetDiscUsageInBlocks(); + // Read in header + file_StreamFormat hdr; + if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + )) + { + // This should never happen, everything has been checked before. + THROW_EXCEPTION(BackupStoreException, Internal) + } + // This tells us nice things + modTime = ntoh64(hdr.mModificationTime); + // And the filename comes next + objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Directory object + BackupStoreDirectory dir; + + // Generate filename + std::string filename; + StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + + // Read it in + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Add a new entry in an appropraite place + dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks, + IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); + + // Fix any flags which have been broken, which there's a good change of going + dir.CheckAndFix(); + + // Write it out + if(mFixErrors) + { + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::FixDirsWithWrongContainerID() +// Purpose: Rewrites container IDs where required +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::FixDirsWithWrongContainerID() +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + // Run through things which need fixing + for(std::vector<BackupStoreCheck_ID_t>::iterator i(mDirsWithWrongContainerID.begin()); + i != mDirsWithWrongContainerID.end(); ++i) + { + int32_t index = 0; + IDBlock *pblock = LookupID(*i, index); + if(pblock == 0) continue; + + // Load in + BackupStoreDirectory dir; + std::string filename; + StoreStructure::MakeObjectFilename(*i, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Adjust container ID + dir.SetContainerID(pblock->mContainer[index]); + + // Write it out + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::FixDirsWithLostDirs() +// Purpose: Fix directories +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::FixDirsWithLostDirs() +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + // Run through things which need fixing + for(std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator i(mDirsWhichContainLostDirs.begin()); + i != mDirsWhichContainLostDirs.end(); ++i) + { + int32_t index = 0; + IDBlock *pblock = LookupID(i->second, index); + if(pblock == 0) continue; + + // Load in + BackupStoreDirectory dir; + std::string filename; + StoreStructure::MakeObjectFilename(i->second, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Delete the dodgy entry + dir.DeleteEntry(i->first); + + // Fix it up + dir.CheckAndFix(); + + // Write it out + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::WriteNewStoreInfo() +// Purpose: Regenerate store info +// Created: 23/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::WriteNewStoreInfo() +{ + // Attempt to load the existing store info file + std::auto_ptr<BackupStoreInfo> poldInfo; + try + { + poldInfo.reset(BackupStoreInfo::Load(mAccountID, mStoreRoot, mDiscSetNumber, true /* read only */).release()); + } + catch(...) + { + ::printf("Load of existing store info failed, regenerating.\n"); + ++mNumberErrorsFound; + } + + // Minimum soft and hard limits + int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024; + int64_t minHard = ((minSoft * 11) / 10) + 1024; + + // Need to do anything? + if(poldInfo.get() != 0 && mNumberErrorsFound == 0 && poldInfo->GetAccountID() == mAccountID) + { + // Leave the store info as it is, no need to alter it because nothing really changed, + // and the only essential thing was that the account ID was correct, which is was. + return; + } + + // NOTE: We will always build a new store info, so the client store marker gets changed. + + // Work out the new limits + int64_t softLimit = minSoft; + int64_t hardLimit = minHard; + if(poldInfo.get() != 0 && poldInfo->GetBlocksSoftLimit() > minSoft) + { + softLimit = poldInfo->GetBlocksSoftLimit(); + } + else + { + ::printf("NOTE: Soft limit for account changed to ensure housekeeping doesn't delete files on next run\n"); + } + if(poldInfo.get() != 0 && poldInfo->GetBlocksHardLimit() > minHard) + { + hardLimit = poldInfo->GetBlocksHardLimit(); + } + else + { + ::printf("NOTE: Hard limit for account changed to ensure housekeeping doesn't delete files on next run\n"); + } + + // Object ID + int64_t lastObjID = mLastIDInInfo; + if(mLostAndFoundDirectoryID != 0) + { + mLastIDInInfo++; + } + + // Build a new store info + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::CreateForRegeneration( + mAccountID, + mStoreRoot, + mDiscSetNumber, + lastObjID, + mBlocksUsed, + mBlocksInOldFiles, + mBlocksInDeletedFiles, + mBlocksInDirectories, + softLimit, + hardLimit)); + + // Save to disc? + if(mFixErrors) + { + info->Save(); + ::printf("New store info file written successfully.\n"); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::CheckAndFix() +// Purpose: Check the directory for obvious logical problems, and fix them. +// Return true if the directory was changed. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreDirectory::CheckAndFix() +{ + bool changed = false; + + // Check that if a file depends on a new version, that version is in this directory + { + std::vector<Entry*>::iterator i(mEntries.begin()); + for(; i != mEntries.end(); ++i) + { + int64_t dependsNewer = (*i)->GetDependsNewer(); + if(dependsNewer != 0) + { + BackupStoreDirectory::Entry *newerEn = FindEntryByID(dependsNewer); + if(newerEn == 0) + { + // Depends on something, but it isn't there. + TRACE2("Entry id %llx removed because depends on newer version %llx which doesn't exist\n", (*i)->GetObjectID(), dependsNewer); + + // Remove + delete *i; + mEntries.erase(i); + + // Start again at the beginning of the vector, the iterator is now invalid + i = mEntries.begin(); + + // Mark as changed + changed = true; + } + else + { + // Check that newerEn has it marked + if(newerEn->GetDependsOlder() != (*i)->GetObjectID()) + { + // Wrong entry + TRACE3("Entry id %llx, correcting DependsOlder to %llx, was %llx\n", dependsNewer, (*i)->GetObjectID(), newerEn->GetDependsOlder()); + newerEn->SetDependsOlder((*i)->GetObjectID()); + // Mark as changed + changed = true; + } + } + } + } + } + + // Check that if a file has a dependency marked, it exists, and remove it if it doesn't + { + std::vector<Entry*>::iterator i(mEntries.begin()); + for(; i != mEntries.end(); ++i) + { + int64_t dependsOlder = (*i)->GetDependsOlder(); + if(dependsOlder != 0 && FindEntryByID(dependsOlder) == 0) + { + // Has an older version marked, but this doesn't exist. Remove this mark + TRACE2("Entry id %llx was marked that %llx depended on it, which doesn't exist, dependency info cleaered\n", (*i)->GetObjectID(), dependsOlder); + + (*i)->SetDependsOlder(0); + + // Mark as changed + changed = true; + } + } + } + + bool ch = false; + do + { + // Reset change marker + ch = false; + + // Search backwards -- so see newer versions first + std::vector<Entry*>::iterator i(mEntries.end()); + if(i == mEntries.begin()) + { + // Directory is empty, stop now + return changed; // changed flag + } + + // Records of things seen + std::set<int64_t> idsEncountered; + std::set<BackupStoreFilename> filenamesEncountered; + + do + { + // Look at previous + --i; + + bool removeEntry = false; + if((*i) == 0) + { + TRACE0("Remove because null pointer found\n"); + removeEntry = true; + } + else + { + bool isDir = (((*i)->GetFlags() & Entry::Flags_Dir) == Entry::Flags_Dir); + + // Check mutually exclusive flags + if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File)) + { + // Bad! Unset the file flag + TRACE1("Entry %llx: File flag set when dir flag set\n", (*i)->GetObjectID()); + (*i)->RemoveFlags(Entry::Flags_File); + changed = true; + } + + // Check... + if(idsEncountered.find((*i)->GetObjectID()) != idsEncountered.end()) + { + // ID already seen, or type doesn't match + TRACE1("Entry %llx: Remove because ID already seen\n", (*i)->GetObjectID()); + removeEntry = true; + } + else + { + // Haven't already seen this ID, remember it + idsEncountered.insert((*i)->GetObjectID()); + + // Check to see if the name has already been encountered -- if not, then it + // needs to have the old version flag set + if(filenamesEncountered.find((*i)->GetName()) != filenamesEncountered.end()) + { + // Seen before -- check old version flag set + if(((*i)->GetFlags() & Entry::Flags_OldVersion) != Entry::Flags_OldVersion + && ((*i)->GetFlags() & Entry::Flags_Deleted) == 0) + { + // Not set, set it + TRACE1("Entry %llx: Set old flag\n", (*i)->GetObjectID()); + (*i)->AddFlags(Entry::Flags_OldVersion); + changed = true; + } + } + else + { + // Check old version flag NOT set + if(((*i)->GetFlags() & Entry::Flags_OldVersion) == Entry::Flags_OldVersion) + { + // Set, unset it + TRACE1("Entry %llx: Old flag unset\n", (*i)->GetObjectID()); + (*i)->RemoveFlags(Entry::Flags_OldVersion); + changed = true; + } + + // Remember filename + filenamesEncountered.insert((*i)->GetName()); + } + } + } + + if(removeEntry) + { + // Mark something as changed, in loop + ch = true; + + // Mark something as globally changed + changed = true; + + // erase the thing from the list + Entry *pentry = (*i); + mEntries.erase(i); + + // And delete the entry object + delete pentry; + + // Stop going around this loop, as the iterator is now invalid + break; + } + } while(i != mEntries.begin()); + + } while(ch != false); + + return changed; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddUnattactedObject(...) +// Purpose: Adds an object which is currently unattached. Assume that CheckAndFix() will be called afterwards. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::AddUnattactedObject(const BackupStoreFilename &rName, + box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags) +{ + Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, + ModificationTime /* use as attr mod time too */); + try + { + // Want to order this just before the first object which has a higher ID, + // which is the place it's most likely to be correct. + std::vector<Entry*>::iterator i(mEntries.begin()); + for(; i != mEntries.end(); ++i) + { + if((*i)->GetObjectID() > ObjectID) + { + // Found a good place to insert it + break; + } + } + if(i == mEntries.end()) + { + mEntries.push_back(pnew); + } + else + { + mEntries.insert(i, 1 /* just the one copy */, pnew); + } + } + catch(...) + { + delete pnew; + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::NameInUse(const BackupStoreFilename &) +// Purpose: Returns true if the name is currently in use in the directory +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreDirectory::NameInUse(const BackupStoreFilename &rName) +{ + for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) + { + if((*i)->GetName() == rName) + { + return true; + } + } + + return false; +} + + diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp new file mode 100644 index 00000000..f22c8339 --- /dev/null +++ b/lib/backupstore/BackupStoreCheckData.cpp @@ -0,0 +1,205 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheckData.cpp +// Purpose: Data handling for store checking +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> +#include <memory> + +#include "BackupStoreCheck.h" +#include "autogen_BackupStoreException.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::FreeInfo() +// Purpose: Free all the data stored +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::FreeInfo() +{ + // Free all the blocks + for(Info_t::iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + ::free(i->second); + } + + // Clear the contents of the map + mInfo.clear(); + + // Reset the last ID, just in case + mpInfoLastBlock = 0; + mInfoLastBlockEntries = 0; + mLastIDInInfo = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::AddID(BackupStoreCheck_ID_t, BackupStoreCheck_ID_t, bool) +// Purpose: Add an ID to the list +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::AddID(BackupStoreCheck_ID_t ID, + BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile) +{ + // Check ID is OK. + if(ID <= mLastIDInInfo) + { + THROW_EXCEPTION(BackupStoreException, InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing) + } + + // Can this go in the current block? + if(mpInfoLastBlock == 0 || mInfoLastBlockEntries >= BACKUPSTORECHECK_BLOCK_SIZE) + { + // No. Allocate a new one + IDBlock *pblk = (IDBlock*)::malloc(sizeof(IDBlock)); + if(pblk == 0) + { + throw std::bad_alloc(); + } + // Zero all the flags entries + for(int z = 0; z < (BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry); ++z) + { + pblk->mFlags[z] = 0; + } + // Store in map + mInfo[ID] = pblk; + // Allocated and stored OK, setup for use + mpInfoLastBlock = pblk; + mInfoLastBlockEntries = 0; + } + ASSERT(mpInfoLastBlock != 0 && mInfoLastBlockEntries < BACKUPSTORECHECK_BLOCK_SIZE); + + // Add to block + mpInfoLastBlock->mID[mInfoLastBlockEntries] = ID; + mpInfoLastBlock->mContainer[mInfoLastBlockEntries] = Container; + mpInfoLastBlock->mObjectSizeInBlocks[mInfoLastBlockEntries] = ObjectSize; + SetFlags(mpInfoLastBlock, mInfoLastBlockEntries, IsFile?(0):(Flags_IsDir)); + + // Increment size + ++mInfoLastBlockEntries; + + // Store last ID + mLastIDInInfo = ID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::LookupID(BackupStoreCheck_ID_t, int32_t +// Purpose: Look up an ID. Return the block it's in, or zero if not found, and the +// index within that block if the thing is found. +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut) +{ + IDBlock *pblock = 0; + + // Find the lower matching block who's first entry is not less than ID + Info_t::const_iterator ib(mInfo.lower_bound(ID)); + + // Was there a block + if(ib == mInfo.end()) + { + // Block wasn't found... could be in last block + pblock = mpInfoLastBlock; + } + else + { + // Found it as first entry? + if(ib->first == ID) + { + rIndexOut = 0; + return ib->second; + } + + // Go back one block as it's not the first entry in this one + if(ib == mInfo.begin()) + { + // Was first block, can't go back + return 0; + } + // Go back... + --ib; + + // So, the ID will be in this block, if it's in anything + pblock = ib->second; + } + + ASSERT(pblock != 0); + if(pblock == 0) return 0; + + // How many entries are there in the block + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + + // Do binary search within block + int high = bentries; + int low = -1; + while(high - low > 1) + { + int i = (high + low) / 2; + if(ID <= pblock->mID[i]) + { + high = i; + } + else + { + low = i; + } + } + if(ID == pblock->mID[high]) + { + // Found + rIndexOut = high; + return pblock; + } + + // Not found + return 0; +} + + +#ifndef NDEBUG +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::DumpObjectInfo() +// Purpose: Debug only. Trace out all object info. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::DumpObjectInfo() +{ + for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + IDBlock *pblock = i->second; + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + TRACE2("BLOCK @ 0x%08x, %d entries\n", pblock, bentries); + + for(int e = 0; e < bentries; ++e) + { + uint8_t flags = GetFlags(pblock, e); + TRACE4("id %llx, c %llx, %s, %s\n", + pblock->mID[e], pblock->mContainer[e], + (flags & Flags_IsDir)?"dir":"file", + (flags & Flags_IsContained)?"contained":"unattached"); + } + } +} +#endif + diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp new file mode 100755 index 00000000..6fa05d06 --- /dev/null +++ b/lib/backupstore/BackupStoreConfigVerify.cpp @@ -0,0 +1,48 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreConfigVerify.h +// Purpose: Configuration definition for the backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreConfigVerify.h" +#include "ServerTLS.h" +#include "BoxPortsAndFiles.h" + +#include "MemLeakFindOn.h" + +static const ConfigurationVerifyKey verifyserverkeys[] = +{ + SERVERTLS_VERIFY_SERVER_KEYS(0) // no default listen addresses +}; + +static const ConfigurationVerify verifyserver[] = +{ + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } +}; + +static const ConfigurationVerifyKey verifyrootkeys[] = +{ + {"AccountDatabase", 0, ConfigTest_Exists, 0}, + {"TimeBetweenHousekeeping", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file + {"RaidFileConf", BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 0} +}; + +const ConfigurationVerify BackupConfigFileVerify = +{ + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 +}; diff --git a/lib/backupstore/BackupStoreConfigVerify.h b/lib/backupstore/BackupStoreConfigVerify.h new file mode 100755 index 00000000..815cfaed --- /dev/null +++ b/lib/backupstore/BackupStoreConfigVerify.h @@ -0,0 +1,18 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreConfigVerify.h +// Purpose: Configuration definition for the backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTORECONFIGVERIFY__H +#define BACKUPSTORECONFIGVERIFY__H + +#include "Configuration.h" + +extern const ConfigurationVerify BackupConfigFileVerify; + +#endif // BACKUPSTORECONFIGVERIFY__H + diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp new file mode 100755 index 00000000..a9effe00 --- /dev/null +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -0,0 +1,592 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreInfo.cpp +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <algorithm> + +#include "BackupStoreInfo.h" +#include "BackupStoreException.h" +#include "RaidFileWrite.h" +#include "RaidFileRead.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +// ****************** +// make sure the defaults in CreateNew are modified! +// ****************** +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mAccountID; + int64_t mClientStoreMarker; + int64_t mLastObjectIDUsed; + int64_t mBlocksUsed; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; + int64_t mBlocksSoftLimit; + int64_t mBlocksHardLimit; + uint32_t mCurrentMarkNumber; + uint32_t mOptionsPresent; // bit mask of optional elements present + int64_t mNumberDeletedDirectories; + // Then loads of int64_t IDs for the deleted directories +} info_StreamFormat; + +#define INFO_MAGIC_VALUE 0x34832476 + +// Use default packing +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +#ifdef NDEBUG + #define NUM_DELETED_DIRS_BLOCK 256 +#else + #define NUM_DELETED_DIRS_BLOCK 2 +#endif + +#define INFO_FILENAME "info" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::BackupStoreInfo() +// Purpose: Default constructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreInfo::BackupStoreInfo() + : mAccountID(-1), + mDiscSet(-1), + mReadOnly(true), + mIsModified(false), + mClientStoreMarker(0), + mLastObjectIDUsed(-1), + mBlocksUsed(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::~BackupStoreInfo +// Purpose: Destructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreInfo::~BackupStoreInfo() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::CreateNew(int32_t, const std::string &, int) +// Purpose: Create a new info file on disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit) +{ + // Initial header (is entire file) + info_StreamFormat hdr = { + htonl(INFO_MAGIC_VALUE), // mMagicValue + htonl(AccountID), // mAccountID + 0, // mClientStoreMarker + hton64(1), // mLastObjectIDUsed (which is the root directory) + 0, // mBlocksUsed + 0, // mBlocksInOldFiles + 0, // mBlocksInDeletedFiles + 0, // mBlocksInDirectories + hton64(BlockSoftLimit), // mBlocksSoftLimit + hton64(BlockHardLimit), // mBlocksHardLimit + 0, // mCurrentMarkNumber + 0, // mOptionsPresent + 0 // mNumberDeletedDirectories + }; + + // Generate the filename + ASSERT(rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); + std::string fn(rRootDir + INFO_FILENAME); + + // Open the file for writing + RaidFileWrite rf(DiscSet, fn); + rf.Open(false); // no overwriting, as this is a new file + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Done. +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::Load(int32_t, const std::string &, int, bool) +// Purpose: Loads the info from disc, given the root information. Can be marked as read only. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID) +{ + // Generate the filename + std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME); + + // Open the file for reading (passing on optional request for revision ID) + std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID)); + + // Read in a header + info_StreamFormat hdr; + if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Check it + if(ntohl(hdr.mMagicValue) != INFO_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != AccountID) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + + // Make new object + std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); + + // Put in basic location info + info->mAccountID = AccountID; + info->mDiscSet = DiscSet; + info->mFilename = fn; + info->mReadOnly = ReadOnly; + + // Insert info from file + info->mClientStoreMarker = ntoh64(hdr.mClientStoreMarker); + info->mLastObjectIDUsed = ntoh64(hdr.mLastObjectIDUsed); + info->mBlocksUsed = ntoh64(hdr.mBlocksUsed); + info->mBlocksInOldFiles = ntoh64(hdr.mBlocksInOldFiles); + info->mBlocksInDeletedFiles = ntoh64(hdr.mBlocksInDeletedFiles); + info->mBlocksInDirectories = ntoh64(hdr.mBlocksInDirectories); + info->mBlocksSoftLimit = ntoh64(hdr.mBlocksSoftLimit); + info->mBlocksHardLimit = ntoh64(hdr.mBlocksHardLimit); + + // Load up array of deleted objects + int64_t numDelObj = ntoh64(hdr.mNumberDeletedDirectories); + + // Then load them in + if(numDelObj > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int64_t toload = numDelObj; + while(toload > 0) + { + // How many in this one? + int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload)); + + if(!rf->ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Add them + for(int t = 0; t < b; ++t) + { + info->mDeletedDirectories.push_back(ntoh64(objs[t])); + } + + // Number loaded + toload -= b; + } + } + + // Final check + if(static_cast<int64_t>(info->mDeletedDirectories.size()) != numDelObj) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + + // return it to caller + return info; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::CreateForRegeneration(...) +// Purpose: Return an object which can be used to save for regeneration. +// Created: 23/4/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration(int32_t AccountID, const std::string &rRootDir, + int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles, + int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit) +{ + // Generate the filename + std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME); + + // Make new object + std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); + + // Put in basic info + info->mAccountID = AccountID; + info->mDiscSet = DiscSet; + info->mFilename = fn; + info->mReadOnly = false; + + // Insert info starting info + info->mClientStoreMarker = 0; + info->mLastObjectIDUsed = LastObjectID; + info->mBlocksUsed = BlocksUsed; + info->mBlocksInOldFiles = BlocksInOldFiles; + info->mBlocksInDeletedFiles = BlocksInDeletedFiles; + info->mBlocksInDirectories = BlocksInDirectories; + info->mBlocksSoftLimit = BlockSoftLimit; + info->mBlocksHardLimit = BlockHardLimit; + + // return it to caller + return info; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::Save() +// Purpose: Save modified info back to disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::Save() +{ + // Make sure we're initialised (although should never come to this) + if(mFilename.empty() || mAccountID == -1 || mDiscSet == -1) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Can we do this? + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Then... open a write file + RaidFileWrite rf(mDiscSet, mFilename); + rf.Open(true); // allow overwriting + + // Make header + info_StreamFormat hdr; + hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); + hdr.mAccountID = htonl(mAccountID); + hdr.mClientStoreMarker = hton64(mClientStoreMarker); + hdr.mLastObjectIDUsed = hton64(mLastObjectIDUsed); + hdr.mBlocksUsed = hton64(mBlocksUsed); + hdr.mBlocksInOldFiles = hton64(mBlocksInOldFiles); + hdr.mBlocksInDeletedFiles = hton64(mBlocksInDeletedFiles); + hdr.mBlocksInDirectories = hton64(mBlocksInDirectories); + hdr.mBlocksSoftLimit = hton64(mBlocksSoftLimit); + hdr.mBlocksHardLimit = hton64(mBlocksHardLimit); + hdr.mCurrentMarkNumber = 0; + hdr.mOptionsPresent = 0; + hdr.mNumberDeletedDirectories = hton64(mDeletedDirectories.size()); + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Write the deleted object list + if(mDeletedDirectories.size() > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int tosave = mDeletedDirectories.size(); + std::vector<int64_t>::iterator i(mDeletedDirectories.begin()); + while(tosave > 0) + { + // How many in this one? + int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); + + // Add them + for(int t = 0; t < b; ++t) + { + ASSERT(i != mDeletedDirectories.end()); + objs[t] = hton64((*i)); + i++; + } + + // Write + rf.Write(objs, b * sizeof(int64_t)); + + // Number saved + tosave -= b; + } + } + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Mark is as not modified + mIsModified = false; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksUsed(int32_t) +// Purpose: Change number of blocks used, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksUsed + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksUsed += Delta; + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksInOldFiles(int32_t) +// Purpose: Change number of blocks in old files, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInOldFiles + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInOldFiles += Delta; + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksInDeletedFiles(int32_t) +// Purpose: Change number of blocks in deleted files, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInDeletedFiles + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInDeletedFiles += Delta; + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksInDirectories(int32_t) +// Purpose: Change number of blocks in directories, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInDirectories + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInDirectories += Delta; + + mIsModified = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::CorrectAllUsedValues(int64_t, int64_t, int64_t, int64_t) +// Purpose: Set all the usage counts to specific values -- use for correcting in housekeeping +// if something when wrong during the backup connection, and the store info wasn't +// saved back to disc. +// Created: 15/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Set the values + mBlocksUsed = Used; + mBlocksInOldFiles = InOldFiles; + mBlocksInDeletedFiles = InDeletedFiles; + mBlocksInDirectories = InDirectories; + + mIsModified = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::AddDeletedDirectory(int64_t) +// Purpose: Add a directory ID to the deleted list +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::AddDeletedDirectory(int64_t DirID) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + mDeletedDirectories.push_back(DirID); + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::RemovedDeletedDirectory(int64_t) +// Purpose: Remove a directory from the deleted list +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::RemovedDeletedDirectory(int64_t DirID) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + std::vector<int64_t>::iterator i(std::find(mDeletedDirectories.begin(), mDeletedDirectories.end(), DirID)); + if(i == mDeletedDirectories.end()) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoDirNotInList) + } + mDeletedDirectories.erase(i); + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeLimits(int64_t, int64_t) +// Purpose: Change the soft and hard limits +// Created: 15/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + mBlocksSoftLimit = BlockSoftLimit; + mBlocksHardLimit = BlockHardLimit; + + mIsModified = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::AllocateObjectID() +// Purpose: Allocate an ID for a new object in the store. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreInfo::AllocateObjectID() +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if(mLastObjectIDUsed < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotInitialised) + } + + // Return the next object ID + return ++mLastObjectIDUsed; + + mIsModified = true; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::SetClientStoreMarker(int64_t) +// Purpose: Sets the client store marker +// Created: 2003/10/29 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + mClientStoreMarker = ClientStoreMarker; + + mIsModified = true; +} + + + diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h new file mode 100755 index 00000000..1aba5edb --- /dev/null +++ b/lib/backupstore/BackupStoreInfo.h @@ -0,0 +1,111 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreInfo.h +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREINFO__H +#define BACKUPSTOREINFO__H + +#include <memory> +#include <string> +#include <vector> + +class BackupStoreCheck; + +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreInfo +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +class BackupStoreInfo +{ + friend class BackupStoreCheck; +public: + ~BackupStoreInfo(); +private: + // Creation through static functions only + BackupStoreInfo(); + // No copying allowed + BackupStoreInfo(const BackupStoreInfo &); + +public: + // Create a New account, saving a blank info object to the disc + static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit); + + // Load it from the store + static std::auto_ptr<BackupStoreInfo> Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0); + + // Has info been modified? + bool IsModified() const {return mIsModified;} + + // Save modified infomation back to store + void Save(); + + // Data access functions + int32_t GetAccountID() const {return mAccountID;} + int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;} + int64_t GetBlocksUsed() const {return mBlocksUsed;} + int64_t GetBlocksInOldFiles() const {return mBlocksInOldFiles;} + int64_t GetBlocksInDeletedFiles() const {return mBlocksInDeletedFiles;} + int64_t GetBlocksInDirectories() const {return mBlocksInDirectories;} + const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;} + int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;} + int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;} + bool IsReadOnly() const {return mReadOnly;} + int GetDiscSetNumber() const {return mDiscSet;} + + // Data modification functions + void ChangeBlocksUsed(int64_t Delta); + void ChangeBlocksInOldFiles(int64_t Delta); + void ChangeBlocksInDeletedFiles(int64_t Delta); + void ChangeBlocksInDirectories(int64_t Delta); + void CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories); + void AddDeletedDirectory(int64_t DirID); + void RemovedDeletedDirectory(int64_t DirID); + void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit); + + // Object IDs + int64_t AllocateObjectID(); + + // Client marker set and get + int64_t GetClientStoreMarker() {return mClientStoreMarker;} + void SetClientStoreMarker(int64_t ClientStoreMarker); + +private: + static std::auto_ptr<BackupStoreInfo> CreateForRegeneration(int32_t AccountID, const std::string &rRootDir, + int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles, + int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit); + +private: + // Location information + int32_t mAccountID; + int mDiscSet; + std::string mFilename; + bool mReadOnly; + bool mIsModified; + + // Client infomation + int64_t mClientStoreMarker; + + // Account information + int64_t mLastObjectIDUsed; + int64_t mBlocksUsed; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; + int64_t mBlocksSoftLimit; + int64_t mBlocksHardLimit; + std::vector<int64_t> mDeletedDirectories; +}; + + +#endif // BACKUPSTOREINFO__H + + diff --git a/lib/backupstore/StoreStructure.cpp b/lib/backupstore/StoreStructure.cpp new file mode 100755 index 00000000..45a1ce91 --- /dev/null +++ b/lib/backupstore/StoreStructure.cpp @@ -0,0 +1,95 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreStructure.cpp +// Purpose: +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "RaidFileController.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StoreStructure::MakeObjectFilename(int64_t, const std::string &, int, std::string &, bool) +// Purpose: Builds the object filename for a given object, given a root. Optionally ensure that the +// directory exists. +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void StoreStructure::MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists) +{ + const static char *hex = "0123456789abcdef"; + + // Set output to root string + rFilenameOut = rStoreRoot; + + // get the id value from the stored object ID so we can do + // bitwise operations on it. + uint64_t id = (uint64_t)ObjectID; + + // get leafname, shift the bits which make up the leafname off + unsigned int leafname(id & STORE_ID_SEGMENT_MASK); + id >>= STORE_ID_SEGMENT_LENGTH; + + // build pathname + while(id != 0) + { + // assumes that the segments are no bigger than 8 bits + int v = id & STORE_ID_SEGMENT_MASK; + rFilenameOut += hex[(v & 0xf0) >> 4]; + rFilenameOut += hex[v & 0xf]; + rFilenameOut += DIRECTORY_SEPARATOR_ASCHAR; + + // shift the bits we used off the pathname + id >>= STORE_ID_SEGMENT_LENGTH; + } + + // Want to make sure this exists? + if(EnsureDirectoryExists) + { + if(!RaidFileRead::DirectoryExists(DiscSet, rFilenameOut)) + { + // Create it + RaidFileWrite::CreateDirectory(DiscSet, rFilenameOut, true /* recusive */); + } + } + + // append the filename + rFilenameOut += 'o'; + rFilenameOut += hex[(leafname & 0xf0) >> 4]; + rFilenameOut += hex[leafname & 0xf]; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StoreStructure::MakeWriteLockFilename(const std::string &, int, std::string &) +// Purpose: Generate the on disc filename of the write lock file +// Created: 15/12/03 +// +// -------------------------------------------------------------------------- +void StoreStructure::MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut) +{ + // Find the disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(DiscSet)); + + // Make the filename + std::string writeLockFile(rdiscSet[0] + DIRECTORY_SEPARATOR + rStoreRoot + "write.lock"); + + // Return it to the caller + rFilenameOut = writeLockFile; +} + + diff --git a/lib/backupstore/StoreStructure.h b/lib/backupstore/StoreStructure.h new file mode 100755 index 00000000..094c0deb --- /dev/null +++ b/lib/backupstore/StoreStructure.h @@ -0,0 +1,32 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreStructure.h +// Purpose: Functions for placing files in the store +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef STORESTRUCTURE__H +#define STORESTRUCTURE__H + +#include <string> + +#ifdef NDEBUG + #define STORE_ID_SEGMENT_LENGTH 8 + #define STORE_ID_SEGMENT_MASK 0xff +#else + // Debug we'll use lots and lots of directories to stress things + #define STORE_ID_SEGMENT_LENGTH 2 + #define STORE_ID_SEGMENT_MASK 0x03 +#endif + + +namespace StoreStructure +{ + void MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists); + void MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut); +}; + +#endif // STORESTRUCTURE__H + diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h new file mode 100755 index 00000000..8c471171 --- /dev/null +++ b/lib/common/BannerText.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BannerText.h +// Purpose: Banner text for daemons and utilities +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BANNERTEXT__H +#define BANNERTEXT__H + +#define BANNER_TEXT(UtilityName) \ + "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers 2003, 2004\n" + +#endif // BANNERTEXT__H + diff --git a/lib/common/BeginStructPackForWire.h b/lib/common/BeginStructPackForWire.h new file mode 100644 index 00000000..f9f8f616 --- /dev/null +++ b/lib/common/BeginStructPackForWire.h @@ -0,0 +1,23 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BeginStructPackForWire.h +// Purpose: Begin structure packing for wire +// Created: 25/11/03 +// +// -------------------------------------------------------------------------- + +// No header guard -- this is intentional + +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS + +#pragma pack(1) + +#else + + logical error -- check BoxPlatform.h and including file + +#endif + + + diff --git a/lib/common/Box.h b/lib/common/Box.h new file mode 100755 index 00000000..19e78ada --- /dev/null +++ b/lib/common/Box.h @@ -0,0 +1,180 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Box.h +// Purpose: Main header file for the Box project +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef BOX__H +#define BOX__H + +// Use the same changes as gcc3 for gcc4 +#ifdef PLATFORM_GCC4 + #define PLATFORM_GCC3 +#endif + +#include "BoxPlatform.h" + +// uncomment this line to enable full memory leak finding on all malloc-ed blocks (at least, ones used by the STL) +//#define MEMLEAKFINDER_FULL_MALLOC_MONITORING + +#ifndef NDEBUG + // not available on OpenBSD... oh well. + //#define SHOW_BACKTRACE_ON_EXCEPTION +#endif + +#ifdef SHOW_BACKTRACE_ON_EXCEPTION + // include "Utils.h" + #define OPTIONAL_DO_BACKTRACE DumpStackBacktrace(); +#else + #define OPTIONAL_DO_BACKTRACE +#endif + +#include "CommonException.h" + +#ifndef NDEBUG + + extern bool AssertFailuresToSyslog; + #define ASSERT_FAILS_TO_SYSLOG_ON {AssertFailuresToSyslog = true;} + void BoxDebugAssertFailed(char *cond, char *file, int line); + #define ASSERT(cond) {if(!(cond)) {BoxDebugAssertFailed(#cond, __FILE__, __LINE__); THROW_EXCEPTION(CommonException, AssertFailed)}} + + // Note that syslog tracing is independent of BoxDebugTraceOn, but stdout tracing is not + extern bool BoxDebugTraceToSyslog; + #define TRACE_TO_SYSLOG(x) {BoxDebugTraceToSyslog = x;} + extern bool BoxDebugTraceToStdout; + #define TRACE_TO_STDOUT(x) {BoxDebugTraceToStdout = x;} + + extern bool BoxDebugTraceOn; + int BoxDebug_printf(const char *format, ...); + int BoxDebugTrace(const char *format, ...); + #define TRACE0(msg) {BoxDebugTrace("%s", msg);} + #define TRACE1(msg, a0) {BoxDebugTrace(msg, a0);} + #define TRACE2(msg, a0, a1) {BoxDebugTrace(msg, a0, a1);} + #define TRACE3(msg, a0, a1, a2) {BoxDebugTrace(msg, a0, a1, a2);} + #define TRACE4(msg, a0, a1, a2, a3) {BoxDebugTrace(msg, a0, a1, a2, a3);} + #define TRACE5(msg, a0, a1, a2, a3, a4) {BoxDebugTrace(msg, a0, a1, a2, a3, a4);} + #define TRACE6(msg, a0, a1, a2, a3, a4, a5) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5);} + #define TRACE7(msg, a0, a1, a2, a3, a4, a5, a6) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6);} + #define TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6, a7);} + + #ifndef PLATFORM_DISABLE_MEM_LEAK_TESTING + #define BOX_MEMORY_LEAK_TESTING + #endif + + // Exception names + #define EXCEPTION_CODENAMES_EXTENDED + +#else + #define ASSERT_FAILS_TO_SYSLOG_ON + #define ASSERT(cond) + + #define TRACE_TO_SYSLOG(x) {} + #define TRACE_TO_STDOUT(x) {} + + #define TRACE0(msg) + #define TRACE1(msg, a0) + #define TRACE2(msg, a0, a1) + #define TRACE3(msg, a0, a1, a2) + #define TRACE4(msg, a0, a1, a2, a3) + #define TRACE5(msg, a0, a1, a2, a3, a4) + #define TRACE6(msg, a0, a1, a2, a3, a4, a5) + #define TRACE7(msg, a0, a1, a2, a3, a4, a5, a6) + #define TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7) + + // Box Backup builds release get extra information for exception logging + #define EXCEPTION_CODENAMES_EXTENDED + #define EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION + +#endif + +#ifdef BOX_MEMORY_LEAK_TESTING + // Memory leak testing + #include "MemLeakFinder.h" + #define MEMLEAKFINDER_NOT_A_LEAK(x) memleakfinder_notaleak(x); + #define MEMLEAKFINDER_START {memleakfinder_global_enable = true;} + #define MEMLEAKFINDER_STOP {memleakfinder_global_enable = false;} +#else + #define DEBUG_NEW new + #define MEMLEAKFINDER_NOT_A_LEAK(x) + #define MEMLEAKFINDER_START + #define MEMLEAKFINDER_STOP +#endif + + +#define THROW_EXCEPTION(type, subtype) \ + { \ + OPTIONAL_DO_BACKTRACE \ + TRACE1("Exception thrown: " #type "(" #subtype ") at " __FILE__ "(%d)\n", __LINE__) \ + throw type(type::subtype); \ + } + +// extra macros for converting to network byte order + +// Always define a swap64 function, as it's useful. +inline uint64_t box_swap64(uint64_t x) +{ + return ((x & 0xff) << 56 | + (x & 0xff00LL) << 40 | + (x & 0xff0000LL) << 24 | + (x & 0xff000000LL) << 8 | + (x & 0xff00000000LL) >> 8 | + (x & 0xff0000000000LL) >> 24 | + (x & 0xff000000000000LL) >> 40 | + (x & 0xff00000000000000LL) >> 56); +} + +// Does the platform provide a built in SWAP64 we can use? +#ifdef PLATFORM_NO_BUILT_IN_SWAP64 + + #define hton64(x) box_swap64(x) + #define ntoh64(x) box_swap64(x) + +#else + + #if BYTE_ORDER == BIG_ENDIAN + + // Less hassle than trying to find some working things + // on Darwin PPC + #define hton64(x) (x) + #define ntoh64(x) (x) + + #else + + #ifdef PLATFORM_LINUX + // On Linux, use some internal kernal stuff to do this + #include <asm/byteorder.h> + #define hton64 __cpu_to_be64 + #define ntoh64 __be64_to_cpu + #else + #define hton64 htobe64 + #define ntoh64 betoh64 + #endif + + // hack to make some of these work + // version in /usr/include/sys/endian.h doesn't include the 'LL' at the end of the constants + // provoking complaints from the compiler + #ifdef __GNUC__ + #undef __swap64gen + #define __swap64gen(x) __extension__({ \ + u_int64_t __swap64gen_x = (x); \ + \ + (u_int64_t)((__swap64gen_x & 0xff) << 56 | \ + (__swap64gen_x & 0xff00LL) << 40 | \ + (__swap64gen_x & 0xff0000LL) << 24 | \ + (__swap64gen_x & 0xff000000LL) << 8 | \ + (__swap64gen_x & 0xff00000000LL) >> 8 | \ + (__swap64gen_x & 0xff0000000000LL) >> 24 | \ + (__swap64gen_x & 0xff000000000000LL) >> 40 | \ + (__swap64gen_x & 0xff00000000000000LL) >> 56); \ + }) + #endif // __GNUC__ + + #endif // n BYTE_ORDER == BIG_ENDIAN + +#endif // PLATFORM_NO_BUILT_IN_SWAP64 + +#endif // BOX__H + diff --git a/lib/common/BoxException.cpp b/lib/common/BoxException.cpp new file mode 100755 index 00000000..2503ca63 --- /dev/null +++ b/lib/common/BoxException.cpp @@ -0,0 +1,21 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxException.cpp +// Purpose: Exception +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BoxException.h" + +#include "MemLeakFindOn.h" + +BoxException::BoxException() +{ +} + +BoxException::~BoxException() throw () +{ +} diff --git a/lib/common/BoxException.h b/lib/common/BoxException.h new file mode 100755 index 00000000..eb992f57 --- /dev/null +++ b/lib/common/BoxException.h @@ -0,0 +1,37 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxException.h +// Purpose: Exception +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- + +#ifndef BOXEXCEPTION__H +#define BOXEXCEPTION__H + +#include <exception> + +// -------------------------------------------------------------------------- +// +// Class +// Name: BoxException +// Purpose: Exception +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +class BoxException : public std::exception +{ +public: + BoxException(); + ~BoxException() throw (); + + virtual unsigned int GetType() const throw() = 0; + virtual unsigned int GetSubType() const throw() = 0; + +private: +}; + + +#endif // BOXEXCEPTION__H + diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h new file mode 100755 index 00000000..716366d2 --- /dev/null +++ b/lib/common/BoxPlatform.h @@ -0,0 +1,236 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxPlatform.h +// Purpose: Specifies what each platform supports in more detail, and includes +// extra files to get basic support for types. +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- + +#ifndef BOXPLATFORM__H +#define BOXPLATFORM__H + +#define DIRECTORY_SEPARATOR "/" +#define DIRECTORY_SEPARATOR_ASCHAR '/' + +#define PLATFORM_DEV_NULL "/dev/null" + + +// Other flags which might be useful... +// +// #define PLATFORM_BERKELEY_DB_NOT_SUPPORTED +// -- dbopen etc not on this platform +// +// #define PLATFORM_REGEX_NOT_SUPPORTED +// -- regex support not available on this platform + + +#ifdef PLATFORM_OPENBSD + + #include <sys/types.h> + + #define PLATFORM_HAVE_setproctitle + + #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp" + + #define PLATFORM_HAVE_getpeereid + + #define PLATFORM_RANDOM_DEVICE "/dev/arandom" + +#endif // PLATFORM_OPENBSD + +#ifdef PLATFORM_NETBSD + + #include <sys/types.h> + + #define PLATFORM_HAVE_setproctitle + + #define PLATFORM_NO_BUILT_IN_SWAP64 + + #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp" + + #define PLATFORM_KQUEUE_NOT_SUPPORTED + + #define PLATFORM_RANDOM_DEVICE "/dev/urandom" + +#endif + +#ifdef PLATFORM_FREEBSD + + #include <sys/types.h> + #include <netinet/in.h> + + #define PLATFORM_HAVE_setproctitle + + #define PLATFORM_NO_BUILT_IN_SWAP64 + + #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp" + + #define PLATFORM_HAVE_getpeereid + + #define PLATFORM_RANDOM_DEVICE "/dev/urandom" + +#endif // PLATFORM_FREEBSD + +#ifdef PLATFORM_DARWIN + + #include <sys/types.h> + #include <netdb.h> + + // types 'missing' + #ifndef _SOCKLEN_T + typedef int socklen_t; + #endif + typedef u_int8_t uint8_t; + typedef signed char int8_t; + typedef u_int64_t uint64_t; + typedef u_int32_t uint32_t; + typedef u_int16_t uint16_t; + + // poll() emulator on Darwin misses this declaration + #define INFTIM -1 + + #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp" + + #define PLATFORM_LCHOWN_NOT_SUPPORTED + + #define PLATFORM_READLINE_NOT_SUPPORTED + + #define PLATFORM_RANDOM_DEVICE "/dev/random" + +#endif // PLATFORM_DARWIN + +#ifdef PLATFORM_LINUX + + #include <sys/types.h> + + // for ntohl etc... + #include <netinet/in.h> + + // types 'missing' + typedef u_int8_t uint8_t; + typedef signed char int8_t; + typedef u_int32_t uint32_t; + typedef u_int16_t uint16_t; + typedef u_int64_t uint64_t; + + // not defined in Linux, a BSD thing + #define INFTIM -1 + + #define LLONG_MAX 9223372036854775807LL + #define LLONG_MIN (-LLONG_MAX - 1LL) + + #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp" + + #define PLATFORM_HAVE_getsockopt_SO_PEERCRED + + // load in installation specific linux configuration + #include "../../local/_linux_platform.h" + + #define PLATFORM_KQUEUE_NOT_SUPPORTED + #define PLATFORM_dirent_BROKEN_d_type + #define PLATFORM_stat_SHORT_mtime + #define PLATFORM_stat_NO_st_flags + #define PLATFORM_USES_MTAB_FILE_FOR_MOUNTS + #define PLATFORM_open_NO_O_EXLOCK + #define PLATFORM_sockaddr_NO_len + + #define PLATFORM_RANDOM_DEVICE "/dev/urandom" + + // If large file support is on, can't do the intercepts in the test/raidfile + #if _FILE_OFFSET_BITS == 64 + #define PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + #endif + +#endif // PLATFORM_LINUX + +#ifdef PLATFORM_CYGWIN + + #define PLATFORM_BERKELEY_DB_NOT_SUPPORTED + + #define PLATFORM_KQUEUE_NOT_SUPPORTED + #define PLATFORM_dirent_BROKEN_d_type + #define PLATFORM_stat_SHORT_mtime + #define PLATFORM_stat_NO_st_flags + #define PLATFORM_USES_MTAB_FILE_FOR_MOUNTS + #define PLATFORM_open_NO_O_EXLOCK + #define PLATFORM_sockaddr_NO_len + #define PLATFORM_NO_BUILT_IN_SWAP64 + + #define PLATFORM_STATIC_TEMP_DIRECTORY_NAME "/tmp" + + #define PLATFORM_READLINE_NOT_SUPPORTED + + #define LLONG_MAX 9223372036854775807LL + #define LLONG_MIN (-LLONG_MAX - 1LL) + + #define INFTIM -1 + + // File listing canonical interesting mount points. + #define MNTTAB _PATH_MNTTAB + + // File listing currently active mount points. + #define MOUNTED _PATH_MOUNTED + + #define __need_FILE + + // Extra includes + #include <stdint.h> + #include <stdlib.h> + #include <netinet/in.h> + #include <sys/socket.h> + #include <sys/stat.h> + #include <sys/types.h> + #include <dirent.h> + #include <stdio.h> + #include <paths.h> + + // No easy random entropy source + #define PLATFORM_RANDOM_DEVICE_NONE + +#endif // PLATFORM_CYGWIN + + +// Find out if credentials on UNIX sockets can be obtained +#ifndef PLATFORM_HAVE_getpeereid + #ifndef PLATFORM_HAVE_getsockopt_SO_PEERCRED + #define PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET + #endif +#endif + + +// Compiler issues +#ifdef __GNUC__ + + #ifdef PLATFORM_GCC3 + + // GCC v3 doesn't like pragmas in #defines + #define STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS + + // But fortunately, the STL allocations are much better behaved. + + #else + + // Force STL to use malloc() for memory allocation + // -- slower, but doesn't gradually use more and more memory + // HOWEVER -- this 'fix' is broken on some platforms. Lots of fun! + #ifndef PLATFORM_STL_USE_MALLOC_BROKEN + #define __USE_MALLOC + #endif + + // set packing to one bytes (can't use push/pop on gcc) + #define BEGIN_STRUCTURE_PACKING_FOR_WIRE #pragma pack(1) + + // Use default packing + #define END_STRUCTURE_PACKING_FOR_WIRE #pragma pack() + + #endif + +#else + compiler not supported! +#endif + + +#endif // BOXPLATFORM__H + diff --git a/lib/common/BoxPortsAndFiles.h b/lib/common/BoxPortsAndFiles.h new file mode 100755 index 00000000..7788537e --- /dev/null +++ b/lib/common/BoxPortsAndFiles.h @@ -0,0 +1,31 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxPortsAndFiles.h +// Purpose: Central list of which tcp/ip ports and hardcoded file locations +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BOXPORTSANDFILES__H +#define BOXPORTSANDFILES__H + +#define BOX_PORT_BASE 2200 + + +// Backup store daemon +#define BOX_PORT_BBSTORED (BOX_PORT_BASE+1) +#define BOX_FILE_BBSTORED_DEFAULT_CONFIG "/etc/box/bbstored.conf" +// directory within the RAIDFILE root for the backup store daemon +#define BOX_RAIDFILE_ROOT_BBSTORED "backup" + +// Backup client daemon +#define BOX_FILE_BBACKUPD_DEFAULT_CONFIG "/etc/box/bbackupd.conf" + + +// RaidFile conf location efault +#define BOX_FILE_RAIDFILE_DEFAULT_CONFIG "/etc/box/raidfile.conf" + + +#endif // BOXPORTSANDFILES__H + diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp new file mode 100755 index 00000000..feada309 --- /dev/null +++ b/lib/common/BoxTime.cpp @@ -0,0 +1,32 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxTime.cpp +// Purpose: Time for the box +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <time.h> + +#include "BoxTime.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: GetCurrentBoxTime() +// Purpose: Returns the current time as a box time. (1 sec precision) +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +box_time_t GetCurrentBoxTime() +{ + ASSERT(sizeof(uint32_t) == sizeof(time_t)); + return SecondsToBoxTime((uint32_t)time(0)); +} + + diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h new file mode 100755 index 00000000..a7a25ac6 --- /dev/null +++ b/lib/common/BoxTime.h @@ -0,0 +1,44 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxTime.h +// Purpose: How time is represented +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BOXTIME__H +#define BOXTIME__H + +// Time is presented as an unsigned 64 bit integer, in microseconds +typedef uint64_t box_time_t; + +#define NANO_SEC_IN_SEC (1000000000LL) +#define NANO_SEC_IN_USEC (1000) +#define NANO_SEC_IN_USEC_LL (1000LL) +#define MICRO_SEC_IN_SEC (1000000) +#define MICRO_SEC_IN_SEC_LL (1000000LL) +#define MILLI_SEC_IN_NANO_SEC (1000) +#define MILLI_SEC_IN_NANO_SEC_LL (1000LL) + +box_time_t GetCurrentBoxTime(); + +inline box_time_t SecondsToBoxTime(uint32_t Seconds) +{ + return ((box_time_t)Seconds * MICRO_SEC_IN_SEC_LL); +} +inline box_time_t SecondsToBoxTime(uint64_t Seconds) +{ + return ((box_time_t)Seconds * MICRO_SEC_IN_SEC_LL); +} +inline int64_t BoxTimeToSeconds(box_time_t Time) +{ + return Time / MICRO_SEC_IN_SEC_LL; +} +inline int64_t BoxTimeToMilliSeconds(box_time_t Time) +{ + return Time / MILLI_SEC_IN_NANO_SEC_LL; +} + +#endif // BOXTIME__H + diff --git a/lib/common/BoxTimeToText.cpp b/lib/common/BoxTimeToText.cpp new file mode 100755 index 00000000..94b0d152 --- /dev/null +++ b/lib/common/BoxTimeToText.cpp @@ -0,0 +1,41 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxTimeToText.cpp +// Purpose: Convert box time to text +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <time.h> +#include <stdio.h> + +#include "BoxTimeToText.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BoxTimeToISO8601String(box_time_t) +// Purpose: Convert a 64 bit box time to a ISO 8601 complient string +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +std::string BoxTimeToISO8601String(box_time_t Time) +{ + time_t timeInSecs = (time_t)BoxTimeToSeconds(Time); + struct tm time; + gmtime_r(&timeInSecs, &time); + + char str[128]; // more than enough space + sprintf(str, "%04d-%02d-%02dT%02d:%02d:%02d", time.tm_year + 1900, + time.tm_mon + 1, time.tm_mday, time.tm_hour, time.tm_min, time.tm_sec); + + return std::string(str); +} + + diff --git a/lib/common/BoxTimeToText.h b/lib/common/BoxTimeToText.h new file mode 100755 index 00000000..a25c70b6 --- /dev/null +++ b/lib/common/BoxTimeToText.h @@ -0,0 +1,19 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxTimeToText.h +// Purpose: Convert box time to text +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#ifndef BOXTIMETOTEXT__H +#define BOXTIMETOTEXT__H + +#include <string> +#include "BoxTime.h" + +std::string BoxTimeToISO8601String(box_time_t Time); + +#endif // BOXTIMETOTEXT__H + diff --git a/lib/common/BoxTimeToUnix.h b/lib/common/BoxTimeToUnix.h new file mode 100755 index 00000000..17e57e27 --- /dev/null +++ b/lib/common/BoxTimeToUnix.h @@ -0,0 +1,30 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxTimeToUnix.h +// Purpose: Convert times in 64 bit values to UNIX structures +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- + +#ifndef FILEMODIFICATIONTIMETOTIMEVAL__H +#define FILEMODIFICATIONTIMETOTIMEVAL__H + +#include <sys/time.h> + +#include "BoxTime.h" + +inline void BoxTimeToTimeval(box_time_t Time, struct timeval &tv) +{ + tv.tv_sec = (long)(Time / MICRO_SEC_IN_SEC_LL); + tv.tv_usec = (long)(Time % MICRO_SEC_IN_SEC_LL); +} + +inline void BoxTimeToTimespec(box_time_t Time, struct timespec &tv) +{ + tv.tv_sec = (time_t)(Time / MICRO_SEC_IN_SEC_LL); + tv.tv_nsec = ((long)(Time % MICRO_SEC_IN_SEC_LL)) * NANO_SEC_IN_USEC; +} + +#endif // FILEMODIFICATIONTIMETOTIMEVAL__H + diff --git a/lib/common/CollectInBufferStream.cpp b/lib/common/CollectInBufferStream.cpp new file mode 100755 index 00000000..90e2e7bc --- /dev/null +++ b/lib/common/CollectInBufferStream.cpp @@ -0,0 +1,274 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CollectInBufferStream.cpp +// Purpose: Collect data in a buffer, and then read it out. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "CollectInBufferStream.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +#define INITIAL_BUFFER_SIZE 1024 +#define MAX_BUFFER_ADDITION (1024*64) + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::CollectInBufferStream() +// Purpose: Constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +CollectInBufferStream::CollectInBufferStream() + : mBuffer(INITIAL_BUFFER_SIZE), + mBufferSize(INITIAL_BUFFER_SIZE), + mBytesInBuffer(0), + mReadPosition(0), + mInWritePhase(true) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::~CollectInBufferStream() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +CollectInBufferStream::~CollectInBufferStream() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::Read(void *, int, int) +// Purpose: As interface. But only works in read phase +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +int CollectInBufferStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + + // Adjust to number of bytes left + if(NBytes > (mBytesInBuffer - mReadPosition)) + { + NBytes = (mBytesInBuffer - mReadPosition); + } + ASSERT(NBytes >= 0); + if(NBytes <= 0) return 0; // careful now + + // Copy in the requested number of bytes and adjust the read pointer + ::memcpy(pBuffer, ((char*)mBuffer) + mReadPosition, NBytes); + mReadPosition += NBytes; + + return NBytes; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::BytesLeftToRead() +// Purpose: As interface. But only works in read phase +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +IOStream::pos_type CollectInBufferStream::BytesLeftToRead() +{ + if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + + return (mBytesInBuffer - mReadPosition); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::Write(void *, int) +// Purpose: As interface. But only works in write phase +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void CollectInBufferStream::Write(const void *pBuffer, int NBytes) +{ + if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + + // Enough space in the buffer + if((mBytesInBuffer + NBytes) > mBufferSize) + { + // Need to reallocate... what's the block size we'll use? + int allocateBlockSize = mBufferSize; + if(allocateBlockSize > MAX_BUFFER_ADDITION) + { + allocateBlockSize = MAX_BUFFER_ADDITION; + } + + // Write it the easy way. Although it's not the most efficient... + int newSize = mBufferSize; + while(newSize < (mBytesInBuffer + NBytes)) + { + newSize += allocateBlockSize; + } + + // Reallocate buffer + mBuffer.Resize(newSize); + + // Store new size + mBufferSize = newSize; + } + + // Copy in data and adjust counter + ::memcpy(((char*)mBuffer) + mBytesInBuffer, pBuffer, NBytes); + mBytesInBuffer += NBytes; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::GetPosition() +// Purpose: In write phase, returns the number of bytes written, in read +// phase, the number of bytes to go +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +IOStream::pos_type CollectInBufferStream::GetPosition() const +{ + return mInWritePhase?mBytesInBuffer:mReadPosition; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::Seek(pos_type, int) +// Purpose: As interface. But read phase only. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void CollectInBufferStream::Seek(pos_type Offset, int SeekType) +{ + if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + + int newPos = 0; + switch(SeekType) + { + case IOStream::SeekType_Absolute: + newPos = Offset; + break; + case IOStream::SeekType_Relative: + newPos = mReadPosition + Offset; + break; + case IOStream::SeekType_End: + newPos = mBytesInBuffer + Offset; + break; + default: + THROW_EXCEPTION(CommonException, IOStreamBadSeekType) + break; + } + + // Make sure it doesn't go over + if(newPos > mBytesInBuffer) + { + newPos = mBytesInBuffer; + } + // or under + if(newPos < 0) + { + newPos = 0; + } + + // Set the new read position + mReadPosition = newPos; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::StreamDataLeft() +// Purpose: As interface +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool CollectInBufferStream::StreamDataLeft() +{ + return mInWritePhase?(false):(mReadPosition < mBytesInBuffer); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::StreamClosed() +// Purpose: As interface +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool CollectInBufferStream::StreamClosed() +{ + return !mInWritePhase; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::SetForReading() +// Purpose: Switch to read phase, after all data written +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void CollectInBufferStream::SetForReading() +{ + if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + + // Move to read phase + mInWritePhase = false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::GetBuffer() +// Purpose: Returns the buffer +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void *CollectInBufferStream::GetBuffer() const +{ + return mBuffer.GetPtr(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::GetSize() +// Purpose: Returns the buffer size +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +int CollectInBufferStream::GetSize() const +{ + return mBytesInBuffer; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::Reset() +// Purpose: Reset the stream, so it is empty and ready to be written to. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void CollectInBufferStream::Reset() +{ + mInWritePhase = true; + mBytesInBuffer = 0; + mReadPosition = 0; +} + diff --git a/lib/common/CollectInBufferStream.h b/lib/common/CollectInBufferStream.h new file mode 100755 index 00000000..d73af8db --- /dev/null +++ b/lib/common/CollectInBufferStream.h @@ -0,0 +1,60 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CollectInBufferStream.h +// Purpose: Collect data in a buffer, and then read it out. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef COLLECTINBUFFERSTREAM__H +#define COLLECTINBUFFERSTREAM__H + +#include "IOStream.h" +#include "Guards.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: CollectInBufferStream +// Purpose: Collect data in a buffer, and then read it out. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class CollectInBufferStream : public IOStream +{ +public: + CollectInBufferStream(); + ~CollectInBufferStream(); +private: + // No copying + CollectInBufferStream(const CollectInBufferStream &); + CollectInBufferStream(const IOStream &); +public: + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(pos_type Offset, int SeekType); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + void SetForReading(); + + void Reset(); + + void *GetBuffer() const; + int GetSize() const; + bool IsSetForReading() const {return !mInWritePhase;} + +private: + MemoryBlockGuard<char*> mBuffer; + int mBufferSize; + int mBytesInBuffer; + int mReadPosition; + bool mInWritePhase; +}; + +#endif // COLLECTINBUFFERSTREAM__H + diff --git a/lib/common/CommonException.h b/lib/common/CommonException.h new file mode 100755 index 00000000..a0eb3bf5 --- /dev/null +++ b/lib/common/CommonException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CommonException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef COMMONEXCEPTION__H +#define COMMONEXCEPTION__H + +// Compatibility header with old non-autogen exception scheme +#include "autogen_CommonException.h" + +#endif // COMMONEXCEPTION__H + diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt new file mode 100644 index 00000000..3875ed71 --- /dev/null +++ b/lib/common/CommonException.txt @@ -0,0 +1,44 @@ + +# NOTE: Exception descriptions are for public distributions of Box Backup only -- do not rely for other applications. + + +EXCEPTION Common 1 + +Internal 0 +AssertFailed 1 +OSFileOpenError 2 Can't open a file -- attempted to load a non-existant config file or bad file referenced within? +OSFileCloseError 3 +FileAlreadyClosed 4 +BadArguments 5 +ConfigNoKey 6 +ConfigNoSubConfig 7 +GetLineNoHandle 8 +OSFileError 9 Error accessing a file. Check permissions. +GetLineEOF 10 +ConfigBadIntValue 11 +GetLineTooLarge 12 Protects against very large lines using up lots of memory. +NotSupported 13 +OSFileReadError 14 +OSFileWriteError 15 +FileClosed 16 +IOStreamBadSeekType 17 +CantWriteToPartialReadStream 18 +CollectInBufferStreamNotInCorrectPhase 19 +NamedLockAlreadyLockingSomething 20 +NamedLockNotHeld 21 +StreamableMemBlockIncompleteRead 22 +MemBlockStreamNotSupported 23 +StreamDoesntHaveRequiredProperty 24 +CannotWriteToReadGatherStream 25 +ReadGatherStreamAddingBadBlock 26 +CouldNotLookUpUsername 27 +CouldNotRestoreProcessUser 28 +CouldNotChangeProcessUser 29 +RegexNotSupportedOnThisPlatform 30 Your platform does not have built in regular expression libraries. +BadRegularExpression 31 +CouldNotCreateKQueue 32 +KEventErrorAdd 33 +KEventErrorWait 34 +KEventErrorRemove 35 +KQueueNotSupportedOnThisPlatform 36 +IOStreamGetLineNotEnoughDataToIgnore 37 Bad value passed to IOStreamGetLine::IgnoreBufferedData() diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp new file mode 100755 index 00000000..def93571 --- /dev/null +++ b/lib/common/Configuration.cpp @@ -0,0 +1,741 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Configuration.cpp +// Purpose: Reading configuration files +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> +#include <limits.h> + +#include "Configuration.h" +#include "CommonException.h" +#include "Guards.h" +#include "FdGetLine.h" + +#include "MemLeakFindOn.h" + +// utility whitespace function +inline bool iw(int c) +{ + return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded +} + +// boolean values +static const char *sValueBooleanStrings[] = {"yes", "true", "no", "false", 0}; +static const bool sValueBooleanValue[] = {true, true, false, false}; + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::Configuration(const std::string &) +// Purpose: Constructor +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +Configuration::Configuration(const std::string &rName) + : mName(rName) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::Configuration(const Configuration &) +// Purpose: Copy constructor +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +Configuration::Configuration(const Configuration &rToCopy) + : mName(rToCopy.mName), + mSubConfigurations(rToCopy.mSubConfigurations), + mKeys(rToCopy.mKeys) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::~Configuration() +// Purpose: Destructor +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +Configuration::~Configuration() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::LoadAndVerify(const std::string &, const ConfigurationVerify *, std::string &) +// Purpose: Loads a configuration file from disc, checks it. Returns NULL if it was faulting, in which +// case they'll be an error message. +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +std::auto_ptr<Configuration> Configuration::LoadAndVerify(const char *Filename, const ConfigurationVerify *pVerify, std::string &rErrorMsg) +{ + // Check arguments + if(Filename == 0) + { + THROW_EXCEPTION(CommonException, BadArguments) + } + + // Just to make sure + rErrorMsg.erase(); + + // Open the file + FileHandleGuard<O_RDONLY> file(Filename); + + // GetLine object + FdGetLine getline(file); + + // Object to create + Configuration *pconfig = new Configuration(std::string("<root>")); + + try + { + // Load + LoadInto(*pconfig, getline, rErrorMsg, true); + + if(!rErrorMsg.empty()) + { + // An error occured, return now + //TRACE1("Error message from LoadInto: %s", rErrorMsg.c_str()); + TRACE0("Error at Configuration::LoadInfo\n"); + delete pconfig; + pconfig = 0; + return std::auto_ptr<Configuration>(0); + } + + // Verify? + if(pVerify) + { + if(!Verify(*pconfig, *pVerify, std::string(), rErrorMsg)) + { + //TRACE1("Error message from Verify: %s", rErrorMsg.c_str()); + TRACE0("Error at Configuration::Verify\n"); + delete pconfig; + pconfig = 0; + return std::auto_ptr<Configuration>(0); + } + } + } + catch(...) + { + // Clean up + delete pconfig; + pconfig = 0; + throw; + } + + // Success. Return result. + return std::auto_ptr<Configuration>(pconfig); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: LoadInto(Configuration &, FdGetLine &, std::string &, bool) +// Purpose: Private. Load configuration information from the file into the config object. +// Returns 'abort' flag, if error, will be appended to rErrorMsg. +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel) +{ + bool startBlockExpected = false; + std::string blockName; + + //TRACE1("BLOCK: |%s|\n", rConfig.mName.c_str()); + + while(!rGetLine.IsEOF()) + { + std::string line(rGetLine.GetLine(true)); /* preprocess out whitespace and comments */ + + if(line.empty()) + { + // Ignore blank lines + continue; + } + + // Line an open block string? + if(line == "{") + { + if(startBlockExpected) + { + // New config object + Configuration config(blockName); + + // Continue processing into this block + if(!LoadInto(config, rGetLine, rErrorMsg, false)) + { + // Abort error + return false; + } + + startBlockExpected = false; + + // Store... + rConfig.mSubConfigurations.push_back(std::pair<std::string, Configuration>(blockName, config)); + } + else + { + rErrorMsg += "Unexpected start block in " + rConfig.mName + "\n"; + } + } + else + { + // Close block? + if(line == "}") + { + if(RootLevel) + { + // error -- root level doesn't have a close + rErrorMsg += "Root level has close block -- forget to terminate subblock?\n"; + // but otherwise ignore + } + else + { + //TRACE0("ENDBLOCK\n"); + return true; // All very good and nice + } + } + // Either a key, or a sub block beginning + else + { + // Can't be a start block + if(startBlockExpected) + { + rErrorMsg += "Block " + blockName + " wasn't started correctly (no '{' on line of it's own)\n"; + startBlockExpected = false; + } + + // Has the line got an = in it? + unsigned int equals = 0; + for(; equals < line.size(); ++equals) + { + if(line[equals] == '=') + { + // found! + break; + } + } + if(equals < line.size()) + { + // Make key value pair + unsigned int keyend = equals; + while(keyend > 0 && iw(line[keyend-1])) + { + keyend--; + } + unsigned int valuestart = equals+1; + while(valuestart < line.size() && iw(line[valuestart])) + { + valuestart++; + } + if(keyend > 0 && valuestart <= line.size()) + { + std::string key(line.substr(0, keyend)); + std::string value(line.substr(valuestart)); + //TRACE2("KEY: |%s|=|%s|\n", key.c_str(), value.c_str()); + + // Check for duplicate values + if(rConfig.mKeys.find(key) != rConfig.mKeys.end()) + { + // Multi-values allowed here, but checked later on + rConfig.mKeys[key] += MultiValueSeparator; + rConfig.mKeys[key] += value; + } + else + { + // Store + rConfig.mKeys[key] = value; + } + } + else + { + rErrorMsg += "Invalid key in block "+rConfig.mName+"\n"; + } + } + else + { + // Start of sub block + blockName = line; + startBlockExpected = true; + } + } + } + } + + // End of file? + if(!RootLevel && rGetLine.IsEOF()) + { + // Error if EOF and this isn't the root level + rErrorMsg += "File ended without terminating all subblocks\n"; + } + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::KeyExists(const char *) +// Purpose: Checks to see if a key exists +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +bool Configuration::KeyExists(const char *pKeyName) const +{ + if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} + + return mKeys.find(pKeyName) != mKeys.end(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetKeyValue(const char *) +// Purpose: Returns the value of a configuration variable +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +const std::string &Configuration::GetKeyValue(const char *pKeyName) const +{ + if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} + + std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName)); + + if(i == mKeys.end()) + { + THROW_EXCEPTION(CommonException, ConfigNoKey) + } + else + { + return i->second; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetKeyValueInt(const char *) +// Purpose: Gets a key value as an integer +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +int Configuration::GetKeyValueInt(const char *pKeyName) const +{ + if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} + + std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName)); + + if(i == mKeys.end()) + { + THROW_EXCEPTION(CommonException, ConfigNoKey) + } + else + { + long value = ::strtol((i->second).c_str(), NULL, 0 /* C style handling */); + if(value == LONG_MAX || value == LONG_MIN) + { + THROW_EXCEPTION(CommonException, ConfigBadIntValue) + } + return (int)value; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetKeyValueBool(const char *) const +// Purpose: Gets a key value as a boolean +// Created: 17/2/04 +// +// -------------------------------------------------------------------------- +bool Configuration::GetKeyValueBool(const char *pKeyName) const +{ + if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} + + std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName)); + + if(i == mKeys.end()) + { + THROW_EXCEPTION(CommonException, ConfigNoKey) + } + else + { + bool value = false; + + // Anything this is called for should have been verified as having a correct + // string in the verification section. However, this does default to false + // if it isn't in the string table. + + for(int l = 0; sValueBooleanStrings[l] != 0; ++l) + { + if(::strcasecmp((i->second).c_str(), sValueBooleanStrings[l]) == 0) + { + // Found. + value = sValueBooleanValue[l]; + break; + } + } + + return value; + } + +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetKeyNames() +// Purpose: Returns list of key names +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +std::vector<std::string> Configuration::GetKeyNames() const +{ + std::map<std::string, std::string>::const_iterator i(mKeys.begin()); + + std::vector<std::string> r; + + for(; i != mKeys.end(); ++i) + { + r.push_back(i->first); + } + + return r; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::SubConfigurationExists(const char *) +// Purpose: Checks to see if a sub configuration exists +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +bool Configuration::SubConfigurationExists(const char *pSubName) const +{ + if(pSubName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} + + // Attempt to find it... + std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin()); + + for(; i != mSubConfigurations.end(); ++i) + { + // This the one? + if(i->first == pSubName) + { + // Yes. + return true; + } + } + + // didn't find it. + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetSubConfiguration(const char *) +// Purpose: Gets a sub configuration +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +const Configuration &Configuration::GetSubConfiguration(const char *pSubName) const +{ + if(pSubName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} + + // Attempt to find it... + std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin()); + + for(; i != mSubConfigurations.end(); ++i) + { + // This the one? + if(i->first == pSubName) + { + // Yes. + return i->second; + } + } + + THROW_EXCEPTION(CommonException, ConfigNoSubConfig) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetSubConfigurationNames() +// Purpose: Return list of sub configuration names +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +std::vector<std::string> Configuration::GetSubConfigurationNames() const +{ + std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin()); + + std::vector<std::string> r; + + for(; i != mSubConfigurations.end(); ++i) + { + r.push_back(i->first); + } + + return r; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::Verify(const Configuration &, const ConfigurationVerify &, const std::string &, std::string &) +// Purpose: Return list of sub configuration names +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg) +{ + bool ok = true; + + // First... check the keys + if(rVerify.mpKeys != 0) + { + const ConfigurationVerifyKey *pvkey = rVerify.mpKeys; + + bool todo = true; + do + { + // Can the key be found? + ASSERT(pvkey->mpName); + if(rConfig.KeyExists(pvkey->mpName)) + { + // Get value + const std::string &rval = rConfig.GetKeyValue(pvkey->mpName); + const char *val = rval.c_str(); + + // Check it's a number? + if((pvkey->Tests & ConfigTest_IsInt) == ConfigTest_IsInt) + { + // Test it... + char *end; + long r = ::strtol(val, &end, 0); + if(r == LONG_MIN || r == LONG_MAX || end != (val + rval.size())) + { + // not a good value + ok = false; + rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) is not a valid integer.\n"; + } + } + + // Check it's a bool? + if((pvkey->Tests & ConfigTest_IsBool) == ConfigTest_IsBool) + { + // See if it's one of the allowed strings. + bool found = false; + for(int l = 0; sValueBooleanStrings[l] != 0; ++l) + { + if(::strcasecmp(val, sValueBooleanStrings[l]) == 0) + { + // Found. + found = true; + break; + } + } + + // Error if it's not one of them. + if(!found) + { + ok = false; + rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) is not a valid boolean value.\n"; + } + } + + // Check for multi valued statments where they're not allowed + if((pvkey->Tests & ConfigTest_MultiValueAllowed) == 0) + { + // Check to see if this key is a multi-value -- it shouldn't be + if(rval.find(MultiValueSeparator) != rval.npos) + { + ok = false; + rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) multi value not allowed (duplicated key?).\n"; + } + } + } + else + { + // Is it required to exist? + if((pvkey->Tests & ConfigTest_Exists) == ConfigTest_Exists) + { + // Should exist, but doesn't. + ok = false; + rErrorMsg += rLevel + rConfig.mName + "." + pvkey->mpName + " (key) is missing.\n"; + } + else if(pvkey->mpDefaultValue) + { + rConfig.mKeys[std::string(pvkey->mpName)] = std::string(pvkey->mpDefaultValue); + } + } + + if((pvkey->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) + { + // No more! + todo = false; + } + + // next + pvkey++; + + } while(todo); + + // Check for additional keys + for(std::map<std::string, std::string>::const_iterator i = rConfig.mKeys.begin(); + i != rConfig.mKeys.end(); ++i) + { + // Is the name in the list? + const ConfigurationVerifyKey *scan = rVerify.mpKeys; + bool found = false; + while(scan) + { + if(scan->mpName == i->first) + { + found = true; + break; + } + + // Next? + if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) + { + break; + } + scan++; + } + + if(!found) + { + // Shouldn't exist, but does. + ok = false; + rErrorMsg += rLevel + rConfig.mName + "." + i->first + " (key) is not a known key. Check spelling and placement.\n"; + } + } + } + + // Then the sub configurations + if(rVerify.mpSubConfigurations) + { + // Find the wildcard entry, if it exists, and check that required subconfigs are there + const ConfigurationVerify *wildcardverify = 0; + + const ConfigurationVerify *scan = rVerify.mpSubConfigurations; + while(scan) + { + ASSERT(scan->mpName); + if(scan->mpName[0] == '*') + { + wildcardverify = scan; + } + + // Required? + if((scan->Tests & ConfigTest_Exists) == ConfigTest_Exists) + { + if(scan->mpName[0] == '*') + { + // Check something exists + if(rConfig.mSubConfigurations.size() < 1) + { + // A sub config should exist, but doesn't. + ok = false; + rErrorMsg += rLevel + rConfig.mName + ".* (block) is missing (a block must be present).\n"; + } + } + else + { + // Check real thing exists + if(!rConfig.SubConfigurationExists(scan->mpName)) + { + // Should exist, but doesn't. + ok = false; + rErrorMsg += rLevel + rConfig.mName + "." + scan->mpName + " (block) is missing.\n"; + } + } + } + + // Next? + if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) + { + break; + } + scan++; + } + + // Go through the sub configurations, one by one + for(std::list<std::pair<std::string, Configuration> >::const_iterator i(rConfig.mSubConfigurations.begin()); + i != rConfig.mSubConfigurations.end(); ++i) + { + // Can this be found? + const ConfigurationVerify *subverify = 0; + + const ConfigurationVerify *scan = rVerify.mpSubConfigurations; + const char *name = i->first.c_str(); + ASSERT(name); + while(scan) + { + if(strcmp(scan->mpName, name) == 0) + { + // found it! + subverify = scan; + } + + // Next? + if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) + { + break; + } + scan++; + } + + // Use wildcard? + if(subverify == 0) + { + subverify = wildcardverify; + } + + // Verify + if(subverify) + { + // override const-ness here... + if(!Verify((Configuration&)i->second, *subverify, rConfig.mName + '.', rErrorMsg)) + { + ok = false; + } + } + } + } + + return ok; +} + + diff --git a/lib/common/Configuration.h b/lib/common/Configuration.h new file mode 100755 index 00000000..4c455b0f --- /dev/null +++ b/lib/common/Configuration.h @@ -0,0 +1,98 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Configuration +// Purpose: Reading configuration files +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- + +#ifndef CONFIGURATION__H +#define CONFIGURATION__H + +#include <map> +#include <list> +#include <vector> +#include <string> +#include <memory> + +// For defining tests +enum +{ + ConfigTest_LastEntry = 1, + ConfigTest_Exists = 2, + ConfigTest_IsInt = 4, + ConfigTest_MultiValueAllowed = 8, + ConfigTest_IsBool = 16 +}; + +class ConfigurationVerifyKey +{ +public: + const char *mpName; // "*" for all other keys (not implemented yet) + const char *mpDefaultValue; // default for when it's not present + int Tests; + void *TestFunction; // set to zero for now, will implement later +}; + +class ConfigurationVerify +{ +public: + const char *mpName; // "*" for all other sub config names + const ConfigurationVerify *mpSubConfigurations; + const ConfigurationVerifyKey *mpKeys; + int Tests; + void *TestFunction; // set to zero for now, will implement later +}; + +class FdGetLine; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Configuration +// Purpose: Loading, checking, and representing configuration files +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +class Configuration +{ +private: + Configuration(const std::string &rName); +public: + Configuration(const Configuration &rToCopy); + ~Configuration(); + + enum + { + // The character to separate multi-values + MultiValueSeparator = '\x01' + }; + + static std::auto_ptr<Configuration> LoadAndVerify(const char *Filename, const ConfigurationVerify *pVerify, std::string &rErrorMsg); + static std::auto_ptr<Configuration> Load(const char *Filename, std::string &rErrorMsg) { return LoadAndVerify(Filename, 0, rErrorMsg); } + + bool KeyExists(const char *pKeyName) const; + const std::string &GetKeyValue(const char *pKeyName) const; + int GetKeyValueInt(const char *pKeyName) const; + bool GetKeyValueBool(const char *pKeyName) const; + std::vector<std::string> GetKeyNames() const; + + bool SubConfigurationExists(const char *pSubName) const; + const Configuration &GetSubConfiguration(const char *pSubName) const; + std::vector<std::string> GetSubConfigurationNames() const; + + std::string mName; + // Order of sub blocks preserved + typedef std::list<std::pair<std::string, Configuration> > SubConfigListType; + SubConfigListType mSubConfigurations; + // Order of keys, not preserved + std::map<std::string, std::string> mKeys; + +private: + static bool LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel); + static bool Verify(Configuration &rConfig, const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg); +}; + +#endif // CONFIGURATION__H + diff --git a/lib/common/Conversion.h b/lib/common/Conversion.h new file mode 100644 index 00000000..cba5bb08 --- /dev/null +++ b/lib/common/Conversion.h @@ -0,0 +1,98 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Conversion.h +// Purpose: Convert between various types +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- + +#ifndef CONVERSION__H +#define CONVERSION__H + +#include <string> + +namespace BoxConvert +{ + // -------------------------------------------------------------------------- + // + // Function + // Name: BoxConvert::Convert<to_type, from_type>(to_type &, from_type) + // Purpose: Convert from types to types + // Created: 9/4/04 + // + // -------------------------------------------------------------------------- + template<typename to_type, typename from_type> + inline to_type Convert(from_type From) + { + // Default conversion, simply use C++ conversion + return From; + } + + // Specialise for string -> integer + int32_t _ConvertStringToInt(const char *pString, int Size); + template<> + inline int32_t Convert<int32_t, const std::string &>(const std::string &rFrom) + { + return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 32); + } + template<> + inline int16_t Convert<int16_t, const std::string &>(const std::string &rFrom) + { + return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 16); + } + template<> + inline int8_t Convert<int8_t, const std::string &>(const std::string &rFrom) + { + return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 8); + } + template<> + inline int32_t Convert<int32_t, const char *>(const char *pFrom) + { + return BoxConvert::_ConvertStringToInt(pFrom, 32); + } + template<> + inline int16_t Convert<int16_t, const char *>(const char *pFrom) + { + return BoxConvert::_ConvertStringToInt(pFrom, 16); + } + template<> + inline int8_t Convert<int8_t, const char *>(const char *pFrom) + { + return BoxConvert::_ConvertStringToInt(pFrom, 8); + } + + // Specialise for integer -> string + void _ConvertIntToString(std::string &rTo, int32_t From); + template<> + inline std::string Convert<std::string, int32_t>(int32_t From) + { + std::string r; + BoxConvert::_ConvertIntToString(r, From); + return r; + } + template<> + inline std::string Convert<std::string, int16_t>(int16_t From) + { + std::string r; + BoxConvert::_ConvertIntToString(r, From); + return r; + } + template<> + inline std::string Convert<std::string, int8_t>(int8_t From) + { + std::string r; + BoxConvert::_ConvertIntToString(r, From); + return r; + } + + // Specialise for bool -> string + template<> + inline std::string Convert<std::string, bool>(bool From) + { + return std::string(From?"true":"false"); + } +}; + +#endif // CONVERSION__H + diff --git a/lib/common/ConversionException.txt b/lib/common/ConversionException.txt new file mode 100644 index 00000000..91b5fa9a --- /dev/null +++ b/lib/common/ConversionException.txt @@ -0,0 +1,8 @@ + +EXCEPTION Conversion 12 + +Internal 0 +CannotConvertEmptyStringToInt 1 +BadStringRepresentationOfInt 2 +IntOverflowInConvertFromString 3 +BadIntSize 4 diff --git a/lib/common/ConversionString.cpp b/lib/common/ConversionString.cpp new file mode 100644 index 00000000..9e237a77 --- /dev/null +++ b/lib/common/ConversionString.cpp @@ -0,0 +1,123 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ConversionString.cpp +// Purpose: Conversions to and from strings +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <stdlib.h> +#include <limits.h> +#include <errno.h> + +#include "Conversion.h" +#include "autogen_ConversionException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BoxConvert::_ConvertStringToInt(const char *, int) +// Purpose: Convert from string to integer, with range checking. +// Always does signed -- no point in unsigned as C++ type checking +// isn't up to handling it properly. +// If a null pointer is passed in, then returns 0. +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- +int32_t BoxConvert::_ConvertStringToInt(const char *pString, int Size) +{ + // Handle null strings gracefully. + if(pString == 0) + { + return 0; + } + + // Check for initial validity + if(*pString == '\0') + { + THROW_EXCEPTION(ConversionException, CannotConvertEmptyStringToInt) + } + + // Convert. + char *numEnd = 0; + errno = 0; // Some platforms don't reset it. + long r = ::strtol(pString, &numEnd, 0); + + // Check that all the characters were used + if(*numEnd != '\0') + { + THROW_EXCEPTION(ConversionException, BadStringRepresentationOfInt) + } + + // Error check + if(r == 0 && errno == EINVAL) + { + THROW_EXCEPTION(ConversionException, BadStringRepresentationOfInt) + } + + // Range check from strtol + if((r == LONG_MIN || r == LONG_MAX) && errno == ERANGE) + { + THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString) + } + + // Check range for size of integer + switch(Size) + { + case 32: + { + // No extra checking needed, if this assert holds true + ASSERT(sizeof(long) == sizeof(int32_t)); + } + break; + + case 16: + { + if(r <= (0 - 0x7fff) || r > 0x7fff) + { + THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString) + } + break; + } + + case 8: + { + if(r <= (0 - 0x7f) || r > 0x7f) + { + THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString) + } + break; + } + + default: + { + THROW_EXCEPTION(ConversionException, BadIntSize) + break; + } + } + + // Return number + return r; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BoxConvert::_ConvertIntToString(std::string &, int32_t) +// Purpose: Convert signed interger to a string +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- +void BoxConvert::_ConvertIntToString(std::string &rTo, int32_t From) +{ + char text[64]; // size more than enough + ::sprintf(text, "%d", From); + rTo = text; +} + diff --git a/lib/common/DebugAssertFailed.cpp b/lib/common/DebugAssertFailed.cpp new file mode 100755 index 00000000..2acf4602 --- /dev/null +++ b/lib/common/DebugAssertFailed.cpp @@ -0,0 +1,32 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: AssertFailed.cpp +// Purpose: Assert failure code +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- + +#ifndef NDEBUG + +#include "Box.h" + +#include <stdio.h> +#include <syslog.h> + +#include "MemLeakFindOn.h" + +bool AssertFailuresToSyslog = false; + +void BoxDebugAssertFailed(char *cond, char *file, int line) +{ + printf("ASSERT FAILED: [%s] at %s(%d)\n", cond, file, line); + if(AssertFailuresToSyslog) + { + ::syslog(LOG_ERR, "ASSERT FAILED: [%s] at %s(%d)\n", cond, file, line); + } +} + + +#endif // NDEBUG + diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp new file mode 100755 index 00000000..e1877a13 --- /dev/null +++ b/lib/common/DebugMemLeakFinder.cpp @@ -0,0 +1,377 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemLeakFinder.cpp +// Purpose: Memory leak finder +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + + +#ifndef NDEBUG + +#include "Box.h" + +#undef malloc +#undef realloc +#undef free + +#include <map> +#include <unistd.h> +#include <stdio.h> +#include <string.h> +#include <set> + +bool memleakfinder_global_enable = false; + +typedef struct +{ + size_t size; + const char *file; + int line; +} MallocBlockInfo; + +typedef struct +{ + size_t size; + const char *file; + int line; + bool array; +} ObjectInfo; + +namespace +{ + static std::map<void *, MallocBlockInfo> sMallocBlocks; + static std::map<void *, ObjectInfo> sObjectBlocks; + + static bool sTrackMallocInSection = false; + static std::set<void *> sSectionMallocBlocks; + static bool sTrackObjectsInSection = false; + static std::map<void *, ObjectInfo> sSectionObjectBlocks; + + static std::set<void *> sNotLeaks; + + void *sNotLeaksPre[1024]; + int sNotLeaksPreNum = 0; +} + +void memleakfinder_malloc_add_block(void *b, size_t size, const char *file, int line) +{ + if(b != 0) + { + MallocBlockInfo i; + i.size = size; + i.file = file; + i.line = line; + sMallocBlocks[b] = i; + + if(sTrackMallocInSection) + { + sSectionMallocBlocks.insert(b); + } + } +} + + +void *memleakfinder_malloc(size_t size, const char *file, int line) +{ + void *b = ::malloc(size); + if(!memleakfinder_global_enable) return b; + + memleakfinder_malloc_add_block(b, size, file, line); + + //TRACE4("malloc(), %d, %s, %d, %08x\n", size, file, line, b); + return b; +} + +void *memleakfinder_realloc(void *ptr, size_t size) +{ + if(!memleakfinder_global_enable) + { + return ::realloc(ptr, size); + } + + // Check it's been allocated + std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); + if(i == sMallocBlocks.end()) + { + TRACE1("Block %x realloc(), but not in list. Error? Or allocated in startup static objects?\n", ptr); + void *b = ::realloc(ptr, size); + memleakfinder_malloc_add_block(b, size, "FOUND-IN-REALLOC", 0); + return b; + } + + void *b = ::realloc(ptr, size); + + // Worked? + if(b != 0) + { + // Update map + MallocBlockInfo inf = i->second; + inf.size = size; + sMallocBlocks.erase(i); + sMallocBlocks[b] = inf; + + if(sTrackMallocInSection) + { + std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr)); + if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si); + sSectionMallocBlocks.insert(b); + } + } + + //TRACE3("realloc(), %d, %08x->%08x\n", size, ptr, b); + return b; +} + +void memleakfinder_free(void *ptr) +{ + if(memleakfinder_global_enable) + { + // Check it's been allocated + std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); + if(i != sMallocBlocks.end()) + { + sMallocBlocks.erase(i); + } + else + { + TRACE1("Block %x freed, but not known. Error? Or allocated in startup static allocation?\n", ptr); + } + + if(sTrackMallocInSection) + { + std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr)); + if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si); + } + } + + //TRACE1("free(), %08x\n", ptr); + ::free(ptr); +} + + +void memleakfinder_notaleak_insert_pre() +{ + if(!memleakfinder_global_enable) return; + for(int l = 0; l < sNotLeaksPreNum; l++) + { + sNotLeaks.insert(sNotLeaksPre[l]); + } + sNotLeaksPreNum = 0; +} + +bool is_leak(void *ptr) +{ + memleakfinder_notaleak_insert_pre(); + return sNotLeaks.find(ptr) == sNotLeaks.end(); +} + +void memleakfinder_notaleak(void *ptr) +{ + memleakfinder_notaleak_insert_pre(); + if(memleakfinder_global_enable) + { + sNotLeaks.insert(ptr); + } + else + { + sNotLeaksPre[sNotLeaksPreNum++] = ptr; + } +/* { + std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); + if(i != sMallocBlocks.end()) sMallocBlocks.erase(i); + } + { + std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr)); + if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si); + } + { + std::map<void *, ObjectInfo>::iterator i(sObjectBlocks.find(ptr)); + if(i != sObjectBlocks.end()) sObjectBlocks.erase(i); + }*/ +} + + + +// start monitoring a section of code +void memleakfinder_startsectionmonitor() +{ + sTrackMallocInSection = true; + sSectionMallocBlocks.clear(); + sTrackObjectsInSection = true; + sSectionObjectBlocks.clear(); +} + +// trace all blocks allocated and still allocated since memleakfinder_startsectionmonitor() called +void memleakfinder_traceblocksinsection() +{ + std::set<void *>::iterator s(sSectionMallocBlocks.begin()); + for(; s != sSectionMallocBlocks.end(); ++s) + { + std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.find(*s)); + if(i == sMallocBlocks.end()) + { + TRACE0("Logical error in section block finding\n"); + } + else + { + TRACE4("Block 0x%08x size %d allocated at %s:%d\n", (int)i->first, i->second.size, i->second.file, i->second.line); + } + } + for(std::map<void *, ObjectInfo>::const_iterator i(sSectionObjectBlocks.begin()); i != sSectionObjectBlocks.end(); ++i) + { + TRACE5("Object%s 0x%08x size %d allocated at %s:%d\n", i->second.array?" []":"", (int)i->first, i->second.size, i->second.file, i->second.line); + } +} + +int memleakfinder_numleaks() +{ + int n = 0; + + for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i) + { + if(is_leak(i->first)) ++n; + } + + for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i) + { + if(is_leak(i->first)) ++n; + } + + return n; +} + +void memleakfinder_reportleaks_file(FILE *file) +{ + for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i) + { + if(is_leak(i->first)) ::fprintf(file, "Block 0x%08x size %d allocated at %s:%d\n", (int)i->first, i->second.size, i->second.file, i->second.line); + } + for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i) + { + if(is_leak(i->first)) ::fprintf(file, "Object%s 0x%08x size %d allocated at %s:%d\n", i->second.array?" []":"", (int)i->first, i->second.size, i->second.file, i->second.line); + } +} + +void memleakfinder_reportleaks() +{ + // report to stdout + memleakfinder_reportleaks_file(stdout); +} + +void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext) +{ + FILE *file = ::fopen(filename, "a"); + if(file != 0) + { + if(memleakfinder_numleaks() > 0) + { + fprintf(file, "MEMORY LEAKS FROM PROCESS %d (%s)\n", getpid(), markertext); + memleakfinder_reportleaks_file(file); + } + + ::fclose(file); + } + else + { + printf("WARNING: Couldn't open memory leak results file %s for appending\n", filename); + } +} + +static char atexit_filename[512]; +static char atexit_markertext[512]; +static bool atexit_registered = false; + +void memleakfinder_atexit() +{ + memleakfinder_reportleaks_appendfile(atexit_filename, atexit_markertext); +} + +void memleakfinder_setup_exit_report(const char *filename, const char *markertext) +{ + ::strcpy(atexit_filename, filename); + ::strcpy(atexit_markertext, markertext); + if(!atexit_registered) + { + atexit(memleakfinder_atexit); + atexit_registered = true; + } +} + + + + +void add_object_block(void *block, size_t size, const char *file, int line, bool array) +{ + if(!memleakfinder_global_enable) return; + + if(block != 0) + { + ObjectInfo i; + i.size = size; + i.file = file; + i.line = line; + i.array = array; + sObjectBlocks[block] = i; + + if(sTrackObjectsInSection) + { + sSectionObjectBlocks[block] = i; + } + } +} + +void remove_object_block(void *block) +{ + if(!memleakfinder_global_enable) return; + + std::map<void *, ObjectInfo>::iterator i(sObjectBlocks.find(block)); + if(i != sObjectBlocks.end()) + { + sObjectBlocks.erase(i); + } + + if(sTrackObjectsInSection) + { + std::map<void *, ObjectInfo>::iterator i(sSectionObjectBlocks.find(block)); + if(i != sSectionObjectBlocks.end()) + { + sSectionObjectBlocks.erase(i); + } + } + + // If it's not in the list, just ignore it, as lots of stuff goes this way... +} + +void *operator new(size_t size, const char *file, int line) +{ + void *r = ::malloc(size); + add_object_block(r, size, file, line, false); + //TRACE4("new(), %d, %s, %d, %08x\n", size, file, line, r); + return r; +} + +void *operator new[](size_t size, const char *file, int line) +{ + void *r = ::malloc(size); + add_object_block(r, size, file, line, true); + //TRACE4("new[](), %d, %s, %d, %08x\n", size, file, line, r); + return r; +} + +void operator delete[](void *ptr) throw () +{ + ::free(ptr); + remove_object_block(ptr); + //TRACE1("delete[]() called, %08x\n", ptr); +} + +void operator delete(void *ptr) throw () +{ + ::free(ptr); + remove_object_block(ptr); + //TRACE1("delete() called, %08x\n", ptr); +} + +#endif // NDEBUG diff --git a/lib/common/DebugPrintf.cpp b/lib/common/DebugPrintf.cpp new file mode 100755 index 00000000..02c25496 --- /dev/null +++ b/lib/common/DebugPrintf.cpp @@ -0,0 +1,72 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: DebugPrintf.cpp +// Purpose: Implementation of a printf function, to avoid a stdio.h include in Box.h +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- + +#ifndef NDEBUG + +#include "Box.h" + +#include <stdio.h> +#include <stdarg.h> +#include <syslog.h> + +#include "MemLeakFindOn.h" + +// Use this apparently superflous printf function to avoid having to +// include stdio.h in every file in the project. + +int BoxDebug_printf(const char *format, ...) +{ + va_list ap; + va_start(ap, format); + int r = vprintf(format, ap); + va_end(ap); + return r; +} + + +bool BoxDebugTraceOn = true; +bool BoxDebugTraceToStdout = true; +bool BoxDebugTraceToSyslog = false; + +int BoxDebugTrace(const char *format, ...) +{ + char text[512]; + int r = 0; + if(BoxDebugTraceOn || BoxDebugTraceToSyslog) + { + va_list ap; + va_start(ap, format); + r = vsnprintf(text, sizeof(text), format, ap); + va_end(ap); + } + + // Send to stdout if trace is on and std out is enabled + if(BoxDebugTraceOn && BoxDebugTraceToStdout) + { + printf("%s", text); + } + + // But tracing to syslog is independent of tracing being on or not + if(BoxDebugTraceToSyslog) + { + // Remove trailing '\n', if it's there + if(r > 0 && text[r] == '\n') + { + text[r] = '\0'; + --r; + } + // Log it + ::syslog(LOG_INFO, "TRACE: %s", text); + } + + return r; +} + + +#endif // NDEBUG diff --git a/lib/common/EndStructPackForWire.h b/lib/common/EndStructPackForWire.h new file mode 100644 index 00000000..c9655cd7 --- /dev/null +++ b/lib/common/EndStructPackForWire.h @@ -0,0 +1,23 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: EndStructPackForWire.h +// Purpose: End structure packing for wire +// Created: 25/11/03 +// +// -------------------------------------------------------------------------- + +// No header guard -- this is intentional + +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS + +#pragma pack() + +#else + + logical error -- check BoxPlatform.h and including file + +#endif + + + diff --git a/lib/common/EventWatchFilesystemObject.cpp b/lib/common/EventWatchFilesystemObject.cpp new file mode 100644 index 00000000..1611d5bd --- /dev/null +++ b/lib/common/EventWatchFilesystemObject.cpp @@ -0,0 +1,99 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: EventWatchFilesystemObject.cpp +// Purpose: WaitForEvent compatible object for watching directories +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <fcntl.h> +#include <unistd.h> + +#include "EventWatchFilesystemObject.h" +#include "autogen_CommonException.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: EventWatchFilesystemObject::EventWatchFilesystemObject(const char *) +// Purpose: Constructor -- opens the file object +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + : mDescriptor(::open(Filename, O_RDONLY /*O_EVTONLY*/, 0)) +#endif +{ +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + if(mDescriptor == -1) + { + THROW_EXCEPTION(CommonException, OSFileOpenError) + } +#else + THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform) +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: EventWatchFilesystemObject::~EventWatchFilesystemObject() +// Purpose: Destructor +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +EventWatchFilesystemObject::~EventWatchFilesystemObject() +{ + if(mDescriptor != -1) + { + ::close(mDescriptor); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesystemObject &) +// Purpose: Copy constructor +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy) + : mDescriptor(::dup(rToCopy.mDescriptor)) +{ + if(mDescriptor == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } +} + + +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED +// -------------------------------------------------------------------------- +// +// Function +// Name: EventWatchFilesystemObject::FillInKEvent(struct kevent &, int) +// Purpose: For WaitForEvent +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +void EventWatchFilesystemObject::FillInKEvent(struct kevent &rEvent, int Flags) const +{ + EV_SET(&rEvent, mDescriptor, EVFILT_VNODE, EV_CLEAR, NOTE_DELETE | NOTE_WRITE, 0, (void*)this); +} +#else +void EventWatchFilesystemObject::FillInPoll(int &fd, short &events, int Flags) const +{ + THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform) +} +#endif + diff --git a/lib/common/EventWatchFilesystemObject.h b/lib/common/EventWatchFilesystemObject.h new file mode 100644 index 00000000..d8a7245b --- /dev/null +++ b/lib/common/EventWatchFilesystemObject.h @@ -0,0 +1,48 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: EventWatchFilesystemObject.h +// Purpose: WaitForEvent compatible object for watching directories +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef EVENTWATCHFILESYSTEMOBJECT__H +#define EVENTWATCHFILESYSTEMOBJECT__H + +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + #include <sys/event.h> +#endif + + +// -------------------------------------------------------------------------- +// +// Class +// Name: EventWatchFilesystemObject +// Purpose: WaitForEvent compatible object for watching files and directories +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +class EventWatchFilesystemObject +{ +public: + EventWatchFilesystemObject(const char *Filename); + ~EventWatchFilesystemObject(); + EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy); +private: + // Assignment not allowed + EventWatchFilesystemObject &operator=(const EventWatchFilesystemObject &); +public: + +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + void FillInKEvent(struct kevent &rEvent, int Flags = 0) const; +#else + void FillInPoll(int &fd, short &events, int Flags = 0) const; +#endif + +private: + int mDescriptor; +}; + +#endif // EventWatchFilesystemObject__H + diff --git a/lib/common/ExcludeList.cpp b/lib/common/ExcludeList.cpp new file mode 100755 index 00000000..6350869d --- /dev/null +++ b/lib/common/ExcludeList.cpp @@ -0,0 +1,219 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ExcludeList.cpp +// Purpose: General purpose exclusion list +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifndef PLATFORM_REGEX_NOT_SUPPORTED + #include <regex.h> + #define EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED +#endif + +#include "ExcludeList.h" +#include "Utils.h" +#include "Configuration.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::ExcludeList() +// Purpose: Constructor. Generates an exclude list which will allow everything +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +ExcludeList::ExcludeList() + : mpAlwaysInclude(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::~ExcludeList() +// Purpose: Destructor +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +ExcludeList::~ExcludeList() +{ +#ifndef PLATFORM_REGEX_NOT_SUPPORTED + // free regex memory + while(mRegex.size() > 0) + { + regex_t *pregex = mRegex.back(); + mRegex.pop_back(); + // Free regex storage, and the structure itself + ::regfree(pregex); + delete pregex; + } +#endif + + // Clean up exceptions list + if(mpAlwaysInclude != 0) + { + delete mpAlwaysInclude; + mpAlwaysInclude = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::AddDefiniteEntries(const std::string &) +// Purpose: Adds a number of definite entries to the exclude list -- ones which +// will be excluded if and only if the test string matches exactly. +// Uses the Configuration classes' multi-value conventions, with +// multiple entires in one string separated by Configuration::MultiValueSeparator +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +void ExcludeList::AddDefiniteEntries(const std::string &rEntries) +{ + // Split strings up + std::vector<std::string> ens; + SplitString(rEntries, Configuration::MultiValueSeparator, ens); + + // Add to set of excluded strings + for(std::vector<std::string>::const_iterator i(ens.begin()); i != ens.end(); ++i) + { + if(i->size() > 0) + { + mDefinite.insert(*i); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::AddRegexEntries(const std::string &) +// Purpose: Adds a number of regular expression entries to the exclude list -- +// if the test expression matches any of these regex, it will be excluded. +// Uses the Configuration classes' multi-value conventions, with +// multiple entires in one string separated by Configuration::MultiValueSeparator +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +void ExcludeList::AddRegexEntries(const std::string &rEntries) +{ +#ifndef PLATFORM_REGEX_NOT_SUPPORTED + + // Split strings up + std::vector<std::string> ens; + SplitString(rEntries, Configuration::MultiValueSeparator, ens); + + // Create and add new regular expressions + for(std::vector<std::string>::const_iterator i(ens.begin()); i != ens.end(); ++i) + { + if(i->size() > 0) + { + // Allocate memory + regex_t *pregex = new regex_t; + + try + { + // Compile + if(::regcomp(pregex, i->c_str(), REG_EXTENDED | REG_NOSUB) != 0) + { + THROW_EXCEPTION(CommonException, BadRegularExpression) + } + + // Store in list of regular expressions + mRegex.push_back(pregex); + } + catch(...) + { + delete pregex; + throw; + } + } + } + +#else + THROW_EXCEPTION(CommonException, RegexNotSupportedOnThisPlatform) +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::IsExcluded(const std::string &) +// Purpose: Returns true if the entry should be excluded +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +bool ExcludeList::IsExcluded(const std::string &rTest) const +{ + // Check against the always include list + if(mpAlwaysInclude != 0) + { + if(mpAlwaysInclude->IsExcluded(rTest)) + { + // Because the "always include" list says it's 'excluded' + // this means it should actually be included. + return false; + } + } + + // Is it in the set of definite entries? + if(mDefinite.find(rTest) != mDefinite.end()) + { + return true; + } + + // Check against regular expressions +#ifndef PLATFORM_REGEX_NOT_SUPPORTED + for(std::vector<regex_t *>::const_iterator i(mRegex.begin()); i != mRegex.end(); ++i) + { + // Test against this expression + if(regexec(*i, rTest.c_str(), 0, 0 /* no match information required */, 0 /* no flags */) == 0) + { + // match happened + return true; + } + // In all other cases, including an error, just continue to the next expression + } +#endif + + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::SetAlwaysIncludeList(ExcludeList *) +// Purpose: Takes ownership of the list, deletes any pre-existing list. +// NULL is acceptable to delete the list. +// The AlwaysInclude list is a list of exceptions to the exclusions. +// Created: 19/2/04 +// +// -------------------------------------------------------------------------- +void ExcludeList::SetAlwaysIncludeList(ExcludeList *pAlwaysInclude) +{ + // Delete old list + if(mpAlwaysInclude != 0) + { + delete mpAlwaysInclude; + mpAlwaysInclude = 0; + } + + // Store the pointer + mpAlwaysInclude = pAlwaysInclude; +} + + + + + diff --git a/lib/common/ExcludeList.h b/lib/common/ExcludeList.h new file mode 100755 index 00000000..a1954044 --- /dev/null +++ b/lib/common/ExcludeList.h @@ -0,0 +1,65 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ExcludeList.h +// Purpose: General purpose exclusion list +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef EXCLUDELIST__H +#define EXCLUDELIST__H + +#include <string> +#include <set> +#include <vector> + +// avoid including regex.h in lots of places +#ifndef EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED + typedef int regex_t; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: ExcludeList +// Purpose: General purpose exclusion list +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +class ExcludeList +{ +public: + ExcludeList(); + ~ExcludeList(); + + void AddDefiniteEntries(const std::string &rEntries); + void AddRegexEntries(const std::string &rEntries); + + // Add exceptions to the exclusions (takes ownership) + void SetAlwaysIncludeList(ExcludeList *pAlwaysInclude); + + // Test function + bool IsExcluded(const std::string &rTest) const; + + // Mainly for tests + unsigned int SizeOfDefiniteList() const {return mDefinite.size();} + unsigned int SizeOfRegexList() const +#ifndef PLATFORM_REGEX_NOT_SUPPORTED + {return mRegex.size();} +#else + {return 0;} +#endif + +private: + std::set<std::string> mDefinite; +#ifndef PLATFORM_REGEX_NOT_SUPPORTED + std::vector<regex_t *> mRegex; +#endif + + // For exceptions to the excludes + ExcludeList *mpAlwaysInclude; +}; + +#endif // EXCLUDELIST__H + diff --git a/lib/common/FdGetLine.cpp b/lib/common/FdGetLine.cpp new file mode 100755 index 00000000..dee02604 --- /dev/null +++ b/lib/common/FdGetLine.cpp @@ -0,0 +1,211 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FdGetLine.cpp +// Purpose: Line based file descriptor reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <unistd.h> + +#include "FdGetLine.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// utility whitespace function +inline bool iw(int c) +{ + return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FdGetLine::FdGetLine(int) +// Purpose: Constructor, taking file descriptor +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +FdGetLine::FdGetLine(int fd) + : mFileHandle(fd), + mLineNumber(0), + mBufferBegin(0), + mBytesInBuffer(0), + mPendingEOF(false), + mEOF(false) +{ + if(mFileHandle < 0) {THROW_EXCEPTION(CommonException, BadArguments)} + //printf("FdGetLine buffer size = %d\n", sizeof(mBuffer)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FdGetLine::~FdGetLine() +// Purpose: Destructor +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +FdGetLine::~FdGetLine() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FdGetLine::GetLine(bool) +// Purpose: Returns a file from the file. If Preprocess is true, leading +// and trailing whitespace is removed, and comments (after #) +// are deleted. +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +std::string FdGetLine::GetLine(bool Preprocess) +{ + if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)} + + // EOF? + if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)} + + std::string r; + + bool foundLineEnd = false; + + while(!foundLineEnd && !mEOF) + { + // Use any bytes left in the buffer + while(mBufferBegin < mBytesInBuffer) + { + int c = mBuffer[mBufferBegin++]; + if(c == '\r') + { + // Ignore nasty Windows line ending extra chars + } + else if(c == '\n') + { + // Line end! + foundLineEnd = true; + break; + } + else + { + // Add to string + r += c; + } + + // Implicit line ending at EOF + if(mBufferBegin >= mBytesInBuffer && mPendingEOF) + { + foundLineEnd = true; + } + } + + // Check size + if(r.size() > FDGETLINE_MAX_LINE_SIZE) + { + THROW_EXCEPTION(CommonException, GetLineTooLarge) + } + + // Read more in? + if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF) + { + int bytes = ::read(mFileHandle, mBuffer, sizeof(mBuffer)); + + // Error? + if(bytes == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Adjust buffer info + mBytesInBuffer = bytes; + mBufferBegin = 0; + + // EOF / closed? + if(bytes == 0) + { + mPendingEOF = true; + } + } + + // EOF? + if(mPendingEOF && mBufferBegin >= mBytesInBuffer) + { + // File is EOF, and now we've depleted the buffer completely, so tell caller as well. + mEOF = true; + } + } + + if(!Preprocess) + { + return r; + } + else + { + // Check for comment char, but char before must be whitespace + int end = 0; + int size = r.size(); + while(end < size) + { + if(r[end] == '#' && (end == 0 || (iw(r[end-1])))) + { + break; + } + end++; + } + + // Remove whitespace + int begin = 0; + while(begin < size && iw(r[begin])) + { + begin++; + } + if(!iw(r[end])) end--; + while(end > begin && iw(r[end])) + { + end--; + } + + // Return a sub string + return r.substr(begin, end - begin + 1); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FdGetLine::DetachFile() +// Purpose: Detaches the file handle, setting the file pointer correctly. +// Probably not good for sockets... +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +void FdGetLine::DetachFile() +{ + if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)} + + // Adjust file pointer + int bytesOver = mBufferBegin - mBufferBegin; + ASSERT(bytesOver >= 0); + if(bytesOver > 0) + { + if(::lseek(mFileHandle, 0 - bytesOver, SEEK_CUR) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + + // Unset file pointer + mFileHandle = -1; +} + + diff --git a/lib/common/FdGetLine.h b/lib/common/FdGetLine.h new file mode 100755 index 00000000..fecb0371 --- /dev/null +++ b/lib/common/FdGetLine.h @@ -0,0 +1,61 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FdGetLine.h +// Purpose: Line based file descriptor reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- + +#ifndef FDGETLINE__H +#define FDGETLINE__H + +#include <string> + +#ifdef NDEBUG + #define FDGETLINE_BUFFER_SIZE 1024 +#else + #define FDGETLINE_BUFFER_SIZE 4 +#endif + +// Just a very large upper bound for line size to avoid +// people sending lots of data over sockets and causing memory problems. +#define FDGETLINE_MAX_LINE_SIZE (1024*256) + +// -------------------------------------------------------------------------- +// +// Class +// Name: FdGetLine +// Purpose: Line based file descriptor reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +class FdGetLine +{ +public: + FdGetLine(int fd); + ~FdGetLine(); +private: + FdGetLine(const FdGetLine &rToCopy); + +public: + std::string GetLine(bool Preprocess = false); + bool IsEOF() {return mEOF;} + int GetLineNumber() {return mLineNumber;} + + // Call to detach, setting file pointer correctly to last bit read. + // Only works for lseek-able file descriptors. + void DetachFile(); + +private: + char mBuffer[FDGETLINE_BUFFER_SIZE]; + int mFileHandle; + int mLineNumber; + int mBufferBegin; + int mBytesInBuffer; + bool mPendingEOF; + bool mEOF; +}; + +#endif // FDGETLINE__H + diff --git a/lib/common/FileModificationTime.h b/lib/common/FileModificationTime.h new file mode 100755 index 00000000..78f5c115 --- /dev/null +++ b/lib/common/FileModificationTime.h @@ -0,0 +1,57 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FileModificationTime.h +// Purpose: Function for getting file modification time. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef FILEMODIFICATIONTIME__H +#define FILEMODIFICATIONTIME__H + +#include <sys/stat.h> + +#include "BoxTime.h" + +inline box_time_t FileModificationTime(struct stat &st) +{ +#ifdef PLATFORM_stat_SHORT_mtime + box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); +#else + box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#endif + + return datamodified; +} + +inline box_time_t FileAttrModificationTime(struct stat &st) +{ +#ifdef PLATFORM_stat_SHORT_mtime + box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL); +#else + box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#endif + + return statusmodified; +} + +inline box_time_t FileModificationTimeMaxModAndAttr(struct stat &st) +{ +#ifdef PLATFORM_stat_SHORT_mtime + box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); + box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL); +#else + box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); + box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#endif + + return (datamodified > statusmodified)?datamodified:statusmodified; +} + +#endif // FILEMODIFICATIONTIME__H + diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp new file mode 100755 index 00000000..ca6894ae --- /dev/null +++ b/lib/common/FileStream.cpp @@ -0,0 +1,245 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FileStream.cpp +// Purpose: IOStream interface to files +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "FileStream.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::FileStream(const char *, int, int) +// Purpose: Constructor, opens file +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +FileStream::FileStream(const char *Filename, int flags, int mode) + : mOSFileHandle(::open(Filename, flags, mode)), + mIsEOF(false) +{ + if(mOSFileHandle < 0) + { + MEMLEAKFINDER_NOT_A_LEAK(this); + THROW_EXCEPTION(CommonException, OSFileOpenError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::FileStream(int) +// Purpose: Constructor, using existing file descriptor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +FileStream::FileStream(int FileDescriptor) + : mOSFileHandle(FileDescriptor), + mIsEOF(false) +{ + if(mOSFileHandle < 0) + { + MEMLEAKFINDER_NOT_A_LEAK(this); + THROW_EXCEPTION(CommonException, OSFileOpenError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::FileStream(const FileStream &) +// Purpose: Copy constructor, creates a duplicate of the file handle +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +FileStream::FileStream(const FileStream &rToCopy) + : mOSFileHandle(::dup(rToCopy.mOSFileHandle)), + mIsEOF(rToCopy.mIsEOF) +{ + if(mOSFileHandle < 0) + { + MEMLEAKFINDER_NOT_A_LEAK(this); + THROW_EXCEPTION(CommonException, OSFileOpenError) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::~FileStream() +// Purpose: Destructor, closes file +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +FileStream::~FileStream() +{ + if(mOSFileHandle >= 0) + { + Close(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::Read(void *, int) +// Purpose: Reads bytes from the file +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +int FileStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)} + int r = ::read(mOSFileHandle, pBuffer, NBytes); + if(r == -1) + { + THROW_EXCEPTION(CommonException, OSFileReadError) + } + if(r == 0) + { + mIsEOF = true; + } + + return r; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::BytesLeftToRead() +// Purpose: Returns number of bytes to read (may not be most efficient function ever) +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +IOStream::pos_type FileStream::BytesLeftToRead() +{ + struct stat st; + if(::fstat(mOSFileHandle, &st) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + return st.st_size - GetPosition(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::Write(void *, int) +// Purpose: Writes bytes to the file +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void FileStream::Write(const void *pBuffer, int NBytes) +{ + if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)} + if(::write(mOSFileHandle, pBuffer, NBytes) != NBytes) + { + THROW_EXCEPTION(CommonException, OSFileWriteError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::GetPosition() +// Purpose: Get position in stream +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type FileStream::GetPosition() const +{ + if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)} + off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR); + if(p == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + return (IOStream::pos_type)p; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::Seek(pos_type, int) +// Purpose: Seeks within file, as lseek +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void FileStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)} + if(::lseek(mOSFileHandle, Offset, ConvertSeekTypeToOSWhence(SeekType)) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Not end of file any more! + mIsEOF = false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::Close() +// Purpose: Closes the underlying file +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void FileStream::Close() +{ + if(mOSFileHandle < 0) + { + THROW_EXCEPTION(CommonException, FileAlreadyClosed) + } + if(::close(mOSFileHandle) != 0) + { + THROW_EXCEPTION(CommonException, OSFileCloseError) + } + mOSFileHandle = -1; + mIsEOF = true; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::StreamDataLeft() +// Purpose: Any data left to write? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool FileStream::StreamDataLeft() +{ + return !mIsEOF; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::StreamClosed() +// Purpose: Is the stream closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool FileStream::StreamClosed() +{ + return mIsEOF; +} + diff --git a/lib/common/FileStream.h b/lib/common/FileStream.h new file mode 100755 index 00000000..bb3cc459 --- /dev/null +++ b/lib/common/FileStream.h @@ -0,0 +1,46 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FileStream.h +// Purpose: FileStream interface to files +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef FILESTREAM__H +#define FILESTREAM__H + +#include "IOStream.h" + +#include <fcntl.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <unistd.h> + +class FileStream : public IOStream +{ +public: + FileStream(const char *Filename, int flags = O_RDONLY, int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + FileStream(int FileDescriptor); + FileStream(const FileStream &rToCopy); + virtual ~FileStream(); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + int mOSFileHandle; + bool mIsEOF; +}; + + +#endif // FILESTREAM__H + + diff --git a/lib/common/Guards.h b/lib/common/Guards.h new file mode 100755 index 00000000..6efc5614 --- /dev/null +++ b/lib/common/Guards.h @@ -0,0 +1,112 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Guards.h +// Purpose: Classes which ensure things are closed/deleted properly when +// going out of scope. Easy exception proof code, etc +// Created: 2003/07/12 +// +// -------------------------------------------------------------------------- + +#ifndef GUARDS__H +#define GUARDS__H + +#include <fcntl.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <unistd.h> +#include <new> + +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +template <int flags = O_RDONLY, int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)> +class FileHandleGuard +{ +public: + FileHandleGuard(const char *filename) + : mOSFileHandle(::open(filename, flags, mode)) + { + if(mOSFileHandle < 0) + { + THROW_EXCEPTION(CommonException, OSFileOpenError) + } + } + + ~FileHandleGuard() + { + if(mOSFileHandle >= 0) + { + Close(); + } + } + + void Close() + { + if(mOSFileHandle < 0) + { + THROW_EXCEPTION(CommonException, FileAlreadyClosed) + } + if(::close(mOSFileHandle) != 0) + { + THROW_EXCEPTION(CommonException, OSFileCloseError) + } + mOSFileHandle = -1; + } + + operator int() const + { + return mOSFileHandle; + } + +private: + int mOSFileHandle; +}; + +template<typename type> +class MemoryBlockGuard +{ +public: + MemoryBlockGuard(int BlockSize) + : mpBlock(::malloc(BlockSize)) + { + if(mpBlock == 0) + { + throw std::bad_alloc(); + } + } + + ~MemoryBlockGuard() + { + free(mpBlock); + } + + operator type() const + { + return (type)mpBlock; + } + + type GetPtr() const + { + return (type)mpBlock; + } + + void Resize(int NewSize) + { + void *ptrn = ::realloc(mpBlock, NewSize); + if(ptrn == 0) + { + throw std::bad_alloc(); + } + mpBlock = ptrn; + } + +private: + void *mpBlock; +}; + +#include "MemLeakFindOff.h" + +#endif // GUARDS__H + diff --git a/lib/common/IOStream.cpp b/lib/common/IOStream.cpp new file mode 100755 index 00000000..024eefcc --- /dev/null +++ b/lib/common/IOStream.cpp @@ -0,0 +1,229 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: IOStream.cpp +// Purpose: I/O Stream abstraction +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "IOStream.h" +#include "CommonException.h" +#include "Guards.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::IOStream() +// Purpose: Constructor +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +IOStream::IOStream() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::IOStream(const IOStream &) +// Purpose: Copy constructor (exceptions) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +IOStream::IOStream(const IOStream &rToCopy) +{ + THROW_EXCEPTION(CommonException, NotSupported) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::~IOStream() +// Purpose: Destructor +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +IOStream::~IOStream() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::Close() +// Purpose: Close the stream +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void IOStream::Close() +{ + // Do nothing by default -- let the destructor clear everything up. +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::Seek(int, int) +// Purpose: Seek in stream (if supported) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void IOStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + THROW_EXCEPTION(CommonException, NotSupported) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::GetPosition() +// Purpose: Returns current position in stream (if supported) +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type IOStream::GetPosition() const +{ + THROW_EXCEPTION(CommonException, NotSupported) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::ConvertSeekTypeToOSWhence(int) +// Purpose: Return an whence arg for lseek given a IOStream seek type +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +int IOStream::ConvertSeekTypeToOSWhence(int SeekType) +{ + // Should be nicely optimised out as values are choosen in header file to match OS values. + int ostype = SEEK_SET; + switch(SeekType) + { + case SeekType_Absolute: + ostype = SEEK_SET; + break; + case SeekType_Relative: + ostype = SEEK_CUR; + break; + case SeekType_End: + ostype = SEEK_END; + break; + + default: + THROW_EXCEPTION(CommonException, IOStreamBadSeekType) + } + + return ostype; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::ReadFullBuffer(void *, int, int) +// Purpose: Reads bytes into buffer, returning whether or not it managed to +// get all the bytes required. Exception and abort use of stream +// if this returns false. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool IOStream::ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout) +{ + int bytesToGo = NBytes; + char *buffer = (char*)pBuffer; + if(pNBytesRead) (*pNBytesRead) = 0; + + while(bytesToGo > 0) + { + int bytesRead = Read(buffer, bytesToGo, Timeout); + if(bytesRead == 0) + { + // Timeout or something + return false; + } + // Increment things + bytesToGo -= bytesRead; + buffer += bytesRead; + if(pNBytesRead) (*pNBytesRead) += bytesRead; + } + + // Got everything + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::WriteAllBuffered() +// Purpose: Ensures that any data which has been buffered is written to the stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void IOStream::WriteAllBuffered() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::BytesLeftToRead() +// Purpose: Numbers of bytes left to read in the stream, or +// IOStream::SizeOfStreamUnknown if this isn't known. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +IOStream::pos_type IOStream::BytesLeftToRead() +{ + return IOStream::SizeOfStreamUnknown; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::CopyStreamTo(IOStream &, int Timeout) +// Purpose: Copies the entire stream to another stream (reading from this, +// writing to rCopyTo). Returns whether the copy completed (ie +// StreamDataLeft() returns false) +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool IOStream::CopyStreamTo(IOStream &rCopyTo, int Timeout, int BufferSize) +{ + // Make sure there's something to do before allocating that buffer + if(!StreamDataLeft()) + { + return true; // complete, even though nothing happened + } + + // Buffer + MemoryBlockGuard<char*> buffer(BufferSize); + + // Get copying! + while(StreamDataLeft()) + { + // Read some data + int bytes = Read(buffer, BufferSize, Timeout); + if(bytes == 0 && StreamDataLeft()) + { + return false; // incomplete, timed out + } + + // Write some data + if(bytes != 0) + { + rCopyTo.Write(buffer, bytes); + } + } + + return true; // completed +} + + diff --git a/lib/common/IOStream.h b/lib/common/IOStream.h new file mode 100755 index 00000000..042ccca4 --- /dev/null +++ b/lib/common/IOStream.h @@ -0,0 +1,67 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: IOStream.h +// Purpose: I/O Stream abstraction +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef IOSTREAM__H +#define IOSTREAM__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: IOStream +// Purpose: Abstract interface to streams of data +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +class IOStream +{ +public: + IOStream(); + IOStream(const IOStream &rToCopy); + virtual ~IOStream(); + + enum + { + TimeOutInfinite = -1, + SizeOfStreamUnknown = -1 + }; + + enum + { + SeekType_Absolute = 0, + SeekType_Relative = 1, + SeekType_End = 2 + }; + + // Timeout in milliseconds + // Read may return 0 -- does not mean end of stream. + typedef int64_t pos_type; + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite) = 0; + virtual pos_type BytesLeftToRead(); // may return IOStream::SizeOfStreamUnknown (and will for most stream types) + virtual void Write(const void *pBuffer, int NBytes) = 0; + virtual void WriteAllBuffered(); + virtual pos_type GetPosition() const; + virtual void Seek(pos_type Offset, int SeekType); + virtual void Close(); + + // Has all data that can be read been read? + virtual bool StreamDataLeft() = 0; + // Has the stream been closed (writing not possible) + virtual bool StreamClosed() = 0; + + // Utility functions + bool ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout = IOStream::TimeOutInfinite); + bool CopyStreamTo(IOStream &rCopyTo, int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024); + + static int ConvertSeekTypeToOSWhence(int SeekType); +}; + + +#endif // IOSTREAM__H + + diff --git a/lib/common/IOStreamGetLine.cpp b/lib/common/IOStreamGetLine.cpp new file mode 100755 index 00000000..27a77c29 --- /dev/null +++ b/lib/common/IOStreamGetLine.cpp @@ -0,0 +1,227 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: IOStreamGetLine.cpp +// Purpose: Line based file descriptor reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "IOStreamGetLine.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// utility whitespace function +inline bool iw(int c) +{ + return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStreamGetLine::IOStreamGetLine(int) +// Purpose: Constructor, taking file descriptor +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +IOStreamGetLine::IOStreamGetLine(IOStream &Stream) + : mrStream(Stream), + mLineNumber(0), + mBufferBegin(0), + mBytesInBuffer(0), + mPendingEOF(false), + mEOF(false) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStreamGetLine::~IOStreamGetLine() +// Purpose: Destructor +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +IOStreamGetLine::~IOStreamGetLine() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStreamGetLine::GetLine(std::string &, bool, int) +// Purpose: Gets a line from the file, returning it in rOutput. If Preprocess is true, leading +// and trailing whitespace is removed, and comments (after #) +// are deleted. +// Returns true if a line is available now, false if retrying may get a line (eg timeout, signal), +// and exceptions if it's EOF. +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +bool IOStreamGetLine::GetLine(std::string &rOutput, bool Preprocess, int Timeout) +{ + // EOF? + if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)} + + // Initialise string to stored into + std::string r(mPendingString); + mPendingString.erase(); + + bool foundLineEnd = false; + + while(!foundLineEnd && !mEOF) + { + // Use any bytes left in the buffer + while(mBufferBegin < mBytesInBuffer) + { + int c = mBuffer[mBufferBegin++]; + if(c == '\r') + { + // Ignore nasty Windows line ending extra chars + } + else if(c == '\n') + { + // Line end! + foundLineEnd = true; + break; + } + else + { + // Add to string + r += c; + } + + // Implicit line ending at EOF + if(mBufferBegin >= mBytesInBuffer && mPendingEOF) + { + foundLineEnd = true; + } + } + + // Check size + if(r.size() > IOSTREAMGETLINE_MAX_LINE_SIZE) + { + THROW_EXCEPTION(CommonException, GetLineTooLarge) + } + + // Read more in? + if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF) + { + int bytes = mrStream.Read(mBuffer, sizeof(mBuffer), Timeout); + + // Adjust buffer info + mBytesInBuffer = bytes; + mBufferBegin = 0; + + // EOF / closed? + if(!mrStream.StreamDataLeft()) + { + mPendingEOF = true; + } + + // No data returned? + if(bytes == 0 && mrStream.StreamDataLeft()) + { + // store string away + mPendingString = r; + // Return false; + return false; + } + } + + // EOF? + if(mPendingEOF && mBufferBegin >= mBytesInBuffer) + { + // File is EOF, and now we've depleted the buffer completely, so tell caller as well. + mEOF = true; + } + } + + if(!Preprocess) + { + rOutput = r; + return true; + } + else + { + // Check for comment char, but char before must be whitespace + int end = 0; + int size = r.size(); + while(end < size) + { + if(r[end] == '#' && (end == 0 || (iw(r[end-1])))) + { + break; + } + end++; + } + + // Remove whitespace + int begin = 0; + while(begin < size && iw(r[begin])) + { + begin++; + } + if(!iw(r[end])) end--; + while(end > begin && iw(r[end])) + { + end--; + } + + // Return a sub string + rOutput = r.substr(begin, end - begin + 1); + return true; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStreamGetLine::DetachFile() +// Purpose: Detaches the file handle, setting the file pointer correctly. +// Probably not good for sockets... +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +void IOStreamGetLine::DetachFile() +{ + // Adjust file pointer + int bytesOver = mBytesInBuffer - mBufferBegin; + ASSERT(bytesOver >= 0); + if(bytesOver > 0) + { + mrStream.Seek(0 - bytesOver, IOStream::SeekType_Relative); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStreamGetLine::IgnoreBufferedData(int) +// Purpose: Ignore buffered bytes (effectively removing them from the +// beginning of the buffered data.) +// Cannot remove more bytes than are currently in the buffer. +// Be careful when this is used! +// Created: 22/12/04 +// +// -------------------------------------------------------------------------- +void IOStreamGetLine::IgnoreBufferedData(int BytesToIgnore) +{ + int bytesInBuffer = mBytesInBuffer - mBufferBegin; + if(BytesToIgnore < 0 || BytesToIgnore > bytesInBuffer) + { + THROW_EXCEPTION(CommonException, IOStreamGetLineNotEnoughDataToIgnore) + } + mBufferBegin += BytesToIgnore; +} + + + diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h new file mode 100755 index 00000000..cf152e5a --- /dev/null +++ b/lib/common/IOStreamGetLine.h @@ -0,0 +1,71 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: IOStreamGetLine.h +// Purpose: Line based file descriptor reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- + +#ifndef IOSTREAMGETLINE__H +#define IOSTREAMGETLINE__H + +#include <string> + +#include "IOStream.h" + +#ifdef NDEBUG + #define IOSTREAMGETLINE_BUFFER_SIZE 1024 +#else + #define IOSTREAMGETLINE_BUFFER_SIZE 4 +#endif + +// Just a very large upper bound for line size to avoid +// people sending lots of data over sockets and causing memory problems. +#define IOSTREAMGETLINE_MAX_LINE_SIZE (1024*256) + +// -------------------------------------------------------------------------- +// +// Class +// Name: IOStreamGetLine +// Purpose: Line based stream reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +class IOStreamGetLine +{ +public: + IOStreamGetLine(IOStream &Stream); + ~IOStreamGetLine(); +private: + IOStreamGetLine(const IOStreamGetLine &rToCopy); + +public: + bool GetLine(std::string &rOutput, bool Preprocess = false, int Timeout = IOStream::TimeOutInfinite); + bool IsEOF() {return mEOF;} + int GetLineNumber() {return mLineNumber;} + + // Call to detach, setting file pointer correctly to last bit read. + // Only works for lseek-able file descriptors. + void DetachFile(); + + // For doing interesting stuff with the remaining data... + // Be careful with this! + const void *GetBufferedData() const {return mBuffer + mBufferBegin;} + int GetSizeOfBufferedData() const {return mBytesInBuffer - mBufferBegin;} + void IgnoreBufferedData(int BytesToIgnore); + IOStream &GetUnderlyingStream() {return mrStream;} + +private: + char mBuffer[IOSTREAMGETLINE_BUFFER_SIZE]; + IOStream &mrStream; + int mLineNumber; + int mBufferBegin; + int mBytesInBuffer; + bool mPendingEOF; + bool mEOF; + std::string mPendingString; +}; + +#endif // IOSTREAMGETLINE__H + diff --git a/lib/common/LinuxWorkaround.cpp b/lib/common/LinuxWorkaround.cpp new file mode 100755 index 00000000..7900fa6e --- /dev/null +++ b/lib/common/LinuxWorkaround.cpp @@ -0,0 +1,73 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: LinuxWorkaround.cpp +// Purpose: Workarounds for Linux +// Created: 2003/10/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/stat.h> +#include <unistd.h> +#include <dirent.h> + +#include <string> + +#include "LinuxWorkaround.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +#ifdef PLATFORM_LINUX + +// -------------------------------------------------------------------------- +// +// Function +// Name: LinuxWorkaround_FinishDirentStruct(struct dirent *, const char *) +// Purpose: Finishes off filling in a dirent structure, which Linux leaves incomplete. +// Created: 2003/10/31 +// +// -------------------------------------------------------------------------- +void LinuxWorkaround_FinishDirentStruct(struct dirent *entry, const char *DirectoryName) +{ + // From man readdir under Linux: + // + // BUGS + // Field d_type is not implemented as of libc6 2.1 and will always return + // DT_UNKNOWN (0). + // + // What kind of an OS is this? + + + // Build filename of this entry + std::string fn(DirectoryName); + fn += '/'; + fn += entry->d_name; + + // Do a stat on it + struct stat st; + if(::lstat(fn.c_str(), &st) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Fill in the d_type field. + if(S_ISREG(st.st_mode)) + { + entry->d_type = DT_REG; + } + else if(S_ISDIR(st.st_mode)) + { + entry->d_type = DT_DIR; + } + else if(S_ISLNK(st.st_mode)) + { + entry->d_type = DT_LNK; + } + // otherwise leave it as we found it +} + +#endif // PLATFORM_LINUX + diff --git a/lib/common/LinuxWorkaround.h b/lib/common/LinuxWorkaround.h new file mode 100755 index 00000000..bcd27495 --- /dev/null +++ b/lib/common/LinuxWorkaround.h @@ -0,0 +1,20 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: LinuxWorkaround.h +// Purpose: Workarounds for Linux +// Created: 2003/10/31 +// +// -------------------------------------------------------------------------- + +#ifndef LINUXWORKAROUND__H +#define LINUXWORKAROUND__H + +#ifdef PLATFORM_LINUX + +void LinuxWorkaround_FinishDirentStruct(struct dirent *entry, const char *DirectoryName); + +#endif // PLATFORM_LINUX + +#endif // LINUXWORKAROUND__H + diff --git a/lib/common/MainHelper.h b/lib/common/MainHelper.h new file mode 100755 index 00000000..ab75af96 --- /dev/null +++ b/lib/common/MainHelper.h @@ -0,0 +1,42 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MainHelper.h +// Purpose: Helper stuff for main() programs +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- + +#ifndef MAINHELPER__H +#define MAINHELPER__H + +#include <stdio.h> + +#include "BoxException.h" + +#define MAINHELPER_START \ + if(argc == 2 && ::strcmp(argv[1], "--version") == 0) \ + { printf(BOX_VERSION "\n"); return 0; } \ + MEMLEAKFINDER_START \ + try { +#define MAINHELPER_END \ + } catch(BoxException &e) { \ + printf("Exception: %s (%d/%d)\n", e.what(), e.GetType(), e.GetSubType()); \ + return 1; \ + } catch(std::exception &e) { \ + printf("Exception: %s\n", e.what()); \ + return 1; \ + } catch(...) { \ + printf("Exception: <UNKNOWN>\n"); \ + return 1; } + +#ifdef BOX_MEMORY_LEAK_TESTING + #define MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, marker) \ + memleakfinder_setup_exit_report(file, marker); +#else + #define MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, marker) +#endif // BOX_MEMORY_LEAK_TESTING + + +#endif // MAINHELPER__H + diff --git a/lib/common/Makefile.extra b/lib/common/Makefile.extra new file mode 100755 index 00000000..92e6fbb3 --- /dev/null +++ b/lib/common/Makefile.extra @@ -0,0 +1,11 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_CommonException.h autogen_CommonException.cpp: $(MAKEEXCEPTION) CommonException.txt + perl $(MAKEEXCEPTION) CommonException.txt + +# AUTOGEN SEEDING +autogen_ConversionException.h autogen_ConversionException.cpp: $(MAKEEXCEPTION) ConversionException.txt + perl $(MAKEEXCEPTION) ConversionException.txt + diff --git a/lib/common/MemBlockStream.cpp b/lib/common/MemBlockStream.cpp new file mode 100755 index 00000000..538a7ef8 --- /dev/null +++ b/lib/common/MemBlockStream.cpp @@ -0,0 +1,235 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemBlockStream.cpp +// Purpose: Stream out data from any memory block +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "MemBlockStream.h" +#include "CommonException.h" +#include "StreamableMemBlock.h" +#include "CollectInBufferStream.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::MemBlockStream() +// Purpose: Constructor (doesn't copy block, careful with lifetimes) +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream(const void *pBuffer, int Size) + : mpBuffer((char*)pBuffer), + mBytesInBuffer(Size), + mReadPosition(0) +{ + ASSERT(pBuffer != 0); + ASSERT(Size >= 0); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::MemBlockStream(const StreamableMemBlock &) +// Purpose: Constructor (doesn't copy block, careful with lifetimes) +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream(const StreamableMemBlock &rBlock) + : mpBuffer((char*)rBlock.GetBuffer()), + mBytesInBuffer(rBlock.GetSize()), + mReadPosition(0) +{ + ASSERT(mpBuffer != 0); + ASSERT(mBytesInBuffer >= 0); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::MemBlockStream(const StreamableMemBlock &) +// Purpose: Constructor (doesn't copy block, careful with lifetimes) +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream(const CollectInBufferStream &rBuffer) + : mpBuffer((char*)rBuffer.GetBuffer()), + mBytesInBuffer(rBuffer.GetSize()), + mReadPosition(0) +{ + ASSERT(mpBuffer != 0); + ASSERT(mBytesInBuffer >= 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::MemBlockStream(const MemBlockStream &) +// Purpose: Copy constructor +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream(const MemBlockStream &rToCopy) + : mpBuffer(rToCopy.mpBuffer), + mBytesInBuffer(rToCopy.mBytesInBuffer), + mReadPosition(0) +{ + ASSERT(mpBuffer != 0); + ASSERT(mBytesInBuffer >= 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::~MemBlockStream() +// Purpose: Destructor +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +MemBlockStream::~MemBlockStream() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::Read(void *, int, int) +// Purpose: As interface. But only works in read phase +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +int MemBlockStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Adjust to number of bytes left + if(NBytes > (mBytesInBuffer - mReadPosition)) + { + NBytes = (mBytesInBuffer - mReadPosition); + } + ASSERT(NBytes >= 0); + if(NBytes <= 0) return 0; // careful now + + // Copy in the requested number of bytes and adjust the read pointer + ::memcpy(pBuffer, mpBuffer + mReadPosition, NBytes); + mReadPosition += NBytes; + + return NBytes; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::BytesLeftToRead() +// Purpose: As interface. But only works in read phase +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +IOStream::pos_type MemBlockStream::BytesLeftToRead() +{ + return (mBytesInBuffer - mReadPosition); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::Write(void *, int) +// Purpose: As interface. But only works in write phase +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void MemBlockStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, MemBlockStreamNotSupported) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::GetPosition() +// Purpose: In write phase, returns the number of bytes written, in read +// phase, the number of bytes to go +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +IOStream::pos_type MemBlockStream::GetPosition() const +{ + return mReadPosition; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::Seek(pos_type, int) +// Purpose: As interface. +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void MemBlockStream::Seek(pos_type Offset, int SeekType) +{ + int newPos = 0; + switch(SeekType) + { + case IOStream::SeekType_Absolute: + newPos = Offset; + break; + case IOStream::SeekType_Relative: + newPos = mReadPosition + Offset; + break; + case IOStream::SeekType_End: + newPos = mBytesInBuffer + Offset; + break; + default: + THROW_EXCEPTION(CommonException, IOStreamBadSeekType) + break; + } + + // Make sure it doesn't go over + if(newPos > mBytesInBuffer) + { + newPos = mBytesInBuffer; + } + // or under + if(newPos < 0) + { + newPos = 0; + } + + // Set the new read position + mReadPosition = newPos; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::StreamDataLeft() +// Purpose: As interface +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +bool MemBlockStream::StreamDataLeft() +{ + return mReadPosition < mBytesInBuffer; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::StreamClosed() +// Purpose: As interface +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +bool MemBlockStream::StreamClosed() +{ + return true; +} + diff --git a/lib/common/MemBlockStream.h b/lib/common/MemBlockStream.h new file mode 100755 index 00000000..f78ff8e6 --- /dev/null +++ b/lib/common/MemBlockStream.h @@ -0,0 +1,52 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemBlockStream.h +// Purpose: Stream out data from any memory block +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- + +#ifndef MEMBLOCKSTREAM__H +#define MEMBLOCKSTREAM__H + +#include "IOStream.h" + +class StreamableMemBlock; +class CollectInBufferStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: MemBlockStream +// Purpose: Stream out data from any memory block -- be careful the lifetime +// of the block is greater than the lifetime of this stream. +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +class MemBlockStream : public IOStream +{ +public: + MemBlockStream(const void *pBuffer, int Size); + MemBlockStream(const StreamableMemBlock &rBlock); + MemBlockStream(const CollectInBufferStream &rBuffer); + MemBlockStream(const MemBlockStream &rToCopy); + ~MemBlockStream(); +public: + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(pos_type Offset, int SeekType); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + const char *mpBuffer; + int mBytesInBuffer; + int mReadPosition; +}; + +#endif // MEMBLOCKSTREAM__H + diff --git a/lib/common/MemLeakFindOff.h b/lib/common/MemLeakFindOff.h new file mode 100755 index 00000000..1cc98bac --- /dev/null +++ b/lib/common/MemLeakFindOff.h @@ -0,0 +1,27 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemLeakFindOff.h +// Purpose: Switch memory leak finding off +// Created: 13/1/04 +// +// -------------------------------------------------------------------------- + +// no header guard + +#ifdef BOX_MEMORY_LEAK_TESTING + +#undef new + +#ifndef MEMLEAKFINDER_FULL_MALLOC_MONITORING + #ifdef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED + #undef malloc + #undef realloc + #undef free + #undef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED + #endif +#endif + +#undef MEMLEAKFINDER_ENABLED + +#endif diff --git a/lib/common/MemLeakFindOn.h b/lib/common/MemLeakFindOn.h new file mode 100755 index 00000000..c20fe25a --- /dev/null +++ b/lib/common/MemLeakFindOn.h @@ -0,0 +1,25 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemLeakFindOn.h +// Purpose: Switch memory leak finding on +// Created: 13/1/04 +// +// -------------------------------------------------------------------------- + +// no header guard + +#ifdef BOX_MEMORY_LEAK_TESTING + +#define new DEBUG_NEW + +#ifndef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED + #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__) + #define realloc memleakfinder_realloc + #define free memleakfinder_free + #define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED +#endif + +#define MEMLEAKFINDER_ENABLED + +#endif diff --git a/lib/common/MemLeakFinder.h b/lib/common/MemLeakFinder.h new file mode 100755 index 00000000..f5887dac --- /dev/null +++ b/lib/common/MemLeakFinder.h @@ -0,0 +1,60 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemLeakFinder.h +// Purpose: Memory leak finder +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef MEMLEAKFINDER__H +#define MEMLEAKFINDER__H + +#define DEBUG_NEW new(__FILE__,__LINE__) + +#ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING + // include stdlib now, to avoid problems with having the macros defined already + #include <stdlib.h> +#endif + +// global enable flag +extern bool memleakfinder_global_enable; + +extern "C" +{ + void *memleakfinder_malloc(size_t size, const char *file, int line); + void *memleakfinder_realloc(void *ptr, size_t size); + void memleakfinder_free(void *ptr); +} + +int memleakfinder_numleaks(); + +void memleakfinder_reportleaks(); + +void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext); + +void memleakfinder_setup_exit_report(const char *filename, const char *markertext); + +void memleakfinder_startsectionmonitor(); + +void memleakfinder_traceblocksinsection(); + +void memleakfinder_notaleak(void *ptr); + +void *operator new(size_t size, const char *file, int line); +void *operator new[](size_t size, const char *file, int line); + +void operator delete(void *ptr) throw (); +void operator delete[](void *ptr) throw (); + +// define the malloc functions now, if required +#ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING + #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__) + #define realloc memleakfinder_realloc + #define free memleakfinder_free + #define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED +#endif + + +#endif // MEMLEAKFINDER__H + diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp new file mode 100755 index 00000000..1f6e038a --- /dev/null +++ b/lib/common/NamedLock.cpp @@ -0,0 +1,144 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: NamedLock.cpp +// Purpose: A global named lock, implemented as a lock file in file system +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#ifdef PLATFORM_LINUX + #include <sys/file.h> +#endif // PLATFORM_LINUX +#ifdef PLATFORM_CYGWIN + #include <sys/file.h> +#endif // PLATFORM_CYGWIN + +#include "NamedLock.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: NamedLock::NamedLock() +// Purpose: Constructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +NamedLock::NamedLock() + : mFileDescriptor(-1) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: NamedLock::~NamedLock() +// Purpose: Destructor (automatically unlocks if locked) +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +NamedLock::~NamedLock() +{ + if(mFileDescriptor != -1) + { + ReleaseLock(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: NamedLock::TryAndGetLock(const char *, int) +// Purpose: Trys to get a lock on the name in the file system. +// IMPORTANT NOTE: If a file exists with this name, it will be deleted. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +bool NamedLock::TryAndGetLock(const char *Filename, int mode) +{ + // Check + if(mFileDescriptor != -1) + { + THROW_EXCEPTION(CommonException, NamedLockAlreadyLockingSomething) + } + + // See if the lock can be got +#ifdef PLATFORM_open_NO_O_EXLOCK + int fd = ::open(Filename, O_WRONLY | O_CREAT | O_TRUNC, mode); + if(fd == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + if(::flock(fd, LOCK_EX | LOCK_NB) != 0) + { + ::close(fd); + if(errno == EWOULDBLOCK) + { + return false; + } + else + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + + // Success + mFileDescriptor = fd; + + return true; +#else + int fd = ::open(Filename, O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode); + if(fd != -1) + { + // Got a lock, lovely + mFileDescriptor = fd; + return true; + } + + // Failed. Why? + if(errno != EWOULDBLOCK) + { + // Not the expected error + THROW_EXCEPTION(CommonException, OSFileError) + } + + return false; +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: NamedLock::ReleaseLock() +// Purpose: Release the lock. Exceptions if the lock is not held +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void NamedLock::ReleaseLock() +{ + // Got a lock? + if(mFileDescriptor == -1) + { + THROW_EXCEPTION(CommonException, NamedLockNotHeld) + } + + // Close the file + if(::close(mFileDescriptor) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + // Mark as unlocked + mFileDescriptor = -1; +} + + + + diff --git a/lib/common/NamedLock.h b/lib/common/NamedLock.h new file mode 100755 index 00000000..5eac712a --- /dev/null +++ b/lib/common/NamedLock.h @@ -0,0 +1,41 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: NamedLock.h +// Purpose: A global named lock, implemented as a lock file in file system +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef NAMEDLOCK__H +#define NAMEDLOCK__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: NamedLock +// Purpose: A global named lock, implemented as a lock file in file system +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +class NamedLock +{ +public: + NamedLock(); + ~NamedLock(); +private: + // No copying allowed + NamedLock(const NamedLock &); + +public: + bool TryAndGetLock(const char *Filename, int mode = 0755); + bool GotLock() {return mFileDescriptor != -1;} + void ReleaseLock(); + + +private: + int mFileDescriptor; +}; + +#endif // NAMEDLOCK__H + diff --git a/lib/common/PartialReadStream.cpp b/lib/common/PartialReadStream.cpp new file mode 100755 index 00000000..0b5c4cf6 --- /dev/null +++ b/lib/common/PartialReadStream.cpp @@ -0,0 +1,135 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: PartialReadStream.h +// Purpose: Read part of another stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "PartialReadStream.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::PartialReadStream(IOStream &, int) +// Purpose: Constructor, taking another stream and the number of bytes +// to be read from it. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +PartialReadStream::PartialReadStream(IOStream &rSource, int BytesToRead) + : mrSource(rSource), + mBytesLeft(BytesToRead) +{ + ASSERT(BytesToRead > 0); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::~PartialReadStream() +// Purpose: Destructor. Won't absorb any unread bytes. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +PartialReadStream::~PartialReadStream() +{ + // Warn in debug mode + if(mBytesLeft != 0) + { + TRACE1("PartialReadStream::~PartialReadStream when mBytesLeft = %d\n", mBytesLeft); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::Read(void *, int, int) +// Purpose: As interface. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +int PartialReadStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Finished? + if(mBytesLeft <= 0) + { + return 0; + } + + // Asking for more than is allowed? + if(NBytes > mBytesLeft) + { + // Adjust downwards + NBytes = mBytesLeft; + } + + // Route the request to the source + int read = mrSource.Read(pBuffer, NBytes, Timeout); + ASSERT(read <= mBytesLeft); + + // Adjust the count + mBytesLeft -= read; + + // Return the number read + return read; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::BytesLeftToRead() +// Purpose: As interface. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +IOStream::pos_type PartialReadStream::BytesLeftToRead() +{ + return mBytesLeft; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::Write(const void *, int) +// Purpose: As interface. But will exception. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void PartialReadStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, CantWriteToPartialReadStream) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::StreamDataLeft() +// Purpose: As interface. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool PartialReadStream::StreamDataLeft() +{ + return mBytesLeft != 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::StreamClosed() +// Purpose: As interface. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool PartialReadStream::StreamClosed() +{ + // always closed + return true; +} + diff --git a/lib/common/PartialReadStream.h b/lib/common/PartialReadStream.h new file mode 100755 index 00000000..42cb7aeb --- /dev/null +++ b/lib/common/PartialReadStream.h @@ -0,0 +1,46 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: PartialReadStream.h +// Purpose: Read part of another stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef PARTIALREADSTREAM__H +#define PARTIALREADSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: PartialReadStream +// Purpose: Read part of another stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class PartialReadStream : public IOStream +{ +public: + PartialReadStream(IOStream &rSource, int BytesToRead); + ~PartialReadStream(); +private: + // no copying allowed + PartialReadStream(const IOStream &); + PartialReadStream(const PartialReadStream &); + +public: + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + IOStream &mrSource; + int mBytesLeft; +}; + +#endif // PARTIALREADSTREAM__H + diff --git a/lib/common/ReadGatherStream.cpp b/lib/common/ReadGatherStream.cpp new file mode 100755 index 00000000..9ccc3a54 --- /dev/null +++ b/lib/common/ReadGatherStream.cpp @@ -0,0 +1,262 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ReadGatherStream.cpp +// Purpose: Build a stream (for reading only) out of a number of other streams. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "ReadGatherStream.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::ReadGatherStream(bool) +// Purpose: Constructor. Args says whether or not all the component streams will be deleted when this +// object is deleted. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +ReadGatherStream::ReadGatherStream(bool DeleteComponentStreamsOnDestruction) + : mDeleteComponentStreamsOnDestruction(DeleteComponentStreamsOnDestruction), + mCurrentPosition(0), + mTotalSize(0), + mCurrentBlock(0), + mPositionInCurrentBlock(0), + mSeekDoneForCurrent(false) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::~ReadGatherStream() +// Purpose: Destructor. Will delete all the stream objects, if required. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +ReadGatherStream::~ReadGatherStream() +{ + // Delete compoenent streams? + if(mDeleteComponentStreamsOnDestruction) + { + for(unsigned int l = 0; l < mComponents.size(); ++l) + { + delete mComponents[l]; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::AddComponent(IOStream *) +// Purpose: Add a component to this stream, returning the index of this component +// in the internal list. Use this with AddBlock() +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +int ReadGatherStream::AddComponent(IOStream *pStream) +{ + ASSERT(pStream != 0); + + // Just add the component to the list, returning it's index. + int index = mComponents.size(); + mComponents.push_back(pStream); + return index; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::AddBlock(int, pos_type, bool, pos_type) +// Purpose: Add a block to the list of blocks being gathered into one stream. +// Length is length of block to read from this component, Seek == true +// if a seek is required, and if true, SeekTo is the position (absolute) +// in the stream to be seeked to when this block is required. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +void ReadGatherStream::AddBlock(int Component, pos_type Length, bool Seek, pos_type SeekTo) +{ + // Check block + if(Component < 0 || Component >= (int)mComponents.size() || Length < 0 || SeekTo < 0) + { + THROW_EXCEPTION(CommonException, ReadGatherStreamAddingBadBlock); + } + + // Add to list + Block b; + b.mLength = Length; + b.mSeekTo = SeekTo; + b.mComponent = Component; + b.mSeek = Seek; + + mBlocks.push_back(b); + + // And update the total size + mTotalSize += Length; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::Read(void *, int, int) +// Purpose: As interface. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +int ReadGatherStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + int bytesToRead = NBytes; + uint8_t *buffer = (uint8_t*)pBuffer; + + while(bytesToRead > 0) + { + // Done? + if(mCurrentBlock >= mBlocks.size()) + { + // Stop now, as have finished the last block + return NBytes - bytesToRead; + } + + // Seek? + if(mPositionInCurrentBlock == 0 && mBlocks[mCurrentBlock].mSeek && !mSeekDoneForCurrent) + { + // Do seeks in this manner so that seeks are done regardless of whether the block + // has length > 0, and it will only be done once, and at as late a stage as possible. + + mComponents[mBlocks[mCurrentBlock].mComponent]->Seek(mBlocks[mCurrentBlock].mSeekTo, IOStream::SeekType_Absolute); + + mSeekDoneForCurrent = true; + } + + // Anything in the current block? + if(mPositionInCurrentBlock < mBlocks[mCurrentBlock].mLength) + { + // Read! + int s = mBlocks[mCurrentBlock].mLength - mPositionInCurrentBlock; + if(s > bytesToRead) s = bytesToRead; + + int r = mComponents[mBlocks[mCurrentBlock].mComponent]->Read(buffer, s, Timeout); + + // update variables + mPositionInCurrentBlock += r; + buffer += r; + bytesToRead -= r; + mCurrentPosition += r; + + if(r != s) + { + // Stream returned less than requested. To avoid blocking when not necessary, + // return now. + return NBytes - bytesToRead; + } + } + else + { + // Move to next block + ++mCurrentBlock; + mPositionInCurrentBlock = 0; + mSeekDoneForCurrent = false; + } + } + + return NBytes - bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::GetPosition() +// Purpose: As interface +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ReadGatherStream::GetPosition() const +{ + return mCurrentPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::BytesLeftToRead() +// Purpose: As interface +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ReadGatherStream::BytesLeftToRead() +{ + return mTotalSize - mCurrentPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::Write(const void *, int) +// Purpose: As interface. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +void ReadGatherStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, CannotWriteToReadGatherStream); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::StreamDataLeft() +// Purpose: As interface. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool ReadGatherStream::StreamDataLeft() +{ + if(mCurrentBlock >= mBlocks.size()) + { + // Done all the blocks + return false; + } + + if(mCurrentBlock == (mBlocks.size() - 1) + && mPositionInCurrentBlock >= mBlocks[mCurrentBlock].mLength) + { + // Are on the last block, and have got all the data from it. + return false; + } + + // Otherwise, there's more data to be read + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::StreamClosed() +// Purpose: As interface. But the stream is always closed. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool ReadGatherStream::StreamClosed() +{ + return true; +} + + diff --git a/lib/common/ReadGatherStream.h b/lib/common/ReadGatherStream.h new file mode 100755 index 00000000..613ede3e --- /dev/null +++ b/lib/common/ReadGatherStream.h @@ -0,0 +1,67 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ReadGatherStream.h +// Purpose: Build a stream (for reading only) out of a number of other streams. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef READGATHERSTREAM_H +#define READGATHERSTREAM_H + +#include "IOStream.h" + +#include <vector> + +// -------------------------------------------------------------------------- +// +// Class +// Name: ReadGatherStream +// Purpose: Build a stream (for reading only) out of a number of other streams. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +class ReadGatherStream : public IOStream +{ +public: + ReadGatherStream(bool DeleteComponentStreamsOnDestruction); + ~ReadGatherStream(); +private: + ReadGatherStream(const ReadGatherStream &); + ReadGatherStream &operator=(const ReadGatherStream &); +public: + + int AddComponent(IOStream *pStream); + void AddBlock(int Component, pos_type Length, bool Seek = false, pos_type SeekTo = 0); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + virtual pos_type GetPosition() const; + +private: + bool mDeleteComponentStreamsOnDestruction; + std::vector<IOStream *> mComponents; + + typedef struct + { + pos_type mLength; + pos_type mSeekTo; + int mComponent; + bool mSeek; + } Block; + + std::vector<Block> mBlocks; + + pos_type mCurrentPosition; + pos_type mTotalSize; + unsigned int mCurrentBlock; + pos_type mPositionInCurrentBlock; + bool mSeekDoneForCurrent; +}; + + +#endif // READGATHERSTREAM_H diff --git a/lib/common/StreamableMemBlock.cpp b/lib/common/StreamableMemBlock.cpp new file mode 100755 index 00000000..4ea9e398 --- /dev/null +++ b/lib/common/StreamableMemBlock.cpp @@ -0,0 +1,365 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StreamableMemBlock.cpp +// Purpose: Memory blocks which can be loaded and saved from streams +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <stdlib.h> +#include <string.h> + +#include "StreamableMemBlock.h" +#include "IOStream.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::StreamableMemBlock() +// Purpose: Constructor, making empty block +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +StreamableMemBlock::StreamableMemBlock() + : mpBuffer(0), + mSize(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::StreamableMemBlock(void *, int) +// Purpose: Create block, copying data from another bit of memory +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +StreamableMemBlock::StreamableMemBlock(void *pBuffer, int Size) + : mpBuffer(0), + mSize(0) +{ + AllocateBlock(Size); + ::memcpy(mpBuffer, pBuffer, Size); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::StreamableMemBlock(int) +// Purpose: Create block, initialising it to all zeros +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +StreamableMemBlock::StreamableMemBlock(int Size) + : mpBuffer(0), + mSize(0) +{ + AllocateBlock(Size); + ::memset(mpBuffer, 0, Size); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::StreamableMemBlock(const StreamableMemBlock &) +// Purpose: Copy constructor +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +StreamableMemBlock::StreamableMemBlock(const StreamableMemBlock &rToCopy) + : mpBuffer(0), + mSize(0) +{ + AllocateBlock(rToCopy.mSize); + ::memcpy(mpBuffer, rToCopy.mpBuffer, mSize); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::Set(void *, int) +// Purpose: Set the contents of the block +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::Set(void *pBuffer, int Size) +{ + FreeBlock(); + AllocateBlock(Size); + ::memcpy(mpBuffer, pBuffer, Size); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::Set(IOStream &) +// Purpose: Set from stream. Stream must support BytesLeftToRead() +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::Set(IOStream &rStream, int Timeout) +{ + // Get size + IOStream::pos_type size = rStream.BytesLeftToRead(); + if(size == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(CommonException, StreamDoesntHaveRequiredProperty) + } + + // Allocate a new block (this way to be exception safe) + char *pblock = (char*)malloc(size); + if(pblock == 0) + { + throw std::bad_alloc(); + } + + try + { + // Read in + if(!rStream.ReadFullBuffer(pblock, size, 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + + // Free the block ready for replacement + FreeBlock(); + } + catch(...) + { + ::free(pblock); + throw; + } + + // store... + ASSERT(mpBuffer == 0); + mpBuffer = pblock; + mSize = size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::Set(const StreamableMemBlock &) +// Purpose: Set from other block. +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::Set(const StreamableMemBlock &rBlock) +{ + Set(rBlock.mpBuffer, rBlock.mSize); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::~StreamableMemBlock() +// Purpose: Destructor +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +StreamableMemBlock::~StreamableMemBlock() +{ + FreeBlock(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::FreeBlock() +// Purpose: Protected. Frees block of memory +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::FreeBlock() +{ + if(mpBuffer != 0) + { + ::free(mpBuffer); + } + mpBuffer = 0; + mSize = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::AllocateBlock(int) +// Purpose: Protected. Allocate the block of memory +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::AllocateBlock(int Size) +{ + ASSERT(mpBuffer == 0); + if(Size > 0) + { + mpBuffer = ::malloc(Size); + if(mpBuffer == 0) + { + throw std::bad_alloc(); + } + } + mSize = Size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::ResizeBlock(int) +// Purpose: Protected. Resizes the allocated block. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::ResizeBlock(int Size) +{ + ASSERT(mpBuffer != 0); + ASSERT(Size > 0); + if(Size > 0) + { + void *pnewBuffer = ::realloc(mpBuffer, Size); + if(pnewBuffer == 0) + { + throw std::bad_alloc(); + } + mpBuffer = pnewBuffer; + } + mSize = Size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::ReadFromStream(IOStream &, int) +// Purpose: Read the block in from a stream +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Get the size of the block + int32_t size_s; + if(!rStream.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + + int size = ntohl(size_s); + + + // Allocate a new block (this way to be exception safe) + char *pblock = (char*)malloc(size); + if(pblock == 0) + { + throw std::bad_alloc(); + } + + try + { + // Read in + if(!rStream.ReadFullBuffer(pblock, size, 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + + // Free the block ready for replacement + FreeBlock(); + } + catch(...) + { + ::free(pblock); + throw; + } + + // store... + ASSERT(mpBuffer == 0); + mpBuffer = pblock; + mSize = size; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::WriteToStream(IOStream &) +// Purpose: Write the block to a stream +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::WriteToStream(IOStream &rStream) const +{ + int32_t sizenbo = htonl(mSize); + // Size + rStream.Write(&sizenbo, sizeof(sizenbo)); + // Buffer + if(mSize > 0) + { + rStream.Write(mpBuffer, mSize); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::WriteEmptyBlockToStream(IOStream &) +// Purpose: Writes an empty block to a stream. +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::WriteEmptyBlockToStream(IOStream &rStream) +{ + int32_t sizenbo = htonl(0); + rStream.Write(&sizenbo, sizeof(sizenbo)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::GetBuffer() +// Purpose: Get pointer to buffer +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void *StreamableMemBlock::GetBuffer() const +{ + if(mSize == 0) + { + // Return something which isn't a null pointer + static const int validptr = 0; + return (void*)&validptr; + } + + // return the buffer + ASSERT(mpBuffer != 0); + return mpBuffer; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::operator==(const StreamableMemBlock &) +// Purpose: Test for equality of memory blocks +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +bool StreamableMemBlock::operator==(const StreamableMemBlock &rCompare) const +{ + if(mSize != rCompare.mSize) return false; + if(mSize == 0 && rCompare.mSize == 0) return true; // without memory comparison! + return ::memcmp(mpBuffer, rCompare.mpBuffer, mSize) == 0; +} + + diff --git a/lib/common/StreamableMemBlock.h b/lib/common/StreamableMemBlock.h new file mode 100755 index 00000000..250c0aea --- /dev/null +++ b/lib/common/StreamableMemBlock.h @@ -0,0 +1,71 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StreamableMemBlock.h +// Purpose: Memory blocks which can be loaded and saved from streams +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- + +#ifndef STREAMABLEMEMBLOCK__H +#define STREAMABLEMEMBLOCK__H + +class IOStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: StreamableMemBlock +// Purpose: Memory blocks which can be loaded and saved from streams +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +class StreamableMemBlock +{ +public: + StreamableMemBlock(); + StreamableMemBlock(int Size); + StreamableMemBlock(void *pBuffer, int Size); + StreamableMemBlock(const StreamableMemBlock &rToCopy); + ~StreamableMemBlock(); + + void Set(const StreamableMemBlock &rBlock); + void Set(void *pBuffer, int Size); + void Set(IOStream &rStream, int Timeout); + StreamableMemBlock &operator=(const StreamableMemBlock &rBlock) + { + Set(rBlock); + return *this; + } + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream) const; + + static void WriteEmptyBlockToStream(IOStream &rStream); + + void *GetBuffer() const; + + // Size of block + int GetSize() const {return mSize;} + + // Buffer empty? + bool IsEmpty() const {return mSize == 0;} + + // Clear the contents of the block + void Clear() {FreeBlock();} + + bool operator==(const StreamableMemBlock &rCompare) const; + + void ResizeBlock(int Size); + +protected: // be careful with these! + void AllocateBlock(int Size); + void FreeBlock(); + +private: + void *mpBuffer; + int mSize; +}; + +#endif // STREAMABLEMEMBLOCK__H + diff --git a/lib/common/TemporaryDirectory.h b/lib/common/TemporaryDirectory.h new file mode 100755 index 00000000..62010f79 --- /dev/null +++ b/lib/common/TemporaryDirectory.h @@ -0,0 +1,26 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: TemporaryDirectory.h +// Purpose: Location of temporary directory +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- + +#ifndef TEMPORARYDIRECTORY__H +#define TEMPORARYDIRECTORY__H + +#include <string> + +#ifdef PLATFORM_STATIC_TEMP_DIRECTORY_NAME + // Prefix name with Box to avoid clashing with OS API names + inline std::string BoxGetTemporaryDirectoryName() + { + return std::string(PLATFORM_STATIC_TEMP_DIRECTORY_NAME); + } +#else + non-static temporary directory names not supported yet +#endif + +#endif // TEMPORARYDIRECTORY__H + diff --git a/lib/common/Test.h b/lib/common/Test.h new file mode 100755 index 00000000..bd69cd5a --- /dev/null +++ b/lib/common/Test.h @@ -0,0 +1,165 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Test.h +// Purpose: Useful stuff for tests +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- + +#ifndef TEST__H +#define TEST__H + +#include <sys/types.h> +#include <sys/stat.h> +#include <stdlib.h> +#include <signal.h> +#include <unistd.h> + +#include <stdio.h> + +extern int failures; + +#define TEST_FAIL_WITH_MESSAGE(msg) {failures++; printf("FAILURE: " msg " at " __FILE__ "(%d)\n", __LINE__);} +#define TEST_ABORT_WITH_MESSAGE(msg) {failures++; printf("FAILURE: " msg " at " __FILE__ "(%d)\n", __LINE__); return 1;} + +#define TEST_THAT(condition) {if(!(condition)) TEST_FAIL_WITH_MESSAGE("Condition [" #condition "] failed")} +#define TEST_THAT_ABORTONFAIL(condition) {if(!(condition)) TEST_ABORT_WITH_MESSAGE("Condition [" #condition "] failed")} + +// NOTE: The 0- bit it to allow this to work with stuff which has negative constants for flags (eg ConnectionException) +#define TEST_CHECK_THROWS(statement, excepttype, subtype) \ + { \ + bool didthrow = false; \ + try \ + { \ + statement; \ + } \ + catch(excepttype &e) \ + { \ + if(e.GetSubType() != ((unsigned int)excepttype::subtype) \ + && e.GetSubType() != (unsigned int)(0-excepttype::subtype)) \ + { \ + throw; \ + } \ + didthrow = true; \ + } \ + catch(...) \ + { \ + throw; \ + } \ + if(!didthrow) \ + { \ + TEST_FAIL_WITH_MESSAGE("Didn't throw exception " #excepttype "(" #subtype ")") \ + } \ + } + +inline bool TestFileExists(const char *Filename) +{ + struct stat st; + return ::stat(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0; +} + +inline bool TestDirExists(const char *Filename) +{ + struct stat st; + return ::stat(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == S_IFDIR; +} + +// -1 if doesn't exist +inline int TestGetFileSize(const char *Filename) +{ + struct stat st; + if(::stat(Filename, &st) == 0) + { + return st.st_size; + } + return -1; +} + +inline int LaunchServer(const char *CommandLine, const char *pidFile) +{ + if(::system(CommandLine) != 0) + { + printf("Server: %s\n", CommandLine); + TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + return -1; + } + // time for it to start up + ::sleep(1); + + // read pid file + if(!TestFileExists(pidFile)) + { + printf("Server: %s\n", CommandLine); + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); + return -1; + } + + FILE *f = fopen(pidFile, "r"); + int pid = -1; + if(f == NULL || fscanf(f, "%d", &pid) != 1) + { + printf("Server: %s (pidfile %s)\n", CommandLine, pidFile); + TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); + return -1; + } + fclose(f); + + return pid; +} + +inline bool ServerIsAlive(int pid) +{ + if(pid == 0) return false; + return ::kill(pid, 0) != -1; +} + +inline bool HUPServer(int pid) +{ + if(pid == 0) return false; + return ::kill(pid, SIGHUP) != -1; +} + +inline bool KillServer(int pid) +{ + if(pid == 0 || pid == -1) return false; + bool KilledOK = ::kill(pid, SIGTERM) != -1; + TEST_THAT(KilledOK); + ::sleep(1); + return !ServerIsAlive(pid); +} + +inline void TestRemoteProcessMemLeaks(const char *filename) +{ +#ifdef BOX_MEMORY_LEAK_TESTING + // Does the file exist? + if(!TestFileExists(filename)) + { + ++failures; + printf("FAILURE: MemLeak report not available (file %s)\n", filename); + } + else + { + // Is it empty? + if(TestGetFileSize(filename) > 0) + { + ++failures; + printf("FAILURE: Memory leaks found in other process (file %s)\n==========\n", filename); + FILE *f = fopen(filename, "r"); + char line[512]; + while(::fgets(line, sizeof(line), f) != 0) + { + printf("%s", line); + } + fclose(f); + printf("==========\n"); + } + + // Delete it + ::unlink(filename); + } +#endif +} + +#endif // TEST__H + diff --git a/lib/common/UnixUser.cpp b/lib/common/UnixUser.cpp new file mode 100755 index 00000000..7a60b263 --- /dev/null +++ b/lib/common/UnixUser.cpp @@ -0,0 +1,121 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: UnixUser.cpp +// Purpose: Interface for managing the UNIX user of the current process +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <pwd.h> +#include <unistd.h> + +#include "UnixUser.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: UnixUser::UnixUser(const char *) +// Purpose: Constructor, initialises to info of given username +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +UnixUser::UnixUser(const char *Username) + : mUID(0), + mGID(0), + mRevertOnDestruction(false) +{ + // Get password info + struct passwd *pwd = ::getpwnam(Username); + if(pwd == 0) + { + THROW_EXCEPTION(CommonException, CouldNotLookUpUsername) + } + + // Store UID and GID + mUID = pwd->pw_uid; + mGID = pwd->pw_gid; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: UnixUser::UnixUser(uid_t, gid_t) +// Purpose: Construct from given UNIX user ID and group ID +// Created: 15/3/04 +// +// -------------------------------------------------------------------------- +UnixUser::UnixUser(uid_t UID, gid_t GID) + : mUID(UID), + mGID(GID), + mRevertOnDestruction(false) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: UnixUser::~UnixUser() +// Purpose: Destructor -- reverts to previous user if the change wasn't perminant +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +UnixUser::~UnixUser() +{ + if(mRevertOnDestruction) + { + // Revert to "real" user and group id of the process + if(::setegid(::getgid()) != 0 + || ::seteuid(::getuid()) != 0) + { + THROW_EXCEPTION(CommonException, CouldNotRestoreProcessUser) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: UnixUser::ChangeProcessUser(bool) +// Purpose: Change the process user and group ID to the user. If Temporary == true +// the process username will be changed back when the object is destructed. +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +void UnixUser::ChangeProcessUser(bool Temporary) +{ + if(Temporary) + { + // Change temporarily (change effective only) + if(::setegid(mGID) != 0 + || ::seteuid(mUID) != 0) + { + THROW_EXCEPTION(CommonException, CouldNotChangeProcessUser) + } + + // Mark for change on destruction + mRevertOnDestruction = true; + } + else + { + // Change perminantely (change all UIDs and GIDs) + if(::setgid(mGID) != 0 + || ::setuid(mUID) != 0) + { + THROW_EXCEPTION(CommonException, CouldNotChangeProcessUser) + } + } +} + + + + diff --git a/lib/common/UnixUser.h b/lib/common/UnixUser.h new file mode 100755 index 00000000..c895eb2a --- /dev/null +++ b/lib/common/UnixUser.h @@ -0,0 +1,37 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: UnixUser.h +// Purpose: Interface for managing the UNIX user of the current process +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef UNIXUSER__H +#define UNIXUSER__H + +class UnixUser +{ +public: + UnixUser(const char *Username); + UnixUser(uid_t UID, gid_t GID); + ~UnixUser(); +private: + // no copying allowed + UnixUser(const UnixUser &); + UnixUser &operator=(const UnixUser &); +public: + + void ChangeProcessUser(bool Temporary = false); + + uid_t GetUID() {return mUID;} + gid_t GetGID() {return mGID;} + +private: + uid_t mUID; + gid_t mGID; + bool mRevertOnDestruction; +}; + +#endif // UNIXUSER__H + diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp new file mode 100755 index 00000000..b4d3144b --- /dev/null +++ b/lib/common/Utils.cpp @@ -0,0 +1,159 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Utils.cpp +// Purpose: Utility function +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> + +#ifdef SHOW_BACKTRACE_ON_EXCEPTION + #include <execinfo.h> + #include <stdlib.h> +#endif + +#include "Utils.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: SplitString(const std::string &, char, std::vector<std::string> &) +// Purpose: Splits a string at a given character +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput) +{ + // Split it up. + std::string::size_type b = 0; + std::string::size_type e = 0; + while(e = String.find_first_of(SplitOn, b), e != String.npos) + { + // Get this string + unsigned int len = e - b; + if(len >= 1) + { + rOutput.push_back(String.substr(b, len)); + } + b = e + 1; + } + // Last string + if(b < String.size()) + { + rOutput.push_back(String.substr(b)); + } +/*#ifndef NDEBUG + TRACE2("Splitting string '%s' on %c\n", String.c_str(), SplitOn); + for(unsigned int l = 0; l < rOutput.size(); ++l) + { + TRACE2("%d = '%s'\n", l, rOutput[l].c_str()); + } +#endif*/ +} + +#ifdef SHOW_BACKTRACE_ON_EXCEPTION +void DumpStackBacktrace() +{ + void *array[10]; + size_t size; + char **strings; + size_t i; + + size = backtrace (array, 10); + strings = backtrace_symbols (array, size); + + printf ("Obtained %zd stack frames.\n", size); + + for(i = 0; i < size; i++) + printf("%s\n", strings[i]); + + free (strings); +} +#endif + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileExists(const char *) +// Purpose: Does a file exist? +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +bool FileExists(const char *Filename, int64_t *pFileSize, bool TreatLinksAsNotExisting) +{ + struct stat st; + if(::stat(Filename, &st) != 0) + { + if(errno == ENOENT) + { + return false; + } + else + { + THROW_EXCEPTION(CommonException, OSFileError); + } + } + + // is it a file? + if((st.st_mode & S_IFDIR) == 0) + { + if(TreatLinksAsNotExisting && ((st.st_mode & S_IFLNK) != 0)) + { + return false; + } + + // Yes. Tell caller the size? + if(pFileSize != 0) + { + *pFileSize = st.st_size; + } + + return true; + } + else + { + return false; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ObjectExists(const char *) +// Purpose: Does a object exist, and if so, is it a file or a directory? +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +int ObjectExists(const char *Filename) +{ + struct stat st; + if(::stat(Filename, &st) != 0) + { + if(errno == ENOENT) + { + return ObjectExists_NoObject; + } + else + { + THROW_EXCEPTION(CommonException, OSFileError); + } + } + + // is it a file or a dir? + return ((st.st_mode & S_IFDIR) == 0)?ObjectExists_File:ObjectExists_Dir; +} + + + + diff --git a/lib/common/Utils.h b/lib/common/Utils.h new file mode 100755 index 00000000..5da84d9a --- /dev/null +++ b/lib/common/Utils.h @@ -0,0 +1,36 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Utils.h +// Purpose: Utility function +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef UTILS__H +#define UTILS__H + +#include <string> +#include <vector> + +#include "MemLeakFindOn.h" + +void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput); + +#ifdef SHOW_BACKTRACE_ON_EXCEPTION + void DumpStackBacktrace(); +#endif + +bool FileExists(const char *Filename, int64_t *pFileSize = 0, bool TreatLinksAsNotExisting = false); + +enum +{ + ObjectExists_NoObject = 0, + ObjectExists_File = 1, + ObjectExists_Dir = 2 +}; +int ObjectExists(const char *Filename); + +#include "MemLeakFindOff.h" + +#endif // UTILS__H diff --git a/lib/common/WaitForEvent.cpp b/lib/common/WaitForEvent.cpp new file mode 100644 index 00000000..e9ee58d2 --- /dev/null +++ b/lib/common/WaitForEvent.cpp @@ -0,0 +1,194 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: WaitForEvent.cpp +// Purpose: Generic waiting for events, using an efficient method (platform dependent) +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <unistd.h> +#include <errno.h> +#include <string.h> + +#include "WaitForEvent.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: WaitForEvent::WaitForEvent() +// Purpose: Constructor +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED +WaitForEvent::WaitForEvent(int Timeout) + : mKQueue(::kqueue()), + mpTimeout(0) +{ + if(mKQueue == -1) + { + THROW_EXCEPTION(CommonException, CouldNotCreateKQueue) + } + + // Set the choosen timeout + SetTimeout(Timeout); +} +#else +WaitForEvent::WaitForEvent(int Timeout) + : mTimeout(Timeout), + mpPollInfo(0) +{ +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: WaitForEvent::~WaitForEvent() +// Purpose: Destructor +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- +WaitForEvent::~WaitForEvent() +{ +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + ::close(mKQueue); + mKQueue = -1; +#else + if(mpPollInfo != 0) + { + ::free(mpPollInfo); + mpPollInfo = 0; + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: WaitForEvent::SetTimeout +// Purpose: Sets the timeout for future wait calls +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- +void WaitForEvent::SetTimeout(int Timeout) +{ +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + // Generate timeout + if(Timeout != TimeoutInfinite) + { + mTimeout.tv_sec = Timeout / 1000; + mTimeout.tv_nsec = (Timeout % 1000) * 1000000; + } + + // Infinite or not? + mpTimeout = (Timeout != TimeoutInfinite)?(&mTimeout):(NULL); +#else + mTimeout = Timeout; +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WaitForEvent::Wait(int) +// Purpose: Wait for an event to take place. Returns a pointer to the object +// which has been signalled, or returns 0 for the timeout condition. +// Timeout specified in milliseconds. +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- +void *WaitForEvent::Wait() +{ +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + // Event return structure + struct kevent e; + ::memset(&e, 0, sizeof(e)); + + switch(::kevent(mKQueue, NULL, 0, &e, 1, mpTimeout)) + { + case 0: + // Timeout + return 0; + break; + + case 1: + // Event happened! + return e.udata; + break; + + default: + // Interrupted system calls aren't an error, just equivalent to a timeout + if(errno != EINTR) + { + THROW_EXCEPTION(CommonException, KEventErrorWait) + } + return 0; + break; + } +#else + // Use poll() instead. + // Need to build the structures? + if(mpPollInfo == 0) + { + // Yes... + mpPollInfo = (struct pollfd *)::malloc((sizeof(struct pollfd) * mItems.size()) + 4); + if(mpPollInfo == 0) + { + throw std::bad_alloc(); + } + + // Build... + for(unsigned int l = 0; l < mItems.size(); ++l) + { + mpPollInfo[l].fd = mItems[l].fd; + mpPollInfo[l].events = mItems[l].events; + mpPollInfo[l].revents = 0; + } + } + + // Make sure everything is reset (don't really have to do this, but don't trust the OS) + for(unsigned int l = 0; l < mItems.size(); ++l) + { + mpPollInfo[l].revents = 0; + } + + // Poll! + switch(::poll(mpPollInfo, mItems.size(), mTimeout)) + { + case -1: + // Interrupted system calls aren't an error, just equivalent to a timeout + if(errno != EINTR) + { + THROW_EXCEPTION(CommonException, KEventErrorWait) + } + return 0; + break; + case 0: // timed out + return 0; + break; + default: // got some thing... + // control flows on... + break; + } + + // Find the item which was ready + for(unsigned int s = 0; s < mItems.size(); ++s) + { + if(mpPollInfo[s].revents & POLLIN) + { + return mItems[s].item; + break; + } + } +#endif + + return 0; +} + diff --git a/lib/common/WaitForEvent.h b/lib/common/WaitForEvent.h new file mode 100644 index 00000000..b8f79da6 --- /dev/null +++ b/lib/common/WaitForEvent.h @@ -0,0 +1,146 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: WaitForEvent.h +// Purpose: Generic waiting for events, using an efficient method (platform dependent) +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef WAITFOREVENT__H +#define WAITFOREVENT__H + +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + #include <sys/event.h> + #include <sys/time.h> +#else + #include <vector> + #include <poll.h> +#endif + +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +class WaitForEvent +{ +public: + WaitForEvent(int Timeout = TimeoutInfinite); + ~WaitForEvent(); +private: + // No copying. + WaitForEvent(const WaitForEvent &); + WaitForEvent &operator=(const WaitForEvent &); +public: + + enum + { + TimeoutInfinite = -1 + }; + + void SetTimeout(int Timeout = TimeoutInfinite); + + void *Wait(); + +#ifdef PLATFORM_KQUEUE_NOT_SUPPORTED + typedef struct + { + int fd; + short events; + void *item; + } ItemInfo; +#endif + + // -------------------------------------------------------------------------- + // + // Function + // Name: WaitForEvent::Add(const Type &, int) + // Purpose: Adds an event to the list of items to wait on. The flags are passed to the object. + // Created: 9/3/04 + // + // -------------------------------------------------------------------------- + template<typename T> + void Add(const T *pItem, int Flags = 0) + { + ASSERT(pItem != 0); +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + struct kevent e; + pItem->FillInKEvent(e, Flags); + // Fill in extra flags to say what to do + e.flags |= EV_ADD; + e.udata = (void*)pItem; + if(::kevent(mKQueue, &e, 1, NULL, 0, NULL) == -1) + { + THROW_EXCEPTION(CommonException, KEventErrorAdd) + } +#else + // Add item + ItemInfo i; + pItem->FillInPoll(i.fd, i.events, Flags); + i.item = (void*)pItem; + mItems.push_back(i); + // Delete any pre-prepared poll info, as it's now out of date + if(mpPollInfo != 0) + { + ::free(mpPollInfo); + mpPollInfo = 0; + } +#endif + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: WaitForEvent::Remove(const Type &, int) + // Purpose: Removes an event from the list of items to wait on. The flags are passed to the object. + // Created: 9/3/04 + // + // -------------------------------------------------------------------------- + template<typename T> + void Remove(const T *pItem, int Flags = 0) + { + ASSERT(pItem != 0); +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + struct kevent e; + pItem->FillInKEvent(e, Flags); + // Fill in extra flags to say what to do + e.flags |= EV_DELETE; + e.udata = (void*)pItem; + if(::kevent(mKQueue, &e, 1, NULL, 0, NULL) == -1) + { + THROW_EXCEPTION(CommonException, KEventErrorRemove) + } +#else + if(mpPollInfo != 0) + { + ::free(mpPollInfo); + mpPollInfo = 0; + } + for(std::vector<ItemInfo>::iterator i(mItems.begin()); i != mItems.end(); ++i) + { + if(i->item == pItem) + { + mItems.erase(i); + return; + } + } +#endif + } + +private: +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + int mKQueue; + struct timespec mTimeout; + struct timespec *mpTimeout; +#else + int mTimeout; + std::vector<ItemInfo> mItems; + struct pollfd *mpPollInfo; +#endif +}; + +#include "MemLeakFindOff.h" + +#endif // WAITFOREVENT__H + + diff --git a/lib/common/makeexception.pl b/lib/common/makeexception.pl new file mode 100755 index 00000000..c0388286 --- /dev/null +++ b/lib/common/makeexception.pl @@ -0,0 +1,277 @@ +#!/usr/bin/perl + +# global exception list file +my $global_list = '../../ExceptionCodes.txt'; + + +my @exception; +my @exception_desc; +my $class; +my $class_number; + +# read the description! + +open EXCEPTION_DESC,$ARGV[0] or die "Can't open $ARGV[0]"; + +while(<EXCEPTION_DESC>) +{ + chomp; s/\A\s+//; s/#.+\Z//; s/\s+\Z//; s/\s+/ /g; + next unless m/\S/; + + if(m/\AEXCEPTION\s+(.+)\s+(\d+)\Z/) + { + $class = $1; + $class_number = $2; + } + else + { + my ($name,$number,$description) = split /\s+/,$_,3; + if($name eq '' || $number =~ m/\D/) + { + die "Bad line '$_'"; + } + if($exception[$number] ne '') + { + die "Duplicate exception number $number"; + } + $exception[$number] = $name; + $exception_desc[$number] = $description; + } +} + +die "Exception class and number not specified" unless $class ne '' && $class_number ne ''; + +close EXCEPTION_DESC; + +# write the code +print "Generating $class exception...\n"; + +open CPP,">autogen_${class}Exception.cpp" or die "Can't open cpp file for writing"; +open H,">autogen_${class}Exception.h" or die "Can't open h file for writing"; + +# write header file +my $guardname = uc 'AUTOGEN_'.$class.'EXCEPTION_H'; +print H <<__E; + +// Auto-generated file -- do not edit + +#ifndef $guardname +#define $guardname + +#include "BoxException.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: ${class}Exception +// Purpose: Exception +// Created: autogen +// +// -------------------------------------------------------------------------- +class ${class}Exception : public BoxException +{ +public: + ${class}Exception(unsigned int SubType) + : mSubType(SubType) + { + } + + ${class}Exception(const ${class}Exception &rToCopy) + : mSubType(rToCopy.mSubType) + { + } + + ~${class}Exception() throw () + { + } + + enum + { + ExceptionType = $class_number + }; + + enum + { +__E + +for(my $e = 0; $e <= $#exception; $e++) +{ + if($exception[$e] ne '') + { + print H "\t\t".$exception[$e].' = '.$e.(($e==$#exception)?'':',')."\n" + } +} + +print H <<__E; + }; + + virtual unsigned int GetType() const throw(); + virtual unsigned int GetSubType() const throw(); + virtual const char *what() const throw(); + +private: + unsigned int mSubType; +}; + +#endif // $guardname +__E + +# ----------------------------------------------------------------------------------------------------------- + +print CPP <<__E; + +// Auto-generated file -- do not edit + +#include "Box.h" +#include "autogen_${class}Exception.h" + +#include "MemLeakFindOn.h" + +#ifdef EXCEPTION_CODENAMES_EXTENDED + #ifdef EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION +static const char *whats[] = { +__E + +my $last_seen = -1; +for(my $e = 0; $e <= $#exception; $e++) +{ + if($exception[$e] ne '') + { + for(my $s = $last_seen + 1; $s < $e; $s++) + { + print CPP "\t\"UNUSED\",\n" + } + my $ext = ($exception_desc[$e] ne '')?" ($exception_desc[$e])":''; + print CPP "\t\"${class} ".$exception[$e].$ext.'"'.(($e==$#exception)?'':',')."\n"; + $last_seen = $e; + } +} + +print CPP <<__E; +}; + #else +static const char *whats[] = { +__E + +$last_seen = -1; +for(my $e = 0; $e <= $#exception; $e++) +{ + if($exception[$e] ne '') + { + for(my $s = $last_seen + 1; $s < $e; $s++) + { + print CPP "\t\"UNUSED\",\n" + } + print CPP "\t\"${class} ".$exception[$e].'"'.(($e==$#exception)?'':',')."\n"; + $last_seen = $e; + } +} + +print CPP <<__E; +}; + #endif +#endif + +unsigned int ${class}Exception::GetType() const throw() +{ + return ${class}Exception::ExceptionType; +} + +unsigned int ${class}Exception::GetSubType() const throw() +{ + return mSubType; +} + +const char *${class}Exception::what() const throw() +{ +#ifdef EXCEPTION_CODENAMES_EXTENDED + if(mSubType < 0 || mSubType > (sizeof(whats) / sizeof(whats[0]))) + { + return "${class}"; + } + return whats[mSubType]; +#else + return "${class}"; +#endif +} + +__E + +close H; +close CPP; + +# update the global exception list +my $list_before; +my $list_after; +my $is_after = 0; +if(open CURRENT,$global_list) +{ + while(<CURRENT>) + { + next if m/\A#/; + + if(m/\AEXCEPTION TYPE (\w+) (\d+)/) + { + # check that the number isn't being reused + if($2 == $class_number && $1 ne $class) + { + die "Class number $class_number is being used by $class and $1 -- correct this.\n"; + } + if($2 > $class_number) + { + # This class comes after the current one (ensures numerical ordering) + $is_after = 1; + } + if($1 eq $class) + { + # skip this entry + while(<CURRENT>) + { + last if m/\AEND TYPE/; + } + $_ = ''; + } + } + + if($is_after) + { + $list_after .= $_; + } + else + { + $list_before .= $_; + } + } + + close CURRENT; +} + +open GLOBAL,">$global_list" or die "Can't open global exception code listing for writing"; + +print GLOBAL <<__E; +# +# automatically generated file, do not edit. +# +# This file lists all the exception codes used by the system. +# Use to look up more detailed descriptions of meanings of errors. +# +__E + +print GLOBAL $list_before; + +print GLOBAL "EXCEPTION TYPE $class $class_number\n"; +for(my $e = 0; $e <= $#exception; $e++) +{ + if($exception[$e] ne '') + { + my $ext = ($exception_desc[$e] ne '')?" - $exception_desc[$e]":''; + print GLOBAL "($class_number/$e) - ${class} ".$exception[$e].$ext."\n"; + } +} +print GLOBAL "END TYPE\n"; + +print GLOBAL $list_after; + +close GLOBAL; + + diff --git a/lib/compress/Compress.h b/lib/compress/Compress.h new file mode 100755 index 00000000..9b609adb --- /dev/null +++ b/lib/compress/Compress.h @@ -0,0 +1,195 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CompressContext.h +// Purpose: Interface to zlib compression +// Created: 5/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef COMPRESSCONTEXT__H +#define COMPRESSCONTEXT__H + +#include <zlib.h> + +#include "CompressException.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: CompressContext +// Purpose: Interface to zlib compression, only very slight wrapper. +// (Use CompressStream for a more friendly interface.) +// Created: 5/12/03 +// +// -------------------------------------------------------------------------- +template<bool Compressing> +class Compress +{ +public: + Compress() + : mFinished(false), + mFlush(Z_NO_FLUSH) + { + // initialise stream + mStream.zalloc = Z_NULL; + mStream.zfree = Z_NULL; + mStream.opaque = Z_NULL; + mStream.data_type = Z_BINARY; + + if((Compressing)?(deflateInit(&mStream, Z_DEFAULT_COMPRESSION)) + :(inflateInit(&mStream)) != Z_OK) + { + THROW_EXCEPTION(CompressException, InitFailed) + } + + mStream.avail_in = 0; + } + + ~Compress() + { + int r = 0; + if((r = ((Compressing)?(deflateEnd(&mStream)) + :(inflateEnd(&mStream)))) != Z_OK) + { + TRACE1("zlib error code = %d\n", r); + if(r == Z_DATA_ERROR) + { + TRACE0("WARNING: End of compress/decompress without all input being consumed -- possible corruption?\n"); + } + else + { + THROW_EXCEPTION(CompressException, EndFailed) + } + } + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: Compress<Function>::InputRequired() + // Purpose: Input required yet? + // Created: 5/12/03 + // + // -------------------------------------------------------------------------- + bool InputRequired() + { + return mStream.avail_in <= 0; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: Compress<Function>::Input(const void *, int) + // Purpose: Set the input buffer ready for next output call. + // Created: 5/12/03 + // + // -------------------------------------------------------------------------- + void Input(const void *pInBuffer, int InLength) + { + // Check usage + if(mStream.avail_in != 0) + { + THROW_EXCEPTION(CompressException, BadUsageInputNotRequired) + } + + // Store info + mStream.next_in = (unsigned char *)pInBuffer; + mStream.avail_in = InLength; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: Compress<Function>::FinishInput() + // Purpose: When compressing, no more input will be given. + // Created: 5/12/03 + // + // -------------------------------------------------------------------------- + void FinishInput() + { + mFlush = Z_FINISH; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: Compress<Function>::Output(void *, int) + // Purpose: Get some output data + // Created: 5/12/03 + // + // -------------------------------------------------------------------------- + int Output(void *pOutBuffer, int OutLength, bool SyncFlush = false) + { + // need more input? + if(mStream.avail_in == 0 && mFlush != Z_FINISH && !SyncFlush) + { + return 0; + } + + // Buffers + mStream.next_out = (unsigned char *)pOutBuffer; + mStream.avail_out = OutLength; + + // Call one of the functions + int flush = mFlush; + if(SyncFlush && mFlush != Z_FINISH) + { + flush = Z_SYNC_FLUSH; + } + int ret = (Compressing)?(deflate(&mStream, flush)):(inflate(&mStream, flush)); + + if(SyncFlush && ret == Z_BUF_ERROR) + { + // No progress possible. Just return 0. + return 0; + } + + // Check errors + if(ret < 0) + { + TRACE1("zlib error code = %d\n", ret); + THROW_EXCEPTION(CompressException, TransformFailed) + } + + // Parse result + if(ret == Z_STREAM_END) + { + mFinished = true; + } + + // Return how much data was output + return OutLength - mStream.avail_out; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: Compress<Function>::OutputHasFinished() + // Purpose: No more output to be recieved + // Created: 5/12/03 + // + // -------------------------------------------------------------------------- + bool OutputHasFinished() + { + return mFinished; + } + + +private: + z_stream mStream; + bool mFinished; + int mFlush; +}; + +template<typename Integer> +Integer Compress_MaxSizeForCompressedData(Integer InLen) +{ + // Conservative rendition of the info found here: http://www.gzip.org/zlib/zlib_tech.html + int blocks = (InLen + 32*1024 - 1) / (32*1024); + return InLen + (blocks * 6) + 8; +} + + +#endif // COMPRESSCONTEXT__H + diff --git a/lib/compress/CompressException.h b/lib/compress/CompressException.h new file mode 100755 index 00000000..7e8c9ca2 --- /dev/null +++ b/lib/compress/CompressException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef COMPRESSEXCEPTION__H +#define COMPRESSEXCEPTION__H + +// Compatibility +#include "autogen_CompressException.h" + +#endif // COMPRESSEXCEPTION__H + diff --git a/lib/compress/CompressException.txt b/lib/compress/CompressException.txt new file mode 100644 index 00000000..278b76d3 --- /dev/null +++ b/lib/compress/CompressException.txt @@ -0,0 +1,12 @@ +EXCEPTION Compress 6 + +Internal 0 +InitFailed 1 +EndFailed 2 +BadUsageInputNotRequired 3 +TransformFailed 4 +CopyCompressStreamNotAllowed 5 +NullPointerPassedToCompressStream 6 +CompressStreamReadSupportNotRequested 7 Specify read in the constructor +CompressStreamWriteSupportNotRequested 8 Specify write in the constructor +CannotWriteToClosedCompressStream 9 diff --git a/lib/compress/CompressStream.cpp b/lib/compress/CompressStream.cpp new file mode 100644 index 00000000..2839b647 --- /dev/null +++ b/lib/compress/CompressStream.cpp @@ -0,0 +1,425 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CompressStream.h +// Purpose: Compressing stream +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> +#include <memory> + +#include "CompressStream.h" +#include "Compress.h" +#include "autogen_CompressException.h" + +#include "MemLeakFindOn.h" + +// How big a buffer to use +#ifndef NDEBUG + // debug! + #define BUFFER_SIZE 256 +#else + #define BUFFER_SIZE (32*1024) +#endif + +#define USE_READ_COMPRESSOR \ + CheckRead(); \ + Compress<false> *pDecompress = (Compress<false> *)mpReadCompressor; + +#define USE_WRITE_COMPRESSOR \ + CheckWrite(); \ + Compress<true> *pCompress = (Compress<true> *)mpWriteCompressor; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::CompressStream(IOStream *, bool, bool, bool, bool) +// Purpose: Constructor +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +CompressStream::CompressStream(IOStream *pStream, bool TakeOwnership, + bool DecompressRead, bool CompressWrite, bool PassThroughWhenNotCompressed) + : mpStream(pStream), + mHaveOwnership(TakeOwnership), + mDecompressRead(DecompressRead), + mCompressWrite(CompressWrite), + mPassThroughWhenNotCompressed(PassThroughWhenNotCompressed), + mpReadCompressor(0), + mpWriteCompressor(0), + mpBuffer(0), + mIsClosed(false) +{ + if(mpStream == 0) + { + THROW_EXCEPTION(CompressException, NullPointerPassedToCompressStream) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::~CompressStream() +// Purpose: Destructor +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +CompressStream::~CompressStream() +{ + // Clean up compressors + if(mpReadCompressor) + { + delete ((Compress<false>*)mpReadCompressor); + mpReadCompressor = 0; + } + if(mpWriteCompressor) + { + delete ((Compress<true>*)mpWriteCompressor); + mpWriteCompressor = 0; + } + + // Delete the stream, if we have ownership + if(mHaveOwnership) + { + delete mpStream; + mpStream = 0; + } + + // Free any buffer + if(mpBuffer != 0) + { + ::free(mpBuffer); + mpBuffer = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::CompressStream(const CompressStream &) +// Purpose: Copy constructor, will exception +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +CompressStream::CompressStream(const CompressStream &) +{ + THROW_EXCEPTION(CompressException, CopyCompressStreamNotAllowed) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::operator=(const CompressStream &) +// Purpose: Assignment operator, will exception +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +CompressStream &CompressStream::operator=(const CompressStream &) +{ + THROW_EXCEPTION(CompressException, CopyCompressStreamNotAllowed) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::Read(void *, int, int) +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +int CompressStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + USE_READ_COMPRESSOR + if(pDecompress == 0) + { + return mpStream->Read(pBuffer, NBytes, Timeout); + } + + // Where is the buffer? (note if writing as well, read buffer is second in a block of two buffer sizes) + void *pbuf = (mDecompressRead && mCompressWrite)?(((uint8_t*)mpBuffer) + BUFFER_SIZE):mpBuffer; + + // Any data left to go? + if(!pDecompress->InputRequired()) + { + // Output some data from the existing data read + return pDecompress->Output(pBuffer, NBytes, true /* write as much as possible */); + } + + // Read data into the buffer -- read as much as possible in one go + int s = mpStream->Read(pbuf, BUFFER_SIZE, Timeout); + if(s == 0) + { + return 0; + } + + // Give input to the compressor + pDecompress->Input(pbuf, s); + + // Output as much as possible + return pDecompress->Output(pBuffer, NBytes, true /* write as much as possible */); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::Write(const void *, int) +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::Write(const void *pBuffer, int NBytes) +{ + USE_WRITE_COMPRESSOR + if(pCompress == 0) + { + mpStream->Write(pBuffer, NBytes); + return; + } + + if(mIsClosed) + { + THROW_EXCEPTION(CompressException, CannotWriteToClosedCompressStream) + } + + // Give the data to the compressor + pCompress->Input(pBuffer, NBytes); + + // Write the data to the stream + WriteCompressedData(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::WriteAllBuffered() +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::WriteAllBuffered() +{ + if(mIsClosed) + { + THROW_EXCEPTION(CompressException, CannotWriteToClosedCompressStream) + } + + // Just ask compressed data to be written out, but with the sync flag set + WriteCompressedData(true); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::Close() +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::Close() +{ + if(mCompressWrite) + { + USE_WRITE_COMPRESSOR + if(pCompress != 0) + { + // Flush anything from the write buffer + pCompress->FinishInput(); + WriteCompressedData(); + + // Mark as definately closed + mIsClosed = true; + } + } + + // Close + mpStream->Close(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::WriteCompressedData(bool) +// Purpose: Private. Writes the output of the compressor to the stream, +// optionally doing a sync flush. +// Created: 28/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::WriteCompressedData(bool SyncFlush) +{ + USE_WRITE_COMPRESSOR + if(pCompress == 0) {THROW_EXCEPTION(CompressException, Internal)} + + int s = 0; + do + { + s = pCompress->Output(mpBuffer, BUFFER_SIZE, SyncFlush); + if(s > 0) + { + mpStream->Write(mpBuffer, s); + } + } while(s > 0); + // Check assumption -- all input has been consumed + if(!pCompress->InputRequired()) {THROW_EXCEPTION(CompressException, Internal)} +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::StreamDataLeft() +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +bool CompressStream::StreamDataLeft() +{ + USE_READ_COMPRESSOR + if(pDecompress == 0) + { + return mpStream->StreamDataLeft(); + } + + // Any bytes left in our buffer? + if(!pDecompress->InputRequired()) + { + // Still buffered data to decompress + return true; + } + + // Otherwise ask the stream + return mpStream->StreamDataLeft(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::StreamClosed() +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +bool CompressStream::StreamClosed() +{ + if(!mIsClosed && mpStream->StreamClosed()) + { + mIsClosed = true; + } + return mIsClosed; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::CheckRead() +// Purpose: Checks that everything is set up to read +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::CheckRead() +{ + // Has the read compressor already been created? + if(mpReadCompressor != 0) + { + return; + } + + // Need to create one? + if(mDecompressRead) + { + mpReadCompressor = new Compress<false>(); + // And make sure there's a buffer + CheckBuffer(); + } + else + { + // Not decompressing. Should be passing through data? + if(!mPassThroughWhenNotCompressed) + { + THROW_EXCEPTION(CompressException, CompressStreamReadSupportNotRequested) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::CheckWrite() +// Purpose: Checks that everything is set up to write +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::CheckWrite() +{ + // Has the read compressor already been created? + if(mpWriteCompressor != 0) + { + return; + } + + // Need to create one? + if(mCompressWrite) + { + mpWriteCompressor = new Compress<true>(); + // And make sure there's a buffer + CheckBuffer(); + } + else + { + // Not decompressing. Should be passing through data? + if(!mPassThroughWhenNotCompressed) + { + THROW_EXCEPTION(CompressException, CompressStreamWriteSupportNotRequested) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::CheckBuffer() +// Purpose: Allocates the buffer for (de)compression operations +// Created: 28/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::CheckBuffer() +{ + // Already done + if(mpBuffer != 0) + { + return; + } + + // Allocate the buffer -- which may actually be two buffers in one + // The temporary use buffer is first (used by write only, so only present if writing) + // and the read buffer follows. + int size = BUFFER_SIZE; + if(mDecompressRead && mCompressWrite) + { + size *= 2; + } + TRACE1("Allocating CompressStream buffer, size %d\n", size); + mpBuffer = ::malloc(size); + if(mpBuffer == 0) + { + throw std::bad_alloc(); + } +} + + diff --git a/lib/compress/CompressStream.h b/lib/compress/CompressStream.h new file mode 100644 index 00000000..7959e3dc --- /dev/null +++ b/lib/compress/CompressStream.h @@ -0,0 +1,62 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CompressStream.h +// Purpose: Compressing stream +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- + +#ifndef COMPRESSSTREAM__H +#define COMPRESSSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: CompressStream +// Purpose: Compressing stream +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +class CompressStream : public IOStream +{ +public: + CompressStream(IOStream *pStream, bool TakeOwnership, + bool DecompressRead, bool CompressWrite, bool PassThroughWhenNotCompressed = false); + ~CompressStream(); +private: + // No copying (have implementations which exception) + CompressStream(const CompressStream &); + CompressStream &operator=(const CompressStream &); +public: + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes); + virtual void WriteAllBuffered(); + virtual void Close(); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +protected: + void CheckRead(); + void CheckWrite(); + void CheckBuffer(); + void WriteCompressedData(bool SyncFlush = false); + +private: + IOStream *mpStream; + bool mHaveOwnership; + bool mDecompressRead; + bool mCompressWrite; + bool mPassThroughWhenNotCompressed; + // Avoid having to include Compress.h + void *mpReadCompressor; + void *mpWriteCompressor; + void *mpBuffer; + bool mIsClosed; +}; + +#endif // COMPRESSSTREAM__H + diff --git a/lib/compress/Makefile.extra b/lib/compress/Makefile.extra new file mode 100755 index 00000000..7ef9930c --- /dev/null +++ b/lib/compress/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_CompressException.h autogen_CompressException.cpp: $(MAKEEXCEPTION) CompressException.txt + perl $(MAKEEXCEPTION) CompressException.txt + diff --git a/lib/crypto/CipherAES.cpp b/lib/crypto/CipherAES.cpp new file mode 100644 index 00000000..b4de6048 --- /dev/null +++ b/lib/crypto/CipherAES.cpp @@ -0,0 +1,163 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherAES.cpp +// Purpose: AES cipher description +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +// Only available in new versions of openssl +#ifndef PLATFORM_OLD_OPENSSL + +#include <openssl/evp.h> + +#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE + +#include "CipherAES.h" +#include "CipherException.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherAES::CipherAES(CipherDescription::CipherMode, const void *, unsigned int, const void *) +// Purpose: Constructor -- note key material and IV are not copied. KeyLength in bytes. +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +CipherAES::CipherAES(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector) + : CipherDescription(), + mMode(Mode), + mpKey(pKey), + mKeyLength(KeyLength), + mpInitialisationVector(pInitialisationVector) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherAES::CipherAES(const CipherAES &) +// Purpose: Copy constructor +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +CipherAES::CipherAES(const CipherAES &rToCopy) + : CipherDescription(rToCopy), + mMode(rToCopy.mMode), + mpKey(rToCopy.mpKey), + mKeyLength(rToCopy.mKeyLength), + mpInitialisationVector(rToCopy.mpInitialisationVector) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ~CipherAES::CipherAES() +// Purpose: Destructor +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +CipherAES::~CipherAES() +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherAES::operator=(const CipherAES &) +// Purpose: Assignment operator +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +CipherAES &CipherAES::operator=(const CipherAES &rToCopy) +{ + CipherDescription::operator=(rToCopy); + + mMode = rToCopy.mMode; + mpKey = rToCopy.mpKey; + mKeyLength = rToCopy.mKeyLength; + mpInitialisationVector = rToCopy.mpInitialisationVector; + + return *this; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherAES::GetCipher() +// Purpose: Returns cipher object +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +const EVP_CIPHER *CipherAES::GetCipher() const +{ + switch(mMode) + { + case CipherDescription::Mode_ECB: + switch(mKeyLength) + { + case (128/8): return EVP_aes_128_ecb(); break; + case (192/8): return EVP_aes_192_ecb(); break; + case (256/8): return EVP_aes_256_ecb(); break; + default: + THROW_EXCEPTION(CipherException, EVPBadKeyLength) + break; + } + break; + + case CipherDescription::Mode_CBC: + switch(mKeyLength) + { + case (128/8): return EVP_aes_128_cbc(); break; + case (192/8): return EVP_aes_192_cbc(); break; + case (256/8): return EVP_aes_256_cbc(); break; + default: + THROW_EXCEPTION(CipherException, EVPBadKeyLength) + break; + } + break; + + default: + break; + } + + // Unknown! + THROW_EXCEPTION(CipherException, UnknownCipherMode) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherAES::SetupParameters(EVP_CIPHER_CTX *) +// Purpose: Set up various parameters for cipher +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +void CipherAES::SetupParameters(EVP_CIPHER_CTX *pCipherContext) const +{ + ASSERT(pCipherContext != 0); + + // Set key (key length is implied) + if(EVP_CipherInit_ex(pCipherContext, NULL, NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + +} + + + +#endif // n PLATFORM_OLD_OPENSSL + diff --git a/lib/crypto/CipherAES.h b/lib/crypto/CipherAES.h new file mode 100644 index 00000000..51fb146a --- /dev/null +++ b/lib/crypto/CipherAES.h @@ -0,0 +1,50 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherAES.h +// Purpose: AES cipher description +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- + +#ifndef CIPHERAES__H +#define CIPHERAES__H + +// Only available in new versions of openssl +#ifndef PLATFORM_OLD_OPENSSL + +#include "CipherDescription.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: CipherAES +// Purpose: AES cipher description +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +class CipherAES : public CipherDescription +{ +public: + CipherAES(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector = 0); + CipherAES(const CipherAES &rToCopy); + virtual ~CipherAES(); + CipherAES &operator=(const CipherAES &rToCopy); + + // Return OpenSSL cipher object + virtual const EVP_CIPHER *GetCipher() const; + + // Setup any other parameters + virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const; + +private: + CipherDescription::CipherMode mMode; + const void *mpKey; + unsigned int mKeyLength; + const void *mpInitialisationVector; +}; + +#endif // n PLATFORM_OLD_OPENSSL + +#endif // CIPHERAES__H + diff --git a/lib/crypto/CipherBlowfish.cpp b/lib/crypto/CipherBlowfish.cpp new file mode 100755 index 00000000..e27e3b0a --- /dev/null +++ b/lib/crypto/CipherBlowfish.cpp @@ -0,0 +1,220 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherBlowfish.cpp +// Purpose: Blowfish cipher description +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <openssl/evp.h> + +#ifdef PLATFORM_OLD_OPENSSL + #include <string.h> + #include <strings.h> +#endif + +#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE + +#include "CipherBlowfish.h" +#include "CipherException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherBlowfish::CipherBlowfish(CipherDescription::CipherMode, const void *, unsigned int, const void *) +// Purpose: Constructor -- note key material and IV are not copied. KeyLength in bytes. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherBlowfish::CipherBlowfish(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector) + : CipherDescription(), + mMode(Mode) +#ifndef PLATFORM_OLD_OPENSSL + , mpKey(pKey), + mKeyLength(KeyLength), + mpInitialisationVector(pInitialisationVector) +{ +} +#else +{ + mKey.assign((const char *)pKey, KeyLength); + if(pInitialisationVector == 0) + { + bzero(mInitialisationVector, sizeof(mInitialisationVector)); + } + else + { + ::memcpy(mInitialisationVector, pInitialisationVector, sizeof(mInitialisationVector)); + } +} +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherBlowfish::CipherBlowfish(const CipherBlowfish &) +// Purpose: Copy constructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherBlowfish::CipherBlowfish(const CipherBlowfish &rToCopy) + : CipherDescription(rToCopy), + mMode(rToCopy.mMode), +#ifndef PLATFORM_OLD_OPENSSL + mpKey(rToCopy.mpKey), + mKeyLength(rToCopy.mKeyLength), + mpInitialisationVector(rToCopy.mpInitialisationVector) +{ +} +#else + mKey(rToCopy.mKey) +{ + ::memcpy(mInitialisationVector, rToCopy.mInitialisationVector, sizeof(mInitialisationVector)); +} +#endif + + +#ifdef PLATFORM_OLD_OPENSSL +// Hack functions to support old OpenSSL API +CipherDescription *CipherBlowfish::Clone() const +{ + return new CipherBlowfish(*this); +} +void CipherBlowfish::SetIV(const void *pIV) +{ + if(pIV == 0) + { + bzero(mInitialisationVector, sizeof(mInitialisationVector)); + } + else + { + ::memcpy(mInitialisationVector, pIV, sizeof(mInitialisationVector)); + } +} +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ~CipherBlowfish::CipherBlowfish() +// Purpose: Destructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherBlowfish::~CipherBlowfish() +{ +#ifdef PLATFORM_OLD_OPENSSL + // Zero copy of key + for(unsigned int l = 0; l < mKey.size(); ++l) + { + mKey[l] = '\0'; + } +#endif +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherBlowfish::operator=(const CipherBlowfish &) +// Purpose: Assignment operator +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherBlowfish &CipherBlowfish::operator=(const CipherBlowfish &rToCopy) +{ + CipherDescription::operator=(rToCopy); + + mMode = rToCopy.mMode; +#ifndef PLATFORM_OLD_OPENSSL + mpKey = rToCopy.mpKey; + mKeyLength = rToCopy.mKeyLength; + mpInitialisationVector = rToCopy.mpInitialisationVector; +#else + mKey = rToCopy.mKey; + ::memcpy(mInitialisationVector, rToCopy.mInitialisationVector, sizeof(mInitialisationVector)); +#endif + + return *this; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherBlowfish::GetCipher() +// Purpose: Returns cipher object +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +const EVP_CIPHER *CipherBlowfish::GetCipher() const +{ + switch(mMode) + { + case CipherDescription::Mode_ECB: + return EVP_bf_ecb(); + break; + + case CipherDescription::Mode_CBC: + return EVP_bf_cbc(); + break; + + case CipherDescription::Mode_CFB: + return EVP_bf_cfb(); + break; + + case CipherDescription::Mode_OFB: + return EVP_bf_ofb(); + break; + + default: + break; + } + + // Unknown! + THROW_EXCEPTION(CipherException, UnknownCipherMode) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherBlowfish::SetupParameters(EVP_CIPHER_CTX *) +// Purpose: Set up various parameters for cipher +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void CipherBlowfish::SetupParameters(EVP_CIPHER_CTX *pCipherContext) const +{ + ASSERT(pCipherContext != 0); + + // Set key length +#ifndef PLATFORM_OLD_OPENSSL + if(EVP_CIPHER_CTX_set_key_length(pCipherContext, mKeyLength) != 1) +#else + if(EVP_CIPHER_CTX_set_key_length(pCipherContext, mKey.size()) != 1) +#endif + { + THROW_EXCEPTION(CipherException, EVPBadKeyLength) + } + // Set key +#ifndef PLATFORM_OLD_OPENSSL + if(EVP_CipherInit_ex(pCipherContext, NULL, NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1) +#else + if(EVP_CipherInit(pCipherContext, NULL, (unsigned char*)mKey.c_str(), (unsigned char*)mInitialisationVector, -1) != 1) +#endif + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + +} + + + diff --git a/lib/crypto/CipherBlowfish.h b/lib/crypto/CipherBlowfish.h new file mode 100755 index 00000000..92db1d85 --- /dev/null +++ b/lib/crypto/CipherBlowfish.h @@ -0,0 +1,60 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherBlowfish.h +// Purpose: Blowfish cipher description +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef CIPHERBLOWFISH__H +#define CIPHERBLOWFISH__H + +#ifdef PLATFORM_OLD_OPENSSL + #include <string> +#endif + +#include "CipherDescription.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: CipherBlowfish +// Purpose: Description of Blowfish cipher parameters -- note that copies are not made of key material and IV, careful with object lifetimes. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +class CipherBlowfish : public CipherDescription +{ +public: + CipherBlowfish(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector = 0); + CipherBlowfish(const CipherBlowfish &rToCopy); + virtual ~CipherBlowfish(); + CipherBlowfish &operator=(const CipherBlowfish &rToCopy); + + // Return OpenSSL cipher object + virtual const EVP_CIPHER *GetCipher() const; + + // Setup any other parameters + virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const; + +#ifdef PLATFORM_OLD_OPENSSL + CipherDescription *Clone() const; + void SetIV(const void *pIV); +#endif + +private: + CipherDescription::CipherMode mMode; +#ifndef PLATFORM_OLD_OPENSSL + const void *mpKey; + unsigned int mKeyLength; + const void *mpInitialisationVector; +#else + std::string mKey; + uint8_t mInitialisationVector[8]; +#endif +}; + + +#endif // CIPHERBLOWFISH__H + diff --git a/lib/crypto/CipherContext.cpp b/lib/crypto/CipherContext.cpp new file mode 100755 index 00000000..42707497 --- /dev/null +++ b/lib/crypto/CipherContext.cpp @@ -0,0 +1,615 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherContext.cpp +// Purpose: Context for symmetric encryption / descryption +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE +#include "CipherContext.h" +#include "CipherDescription.h" +#include "CipherException.h" +#include "Random.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::CipherContext() +// Purpose: Constructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherContext::CipherContext() + : mInitialised(false), + mWithinTransform(false), + mPaddingOn(true) +#ifdef PLATFORM_OLD_OPENSSL + , mFunction(Decrypt), + mpDescription(0) +#endif +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::~CipherContext() +// Purpose: Destructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherContext::~CipherContext() +{ + if(mInitialised) + { + // Clean up + EVP_CIPHER_CTX_cleanup(&ctx); + mInitialised = false; + } +#ifdef PLATFORM_OLD_OPENSSL + if(mpDescription != 0) + { + delete mpDescription; + mpDescription = 0; + } +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::Init(CipherContext::CipherFunction, const CipherDescription &) +// Purpose: Initialises the context, specifying the direction for the encryption, and a +// description of the cipher to use, it's keys, etc +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDescription &rDescription) +{ + // Check for bad usage + if(mInitialised) + { + THROW_EXCEPTION(CipherException, AlreadyInitialised) + } + if(Function != Decrypt && Function != Encrypt) + { + THROW_EXCEPTION(CipherException, BadArguments) + } + + // Initialise the cipher +#ifndef PLATFORM_OLD_OPENSSL + EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does + + if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL, Function) != 1) +#else + // Store function for later + mFunction = Function; + + // Use old version of init call + if(EVP_CipherInit(&ctx, rDescription.GetCipher(), NULL, NULL, Function) != 1) +#endif + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + + try + { +#ifndef PLATFORM_OLD_OPENSSL + // Let the description set up everything else + rDescription.SetupParameters(&ctx); +#else + // With the old version, a copy needs to be taken first. + mpDescription = rDescription.Clone(); + // Mark it as not a leak, otherwise static cipher contexts cause supriously memory leaks to be reported + MEMLEAKFINDER_NOT_A_LEAK(mpDescription); + mpDescription->SetupParameters(&ctx); +#endif + } + catch(...) + { + EVP_CIPHER_CTX_cleanup(&ctx); + throw; + } + + // mark as initialised + mInitialised = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::Reset() +// Purpose: Reset the context, so it can be initialised again with a different key +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void CipherContext::Reset() +{ + if(mInitialised) + { + // Clean up + EVP_CIPHER_CTX_cleanup(&ctx); + mInitialised = false; + } +#ifdef PLATFORM_OLD_OPENSSL + if(mpDescription != 0) + { + delete mpDescription; + mpDescription = 0; + } +#endif + mWithinTransform = false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::Begin() +// Purpose: Begin a transformation +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void CipherContext::Begin() +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Warn if in a transformation (not an error, because a context might not have been finalised if an exception occured) + if(mWithinTransform) + { + TRACE0("CipherContext::Begin called when context flagged as within a transform\n"); + } + + // Initialise the cipher context again + if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + + // Mark as being within a transform + mWithinTransform = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::Transform(void *, int, const void *, int) +// Purpose: Transforms the data in the in buffer to the out buffer. If pInBuffer == 0 && InLength == 0 +// then Final() is called instead. +// Returns the number of bytes placed in the out buffer. +// There must be room in the out buffer for all the data in the in buffer. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::Transform(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + if(!mWithinTransform) + { + THROW_EXCEPTION(CipherException, BeginNotCalled) + } + + // Check parameters + if(pOutBuffer == 0 || OutLength < 0 || (pInBuffer != 0 && InLength <= 0) || (pInBuffer == 0 && InLength != 0)) + { + THROW_EXCEPTION(CipherException, BadArguments) + } + + // Is this the final call? + if(pInBuffer == 0) + { + return Final(pOutBuffer, OutLength); + } + + // Check output buffer size + if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx))) + { + THROW_EXCEPTION(CipherException, OutputBufferTooSmall); + } + + // Do the transform + int outLength = OutLength; + if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) + { + THROW_EXCEPTION(CipherException, EVPUpdateFailure) + } + + return outLength; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::Final(void *, int) +// Purpose: Transforms the data as per Transform, and returns the final data in the out buffer. +// Returns the number of bytes written in the out buffer. +// Two main causes of exceptions being thrown: 1) Data is corrupt, and so the end isn't +// padded properly. 2) Padding is off, and the data to be encrypted isn't a multiple +// of a block long. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::Final(void *pOutBuffer, int OutLength) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + if(!mWithinTransform) + { + THROW_EXCEPTION(CipherException, BeginNotCalled) + } + + // Check parameters + if(pOutBuffer == 0 || OutLength < 0) + { + THROW_EXCEPTION(CipherException, BadArguments) + } + + // Check output buffer size + if(OutLength < (2 * EVP_CIPHER_CTX_block_size(&ctx))) + { + THROW_EXCEPTION(CipherException, OutputBufferTooSmall); + } + + // Do the transform + int outLength = OutLength; +#ifndef PLATFORM_OLD_OPENSSL + if(EVP_CipherFinal_ex(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } +#else + OldOpenSSLFinal((unsigned char*)pOutBuffer, outLength); +#endif + + mWithinTransform = false; + + return outLength; +} + + +#ifdef PLATFORM_OLD_OPENSSL +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::OldOpenSSLFinal(unsigned char *, int &) +// Purpose: The old version of OpenSSL needs more work doing to finalise the cipher, +// and reset it so that it's ready for another go. +// Created: 27/3/04 +// +// -------------------------------------------------------------------------- +void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) +{ + // Old version needs to use a different form, and then set up the cipher again for next time around + int outLength = rOutLengthOut; + // Have to emulate padding off... + int blockSize = EVP_CIPHER_CTX_block_size(&ctx); + if(mPaddingOn) + { + // Just use normal final call + if(EVP_CipherFinal(&ctx, Buffer, &outLength) != 1) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } + } + else + { + // Padding is off. OpenSSL < 0.9.7 doesn't support this, so it has to be + // bodged in there. Which isn't nice. + if(mFunction == Decrypt) + { + // NASTY -- fiddling around with internals like this is bad. + // But only way to get this working on old versions of OpenSSL. + if(!EVP_EncryptUpdate(&ctx,Buffer,&outLength,ctx.buf,0) + || outLength != blockSize) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } + // Clean up + EVP_CIPHER_CTX_cleanup(&ctx); + } + else + { + // Check that the length is correct + if((ctx.buf_len % blockSize) != 0) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } + // For encryption, assume that the last block entirely is + // padding, and remove it. + char temp[1024]; + outLength = sizeof(temp); + if(EVP_CipherFinal(&ctx, Buffer, &outLength) != 1) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } + // Remove last block, assuming it's full of padded bytes only. + outLength -= blockSize; + // Copy anything to the main buffer + // (can't just use main buffer, because it might overwrite something important) + if(outLength > 0) + { + ::memcpy(Buffer, temp, outLength); + } + } + } + // Reinitialise the cipher for the next time around + if(EVP_CipherInit(&ctx, mpDescription->GetCipher(), NULL, NULL, mFunction) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + mpDescription->SetupParameters(&ctx); + + // Update length for caller + rOutLengthOut = outLength; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::InSizeForOutBufferSize(int) +// Purpose: Returns the maximum amount of data that can be sent in +// given a output buffer size. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::InSizeForOutBufferSize(int OutLength) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Strictly speaking, the *2 is unnecessary. However... + // Final() is paranoid, and requires two input blocks of space to work. + return OutLength - (EVP_CIPHER_CTX_block_size(&ctx) * 2); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::MaxOutSizeForInBufferSize(int) +// Purpose: Returns the maximum output size for an input of a given length. +// Will tend to over estimate, as it needs to allow space for Final() to be called. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::MaxOutSizeForInBufferSize(int InLength) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Final() is paranoid, and requires two input blocks of space to work, and so we need to add + // three blocks on to be absolutely sure. + return InLength + (EVP_CIPHER_CTX_block_size(&ctx) * 3); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::TransformBlock(void *, int, const void *, int) +// Purpose: Transform one block to another all in one go, no Final required. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Warn if in a transformation + if(mWithinTransform) + { + TRACE0("CipherContext::TransformBlock called when context flagged as within a transform\n"); + } + + // Check output buffer size + if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx))) + { + // Check if padding is off, in which case the buffer can be smaller + if(!mPaddingOn && OutLength <= InLength) + { + // This is OK. + } + else + { + THROW_EXCEPTION(CipherException, OutputBufferTooSmall); + } + } + + // Initialise the cipher context again + if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + + // Do the entire block + int outLength = 0; + try + { + // Update + outLength = OutLength; + if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) + { + THROW_EXCEPTION(CipherException, EVPUpdateFailure) + } + // Finalise + int outLength2 = OutLength - outLength; +#ifndef PLATFORM_OLD_OPENSSL + if(EVP_CipherFinal_ex(&ctx, ((unsigned char*)pOutBuffer) + outLength, &outLength2) != 1) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } +#else + OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2); +#endif + outLength += outLength2; + } + catch(...) + { + // Finalise the context, so definately ready for the next caller + int outs = OutLength; +#ifndef PLATFORM_OLD_OPENSSL + EVP_CipherFinal_ex(&ctx, (unsigned char*)pOutBuffer, &outs); +#else + OldOpenSSLFinal((unsigned char*)pOutBuffer, outs); +#endif + throw; + } + + return outLength; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::GetIVLength() +// Purpose: Returns the size of the IV for this context +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::GetIVLength() +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + return EVP_CIPHER_CTX_iv_length(&ctx); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::SetIV(const void *) +// Purpose: Sets the IV for this context (must be correctly sized, use GetIVLength) +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void CipherContext::SetIV(const void *pIV) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Warn if in a transformation + if(mWithinTransform) + { + TRACE0("CipherContext::SetIV called when context flagged as within a transform\n"); + } + + // Set IV + if(EVP_CipherInit(&ctx, NULL, NULL, (unsigned char *)pIV, -1) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + +#ifdef PLATFORM_OLD_OPENSSL + // Update description + if(mpDescription != 0) + { + mpDescription->SetIV(pIV); + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::SetRandomIV(int &) +// Purpose: Set a random IV for the context, and return a pointer to the IV used, +// and the length of this IV in the rLengthOut arg. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +const void *CipherContext::SetRandomIV(int &rLengthOut) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Warn if in a transformation + if(mWithinTransform) + { + TRACE0("CipherContext::SetRandomIV called when context flagged as within a transform\n"); + } + + // Get length of IV + unsigned int ivLen = EVP_CIPHER_CTX_iv_length(&ctx); + if(ivLen > sizeof(mGeneratedIV)) + { + THROW_EXCEPTION(CipherException, IVSizeImplementationLimitExceeded) + } + + // Generate some random data + Random::Generate(mGeneratedIV, ivLen); + + // Set IV + if(EVP_CipherInit(&ctx, NULL, NULL, mGeneratedIV, -1) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + +#ifdef PLATFORM_OLD_OPENSSL + // Update description + if(mpDescription != 0) + { + mpDescription->SetIV(mGeneratedIV); + } +#endif + + // Return the IV and it's length + rLengthOut = ivLen; + return mGeneratedIV; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::UsePadding(bool) +// Purpose: Set whether or not the context uses padding. +// Created: 12/12/03 +// +// -------------------------------------------------------------------------- +void CipherContext::UsePadding(bool Padding) +{ +#ifndef PLATFORM_OLD_OPENSSL + if(EVP_CIPHER_CTX_set_padding(&ctx, Padding) != 1) + { + THROW_EXCEPTION(CipherException, EVPSetPaddingFailure) + } +#endif + mPaddingOn = Padding; +} + + + diff --git a/lib/crypto/CipherContext.h b/lib/crypto/CipherContext.h new file mode 100755 index 00000000..f9dcd022 --- /dev/null +++ b/lib/crypto/CipherContext.h @@ -0,0 +1,83 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherContext.h +// Purpose: Context for symmetric encryption / descryption +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef CIPHERCONTEXT__H +#define CIPHERCONTEXT__H + +#ifdef BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_FALSE + always include CipherContext.h first in any .cpp file +#endif +#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE +#include <openssl/evp.h> +class CipherDescription; + +#define CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH 32 + +// -------------------------------------------------------------------------- +// +// Class +// Name: CipherContext +// Purpose: Context for symmetric encryption / descryption +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +class CipherContext +{ +public: + CipherContext(); + ~CipherContext(); +private: + CipherContext(const CipherContext &); // no copying + CipherContext &operator=(const CipherContext &); // no assignment +public: + + typedef enum + { + Decrypt = 0, + Encrypt = 1 + } CipherFunction; + + void Init(CipherContext::CipherFunction Function, const CipherDescription &rDescription); + void Reset(); + + void Begin(); + int Transform(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength); + int Final(void *pOutBuffer, int OutLength); + int InSizeForOutBufferSize(int OutLength); + int MaxOutSizeForInBufferSize(int InLength); + + int TransformBlock(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength); + + bool IsInitialised() {return mInitialised;} + + int GetIVLength(); + void SetIV(const void *pIV); + const void *SetRandomIV(int &rLengthOut); + + void UsePadding(bool Padding = true); + +#ifdef PLATFORM_OLD_OPENSSL + void OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut); +#endif + +private: + EVP_CIPHER_CTX ctx; + bool mInitialised; + bool mWithinTransform; + bool mPaddingOn; + uint8_t mGeneratedIV[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH]; +#ifdef PLATFORM_OLD_OPENSSL + CipherFunction mFunction; + CipherDescription *mpDescription; +#endif +}; + + +#endif // CIPHERCONTEXT__H + diff --git a/lib/crypto/CipherDescription.cpp b/lib/crypto/CipherDescription.cpp new file mode 100755 index 00000000..f0ba6ec8 --- /dev/null +++ b/lib/crypto/CipherDescription.cpp @@ -0,0 +1,73 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherDescription.cpp +// Purpose: Pure virtual base class for describing ciphers +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <openssl/evp.h> + +#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE + +#include "CipherDescription.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherDescription::CipherDescription() +// Purpose: Constructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherDescription::CipherDescription() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherDescription::CipherDescription(const CipherDescription &) +// Purpose: Copy constructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherDescription::CipherDescription(const CipherDescription &rToCopy) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ~CipherDescription::CipherDescription() +// Purpose: Destructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherDescription::~CipherDescription() +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherDescription::operator=(const CipherDescription &) +// Purpose: Assignment operator +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherDescription &CipherDescription::operator=(const CipherDescription &rToCopy) +{ + return *this; +} + + diff --git a/lib/crypto/CipherDescription.h b/lib/crypto/CipherDescription.h new file mode 100755 index 00000000..2977f7da --- /dev/null +++ b/lib/crypto/CipherDescription.h @@ -0,0 +1,59 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherDescription.h +// Purpose: Pure virtual base class for describing ciphers +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef CIPHERDESCRIPTION__H +#define CIPHERDESCRIPTION__H + +#ifndef BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE + #define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_FALSE + class EVP_CIPHER; + class EVP_CIPHER_CTX; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: CipherDescription +// Purpose: Describes a cipher +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +class CipherDescription +{ +public: + CipherDescription(); + CipherDescription(const CipherDescription &rToCopy); + virtual ~CipherDescription(); + CipherDescription &operator=(const CipherDescription &rToCopy); + + // Return OpenSSL cipher object + virtual const EVP_CIPHER *GetCipher() const = 0; + + // Setup any other parameters + virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const = 0; + + // Mode parameter for cipher -- used in derived classes + typedef enum + { + Mode_ECB = 0, + Mode_CBC = 1, + Mode_CFB = 2, + Mode_OFB = 3 + } CipherMode; + +#ifdef PLATFORM_OLD_OPENSSL + // For the old version of OpenSSL, we need to be able to store cipher descriptions. + virtual CipherDescription *Clone() const = 0; + // And to be able to store new IVs + virtual void SetIV(const void *pIV) = 0; +#endif +}; + +#endif // CIPHERDESCRIPTION__H + diff --git a/lib/crypto/CipherException.h b/lib/crypto/CipherException.h new file mode 100755 index 00000000..b0c9f8cd --- /dev/null +++ b/lib/crypto/CipherException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef CIPHEREXCEPTION__H +#define CIPHEREXCEPTION__H + +// Compatibility +#include "autogen_CipherException.h" + +#endif // CIPHEREXCEPTION__H + diff --git a/lib/crypto/CipherException.txt b/lib/crypto/CipherException.txt new file mode 100644 index 00000000..abdbac87 --- /dev/null +++ b/lib/crypto/CipherException.txt @@ -0,0 +1,18 @@ +EXCEPTION Cipher 5 + +Internal 0 +UnknownCipherMode 1 +AlreadyInitialised 2 +BadArguments 3 +EVPInitFailure 4 +EVPUpdateFailure 5 +EVPFinalFailure 6 +NotInitialised 7 +OutputBufferTooSmall 8 +EVPBadKeyLength 9 +BeginNotCalled 10 +IVSizeImplementationLimitExceeded 11 +PseudoRandNotAvailable 12 +EVPSetPaddingFailure 13 +RandomInitFailed 14 Failed to read from random device +LengthRequestedTooLongForRandomHex 15 diff --git a/lib/crypto/MD5Digest.cpp b/lib/crypto/MD5Digest.cpp new file mode 100755 index 00000000..58cc90ee --- /dev/null +++ b/lib/crypto/MD5Digest.cpp @@ -0,0 +1,82 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MD5Digest.cpp +// Purpose: Simple interface for creating MD5 digests +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- + + +#include "Box.h" + +#include "MD5Digest.h" + +#include "MemLeakFindOn.h" + + +MD5Digest::MD5Digest() +{ + MD5_Init(&md5); + for(unsigned int l = 0; l < sizeof(mDigest); ++l) + { + mDigest[l] = 0; + } +} + +MD5Digest::~MD5Digest() +{ +} + +void MD5Digest::Add(const std::string &rString) +{ + MD5_Update(&md5, rString.c_str(), rString.size()); +} + +void MD5Digest::Add(const void *pData, int Length) +{ + MD5_Update(&md5, pData, Length); +} + +void MD5Digest::Finish() +{ + MD5_Final(mDigest, &md5); +} + +std::string MD5Digest::DigestAsString() +{ + std::string r; + + static const char *hex = "0123456789abcdef"; + + for(unsigned int l = 0; l < sizeof(mDigest); ++l) + { + r += hex[(mDigest[l] & 0xf0) >> 4]; + r += hex[(mDigest[l] & 0x0f)]; + } + + return r; +} + +int MD5Digest::CopyDigestTo(uint8_t *to) +{ + for(int l = 0; l < MD5_DIGEST_LENGTH; ++l) + { + to[l] = mDigest[l]; + } + + return MD5_DIGEST_LENGTH; +} + + +bool MD5Digest::DigestMatches(uint8_t *pCompareWith) const +{ + for(int l = 0; l < MD5_DIGEST_LENGTH; ++l) + { + if(pCompareWith[l] != mDigest[l]) + return false; + } + + return true; +} + diff --git a/lib/crypto/MD5Digest.h b/lib/crypto/MD5Digest.h new file mode 100755 index 00000000..1be01ea9 --- /dev/null +++ b/lib/crypto/MD5Digest.h @@ -0,0 +1,57 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MD5Digest.h +// Purpose: Simple interface for creating MD5 digests +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef MD5DIGEST_H +#define MD5DIGEST_H + +#include <openssl/md5.h> +#include <string> + +// -------------------------------------------------------------------------- +// +// Function +// Name: MD5Digest +// Purpose: Simple interface for creating MD5 digests +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +class MD5Digest +{ +public: + MD5Digest(); + virtual ~MD5Digest(); + + void Add(const std::string &rString); + void Add(const void *pData, int Length); + + void Finish(); + + std::string DigestAsString(); + uint8_t *DigestAsData(int *pLength = 0) + { + if(pLength) *pLength = sizeof(mDigest); + return mDigest; + } + + enum + { + DigestLength = MD5_DIGEST_LENGTH + }; + + int CopyDigestTo(uint8_t *to); + + bool DigestMatches(uint8_t *pCompareWith) const; + +private: + MD5_CTX md5; + uint8_t mDigest[MD5_DIGEST_LENGTH]; +}; + +#endif // MD5DIGEST_H + diff --git a/lib/crypto/Makefile.extra b/lib/crypto/Makefile.extra new file mode 100755 index 00000000..a7e42000 --- /dev/null +++ b/lib/crypto/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_CipherException.cpp autogen_CipherException.h: $(MAKEEXCEPTION) CipherException.txt + perl $(MAKEEXCEPTION) CipherException.txt + diff --git a/lib/crypto/Random.cpp b/lib/crypto/Random.cpp new file mode 100755 index 00000000..3df5fc5e --- /dev/null +++ b/lib/crypto/Random.cpp @@ -0,0 +1,127 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Random.cpp +// Purpose: Random numbers +// Created: 31/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <openssl/rand.h> +#include <stdio.h> + +#include "Random.h" +#include "CipherException.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Random::Initialise() +// Purpose: Add additional randomness to the standard library initialisation +// Created: 18/6/04 +// +// -------------------------------------------------------------------------- +void Random::Initialise() +{ +#ifndef PLATFORM_RANDOM_DEVICE_NONE + if(::RAND_load_file(PLATFORM_RANDOM_DEVICE, 1024) != 1024) + { + THROW_EXCEPTION(CipherException, RandomInitFailed) + } +#else + ::fprintf(stderr, "No random device -- additional seeding of random number generator not performed.\n"); +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Random::Generate(void *, int) +// Purpose: Generate Length bytes of random data +// Created: 31/12/03 +// +// -------------------------------------------------------------------------- +void Random::Generate(void *pOutput, int Length) +{ + if(RAND_pseudo_bytes((uint8_t*)pOutput, Length) == -1) + { + THROW_EXCEPTION(CipherException, PseudoRandNotAvailable) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Random::GenerateHex(int) +// Purpose: Generate Length bytes of hex encoded data. Note that the +// maximum length requested is limited. (Returns a string +// 2 x Length characters long.) +// Created: 1/11/04 +// +// -------------------------------------------------------------------------- +std::string Random::GenerateHex(int Length) +{ + uint8_t r[256]; + if(Length > sizeof(r)) + { + THROW_EXCEPTION(CipherException, LengthRequestedTooLongForRandomHex) + } + Random::Generate(r, Length); + + std::string o; + static const char *h = "0123456789abcdef"; + for(int l = 0; l < Length; ++l) + { + o += h[r[l] >> 4]; + o += h[r[l] & 0xf]; + } + + return o; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Random::RandomInt(int) +// Purpose: Return a random integer between 0 and MaxValue inclusive. +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +uint32_t Random::RandomInt(uint32_t MaxValue) +{ + uint32_t v = 0; + + // Generate a mask + uint32_t mask = 0; + while(mask < MaxValue) + { + mask = (mask << 1) | 1; + } + + do + { + // Generate a random number + uint32_t r = 0; + Random::Generate(&r, sizeof(r)); + + // Mask off relevant bits + v = r & mask; + + // Check that it's in the right range. + } while(v > MaxValue); + + // NOTE: don't do a mod, because this doesn't give a correct random distribution + + return v; +} + + + diff --git a/lib/crypto/Random.h b/lib/crypto/Random.h new file mode 100755 index 00000000..da3e4335 --- /dev/null +++ b/lib/crypto/Random.h @@ -0,0 +1,25 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Random.h +// Purpose: Random numbers +// Created: 31/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef RANDOM__H +#define RANDOM__H + +#include <string> + +namespace Random +{ + void Initialise(); + void Generate(void *pOutput, int Length); + std::string GenerateHex(int Length); + uint32_t RandomInt(uint32_t MaxValue); +}; + + +#endif // RANDOM__H + diff --git a/lib/crypto/RollingChecksum.cpp b/lib/crypto/RollingChecksum.cpp new file mode 100755 index 00000000..75bad7df --- /dev/null +++ b/lib/crypto/RollingChecksum.cpp @@ -0,0 +1,38 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RollingChecksum.cpp +// Purpose: A simple rolling checksum over a block of data +// Created: 6/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "RollingChecksum.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: RollingChecksum::RollingChecksum(const void *, unsigned int) +// Purpose: Constructor -- does initial computation of the checksum. +// Created: 6/12/03 +// +// -------------------------------------------------------------------------- +RollingChecksum::RollingChecksum(const void *data, unsigned int Length) + : a(0), + b(0) +{ + uint8_t *block = (uint8_t *)data; + for(unsigned int x = Length; x >= 1; --x) + { + a += (*block); + b += x * (*block); + + ++block; + } +} + + + diff --git a/lib/crypto/RollingChecksum.h b/lib/crypto/RollingChecksum.h new file mode 100755 index 00000000..99d116b9 --- /dev/null +++ b/lib/crypto/RollingChecksum.h @@ -0,0 +1,96 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RollingChecksum.h +// Purpose: A simple rolling checksum over a block of data +// Created: 6/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef ROLLINGCHECKSUM__H +#define ROLLINGCHECKSUM__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: RollingChecksum +// Purpose: A simple rolling checksum over a block of data -- can move the block +// "forwards" in memory and get the next checksum efficiently. +// +// Implementation of http://rsync.samba.org/tech_report/node3.html +// Created: 6/12/03 +// +// -------------------------------------------------------------------------- +class RollingChecksum +{ +public: + RollingChecksum(const void *data, unsigned int Length); + + // -------------------------------------------------------------------------- + // + // Function + // Name: RollingChecksum::RollForward(uint8_t, uint8_t, unsigned int) + // Purpose: Move the checksum forward a block, given the first byte of the current block, + // last byte of the next block (it's rolling forward to) and the length of the block. + // Created: 6/12/03 + // + // -------------------------------------------------------------------------- + inline void RollForward(uint8_t StartOfThisBlock, uint8_t LastOfNextBlock, unsigned int Length) + { + // IMPLEMENTATION NOTE: Everything is implicitly mod 2^16 -- uint16_t's will overflow nicely. + a -= StartOfThisBlock; + a += LastOfNextBlock; + b -= Length * StartOfThisBlock; + b += a; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: RollingChecksum::GetChecksum() + // Purpose: Returns the checksum + // Created: 6/12/03 + // + // -------------------------------------------------------------------------- + inline uint32_t GetChecksum() + { + return ((uint32_t)a) | (((uint32_t)b) << 16); + } + + // Components, just in case they're handy + inline uint16_t GetComponent1() {return a;} + inline uint16_t GetComponent2() {return b;} + + // -------------------------------------------------------------------------- + // + // Function + // Name: RollingChecksum::GetComponentForHashing() + // Purpose: Return the 16 bit component used for hashing and/or quick checks + // Created: 6/12/03 + // + // -------------------------------------------------------------------------- + inline uint16_t GetComponentForHashing() + { + return b; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: RollingChecksum::ExtractHashingComponent(uint32_t) + // Purpose: Static. Given a full checksum, extract the component used in the hashing table. + // Created: 14/1/04 + // + // -------------------------------------------------------------------------- + static inline uint16_t ExtractHashingComponent(uint32_t Checksum) + { + return Checksum >> 16; + } + +private: + uint16_t a; + uint16_t b; +}; + +#endif // ROLLINGCHECKSUM__H + diff --git a/lib/raidfile/Makefile.extra b/lib/raidfile/Makefile.extra new file mode 100755 index 00000000..8d036633 --- /dev/null +++ b/lib/raidfile/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_RaidFileException.h autogen_RaidFileException.cpp: $(MAKEEXCEPTION) RaidFileException.txt + perl $(MAKEEXCEPTION) RaidFileException.txt + diff --git a/lib/raidfile/RaidFileController.cpp b/lib/raidfile/RaidFileController.cpp new file mode 100755 index 00000000..89ae6791 --- /dev/null +++ b/lib/raidfile/RaidFileController.cpp @@ -0,0 +1,217 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileController.cpp +// Purpose: Controls config and daemon comms for RaidFile classes +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> + +#include "RaidFileController.h" +#include "RaidFileException.h" +#include "Configuration.h" + +#include "MemLeakFindOn.h" + +RaidFileController RaidFileController::mController; + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::RaidFileController() +// Purpose: Constructor +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +RaidFileController::RaidFileController() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::~RaidFileController() +// Purpose: Destructor +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +RaidFileController::~RaidFileController() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::RaidFileController() +// Purpose: Copy constructor +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +RaidFileController::RaidFileController(const RaidFileController &rController) +{ + THROW_EXCEPTION(RaidFileException, Internal) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::Initialise(const char *) +// Purpose: Initialises the system, loading the configuration file. +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +void RaidFileController::Initialise(const char *ConfigFilename) +{ + static const ConfigurationVerifyKey verifykeys[] = + { + {"SetNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"BlockSize", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"Dir0", 0, ConfigTest_Exists, 0}, + {"Dir1", 0, ConfigTest_Exists, 0}, + {"Dir2", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + }; + + static const ConfigurationVerify subverify = + { + "*", + 0, + verifykeys, + ConfigTest_LastEntry, + 0 + }; + + static const ConfigurationVerify verify = + { + "RAID FILE CONFIG", + &subverify, + 0, + ConfigTest_LastEntry, + 0 + }; + + // Load the configuration + std::string err; + std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(ConfigFilename, &verify, err); + + if(pconfig.get() == 0 || !err.empty()) + { + fprintf(stderr, "RaidFile configuation file errors:\n%s", err.c_str()); + THROW_EXCEPTION(RaidFileException, BadConfigFile) + } + + // Use the values + int expectedSetNum = 0; + std::vector<std::string> confdiscs(pconfig->GetSubConfigurationNames()); + for(std::vector<std::string>::const_iterator i(confdiscs.begin()); i != confdiscs.end(); ++i) + { + const Configuration &disc(pconfig->GetSubConfiguration((*i).c_str())); + + int setNum = disc.GetKeyValueInt("SetNumber"); + if(setNum != expectedSetNum) + { + THROW_EXCEPTION(RaidFileException, BadConfigFile) + } + RaidFileDiscSet set(setNum, (unsigned int)disc.GetKeyValueInt("BlockSize")); + // Get the values of the directory keys + std::string d0(disc.GetKeyValue("Dir0")); + std::string d1(disc.GetKeyValue("Dir1")); + std::string d2(disc.GetKeyValue("Dir2")); + // Are they all different (using RAID) or all the same (not using RAID) + if(d0 != d1 && d1 != d2 && d0 != d2) + { + set.push_back(d0); + set.push_back(d1); + set.push_back(d2); + } + else if(d0 == d1 && d0 == d2) + { + // Just push the first one, which is the non-RAID place to store files + set.push_back(d0); + } + else + { + // One must be the same as another! Which is bad. + THROW_EXCEPTION(RaidFileException, BadConfigFile) + } + mSetList.push_back(set); + expectedSetNum++; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::GetDiscSet(int) +// Purpose: Returns the numbered disc set +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +RaidFileDiscSet &RaidFileController::GetDiscSet(unsigned int DiscSetNum) +{ + if(DiscSetNum < 0 || DiscSetNum >= mSetList.size()) + { + THROW_EXCEPTION(RaidFileException, NoSuchDiscSet) + } + + return mSetList[DiscSetNum]; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileDiscSet::GetSetNumForWriteFiles(const std::string &) +// Purpose: Returns the set number the 'temporary' written files should +// be stored on, given a filename. +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +int RaidFileDiscSet::GetSetNumForWriteFiles(const std::string &rFilename) const +{ + // Simple hash function, add up the ASCII values of all the characters, + // and get modulo number of partitions in the set. + std::string::const_iterator i(rFilename.begin()); + int h = 0; + for(; i != rFilename.end(); ++i) + { + h += (*i); + } + return h % size(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::DiscSetPathToFileSystemPath(unsigned int, const std::string &, int) +// Purpose: Given a Raid File style file name, return a filename for the physical filing system. +// DiscOffset is effectively the disc number (but remember files are rotated around the +// discs in a disc set) +// Created: 19/1/04 +// +// -------------------------------------------------------------------------- +std::string RaidFileController::DiscSetPathToFileSystemPath(unsigned int DiscSetNum, const std::string &rFilename, int DiscOffset) +{ + if(DiscSetNum < 0 || DiscSetNum >= mController.mSetList.size()) + { + THROW_EXCEPTION(RaidFileException, NoSuchDiscSet) + } + + // Work out which disc it's to be on + int disc = (mController.mSetList[DiscSetNum].GetSetNumForWriteFiles(rFilename) + DiscOffset) + % mController.mSetList[DiscSetNum].size(); + + // Make the string + std::string r((mController.mSetList[DiscSetNum])[disc]); + r += DIRECTORY_SEPARATOR_ASCHAR; + r += rFilename; + return r; +} + + + diff --git a/lib/raidfile/RaidFileController.h b/lib/raidfile/RaidFileController.h new file mode 100755 index 00000000..4962d236 --- /dev/null +++ b/lib/raidfile/RaidFileController.h @@ -0,0 +1,107 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileController.h +// Purpose: Controls config and daemon comms for RaidFile classes +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +/* NOTE: will log to local5: include a line like + local5.info /var/log/raidfile + in /etc/syslog.conf +*/ + +#ifndef RAIDFILECONTROLLER__H +#define RAIDFILECONTROLLER__H + +#include <string> +#include <vector> + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileDiscSet +// Purpose: Describes a set of paritions for RAID like files. +// Use as list of directories containing the files. +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +class RaidFileDiscSet : public std::vector<std::string> +{ +public: + RaidFileDiscSet(int SetID, unsigned int BlockSize) + : mSetID(SetID), + mBlockSize(BlockSize) + { + } + RaidFileDiscSet(const RaidFileDiscSet &rToCopy) + : std::vector<std::string>(rToCopy), + mSetID(rToCopy.mSetID), + mBlockSize(rToCopy.mBlockSize) + { + } + + ~RaidFileDiscSet() + { + } + + int GetSetID() const {return mSetID;} + + int GetSetNumForWriteFiles(const std::string &rFilename) const; + + unsigned int GetBlockSize() const {return mBlockSize;} + + // Is this disc set a non-RAID disc set? (ie files never get transformed to raid storage) + bool IsNonRaidSet() const {return 1 == size();} + +private: + int mSetID; + unsigned int mBlockSize; +}; + +class _RaidFileController; // compiler warning avoidance + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileController +// Purpose: Manages the configuration of the RaidFile system, handles +// communication with the daemon. +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +class RaidFileController +{ + friend class _RaidFileController; // to avoid compiler warning +private: + RaidFileController(); + RaidFileController(const RaidFileController &rController); +public: + ~RaidFileController(); + +public: + void Initialise(const char *ConfigFilename = "/etc/box/raidfile.conf"); + int GetNumDiscSets() {return mSetList.size();} + + // -------------------------------------------------------------------------- + // + // Function + // Name: RaidFileController::GetController() + // Purpose: Gets the one and only controller object. + // Created: 2003/07/08 + // + // -------------------------------------------------------------------------- + static RaidFileController &GetController() {return mController;} + RaidFileDiscSet &GetDiscSet(unsigned int DiscSetNum); + + static std::string DiscSetPathToFileSystemPath(unsigned int DiscSetNum, const std::string &rFilename, int DiscOffset); + +private: + std::vector<RaidFileDiscSet> mSetList; + + static RaidFileController mController; +}; + +#endif // RAIDFILECONTROLLER__H + diff --git a/lib/raidfile/RaidFileException.h b/lib/raidfile/RaidFileException.h new file mode 100755 index 00000000..809d4d5a --- /dev/null +++ b/lib/raidfile/RaidFileException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef RAIDFILEEXCEPTION__H +#define RAIDFILEEXCEPTION__H + +// Compatibility +#include "autogen_RaidFileException.h" + +#endif // RAIDFILEEXCEPTION__H + diff --git a/lib/raidfile/RaidFileException.txt b/lib/raidfile/RaidFileException.txt new file mode 100644 index 00000000..6ad74563 --- /dev/null +++ b/lib/raidfile/RaidFileException.txt @@ -0,0 +1,25 @@ +EXCEPTION RaidFile 2 + +Internal 0 +CantOpenConfigFile 1 The raidfile.conf file is not accessible. Check that it is present in the default location or daemon configuration files point to the correct location. +BadConfigFile 2 +NoSuchDiscSet 3 +CannotOverwriteExistingFile 4 +AlreadyOpen 5 +ErrorOpeningWriteFile 6 +NotOpen 7 +OSError 8 Error when accessing an underlying file. Check file permissions allow files to be read and written in the configured raid directories. +WriteFileOpenOnTransform 9 +WrongNumberOfDiscsInSet 10 There should be three directories in each disc set. +RaidFileDoesntExist 11 +ErrorOpeningFileForRead 12 +FileIsDamagedNotRecoverable 13 +InvalidRaidFile 14 +DirectoryIncomplete 15 +UnexpectedFileInDirPlace 16 +FileExistsInDirectoryCreation 17 +UnsupportedReadWriteOrClose 18 +CanOnlyGetUsageBeforeCommit 19 +CanOnlyGetFileSizeBeforeCommit 20 +ErrorOpeningWriteFileOnTruncate 21 +FileIsCurrentlyOpenForWriting 22 diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp new file mode 100755 index 00000000..fed22ac0 --- /dev/null +++ b/lib/raidfile/RaidFileRead.cpp @@ -0,0 +1,1701 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileRead.cpp +// Purpose: Read Raid like Files +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> +#include <sys/stat.h> +#include <sys/uio.h> +#include <syslog.h> +#include <stdarg.h> +#include <dirent.h> + +#include <stdio.h> +#include <string.h> +#include <memory> +#include <map> + +#include "RaidFileRead.h" +#include "RaidFileException.h" +#include "RaidFileController.h" +#include "RaidFileUtil.h" + +#ifdef PLATFORM_LINUX + #include "LinuxWorkaround.h" +#endif + +#include "MemLeakFindOn.h" + +#define READ_NUMBER_DISCS_REQUIRED 3 +#define READV_MAX_BLOCKS 64 + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileRead_NonRaid +// Purpose: Internal class for reading RaidFiles which haven't been transformed +// into the RAID like form yet. +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +class RaidFileRead_NonRaid : public RaidFileRead +{ +public: + RaidFileRead_NonRaid(int SetNumber, const std::string &Filename, int OSFileHandle); + virtual ~RaidFileRead_NonRaid(); +private: + RaidFileRead_NonRaid(const RaidFileRead_NonRaid &rToCopy); + +public: + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + virtual pos_type GetFileSize() const; + virtual bool StreamDataLeft(); + +private: + int mOSFileHandle; + bool mEOF; +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid(int, const std::string &, const std::string &) +// Purpose: Constructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead_NonRaid::RaidFileRead_NonRaid(int SetNumber, const std::string &Filename, int OSFileHandle) + : RaidFileRead(SetNumber, Filename), + mOSFileHandle(OSFileHandle), + mEOF(false) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::~RaidFileRead_NonRaid() +// Purpose: Destructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead_NonRaid::~RaidFileRead_NonRaid() +{ + if(mOSFileHandle != -1) + { + Close(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::Read(const void *, int) +// Purpose: Reads bytes from the file +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +int RaidFileRead_NonRaid::Read(void *pBuffer, int NBytes, int Timeout) +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Read data + int bytesRead = ::read(mOSFileHandle, pBuffer, NBytes); + if(bytesRead == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Check for EOF + if(bytesRead == 0) + { + mEOF = true; + } + + return bytesRead; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::GetPosition() +// Purpose: Returns current position +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead::pos_type RaidFileRead_NonRaid::GetPosition() const +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Use lseek to find the current file position + off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR); + if(p == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + return p; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::Seek(pos_type, int) +// Purpose: Seek within the file +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +void RaidFileRead_NonRaid::Seek(IOStream::pos_type Offset, int SeekType) +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Seek... + if(::lseek(mOSFileHandle, Offset, ConvertSeekTypeToOSWhence(SeekType)) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Not EOF any more + mEOF = false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::Close() +// Purpose: Close the file (automatically done by destructor) +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +void RaidFileRead_NonRaid::Close() +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Close file... + if(::close(mOSFileHandle) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + mOSFileHandle = -1; + mEOF = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::GetFileSize() +// Purpose: Returns file size. +// Created: 2003/07/14 +// +// -------------------------------------------------------------------------- +RaidFileRead::pos_type RaidFileRead_NonRaid::GetFileSize() const +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // stat the file + struct stat st; + if(::fstat(mOSFileHandle, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + return st.st_size; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::StreamDataLeft() +// Purpose: Any data left? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool RaidFileRead_NonRaid::StreamDataLeft() +{ + return !mEOF; +} + +// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileRead_Raid +// Purpose: Internal class for reading RaidFiles have been transformed. +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +class RaidFileRead_Raid : public RaidFileRead +{ +public: + friend class RaidFileRead; + RaidFileRead_Raid(int SetNumber, const std::string &Filename, int Stripe1Handle, + int Stripe2Handle, int ParityHandle, pos_type FileSize, unsigned int BlockSize, + bool LastBlockHasSize); + virtual ~RaidFileRead_Raid(); +private: + RaidFileRead_Raid(const RaidFileRead_Raid &rToCopy); + +public: + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + virtual pos_type GetFileSize() const; + virtual bool StreamDataLeft(); + +private: + int ReadRecovered(void *pBuffer, int NBytes); + void AttemptToRecoverFromIOError(bool Stripe1); + void SetPosition(pos_type FilePosition); + static void MoveDamagedFileAlertDaemon(int SetNumber, const std::string &Filename, bool Stripe1); + +private: + int mStripe1Handle; + int mStripe2Handle; + int mParityHandle; + pos_type mFileSize; + unsigned int mBlockSize; + pos_type mCurrentPosition; + char *mRecoveryBuffer; + pos_type mRecoveryBufferStart; + bool mLastBlockHasSize; + bool mEOF; +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid(int, const std::string &, const std::string &) +// Purpose: Constructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead_Raid::RaidFileRead_Raid(int SetNumber, const std::string &Filename, int Stripe1Handle, int Stripe2Handle, int ParityHandle, pos_type FileSize, unsigned int BlockSize, bool LastBlockHasSize) + : RaidFileRead(SetNumber, Filename), + mStripe1Handle(Stripe1Handle), + mStripe2Handle(Stripe2Handle), + mParityHandle(ParityHandle), + mFileSize(FileSize), + mBlockSize(BlockSize), + mCurrentPosition(0), + mRecoveryBuffer(0), + mRecoveryBufferStart(-1), + mLastBlockHasSize(LastBlockHasSize), + mEOF(false) +{ + // Make sure size of the IOStream::pos_type matches the pos_type used +#ifdef PLATFORM_LINUX + ASSERT(sizeof(pos_type) >= sizeof(off_t)); +#else + ASSERT(sizeof(pos_type) == sizeof(off_t)); +#endif + + // Sanity check handles + if(mStripe1Handle != -1 && mStripe2Handle != -1) + { + // Everything is lovely, got two perfect files + } + else + { + // Check we have at least one stripe and a parity file + if((mStripe1Handle == -1 && mStripe2Handle == -1) || mParityHandle == -1) + { + // Should never have got this far + THROW_EXCEPTION(RaidFileException, Internal) + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::~RaidFileRead_Raid() +// Purpose: Destructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead_Raid::~RaidFileRead_Raid() +{ + Close(); + if(mRecoveryBuffer != 0) + { + ::free(mRecoveryBuffer); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::Read(const void *, int) +// Purpose: Reads bytes from the file +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +int RaidFileRead_Raid::Read(void *pBuffer, int NBytes, int Timeout) +{ + // How many more bytes could we read? + unsigned int maxRead = mFileSize - mCurrentPosition; + if((unsigned int)NBytes > maxRead) + { + NBytes = maxRead; + } + + // Return immediately if there's nothing to read, and set EOF + if(NBytes == 0) + { + mEOF = true; + return 0; + } + + // Can we use the normal file reading routine? + if(mStripe1Handle == -1 || mStripe2Handle == -1) + { + // File is damaged, try a the recovery read function + return ReadRecovered(pBuffer, NBytes); + } + + // Vectors for reading stuff from the files + struct iovec stripe1Reads[READV_MAX_BLOCKS]; + struct iovec stripe2Reads[READV_MAX_BLOCKS]; + struct iovec *stripeReads[2] = {stripe1Reads, stripe2Reads}; + unsigned int stripeReadsDataSize[2] = {0, 0}; + unsigned int stripeReadsSize[2] = {0, 0}; + int stripeHandles[2] = {mStripe1Handle, mStripe2Handle}; + + // Which block are we doing? + unsigned int currentBlock = mCurrentPosition / mBlockSize; + unsigned int bytesLeftInCurrentBlock = mBlockSize - (mCurrentPosition % mBlockSize); + ASSERT(bytesLeftInCurrentBlock > 0) + unsigned int leftToRead = NBytes; + char *bufferPtr = (char*)pBuffer; + + // Now... add some whole block entries in... + try + { + while(leftToRead > 0) + { + int whichStripe = (currentBlock & 1); + size_t rlen = mBlockSize; + // Adjust if it's the first block + if(bytesLeftInCurrentBlock != 0) + { + rlen = bytesLeftInCurrentBlock; + bytesLeftInCurrentBlock = 0; + } + // Adjust if we're out of bytes + if(rlen > leftToRead) + { + rlen = leftToRead; + } + stripeReads[whichStripe][stripeReadsSize[whichStripe]].iov_base = bufferPtr; + stripeReads[whichStripe][stripeReadsSize[whichStripe]].iov_len = rlen; + stripeReadsSize[whichStripe]++; + stripeReadsDataSize[whichStripe] += rlen; + leftToRead -= rlen; + bufferPtr += rlen; + currentBlock++; + + // Read data? + for(int s = 0; s < 2; ++s) + { + if((leftToRead == 0 || stripeReadsSize[s] >= READV_MAX_BLOCKS) && stripeReadsSize[s] > 0) + { + int r = ::readv(stripeHandles[s], stripeReads[s], stripeReadsSize[s]); + if(r == -1) + { + // Bad news... IO error? + if(errno == EIO) + { + // Attempt to recover from this failure + AttemptToRecoverFromIOError((s == 0) /* is stripe 1 */); + // Retry + return Read(pBuffer, NBytes, Timeout); + } + else + { + // Can't do anything, throw + THROW_EXCEPTION(RaidFileException, OSError) + } + } + else if(r != (int)stripeReadsDataSize[s]) + { + // Got the file sizes wrong/logic error! + THROW_EXCEPTION(RaidFileException, Internal) + } + stripeReadsSize[s] = 0; + stripeReadsDataSize[s] = 0; + } + } + } + } + catch(...) + { + // Get file pointers to right place (to meet exception safe stuff) + SetPosition(mCurrentPosition); + + throw; + } + + // adjust current position + mCurrentPosition += NBytes; + + return NBytes; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::MoveDamagedFileAlertDaemon(bool) +// Purpose: Moves a file into the damaged directory, and alerts the Daemon to recover it properly later. +// Created: 2003/07/22 +// +// -------------------------------------------------------------------------- +void RaidFileRead_Raid::MoveDamagedFileAlertDaemon(int SetNumber, const std::string &Filename, bool Stripe1) +{ + // Move the dodgy file away + // Get the controller and the disc set we're on + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size()) + { + THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet) + } + // Start disc + int startDisc = rdiscSet.GetSetNumForWriteFiles(Filename); + int errOnDisc = (startDisc + (Stripe1?0:1)) % READ_NUMBER_DISCS_REQUIRED; + + // Make a munged filename for renaming + std::string mungeFn(Filename + RAIDFILE_EXTENSION); + std::string awayName; + for(std::string::const_iterator i = mungeFn.begin(); i != mungeFn.end(); ++i) + { + char c = (*i); + if(c == DIRECTORY_SEPARATOR_ASCHAR) + { + awayName += '_'; + } + else if(c == '_') + { + awayName += "__"; + } + else + { + awayName += c; + } + } + // Make sure the error files directory exists + std::string dirname(rdiscSet[errOnDisc] + DIRECTORY_SEPARATOR ".raidfile-unreadable"); + int mdr = ::mkdir(dirname.c_str(), 0750); + if(mdr != 0 && errno != EEXIST) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Attempt to rename the file there -- ignore any return code here, as it's dubious anyway + std::string errorFile(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, errOnDisc)); + ::rename(errorFile.c_str(), (dirname + DIRECTORY_SEPARATOR_ASCHAR + awayName).c_str()); + + // TODO: Inform the recovery daemon +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::AttemptToRecoverFromIOError(bool) +// Purpose: Attempt to recover from an IO error, setting up to read from parity instead. +// Will exception if this isn't possible. +// Created: 2003/07/14 +// +// -------------------------------------------------------------------------- +void RaidFileRead_Raid::AttemptToRecoverFromIOError(bool Stripe1) +{ + TRACE3("Attempting to recover from I/O error: %d %s, on stripe %d\n", mSetNumber, mFilename.c_str(), Stripe1?1:2); + ::syslog(LOG_ERR | LOG_LOCAL5, "Attempting to recover from I/O error: %d %s, on stripe %d\n", mSetNumber, mFilename.c_str(), Stripe1?1:2); + + // Close offending file + if(Stripe1) + { + if(mStripe1Handle != -1) + { + ::close(mStripe1Handle); + mStripe1Handle = -1; + } + } + else + { + if(mStripe2Handle != -1) + { + ::close(mStripe2Handle); + mStripe2Handle = -1; + } + } + + // Check... + ASSERT((Stripe1?mStripe2Handle:mStripe1Handle) != -1); + + // Get rid of the damaged file + MoveDamagedFileAlertDaemon(mSetNumber, mFilename, Stripe1); + + // Get the controller and the disc set we're on + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size()) + { + THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet) + } + // Start disc + int startDisc = rdiscSet.GetSetNumForWriteFiles(mFilename); + + // Mark as nothing in recovery buffer + mRecoveryBufferStart = -1; + + // Seek to zero on the remaining file -- get to nice state + if(::lseek(Stripe1?mStripe2Handle:mStripe1Handle, 0, SEEK_SET) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Open the parity file + std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (2 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + mParityHandle = ::open(parityFilename.c_str(), O_RDONLY, 0555); + if(mParityHandle == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Work out whether or not there's a size XORed into the last block + unsigned int bytesInLastTwoBlocks = mFileSize % (mBlockSize * 2); + if(bytesInLastTwoBlocks > mBlockSize && bytesInLastTwoBlocks < ((mBlockSize * 2) - sizeof(FileSizeType))) + { + // Yes, there's something to XOR in the last block + mLastBlockHasSize = true; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::ReadRecovered(const void *, int) +// Purpose: Reads data recreating from the parity stripe +// Created: 2003/07/14 +// +// -------------------------------------------------------------------------- +int RaidFileRead_Raid::ReadRecovered(void *pBuffer, int NBytes) +{ + // Note: NBytes has been adjusted to definately be a range + // inside the given file length. + + // Make sure a buffer is allocated + if(mRecoveryBuffer == 0) + { + mRecoveryBuffer = (char*)::malloc(mBlockSize * 2); + if(mRecoveryBuffer == 0) + { + throw std::bad_alloc(); + } + } + + // Which stripe? + int stripe = (mStripe1Handle != -1)?mStripe1Handle:mStripe2Handle; + if(stripe == -1) + { + // Not enough file handles around + THROW_EXCEPTION(RaidFileException, FileIsDamagedNotRecoverable) + } + + char *outptr = (char*)pBuffer; + int bytesToGo = NBytes; + + pos_type preservedCurrentPosition = mCurrentPosition; + + try + { + // Start offset within buffer + int offset = (mCurrentPosition - mRecoveryBufferStart); + // Let's go! + while(bytesToGo > 0) + { + int bytesLeftInBuffer = 0; + if(mRecoveryBufferStart != -1) + { + bytesLeftInBuffer = (mRecoveryBufferStart + (mBlockSize*2)) - mCurrentPosition; + ASSERT(bytesLeftInBuffer >= 0); + } + + // How many bytes can be copied out? + int toCopy = bytesLeftInBuffer; + if(toCopy > bytesToGo) toCopy = bytesToGo; + //printf("offset = %d, tocopy = %d, bytestogo = %d, leftinbuffer = %d\n", (int)offset, toCopy, bytesToGo, bytesLeftInBuffer); + if(toCopy > 0) + { + for(int l = 0; l < toCopy; ++l) + { + *(outptr++) = mRecoveryBuffer[offset++]; + } + bytesToGo -= toCopy; + mCurrentPosition += toCopy; + } + + // Load in the next buffer? + if(bytesToGo > 0) + { + // Calculate the blocks within the file that are needed to be loaded. + pos_type fileBlock = mCurrentPosition / (mBlockSize * 2); + // Is this the last block + bool isLastBlock = (fileBlock == (mFileSize / (mBlockSize * 2))); + + // Need to reposition file pointers? + if(mRecoveryBufferStart == -1) + { + // Yes! + // And the offset from which to read it + pos_type filePos = fileBlock * mBlockSize; + // Then seek + if(::lseek(stripe, filePos, SEEK_SET) == -1 + || ::lseek(mParityHandle, filePos, SEEK_SET) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + // Load a block from each file, getting the ordering the right way round + int r1 = ::read((mStripe1Handle != -1)?stripe:mParityHandle, mRecoveryBuffer, mBlockSize); + int r2 = ::read((mStripe1Handle != -1)?mParityHandle:stripe, mRecoveryBuffer + mBlockSize, mBlockSize); + if(r1 == -1 || r2 == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // error checking and manipulation + if(isLastBlock) + { + // Allow not full reads, and append zeros if necessary to fill the space. + int r1zeros = mBlockSize - r1; + if(r1zeros > 0) + { + ::memset(mRecoveryBuffer + r1, 0, r1zeros); + } + int r2zeros = mBlockSize - r2; + if(r2zeros > 0) + { + ::memset(mRecoveryBuffer + mBlockSize + r2, 0, r2zeros); + } + + // if it's got the file size in it, XOR it off + if(mLastBlockHasSize) + { + int sizeXorOffset = (mBlockSize - sizeof(FileSizeType)) + ((mStripe1Handle != -1)?mBlockSize:0); + *((FileSizeType*)(mRecoveryBuffer + sizeXorOffset)) ^= ntoh64(mFileSize); + } + } + else + { + // Must have got a full block, otherwise things are a bit bad here. + if(r1 != (int)mBlockSize || r2 != (int)mBlockSize) + { + THROW_EXCEPTION(RaidFileException, InvalidRaidFile) + } + } + + // Go XORing! + unsigned int *b1 = (unsigned int*)mRecoveryBuffer; + unsigned int *b2 = (unsigned int *)(mRecoveryBuffer + mBlockSize); + if((mStripe1Handle == -1)) + { + b1 = b2; + b2 = (unsigned int*)mRecoveryBuffer; + } + for(int x = ((mBlockSize/sizeof(unsigned int)) - 1); x >= 0; --x) + { + *b2 = (*b1) ^ (*b2); + ++b1; + ++b2; + } + + // New block location + mRecoveryBufferStart = fileBlock * (mBlockSize * 2); + + // New offset withing block + offset = (mCurrentPosition - mRecoveryBufferStart); + ASSERT(offset >= 0); + } + } + } + catch(...) + { + // Change variables so 1) buffer is invalidated and 2) the file will be seeked properly the next time round + mRecoveryBufferStart = -1; + mCurrentPosition = preservedCurrentPosition; + throw; + } + + return NBytes; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::GetPosition() +// Purpose: Returns current position +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileRead_Raid::GetPosition() const +{ + return mCurrentPosition; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::Seek(RaidFileRead::pos_type, bool) +// Purpose: Seek within the file +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +void RaidFileRead_Raid::Seek(IOStream::pos_type Offset, int SeekType) +{ + pos_type newpos = mCurrentPosition; + switch(SeekType) + { + case IOStream::SeekType_Absolute: + newpos = Offset; + break; + + case IOStream::SeekType_Relative: + newpos += Offset; + break; + + case IOStream::SeekType_End: + newpos = mFileSize + Offset; + break; + + default: + THROW_EXCEPTION(CommonException, IOStreamBadSeekType) + } + + if(newpos != mCurrentPosition) + { + SetPosition(newpos); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::SetPosition(pos_type) +// Purpose: Move the file pointers +// Created: 2003/07/14 +// +// -------------------------------------------------------------------------- +void RaidFileRead_Raid::SetPosition(pos_type FilePosition) +{ + if(FilePosition > mFileSize) + { + FilePosition = mFileSize; + } + + if(mStripe1Handle != -1 && mStripe2Handle != -1) + { + // right then... which block is it in? + pos_type block = FilePosition / mBlockSize; + pos_type offset = FilePosition % mBlockSize; + + // Calculate offsets for each file + pos_type basepos = (block / 2) * mBlockSize; + pos_type s1p, s2p; + if((block & 1) == 0) + { + s1p = basepos + offset; + s2p = basepos; + } + else + { + s1p = basepos + mBlockSize; + s2p = basepos + offset; + } + // Note: lseek isn't in the man pages to return EIO, but assuming that it can return this, + // as it calls various OS bits and returns their error codes, and those fns look like they might. + if(::lseek(mStripe1Handle, s1p, SEEK_SET) == -1) + { + if(errno == EIO) + { + TRACE3("I/O error when seeking in %d %s (to %d), stripe 1\n", mSetNumber, mFilename.c_str(), (int)FilePosition); + ::syslog(LOG_ERR | LOG_LOCAL5, "I/O error when seeking in %d %s (to %d), stripe 1\n", mSetNumber, mFilename.c_str(), (int)FilePosition); + // Attempt to recover + AttemptToRecoverFromIOError(true /* is stripe 1 */); + ASSERT(mStripe1Handle == -1); + // Retry + SetPosition(FilePosition); + return; + } + else + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + if(::lseek(mStripe2Handle, s2p, SEEK_SET) == -1) + { + if(errno == EIO) + { + TRACE3("I/O error when seeking in %d %s (to %d), stripe 2\n", mSetNumber, mFilename.c_str(), (int)FilePosition); + ::syslog(LOG_ERR | LOG_LOCAL5, "I/O error when seeking in %d %s (to %d), stripe 2\n", mSetNumber, mFilename.c_str(), (int)FilePosition); + // Attempt to recover + AttemptToRecoverFromIOError(false /* is stripe 2 */); + ASSERT(mStripe2Handle == -1); + // Retry + SetPosition(FilePosition); + return; + } + else + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + // Store position + mCurrentPosition = FilePosition; + } + else + { + // Simply store, and mark the recovery buffer invalid + mCurrentPosition = FilePosition; + mRecoveryBufferStart = -1; + } + + // not EOF any more + mEOF = false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::Close() +// Purpose: Close the file (automatically done by destructor) +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +void RaidFileRead_Raid::Close() +{ + if(mStripe1Handle != -1) + { + ::close(mStripe1Handle); + mStripe1Handle = -1; + } + if(mStripe2Handle != -1) + { + ::close(mStripe2Handle); + mStripe2Handle = -1; + } + if(mParityHandle != -1) + { + ::close(mParityHandle); + mParityHandle = -1; + } + + mEOF = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::StreamDataLeft() +// Purpose: Any data left? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool RaidFileRead_Raid::StreamDataLeft() +{ + return !mEOF; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::GetFileSize() +// Purpose: Returns file size. +// Created: 2003/07/14 +// +// -------------------------------------------------------------------------- +RaidFileRead::pos_type RaidFileRead_Raid::GetFileSize() const +{ + return mFileSize; +} + + +// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::RaidFileRead(int, const std::string &) +// Purpose: Constructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead::RaidFileRead(int SetNumber, const std::string &Filename) + : mSetNumber(SetNumber), + mFilename(Filename) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::~RaidFileRead() +// Purpose: Destructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead::~RaidFileRead() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::Open(int, const std::string &, int) +// Purpose: Opens a RaidFile for reading. +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string &Filename, int64_t *pRevisionID, int BufferSizeHint) +{ + // See what's available... + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size() && 1 != rdiscSet.size()) // allow non-RAID configurations + { + THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet) + } + + // See if the file exists + int startDisc = 0, existingFiles = 0; + RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, Filename, &startDisc, &existingFiles, pRevisionID); + if(existance == RaidFileUtil::NoFile) + { + THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist) + } + else if(existance == RaidFileUtil::NonRaid) + { + // Simple non-RAID file so far... + + // Get the filename for the write file + std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, Filename)); + + // Attempt to open + int osFileHandle = ::open(writeFilename.c_str(), O_RDONLY, 0); + if(osFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, ErrorOpeningFileForRead) + } + + // Return a read object for this file + try + { + return std::auto_ptr<RaidFileRead>(new RaidFileRead_NonRaid(SetNumber, Filename, osFileHandle)); + } + catch(...) + { + ::close(osFileHandle); + throw; + } + } + else if(existance == RaidFileUtil::AsRaid + || ((existingFiles & RaidFileUtil::Stripe1Exists) && (existingFiles & RaidFileUtil::Stripe2Exists))) + { + if(existance != RaidFileUtil::AsRaid) + { + TRACE2("Opening %d %s in normal mode, but parity file doesn't exist\n", SetNumber, Filename.c_str()); + ::syslog(LOG_ERR | LOG_LOCAL5, "Opening %d %s in normal mode, but parity file doesn't exist\n", SetNumber, Filename.c_str()); + // TODO: Alert recovery daemon + } + + // Open the two stripe files + std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (0 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (1 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + int stripe1 = -1; + int stripe1errno = 0; + int stripe2 = -1; + int stripe2errno = 0; + + try + { + // Open stripe1 + stripe1 = ::open(stripe1Filename.c_str(), O_RDONLY, 0555); + if(stripe1 == -1) + { + stripe1errno = errno; + } + // Open stripe2 + stripe2 = ::open(stripe2Filename.c_str(), O_RDONLY, 0555); + if(stripe2 == -1) + { + stripe2errno = errno; + } + if(stripe1errno != 0 || stripe2errno != 0) + { + THROW_EXCEPTION(RaidFileException, ErrorOpeningFileForRead) + } + + // stat stripe 1 to find ('half' of) length... + struct stat st; + if(::fstat(stripe1, &st) != 0) + { + stripe1errno = errno; + } + pos_type length = st.st_size; + + // stat stripe2 to find (other 'half' of) length... + if(::fstat(stripe2, &st) != 0) + { + stripe2errno = errno; + } + length += st.st_size; + + // Handle errors + if(stripe1errno != 0 || stripe2errno != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Make a nice object to represent this file + return std::auto_ptr<RaidFileRead>(new RaidFileRead_Raid(SetNumber, Filename, stripe1, stripe2, -1, length, rdiscSet.GetBlockSize(), false /* actually we don't know */)); + } + catch(...) + { + // Close open files + if(stripe1 != -1) + { + ::close(stripe1); + stripe1 = -1; + } + if(stripe2 != -1) + { + ::close(stripe2); + stripe2 = -1; + } + + // Now... maybe we can try again with one less file? + bool oktotryagain = true; + if(stripe1errno == EIO) + { + TRACE2("I/O error on opening %d %s stripe 1, trying recovery mode\n", SetNumber, Filename.c_str()); + ::syslog(LOG_ERR | LOG_LOCAL5, "I/O error on opening %d %s stripe 1, trying recovery mode\n", SetNumber, Filename.c_str()); + RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, true /* is stripe 1 */); + + existingFiles = existingFiles & ~RaidFileUtil::Stripe1Exists; + existance = (existance == RaidFileUtil::AsRaidWithMissingReadable) + ?RaidFileUtil::AsRaidWithMissingNotRecoverable + :RaidFileUtil::AsRaidWithMissingReadable; + } + else if(stripe1errno != 0) + { + oktotryagain = false; + } + + if(stripe2errno == EIO) + { + TRACE2("I/O error on opening %d %s stripe 2, trying recovery mode\n", SetNumber, Filename.c_str()); + ::syslog(LOG_ERR | LOG_LOCAL5, "I/O error on opening %d %s stripe 2, trying recovery mode\n", SetNumber, Filename.c_str()); + RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, false /* is stripe 2 */); + + existingFiles = existingFiles & ~RaidFileUtil::Stripe2Exists; + existance = (existance == RaidFileUtil::AsRaidWithMissingReadable) + ?RaidFileUtil::AsRaidWithMissingNotRecoverable + :RaidFileUtil::AsRaidWithMissingReadable; + } + else if(stripe2errno != 0) + { + oktotryagain = false; + } + + if(!oktotryagain) + { + throw; + } + } + } + + if(existance == RaidFileUtil::AsRaidWithMissingReadable) + { + TRACE3("Attempting to open RAID file %d %s in recovery mode (stripe %d present)\n", SetNumber, Filename.c_str(), (existingFiles & RaidFileUtil::Stripe1Exists)?1:2); + ::syslog(LOG_ERR | LOG_LOCAL5, "Attempting to open RAID file %d %s in recovery mode (stripe %d present)\n", SetNumber, Filename.c_str(), (existingFiles & RaidFileUtil::Stripe1Exists)?1:2); + + // Generate the filenames of all the lovely files + std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (0 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (1 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (2 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + + int stripe1 = -1; + int stripe2 = -1; + int parity = -1; + + try + { + // Open stripe1? + if(existingFiles & RaidFileUtil::Stripe1Exists) + { + stripe1 = ::open(stripe1Filename.c_str(), O_RDONLY, 0555); + if(stripe1 == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + // Open stripe2? + if(existingFiles & RaidFileUtil::Stripe2Exists) + { + stripe2 = ::open(stripe2Filename.c_str(), O_RDONLY, 0555); + if(stripe2 == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + // Open parity + parity = ::open(parityFilename.c_str(), O_RDONLY, 0555); + if(parity == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Find the length. This is slightly complex. + unsigned int blockSize = rdiscSet.GetBlockSize(); + pos_type length = 0; + + // The easy one... if the parity file is of an integral block size + sizeof(FileSizeType) + // then it's stored at the end of the parity file + struct stat st; + if(::fstat(parity, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + pos_type paritySize = st.st_size; + FileSizeType parityLastData = 0; + bool parityIntegralPlusOffT = ((paritySize % blockSize) == sizeof(FileSizeType)); + if(paritySize >= static_cast<pos_type>(sizeof(parityLastData)) && (parityIntegralPlusOffT || stripe1 != -1)) + { + // Seek to near the end + ASSERT(sizeof(FileSizeType) == 8); // compiler bug (I think) prevents from using 0 - sizeof(FileSizeType)... + if(::lseek(parity, -8 /*(0 - sizeof(FileSizeType))*/, SEEK_END) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Read it in + if(::read(parity, &parityLastData, sizeof(parityLastData)) != sizeof(parityLastData)) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Set back to beginning of file + if(::lseek(parity, 0, SEEK_SET) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + bool lastBlockHasSize = false; + if(parityIntegralPlusOffT) + { + // Wonderful! Have the value + length = ntoh64(parityLastData); + } + else + { + // Have to resort to more devious means. + if(existingFiles & RaidFileUtil::Stripe1Exists) + { + // Procedure for stripe 1 existence... + // Get size of stripe1. + // If this is not an integral block size, then size can use this + // to work out the size of the file. + // Otherwise, read in the end of the last block, and use a bit of XORing + // to get the size from the FileSizeType value at end of the file. + if(::fstat(stripe1, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + pos_type stripe1Size = st.st_size; + // Is size integral? + if((stripe1Size % ((pos_type)blockSize)) != 0) + { + // No, so know the size. + length = stripe1Size + ((stripe1Size / blockSize) * blockSize); + } + else + { + // Must read the last bit of data from the block and XOR. + FileSizeType stripe1LastData = 0; // initialise to zero, as we may not read everything from it + + // Work out how many bytes to read + int btr = 0; // bytes to read off end + unsigned int lbs = stripe1Size % blockSize; + if(lbs == 0 && stripe1Size > 0) + { + // integral size, need the entire bit + btr = sizeof(FileSizeType); + } + else if(lbs > (blockSize - sizeof(FileSizeType))) + { + btr = lbs - (blockSize - sizeof(FileSizeType)); + } + + // Seek to near the end + if(btr > 0) + { + ASSERT(sizeof(FileSizeType) == 8); // compiler bug (I think) prevents from using 0 - sizeof(FileSizeType)... + ASSERT(btr <= (int)sizeof(FileSizeType)); + if(::lseek(stripe1, 0 - btr, SEEK_END) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Read it in + if(::read(stripe1, &stripe1LastData, btr) != btr) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Set back to beginning of file + if(::lseek(stripe1, 0, SEEK_SET) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + // Lovely! + length = stripe1LastData ^ parityLastData; + // Convert to host byte order + length = ntoh64(length); + ASSERT(length <= (paritySize + stripe1Size)); + // Mark is as having this to aid code later + lastBlockHasSize = true; + } + } + else + { + ASSERT(existingFiles & RaidFileUtil::Stripe2Exists); + } + + if(existingFiles & RaidFileUtil::Stripe2Exists) + { + // Get size of stripe2 file + if(::fstat(stripe2, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + pos_type stripe2Size = st.st_size; + + // Is it an integral size? + if(stripe2Size % blockSize != 0) + { + // No. Working out the size is easy. + length = stripe2Size + (((stripe2Size / blockSize)+1) * blockSize); + // Got last block size in there? + if((stripe2Size % blockSize) <= static_cast<pos_type>((blockSize - sizeof(pos_type)))) + { + // Yes... + lastBlockHasSize = true; + } + } + else + { + // Yes. So we need to compare with the parity file to get a clue... + pos_type stripe2Blocks = stripe2Size / blockSize; + pos_type parityBlocks = paritySize / blockSize; + if(stripe2Blocks == parityBlocks) + { + // Same size, so stripe1 must be the same size + length = (stripe2Blocks * 2) * blockSize; + } + else + { + // Different size, so stripe1 must be one block bigger + ASSERT(stripe2Blocks < parityBlocks); + length = ((stripe2Blocks * 2)+1) * blockSize; + } + + // Then... add in the extra bit of the parity length + unsigned int lastBlockSize = paritySize % blockSize; + length += lastBlockSize; + } + } + else + { + ASSERT(existingFiles & RaidFileUtil::Stripe1Exists); + } + } + + // Create a lovely object to return + return std::auto_ptr<RaidFileRead>(new RaidFileRead_Raid(SetNumber, Filename, stripe1, stripe2, parity, length, blockSize, lastBlockHasSize)); + } + catch(...) + { + // Close open files + if(stripe1 != -1) + { + ::close(stripe1); + stripe1 = -1; + } + if(stripe2 != -1) + { + ::close(stripe2); + stripe2 = -1; + } + if(parity != -1) + { + ::close(parity); + parity = -1; + } + throw; + } + } + + THROW_EXCEPTION(RaidFileException, FileIsDamagedNotRecoverable) + + // Avoid compiler warning -- it'll never get here... + return std::auto_ptr<RaidFileRead>(); +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::DirectoryExists(int, const std::string &) +// Purpose: Returns true if the directory exists. Throws exception if it's partially in existence. +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +bool RaidFileRead::DirectoryExists(int SetNumber, const std::string &rDirName) +{ + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + + return DirectoryExists(rdiscSet, rDirName); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::DirectoryExists(const RaidFileDiscSet &, const std::string &) +// Purpose: Returns true if the directory exists. Throws exception if it's partially in existence. +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +bool RaidFileRead::DirectoryExists(const RaidFileDiscSet &rSet, const std::string &rDirName) +{ + // For each directory, test to see if it exists + unsigned int nexist = 0; + for(unsigned int l = 0; l < rSet.size(); ++l) + { + // build name + std::string dn(rSet[l] + DIRECTORY_SEPARATOR + rDirName); + + // check for existence + struct stat st; + if(::stat(dn.c_str(), &st) == 0) + { + // Directory? + if(st.st_mode & S_IFDIR) + { + // yes + nexist++; + } + else + { + // No. It's a file. Bad! + THROW_EXCEPTION(RaidFileException, UnexpectedFileInDirPlace) + } + } + else + { + // Was it a non-exist error? + if(errno != ENOENT) + { + // No. Bad things. + THROW_EXCEPTION(RaidFileException, OSError) + } + } + } + + // Were all of them found? + if(nexist == 0) + { + // None. + return false; + } + else if(nexist == rSet.size()) + { + // All + return true; + } + + // Some exist. We don't like this -- it shows something bad happened before + // TODO: notify recovery daemon + THROW_EXCEPTION(RaidFileException, DirectoryIncomplete) + return false; // avoid compiler warning +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::FileExists(int, const std::string &, int64_t *) +// Purpose: Does a Raid file exist? Optionally return a revision number, which is +// unique to this saving of the file. (revision number may change +// after transformation to RAID -- so only use for cache control, +// not detecting changes to content). +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +bool RaidFileRead::FileExists(int SetNumber, const std::string &rFilename, int64_t *pRevisionID) +{ + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + + return RaidFileUtil::RaidFileExists(rdiscSet, rFilename, 0, 0, pRevisionID) != RaidFileUtil::NoFile; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadDirectoryContents(int, const std::string &, int, std::vector<std::string> &) +// Purpose: Read directory contents, returning whether or not all entries are likely to be readable or not +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector<std::string> &rOutput) +{ + // Remove anything in the vector to begin with. + rOutput.clear(); + + // Controller and set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + + // Collect the directory listings + std::map<std::string, unsigned int> counts; + + unsigned int numDiscs = rdiscSet.size(); + + for(unsigned int l = 0; l < numDiscs; ++l) + { + // build name + std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName); + + // read the contents... + DIR *dirHandle = 0; + try + { + dirHandle = ::opendir(dn.c_str()); + if(dirHandle == 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + struct dirent *en = 0; + while((en = ::readdir(dirHandle)) != 0) + { +#ifdef PLATFORM_LINUX + LinuxWorkaround_FinishDirentStruct(en, dn.c_str()); +#endif + + if(en->d_name[0] == '.' && + (en->d_name[1] == '\0' || (en->d_name[1] == '.' && en->d_name[2] == '\0'))) + { + // ignore, it's . or .. + continue; + } + + // stat the file to find out what type it is +/* struct stat st; + std::string fullName(dn + DIRECTORY_SEPARATOR + en->d_name); + if(::stat(fullName.c_str(), &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + }*/ + + // Entry... + std::string name; + unsigned int countToAdd = 1; + if(DirReadType == DirReadType_FilesOnly && en->d_type == DT_REG) // (st.st_mode & S_IFDIR) == 0) + { + // File. Complex, need to check the extension + int dot = -1; + int p = 0; + while(en->d_name[p] != '\0') + { + if(en->d_name[p] == '.') + { + // store location of dot + dot = p; + } + ++p; + } + // p is length of string + if(dot != -1 && ((p - dot) == 3 || (p - dot) == 4) + && en->d_name[dot+1] == 'r' && en->d_name[dot+2] == 'f' + && (en->d_name[dot+3] == 'w' || en->d_name[dot+3] == '\0')) + { + // so has right extension + name.assign(en->d_name, dot); /* get name up to last . */ + // Was it a write file (which counts as everything) + if(en->d_name[dot+3] == 'w') + { + countToAdd = numDiscs; + } + } + } + if(DirReadType == DirReadType_DirsOnly && en->d_type == DT_DIR) // (st.st_mode & S_IFDIR)) + { + // Directory, and we want directories + name = en->d_name; + } + // Eligable for entry? + if(!name.empty()) + { + // add to map... + std::map<std::string, unsigned int>::iterator i = counts.find(name); + if(i != counts.end()) + { + // add to count + i->second += countToAdd; + } + else + { + // insert into map + counts[name] = countToAdd; + } + } + } + + if(::closedir(dirHandle) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + dirHandle = 0; + } + catch(...) + { + if(dirHandle != 0) + { + ::closedir(dirHandle); + } + throw; + } + } + + // Now go through the map, adding in entries + bool everythingReadable = true; + + for(std::map<std::string, unsigned int>::const_iterator i = counts.begin(); i != counts.end(); ++i) + { + if(i->second < (numDiscs - 1)) + { + // Too few discs to be confident of reading everything + everythingReadable = false; + } + + // Add name to vector + rOutput.push_back(i->first); + } + + return everythingReadable; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::Write(const void *, int) +// Purpose: Not support, throws exception +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void RaidFileRead::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(RaidFileException, UnsupportedReadWriteOrClose) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::StreamClosed() +// Purpose: Never any data to write +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool RaidFileRead::StreamClosed() +{ + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::BytesLeftToRead() +// Purpose: Can tell how many bytes there are to go +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileRead::BytesLeftToRead() +{ + return GetFileSize() - GetPosition(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::GetDiscUsageInBlocks() +// Purpose: Return how many blocks are used. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileRead::GetDiscUsageInBlocks() +{ + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + return RaidFileUtil::DiscUsageInBlocks(GetFileSize(), rdiscSet); +} + + + + diff --git a/lib/raidfile/RaidFileRead.h b/lib/raidfile/RaidFileRead.h new file mode 100755 index 00000000..93bf7388 --- /dev/null +++ b/lib/raidfile/RaidFileRead.h @@ -0,0 +1,72 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileRead.h +// Purpose: Read Raid like Files +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- + +#ifndef RAIDFILEREAD__H +#define RAIDFILEREAD__H + +#include <string> +#include <memory> +#include <vector> + +#include "IOStream.h" + +class RaidFileDiscSet; + + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileRead +// Purpose: Read RAID like files +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +class RaidFileRead : public IOStream +{ +protected: + RaidFileRead(int SetNumber, const std::string &Filename); +public: + virtual ~RaidFileRead(); +private: + RaidFileRead(const RaidFileRead &rToCopy); + +public: + // Open a raid file + static std::auto_ptr<RaidFileRead> Open(int SetNumber, const std::string &Filename, int64_t *pRevisionID = 0, int BufferSizeHint = 4096); + + // Extra info + virtual pos_type GetFileSize() const = 0; + + // Utility functions + static bool FileExists(int SetNumber, const std::string &rFilename, int64_t *pRevisionID = 0); + static bool DirectoryExists(const RaidFileDiscSet &rSet, const std::string &rDirName); + static bool DirectoryExists(int SetNumber, const std::string &rDirName); + enum + { + DirReadType_FilesOnly = 0, + DirReadType_DirsOnly = 1 + }; + static bool ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector<std::string> &rOutput); + + // Common IOStream interface implementation + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamClosed(); + virtual pos_type BytesLeftToRead(); + + pos_type GetDiscUsageInBlocks(); + + typedef int64_t FileSizeType; + +protected: + int mSetNumber; + std::string mFilename; +}; + +#endif // RAIDFILEREAD__H + diff --git a/lib/raidfile/RaidFileUtil.cpp b/lib/raidfile/RaidFileUtil.cpp new file mode 100755 index 00000000..9177c8ba --- /dev/null +++ b/lib/raidfile/RaidFileUtil.cpp @@ -0,0 +1,188 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileUtil.cpp +// Purpose: Utilities for raid files +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <sys/stat.h> + +#include "RaidFileUtil.h" +#include "FileModificationTime.h" +#include "RaidFileRead.h" // for type definition + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileUtil::RaidFileExists(RaidFileDiscSet &, const std::string &) +// Purpose: Check to see the state of a RaidFile on disc (doesn't look at contents, +// just at existense of files) +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- +RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pStartDisc, int *pExisitingFiles, int64_t *pRevisionID) +{ + if(pExisitingFiles) + { + *pExisitingFiles = 0; + } + + // For stat call, although the results are not examined + struct stat st; + + // check various files + int startDisc = 0; + { + std::string writeFile(RaidFileUtil::MakeWriteFileName(rDiscSet, rFilename, &startDisc)); + if(pStartDisc) + { + *pStartDisc = startDisc; + } + if(::stat(writeFile.c_str(), &st) == 0) + { + // write file exists, use that + + // Get unique ID + if(pRevisionID != 0) + { + (*pRevisionID) = FileModificationTime(st); +#ifdef PLATFORM_LINUX + // On linux, the time resolution is very low for modification times. + // So add the size to it to give a bit more chance of it changing. + // TODO: Make this better. + (*pRevisionID) += st.st_size; +#endif + } + + // return non-raid file + return NonRaid; + } + } + + // Now see how many of the raid components exist + int64_t revisionID = 0; + int setSize = rDiscSet.size(); + int rfCount = 0; +#ifdef PLATFORM_LINUX + // TODO: replace this with better linux revision ID detection + int64_t revisionIDplus = 0; +#endif + for(int f = 0; f < setSize; ++f) + { + std::string componentFile(RaidFileUtil::MakeRaidComponentName(rDiscSet, rFilename, (f + startDisc) % setSize)); + if(::stat(componentFile.c_str(), &st) == 0) + { + // Component file exists, add to count + rfCount++; + // Set flags for existance? + if(pExisitingFiles) + { + (*pExisitingFiles) |= (1 << f); + } + // Revision ID + if(pRevisionID != 0) + { + int64_t rid = FileModificationTime(st); + if(rid > revisionID) revisionID = rid; +#ifdef PLATFORM_LINUX + revisionIDplus += st.st_size; +#endif + } + } + } + if(pRevisionID != 0) + { + (*pRevisionID) = revisionID; +#ifdef PLATFORM_LINUX + (*pRevisionID) += revisionIDplus; +#endif + } + + // Return a status based on how many parts are available + if(rfCount == setSize) + { + return AsRaid; + } + else if((setSize > 1) && rfCount == (setSize - 1)) + { + return AsRaidWithMissingReadable; + } + else if(rfCount > 0) + { + return AsRaidWithMissingNotRecoverable; + } + + return NoFile; // Obviously doesn't exist +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileUtil::DiscUsageInBlocks(int64_t, const RaidFileDiscSet &) +// Purpose: Returns the size of the file in blocks, given the file size and disc set +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +int64_t RaidFileUtil::DiscUsageInBlocks(int64_t FileSize, const RaidFileDiscSet &rDiscSet) +{ + // Get block size + int blockSize = rDiscSet.GetBlockSize(); + + // OK... so as the size of the file is always sizes of stripe1 + stripe2, we can + // do a very simple calculation for the main data. + int64_t blocks = (FileSize + (((int64_t)blockSize) - 1)) / ((int64_t)blockSize); + + // It's just that simple calculation for non-RAID disc sets + if(rDiscSet.IsNonRaidSet()) + { + return blocks; + } + + // It's the parity which is mildly complex. + // First of all, add in size for all but the last two blocks. + int64_t parityblocks = (FileSize / ((int64_t)blockSize)) / 2; + blocks += parityblocks; + + // Work out how many bytes are left + int bytesOver = (int)(FileSize - (parityblocks * ((int64_t)(blockSize*2)))); + + // Then... (let compiler optimise this out) + if(bytesOver == 0) + { + // Extra block for the size info + blocks++; + } + else if(bytesOver == sizeof(RaidFileRead::FileSizeType)) + { + // For last block of parity, plus the size info + blocks += 2; + } + else if(bytesOver < blockSize) + { + // Just want the parity block + blocks += 1; + } + else if(bytesOver == blockSize || bytesOver >= ((blockSize*2)-((int)sizeof(RaidFileRead::FileSizeType)))) + { + // Last block, plus size info + blocks += 2; + } + else + { + // Just want parity block + blocks += 1; + } + + return blocks; +} + + diff --git a/lib/raidfile/RaidFileUtil.h b/lib/raidfile/RaidFileUtil.h new file mode 100755 index 00000000..16670bf1 --- /dev/null +++ b/lib/raidfile/RaidFileUtil.h @@ -0,0 +1,97 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileUtil.h +// Purpose: Utilities for the raid file classes +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- + +#ifndef RAIDFILEUTIL__H +#define RAIDFILEUTIL__H + +#include "RaidFileController.h" +#include "RaidFileException.h" + +// note: these are hardcoded into the directory searching code +#define RAIDFILE_EXTENSION ".rf" +#define RAIDFILE_WRITE_EXTENSION ".rfw" + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileUtil +// Purpose: Utility functions for RaidFile classes +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- +class RaidFileUtil +{ +public: + typedef enum + { + NoFile = 0, + NonRaid = 1, + AsRaid = 2, + AsRaidWithMissingReadable = 3, + AsRaidWithMissingNotRecoverable = 4 + } ExistType; + + typedef enum + { + Stripe1Exists = 1, + Stripe2Exists = 2, + ParityExists = 4 + }; + + static ExistType RaidFileExists(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pStartDisc = 0, int *pExisitingFiles = 0, int64_t *pRevisionID = 0); + + static int64_t DiscUsageInBlocks(int64_t FileSize, const RaidFileDiscSet &rDiscSet); + + // -------------------------------------------------------------------------- + // + // Function + // Name: std::string MakeRaidComponentName(RaidFileDiscSet &, const std::string &, int) + // Purpose: Returns the OS filename for a file of part of a disc set + // Created: 2003/07/11 + // + // -------------------------------------------------------------------------- + static inline std::string MakeRaidComponentName(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int Disc) + { + if(Disc < 0 || Disc >= (int)rDiscSet.size()) + { + THROW_EXCEPTION(RaidFileException, NoSuchDiscSet) + } + std::string r(rDiscSet[Disc]); + r += DIRECTORY_SEPARATOR_ASCHAR; + r += rFilename; + r += RAIDFILE_EXTENSION; + return r; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: std::string MakeWriteFileName(RaidFileDiscSet &, const std::string &) + // Purpose: Returns the OS filename for the temporary write file + // Created: 2003/07/11 + // + // -------------------------------------------------------------------------- + static inline std::string MakeWriteFileName(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pOnDiscSet = 0) + { + int livesOnSet = rDiscSet.GetSetNumForWriteFiles(rFilename); + + // does the caller want to know which set it's on? + if(pOnDiscSet) *pOnDiscSet = livesOnSet; + + // Make the string + std::string r(rDiscSet[livesOnSet]); + r += DIRECTORY_SEPARATOR_ASCHAR; + r += rFilename; + r += RAIDFILE_WRITE_EXTENSION; + return r; + } +}; + +#endif // RAIDFILEUTIL__H + diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp new file mode 100755 index 00000000..414f24e6 --- /dev/null +++ b/lib/raidfile/RaidFileWrite.cpp @@ -0,0 +1,817 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileWrite.cpp +// Purpose: Writing RAID like files +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <errno.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <sys/file.h> + +#include <stdio.h> +#include <string.h> + +#include "Guards.h" +#include "RaidFileWrite.h" +#include "RaidFileController.h" +#include "RaidFileException.h" +#include "RaidFileUtil.h" +#include "Utils.h" +// For DirectoryExists fn +#include "RaidFileRead.h" + +#include "MemLeakFindOn.h" + +// should be a multiple of 2 +#define TRANSFORM_BLOCKS_TO_LOAD 4 +// Must have this number of discs in the set +#define TRANSFORM_NUMBER_DISCS_REQUIRED 3 + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::RaidFileWrite(int, const std::string &) +// Purpose: Construtor, just stores requried details +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +RaidFileWrite::RaidFileWrite(int SetNumber, const std::string &Filename) + : mSetNumber(SetNumber), + mFilename(Filename), + mOSFileHandle(-1) // not valid file handle +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::~RaidFileWrite() +// Purpose: Destructor (will discard written file if not commited) +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +RaidFileWrite::~RaidFileWrite() +{ + if(mOSFileHandle != -1) + { + Discard(); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Open() +// Purpose: Opens the file for writing +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Open(bool AllowOverwrite) +{ + if(mOSFileHandle != -1) + { + THROW_EXCEPTION(RaidFileException, AlreadyOpen) + } + + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + + // Check for overwriting? (step 1) + if(!AllowOverwrite) + { + // See if the file exists already -- can't overwrite existing files + RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename); + if(existance != RaidFileUtil::NoFile) + { + TRACE2("Trying to overwrite raidfile %d %s\n", mSetNumber, mFilename.c_str()); + THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile) + } + } + + // Get the filename for the write file + std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename)); + // Add on a temporary extension + writeFilename += 'X'; + + // Attempt to open + mOSFileHandle = ::open(writeFilename.c_str(), O_WRONLY | O_CREAT, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFile) + } + + // Get a lock on the write file + if(::flock(mOSFileHandle, LOCK_EX | LOCK_NB) != 0) + { + // Lock was not obtained. + bool wasLocked = (errno == EWOULDBLOCK); + // Close the file + ::close(mOSFileHandle); + mOSFileHandle = -1; + // Report an exception? + if(wasLocked) + { + THROW_EXCEPTION(RaidFileException, FileIsCurrentlyOpenForWriting) + } + else + { + // Random error occured + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + // Truncate it to size zero + if(::ftruncate(mOSFileHandle, 0) != 0) + { + THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFileOnTruncate) + } + + // Done! +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Write(const void *, int) +// Purpose: Writes a block of data +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Write(const void *pBuffer, int Length) +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Write data + int written = ::write(mOSFileHandle, pBuffer, Length); + if(written != Length) + { + TRACE3("RaidFileWrite::Write: Write failure, Length = %d, written = %d, errno = %d\n", Length, written, errno); + THROW_EXCEPTION(RaidFileException, OSError) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::GetPosition() +// Purpose: Returns current position in file +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileWrite::GetPosition() const +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Use lseek to find the current file position + off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR); + if(p == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + return p; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Seek(RaidFileWrite::pos_type, bool) +// Purpose: Seeks in the file, relative to current position if Relative is true. +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Seek(IOStream::pos_type SeekTo, int SeekType) +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Seek... + if(::lseek(mOSFileHandle, SeekTo, ConvertSeekTypeToOSWhence(SeekType)) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Commit(bool) +// Purpose: Closes, and commits the written file +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Commit(bool ConvertToRaidNow) +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Rename it into place -- BEFORE it's closed so lock remains + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + // Get the filename for the write file + std::string renameTo(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename)); + // And the current name + std::string renameFrom(renameTo + 'X'); + if(::rename(renameFrom.c_str(), renameTo.c_str()) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Close file... + if(::close(mOSFileHandle) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + mOSFileHandle = -1; + + // Raid it? + if(ConvertToRaidNow) + { + TransformToRaidStorage(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Discard() +// Purpose: Closes, discarding the data written. +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Discard() +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + + // Get the filename for the write file (temporary) + std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename)); + writeFilename += 'X'; + + // Unlink and close it + if((::unlink(writeFilename.c_str()) != 0) + || (::close(mOSFileHandle) != 0)) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // reset file handle + mOSFileHandle = -1; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::TransformToRaidStorage() +// Purpose: Turns the file into the RAID storage form +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::TransformToRaidStorage() +{ + // open? + if(mOSFileHandle != -1) + { + THROW_EXCEPTION(RaidFileException, WriteFileOpenOnTransform) + } + + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + if(rdiscSet.IsNonRaidSet()) + { + // Not in RAID mode -- do nothing + return; + } + // Otherwise check that it's the right sized set + if(TRANSFORM_NUMBER_DISCS_REQUIRED != rdiscSet.size()) + { + THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet) + } + unsigned int blockSize = rdiscSet.GetBlockSize(); + + // Get the filename for the write file (and get the disc set name for the start disc) + int startDisc = 0; + std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename, &startDisc)); + + // Open it + FileHandleGuard<> writeFile(writeFilename.c_str()); + + // Get file information for write file + struct stat writeFileStat; + if(::fstat(writeFile, &writeFileStat) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } +// // DEBUG MODE -- check file system size block size is same as block size for files +// // doesn't really apply, as space benefits of using fragment size are worth efficiency, +// // and anyway, it'll be buffered eventually so it won't matter. +// #ifndef NDEBUG +// { +// if(writeFileStat.st_blksize != blockSize) +// { +// TRACE2("TransformToRaidStorage: optimal block size of file = %d, of set = %d, MISMATCH\n", +// writeFileStat.st_blksize, blockSize); +// } +// } +// #endif + + // How many blocks is the file? (rounding up) + int writeFileSizeInBlocks = (writeFileStat.st_size + (blockSize - 1)) / blockSize; + // And how big should the buffer be? (round up to multiple of 2, and no bigger than the preset limit) + int bufferSizeBlocks = (writeFileSizeInBlocks + 1) & ~1; + if(bufferSizeBlocks > TRANSFORM_BLOCKS_TO_LOAD) bufferSizeBlocks = TRANSFORM_BLOCKS_TO_LOAD; + // How big should the buffer be? + int bufferSize = (TRANSFORM_BLOCKS_TO_LOAD * blockSize); + + // Allocate buffer... + MemoryBlockGuard<char*> buffer(bufferSize); + + // Allocate buffer for parity file + MemoryBlockGuard<char*> parityBuffer(blockSize); + + // Get filenames of eventual files + std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 0) % TRANSFORM_NUMBER_DISCS_REQUIRED)); + std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 1) % TRANSFORM_NUMBER_DISCS_REQUIRED)); + std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 2) % TRANSFORM_NUMBER_DISCS_REQUIRED)); + // Make write equivalents + std::string stripe1FilenameW(stripe1Filename + 'P'); + std::string stripe2FilenameW(stripe2Filename + 'P'); + std::string parityFilenameW(parityFilename + 'P'); + + + // Then open them all for writing (in strict order) + try + { +#ifdef PLATFORM_LINUX + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> stripe1(stripe1FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> stripe2(stripe2FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> parity(parityFilenameW.c_str()); +#else + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> stripe1(stripe1FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> stripe2(stripe2FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> parity(parityFilenameW.c_str()); +#endif + + // Then... read in data... + int bytesRead = -1; + bool sizeRecordRequired = false; + int blocksDone = 0; + while((bytesRead = ::read(writeFile, buffer, bufferSize)) > 0) + { + // Blocks to do... + int blocksToDo = (bytesRead + (blockSize - 1)) / blockSize; + + // Need to add zeros to end? + int blocksRoundUp = (blocksToDo + 1) & ~1; + int zerosEnd = (blocksRoundUp * blockSize); + if(bytesRead != zerosEnd) + { + // Set the end of the blocks to zero + ::memset(buffer + bytesRead, 0, zerosEnd - bytesRead); + } + + // number of int's to XOR + unsigned int num = blockSize / sizeof(unsigned int); + + // Then... calculate and write parity data + for(int b = 0; b < blocksToDo; b += 2) + { + // Calculate int pointers + unsigned int *pstripe1 = (unsigned int *)(buffer + (b * blockSize)); + unsigned int *pstripe2 = (unsigned int *)(buffer + ((b+1) * blockSize)); + unsigned int *pparity = (unsigned int *)((char*)parityBuffer); + + // Do XOR + for(unsigned int n = 0; n < num; ++n) + { + pparity[n] = pstripe1[n] ^ pstripe2[n]; + } + + // Size of parity to write... + int parityWriteSize = blockSize; + + // Adjust if it's the last block + if((blocksDone + (b + 2)) >= writeFileSizeInBlocks) + { + // Yes... + unsigned int bytesInLastTwoBlocks = bytesRead - (b * blockSize); + + // Some special cases... + // Zero will never happen... but in the (imaginary) case it does, the file size will be appended + // by the test at the end. + if(bytesInLastTwoBlocks == sizeof(RaidFileRead::FileSizeType) + || bytesInLastTwoBlocks == blockSize) + { + // Write the entire block, and put the file size at end + sizeRecordRequired = true; + } + else if(bytesInLastTwoBlocks < blockSize) + { + // write only these bits + parityWriteSize = bytesInLastTwoBlocks; + } + else if(bytesInLastTwoBlocks < ((blockSize * 2) - sizeof(RaidFileRead::FileSizeType))) + { + // XOR in the size at the end of the parity block + ASSERT(sizeof(RaidFileRead::FileSizeType) == (2*sizeof(unsigned int))); +#ifdef PLATFORM_LINUX + ASSERT(sizeof(RaidFileRead::FileSizeType) >= sizeof(off_t)); +#else + ASSERT(sizeof(RaidFileRead::FileSizeType) == sizeof(off_t)); +#endif + int sizePos = (blockSize/sizeof(unsigned int)) - 2; + RaidFileRead::FileSizeType sw = hton64(writeFileStat.st_size); + unsigned int *psize = (unsigned int *)(&sw); + pparity[sizePos+0] = pstripe1[sizePos+0] ^ psize[0]; + pparity[sizePos+1] = pstripe1[sizePos+1] ^ psize[1]; + } + else + { + // Write the entire block, and put the file size at end + sizeRecordRequired = true; + } + } + + // Write block + if(::write(parity, parityBuffer, parityWriteSize) != parityWriteSize) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + // Write stripes + char *writeFrom = buffer; + for(int l = 0; l < blocksToDo; ++l) + { + // Write the block + int toWrite = (l == (blocksToDo - 1)) + ?(bytesRead - ((blocksToDo-1)*blockSize)) + :blockSize; + if(::write(((l&1)==0)?stripe1:stripe2, writeFrom, toWrite) != toWrite) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Next block + writeFrom += blockSize; + } + + // Count of blocks done + blocksDone += blocksToDo; + } + // Error on read? + if(bytesRead == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Special case for zero length files + if(writeFileStat.st_size == 0) + { + sizeRecordRequired = true; + } + + // Might need to write the file size to the end of the parity file + // if it can't be worked out some other means -- size is required to rebuild the file if one of the stripe files is missing + if(sizeRecordRequired) + { +#ifdef PLATFORM_LINUX + ASSERT(sizeof(writeFileStat.st_size) <= sizeof(RaidFileRead::FileSizeType)); +#else + ASSERT(sizeof(writeFileStat.st_size) == sizeof(RaidFileRead::FileSizeType)); +#endif + RaidFileRead::FileSizeType sw = hton64(writeFileStat.st_size); + ASSERT((::lseek(parity, 0, SEEK_CUR) % blockSize) == 0); + if(::write(parity, &sw, sizeof(sw)) != sizeof(sw)) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + // Then close the written files (note in reverse order of opening) + parity.Close(); + stripe2.Close(); + stripe1.Close(); + + // Rename them into place + if(::rename(stripe1FilenameW.c_str(), stripe1Filename.c_str()) != 0 + || ::rename(stripe2FilenameW.c_str(), stripe2Filename.c_str()) != 0 + || ::rename(parityFilenameW.c_str(), parityFilename.c_str()) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Close the write file + writeFile.Close(); + + // Finally delete the write file + if(::unlink(writeFilename.c_str()) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + catch(...) + { + // Unlink all the dodgy files + ::unlink(stripe1Filename.c_str()); + ::unlink(stripe2Filename.c_str()); + ::unlink(parityFilename.c_str()); + ::unlink(stripe1FilenameW.c_str()); + ::unlink(stripe2FilenameW.c_str()); + ::unlink(parityFilenameW.c_str()); + + // and send the error on its way + throw; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Delete() +// Purpose: Deletes a RAID file +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Delete() +{ + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + + // See if the file exists already -- can't delete files which don't exist + RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename); + if(existance == RaidFileUtil::NoFile) + { + THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist) + } + + // Get the filename for the write file + std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename)); + + // Attempt to delete it + bool deletedSomething = false; + if(::unlink(writeFilename.c_str()) == 0) + { + deletedSomething = true; + } + + // If we're not running in RAID mode, stop now + if(rdiscSet.size() == 1) + { + return; + } + + // Now the other files + std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 0 % TRANSFORM_NUMBER_DISCS_REQUIRED)); + std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 1 % TRANSFORM_NUMBER_DISCS_REQUIRED)); + std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 2 % TRANSFORM_NUMBER_DISCS_REQUIRED)); + if(::unlink(stripe1Filename.c_str()) == 0) + { + deletedSomething = true; + } + if(::unlink(stripe2Filename.c_str()) == 0) + { + deletedSomething = true; + } + if(::unlink(parityFilename.c_str()) == 0) + { + deletedSomething = true; + } + + // Check something happened + if(!deletedSomething) + { + THROW_EXCEPTION(RaidFileException, OSError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::CreateDirectory(int, const std::string &, bool, int) +// Purpose: Creates a directory within the raid file directories with the given name. +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::CreateDirectory(int SetNumber, const std::string &rDirName, bool Recursive, int mode) +{ + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + // Pass on... + CreateDirectory(rdiscSet, rDirName, Recursive, mode); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::CreateDirectory(const RaidFileDiscSet &, const std::string &, bool, int) +// Purpose: Creates a directory within the raid file directories with the given name. +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::CreateDirectory(const RaidFileDiscSet &rSet, const std::string &rDirName, bool Recursive, int mode) +{ + if(Recursive) + { + // split up string + std::vector<std::string> elements; + SplitString(rDirName, DIRECTORY_SEPARATOR_ASCHAR, elements); + + // Do each element in turn + std::string pn; + for(unsigned int e = 0; e < elements.size(); ++e) + { + // Only do this if the element has some text in it + if(elements[e].size() > 0) + { + pn += elements[e]; + if(!RaidFileRead::DirectoryExists(rSet, pn)) + { + CreateDirectory(rSet, pn, false, mode); + } + + // add separator + pn += DIRECTORY_SEPARATOR_ASCHAR; + } + } + + return; + } + + // Create a directory in every disc of the set + for(unsigned int l = 0; l < rSet.size(); ++l) + { + // build name + std::string dn(rSet[l] + DIRECTORY_SEPARATOR + rDirName); + + // attempt to create + if(::mkdir(dn.c_str(), mode) != 0) + { + if(errno == EEXIST) + { + // No. Bad things. + THROW_EXCEPTION(RaidFileException, FileExistsInDirectoryCreation) + } + else + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Read(void *, int, int) +// Purpose: Unsupported, will exception +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +int RaidFileWrite::Read(void *pBuffer, int NBytes, int Timeout) +{ + THROW_EXCEPTION(RaidFileException, UnsupportedReadWriteOrClose) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Close() +// Purpose: Close, discarding file. +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Close() +{ + TRACE0("Warning: RaidFileWrite::Close() called, discarding file\n"); + if(mOSFileHandle != -1) + { + Discard(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::StreamDataLeft() +// Purpose: Never any data left to read! +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool RaidFileWrite::StreamDataLeft() +{ + return false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::StreamClosed() +// Purpose: Is stream closed for writing? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool RaidFileWrite::StreamClosed() +{ + return mOSFileHandle == -1; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::GetFileSize() +// Purpose: Returns the size of the file written. +// Can only be used before the file is commited. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileWrite::GetFileSize() +{ + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, CanOnlyGetFileSizeBeforeCommit) + } + + // Stat to get size + struct stat st; + if(fstat(mOSFileHandle, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + return st.st_size; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::GetDiscUsageInBlocks() +// Purpose: Returns the amount of disc space used, in blocks. +// Can only be used before the file is commited. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileWrite::GetDiscUsageInBlocks() +{ + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, CanOnlyGetUsageBeforeCommit) + } + + // Stat to get size + struct stat st; + if(fstat(mOSFileHandle, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Then return calculation + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + return RaidFileUtil::DiscUsageInBlocks(st.st_size, rdiscSet); +} + + diff --git a/lib/raidfile/RaidFileWrite.h b/lib/raidfile/RaidFileWrite.h new file mode 100755 index 00000000..d7e51f21 --- /dev/null +++ b/lib/raidfile/RaidFileWrite.h @@ -0,0 +1,66 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileWrite.h +// Purpose: Writing RAID like files +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- + +#ifndef RAIDFILEWRITE__H +#define RAIDFILEWRITE__H + +#include <string> + +#include "IOStream.h" + +class RaidFileDiscSet; + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileWrite +// Purpose: Writing RAID like files +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +class RaidFileWrite : public IOStream +{ +public: + RaidFileWrite(int SetNumber, const std::string &Filename); + ~RaidFileWrite(); +private: + RaidFileWrite(const RaidFileWrite &rToCopy); + +public: + // IOStream interface + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); // will exception + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(pos_type Offset, int SeekType); + virtual void Close(); // will discard the file! Use commit instead. + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + // Extra bits + void Open(bool AllowOverwrite = false); + void Commit(bool ConvertToRaidNow = false); + void Discard(); + void TransformToRaidStorage(); + void Delete(); + pos_type GetFileSize(); + pos_type GetDiscUsageInBlocks(); + + static void CreateDirectory(int SetNumber, const std::string &rDirName, bool Recursive = false, int mode = 0777); + static void CreateDirectory(const RaidFileDiscSet &rSet, const std::string &rDirName, bool Recursive = false, int mode = 0777); + +private: + +private: + int mSetNumber; + std::string mFilename; + int mOSFileHandle; +}; + +#endif // RAIDFILEWRITE__H + diff --git a/lib/raidfile/raidfile-config b/lib/raidfile/raidfile-config new file mode 100755 index 00000000..b3f31077 --- /dev/null +++ b/lib/raidfile/raidfile-config @@ -0,0 +1,97 @@ +#!/usr/bin/perl +use strict; + +# should be running as root +if($> != 0) +{ + printf "\nWARNING: this should be run as root\n\n" +} + +# check and get command line parameters +if($#ARGV != 4 && $#ARGV != 2) +{ + print <<__E; + +Setup raidfile config utility. + +Bad command line parameters. +Usage: + raidfile-config config-dir block-size dir0 [dir1 dir2] + +config-dir usually /etc/box +block-size must be a power of two, and usually the block or fragment size of your filing system +dir0, dir1, dir2 are the directories used as the root of the raid file system + +If only one directory is specified, then userland RAID is disabled. Specifying three directories +enables it. + +__E + exit(1); +} + +my ($config_dir,$block_size,@dirs) = @ARGV; + +my $conf = $config_dir . '/raidfile.conf'; + +# check dirs are unique, and exist +my %d; +for(@dirs) +{ + die "$_ is used twice" if exists $d{$_}; + die "$_ is not a directory" unless -d $_; + die "$_ should be an absolute path" unless m/\A\//; + $d{$_} = 1; +} + +# check block size is OK +$block_size = int($block_size); +die "Bad block size" if $block_size <= 0; +my $c = 1; +while(1) +{ + last if $c == $block_size; + die "Block size $block_size is not a power of two" if $c > $block_size; + $c = $c * 2; +} + +# check that it doesn't already exist +if(-f $conf) +{ + die "$conf already exists. Delete and try again" +} + +# create directory +if(!-d $config_dir) +{ + print "Creating $config_dir...\n"; + mkdir $config_dir,0755 or die "Can't create $config_dir"; +} + +# adjust if userland RAID is disabled +if($#dirs == 0) +{ + $dirs[1] = $dirs[0]; + $dirs[2] = $dirs[0]; + print "WARNING: userland RAID is disabled.\n" +} + +# write the file +open CONFIG,">$conf" or die "Can't open $conf for writing"; + +print CONFIG <<__E; + +disc0 +{ + SetNumber = 0 + BlockSize = $block_size + Dir0 = $dirs[0] + Dir1 = $dirs[1] + Dir2 = $dirs[2] +} + +__E + +close CONFIG; + +print "Config file written.\n"; + diff --git a/lib/server/ConnectionException.txt b/lib/server/ConnectionException.txt new file mode 100644 index 00000000..5056754f --- /dev/null +++ b/lib/server/ConnectionException.txt @@ -0,0 +1,27 @@ +EXCEPTION Connection 7 + +# for historic reasons not all numbers are used + +SocketWriteError 6 Probably a network issue between client and server. +SocketReadError 7 Probably a network issue between client and server. +SocketNameLookupError 9 Check hostname specified. +SocketShutdownError 12 +SocketConnectError 15 Probably a network issue between client and server, bad hostname, or server not running. +TLSHandshakeFailed 30 +TLSShutdownFailed 32 +TLSWriteFailed 33 Probably a network issue between client and server. +TLSReadFailed 34 Probably a network issue between client and server. +TLSNoPeerCertificate 36 +TLSPeerCertificateInvalid 37 Check certification process +TLSClosedWhenWriting 38 +TLSHandshakeTimedOut 39 +Protocol_Timeout 41 Probably a network issue between client and server. +Protocol_ObjTooBig 42 +Protocol_BadCommandRecieved 44 +Protocol_UnknownCommandRecieved 45 +Protocol_TriedToExecuteReplyCommand 46 +Protocol_UnexpectedReply 47 Server probably reported an error. +Protocol_HandshakeFailed 48 +Protocol_StreamWhenObjExpected 49 +Protocol_ObjWhenStreamExpected 50 +Protocol_TimeOutWhenSendingStream 52 Probably a network issue between client and server. diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp new file mode 100755 index 00000000..31997fb6 --- /dev/null +++ b/lib/server/Daemon.cpp @@ -0,0 +1,530 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Daemon.cpp +// Purpose: Basic daemon functionality +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <unistd.h> +#include <syslog.h> +#include <signal.h> +#include <string.h> +#include <stdarg.h> + +#include "Daemon.h" +#include "Configuration.h" +#include "ServerException.h" +#include "Guards.h" +#include "UnixUser.h" + +#include "MemLeakFindOn.h" + +Daemon *Daemon::spDaemon = 0; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Daemon() +// Purpose: Constructor +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +Daemon::Daemon() + : mpConfiguration(0), + mReloadConfigWanted(false), + mTerminateWanted(false) +{ + if(spDaemon != 0) + { + THROW_EXCEPTION(ServerException, AlreadyDaemonConstructed) + } + spDaemon = this; + + // And in debug builds, we'll switch on assert failure logging to syslog + ASSERT_FAILS_TO_SYSLOG_ON + // And trace goes to syslog too + TRACE_TO_SYSLOG(true) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::~Daemon() +// Purpose: Destructor +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +Daemon::~Daemon() +{ + if(mpConfiguration) + { + delete mpConfiguration; + mpConfiguration = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Main(const char *, int, const char *[]) +// Purpose: Starts the daemon off -- equivalent of C main() function +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) +{ + // Banner (optional) + { + const char *banner = DaemonBanner(); + if(banner != 0) + { + printf("%s", banner); + } + } + + std::string pidFileName; + const char *configfile = 0; + + try + { + // Find filename of config file + configfile = DefaultConfigFile; + if(argc >= 2) + { + // First argument is config file, or it's -c and the next arg is the config file + if(::strcmp(argv[1], "-c") == 0 && argc >= 3) + { + configfile = argv[2]; + } + else + { + configfile = argv[1]; + } + } + + // Test mode with no daemonisation? + bool asDaemon = true; + if(argc >= 3) + { + if(::strcmp(argv[2], "SINGLEPROCESS") == 0) + { + asDaemon = false; + } + } + + // Load the configuration file. + std::string errors; + std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(configfile, GetConfigVerify(), errors); + + // Got errors? + if(pconfig.get() == 0 || !errors.empty()) + { + // Tell user about errors + fprintf(stderr, "%s: Errors in config file %s:\n%s", DaemonName(), configfile, errors.c_str()); + // And give up + return 1; + } + + // Store configuration + mpConfiguration = pconfig.release(); + + // Server configuration + const Configuration &serverConfig(mpConfiguration->GetSubConfiguration("Server")); + + // Let the derived class have a go at setting up stuff in the initial process + SetupInInitialProcess(); + + // Set signal handler + if(::signal(SIGHUP, SignalHandler) == SIG_ERR || ::signal(SIGTERM, SignalHandler) == SIG_ERR) + { + THROW_EXCEPTION(ServerException, DaemoniseFailed) + } + + // Open PID file for writing + pidFileName = serverConfig.GetKeyValue("PidFile"); + FileHandleGuard<(O_WRONLY | O_CREAT | O_TRUNC), (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)> pidFile(pidFileName.c_str()); + + // Handle changing to a different user + if(serverConfig.KeyExists("User")) + { + // Config file specifies an user -- look up + UnixUser daemonUser(serverConfig.GetKeyValue("User").c_str()); + + // Change the owner on the PID file, so it can be deleted properly on termination + if(::fchown(pidFile, daemonUser.GetUID(), daemonUser.GetGID()) != 0) + { + THROW_EXCEPTION(ServerException, CouldNotChangePIDFileOwner) + } + + // Change the process ID + daemonUser.ChangeProcessUser(); + } + + if(asDaemon) + { + // Let's go... Daemonise... + switch(::fork()) + { + case -1: + // error + THROW_EXCEPTION(ServerException, DaemoniseFailed) + break; + + default: + // parent + _exit(0); + return 0; + break; + + case 0: + // child + break; + } + + // In child + + // Set new session + if(::setsid() == -1) + { + ::syslog(LOG_ERR, "can't setsid"); + THROW_EXCEPTION(ServerException, DaemoniseFailed) + } + + // Fork again... + switch(::fork()) + { + case -1: + // error + THROW_EXCEPTION(ServerException, DaemoniseFailed) + break; + + default: + // parent + _exit(0); + return 0; + break; + + case 0: + // child + break; + } + } + + // open the log + ::openlog(DaemonName(), LOG_PID, LOG_LOCAL6); + // Log the start message + ::syslog(LOG_INFO, "Starting daemon (config: %s) (version " BOX_VERSION ")", configfile); + + // Write PID to file + char pid[32]; + int pidsize = sprintf(pid, "%d", (int)getpid()); + if(::write(pidFile, pid, pidsize) != pidsize) + { + ::syslog(LOG_ERR, "can't write pid file"); + THROW_EXCEPTION(ServerException, DaemoniseFailed) + } + + // Set up memory leak reporting + #ifdef BOX_MEMORY_LEAK_TESTING + { + char filename[256]; + sprintf(filename, "%s.memleaks", DaemonName()); + memleakfinder_setup_exit_report(filename, DaemonName()); + } + #endif // BOX_MEMORY_LEAK_TESTING + + if(asDaemon) + { + // Close standard streams + ::close(0); + ::close(1); + ::close(2); + + // Open and redirect them into /dev/null + int devnull = ::open(PLATFORM_DEV_NULL, O_RDWR, 0); + if(devnull == -1) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + // Then duplicate them to all three handles + if(devnull != 0) dup2(devnull, 0); + if(devnull != 1) dup2(devnull, 1); + if(devnull != 2) dup2(devnull, 2); + // Close the original handle if it was opened above the std* range + if(devnull > 2) + { + ::close(devnull); + } + + // And definately don't try and send anything to those file descriptors + // -- this has in the past sent text to something which isn't expecting it. + TRACE_TO_STDOUT(false); + } + } + catch(BoxException &e) + { + fprintf(stderr, "%s: exception %s (%d/%d)\n", DaemonName(), e.what(), e.GetType(), e.GetSubType()); + return 1; + } + catch(std::exception &e) + { + fprintf(stderr, "%s: exception %s\n", DaemonName(), e.what()); + return 1; + } + catch(...) + { + fprintf(stderr, "%s: unknown exception\n", DaemonName()); + return 1; + } + + // Main Daemon running + try + { + while(!mTerminateWanted) + { + Run(); + + if(mReloadConfigWanted && !mTerminateWanted) + { + // Need to reload that config file... + ::syslog(LOG_INFO, "Reloading configuration (config: %s)", configfile); + std::string errors; + std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(configfile, GetConfigVerify(), errors); + + // Got errors? + if(pconfig.get() == 0 || !errors.empty()) + { + // Tell user about errors + ::syslog(LOG_ERR, "Errors in config file %s:\n%s", configfile, errors.c_str()); + // And give up + return 1; + } + + // delete old configuration + delete mpConfiguration; + mpConfiguration = 0; + + // Store configuration + mpConfiguration = pconfig.release(); + + // Stop being marked for loading config again + mReloadConfigWanted = false; + } + } + + // Delete the PID file + ::unlink(pidFileName.c_str()); + + // Log + ::syslog(LOG_INFO, "Terminating daemon"); + } + catch(BoxException &e) + { + ::syslog(LOG_ERR, "exception %s (%d/%d) -- terminating", e.what(), e.GetType(), e.GetSubType()); + return 1; + } + catch(std::exception &e) + { + ::syslog(LOG_ERR, "exception %s -- terminating", e.what()); + return 1; + } + catch(...) + { + ::syslog(LOG_ERR, "unknown exception -- terminating"); + return 1; + } + + return 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::EnterChild() +// Purpose: Sets up for a child task of the main server. Call just after fork() +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void Daemon::EnterChild() +{ + // Unset signal handlers + ::signal(SIGHUP, SIG_DFL); + ::signal(SIGTERM, SIG_DFL); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::SignalHandler(int) +// Purpose: Signal handler +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +void Daemon::SignalHandler(int sigraised) +{ + if(spDaemon != 0) + { + switch(sigraised) + { + case SIGHUP: + spDaemon->mReloadConfigWanted = true; + break; + + case SIGTERM: + spDaemon->mTerminateWanted = true; + break; + + default: + break; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::DaemonName() +// Purpose: Returns name of the daemon +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +const char *Daemon::DaemonName() const +{ + return "generic-daemon"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::DaemonBanner() +// Purpose: Returns the text banner for this daemon's startup +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +const char *Daemon::DaemonBanner() const +{ + return 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Run() +// Purpose: Main run function after basic Daemon initialisation +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +void Daemon::Run() +{ + while(!StopRun()) + { + ::sleep(10); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::GetConfigVerify() +// Purpose: Returns the configuration file verification structure for this daemon +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify *Daemon::GetConfigVerify() const +{ + static ConfigurationVerifyKey verifyserverkeys[] = + { + DAEMON_VERIFY_SERVER_KEYS + }; + + static ConfigurationVerify verifyserver[] = + { + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } + }; + + static ConfigurationVerify verify = + { + "root", + verifyserver, + 0, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + }; + + return &verify; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::GetConfiguration() +// Purpose: Returns the daemon configuration object +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +const Configuration &Daemon::GetConfiguration() const +{ + if(mpConfiguration == 0) + { + // Shouldn't get anywhere near this if a configuration file can't be loaded + THROW_EXCEPTION(ServerException, Internal) + } + + return *mpConfiguration; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::SetupInInitialProcess() +// Purpose: A chance for the daemon to do something initial setting up in the process which +// initiates everything, and after the configuration file has been read and verified. +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +void Daemon::SetupInInitialProcess() +{ + // Base class doesn't do anything. +} + + +void Daemon::SetProcessTitle(const char *format, ...) +{ + // On OpenBSD, setproctitle() sets the process title to imagename: <text> (imagename) + // -- make sure other platforms include the image name somewhere so ps listings give + // useful information. + +#ifdef PLATFORM_HAVE_setproctitle + // optional arguments + va_list args; + va_start(args, format); + + // Make the string + char title[256]; + ::vsnprintf(title, sizeof(title), format, args); + + // Set process title + ::setproctitle("%s", title); + +#endif // PLATFORM_HAVE_setproctitle +} + + diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h new file mode 100755 index 00000000..a7b9488b --- /dev/null +++ b/lib/server/Daemon.h @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Daemon.h +// Purpose: Basic daemon functionality +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- + +/* NOTE: will log to local6: include a line like + local6.info /var/log/box + in /etc/syslog.conf +*/ + + +#ifndef DAEMON__H +#define DAEMON__H + +class Configuration; +class ConfigurationVerify; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Daemon +// Purpose: Basic daemon functionality +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +class Daemon +{ +public: + Daemon(); + virtual ~Daemon(); +private: + Daemon(const Daemon &rToCopy); +public: + + int Main(const char *DefaultConfigFile, int argc, const char *argv[]); + + virtual void Run(); + const Configuration &GetConfiguration() const; + + virtual const char *DaemonName() const; + virtual const char *DaemonBanner() const; + virtual const ConfigurationVerify *GetConfigVerify() const; + + bool StopRun() {return mReloadConfigWanted | mTerminateWanted;} + bool IsReloadConfigWanted() {return mReloadConfigWanted;} + bool IsTerminateWanted() {return mTerminateWanted;} + + // To allow derived classes to get these signals in other ways + void SetReloadConfigWanted() {mReloadConfigWanted = true;} + void SetTerminateWanted() {mTerminateWanted = true;} + + virtual void SetupInInitialProcess(); + virtual void EnterChild(); + + static void SetProcessTitle(const char *format, ...); + +private: + static void SignalHandler(int sigraised); + +private: + Configuration *mpConfiguration; + bool mReloadConfigWanted; + bool mTerminateWanted; + static Daemon *spDaemon; +}; + +#define DAEMON_VERIFY_SERVER_KEYS {"PidFile", 0, ConfigTest_Exists, 0}, \ + {"User", 0, ConfigTest_LastEntry, 0} + +#endif // DAEMON__H + diff --git a/lib/server/LocalProcessStream.cpp b/lib/server/LocalProcessStream.cpp new file mode 100644 index 00000000..f2a97c56 --- /dev/null +++ b/lib/server/LocalProcessStream.cpp @@ -0,0 +1,101 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: LocalProcessStream.cpp +// Purpose: Opens a process, and presents stdin/stdout as a stream. +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/socket.h> +#include <unistd.h> + +#include "LocalProcessStream.h" +#include "SocketStream.h" +#include "autogen_ServerException.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +#define MAX_ARGUMENTS 64 + +// -------------------------------------------------------------------------- +// +// Function +// Name: LocalProcessStream(const char *, pid_t &) +// Purpose: Run a new process, and return a stream giving access to it's +// stdin and stdout. Returns the PID of the new process -- this +// must be waited on at some point to avoid zombies. +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> LocalProcessStream(const char *CommandLine, pid_t &rPidOut) +{ + // Split up command + std::vector<std::string> command; + SplitString(std::string(CommandLine), ' ', command); + // Build arguments + char *args[MAX_ARGUMENTS + 4]; + { + int a = 0; + std::vector<std::string>::const_iterator i(command.begin()); + while(a < MAX_ARGUMENTS && i != command.end()) + { + args[a++] = (char*)(*(i++)).c_str(); + } + args[a] = NULL; + } + + // Create a socket pair to communicate over. + int sv[2] = {-1,-1}; + if(::socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, sv) != 0) + { + THROW_EXCEPTION(ServerException, SocketPairFailed) + } + + std::auto_ptr<IOStream> stream(new SocketStream(sv[0])); + + // Fork + pid_t pid = 0; + switch(pid = vfork()) + { + case -1: // error + ::close(sv[0]); + ::close(sv[1]); + THROW_EXCEPTION(ServerException, ServerForkError) + break; + + case 0: // child + // Close end of the socket not being used + ::close(sv[0]); + // Duplicate the file handles to stdin and stdout + if(sv[1] != 0) ::dup2(sv[1], 0); + if(sv[1] != 1) ::dup2(sv[1], 1); + // Close the now redundant socket + if(sv[1] != 0 && sv[1] != 1) + { + ::close(sv[1]); + } + // Execute command! + ::execv(args[0], args); + ::_exit(127); // report error + break; + + default: + // just continue... + break; + } + + // Close the file descriptor not being used + ::close(sv[1]); + + // Return the stream object and PID + rPidOut = pid; + return stream; +} + + + + diff --git a/lib/server/LocalProcessStream.h b/lib/server/LocalProcessStream.h new file mode 100644 index 00000000..490c0f45 --- /dev/null +++ b/lib/server/LocalProcessStream.h @@ -0,0 +1,19 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: LocalProcessStream.h +// Purpose: Opens a process, and presents stdin/stdout as a stream. +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef LOCALPROCESSSTREAM__H +#define LOCALPROCESSSTREAM__H + +#include <memory> +#include "IOStream.h" + +std::auto_ptr<IOStream> LocalProcessStream(const char *CommandLine, pid_t &rPidOut); + +#endif // LOCALPROCESSSTREAM__H + diff --git a/lib/server/Makefile.extra b/lib/server/Makefile.extra new file mode 100755 index 00000000..6cc0de2e --- /dev/null +++ b/lib/server/Makefile.extra @@ -0,0 +1,11 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_ServerException.h autogen_ServerException.cpp: $(MAKEEXCEPTION) ServerException.txt + perl $(MAKEEXCEPTION) ServerException.txt + +# AUTOGEN SEEDING +autogen_ConnectionException.h autogen_ConnectionException.cpp: $(MAKEEXCEPTION) ConnectionException.txt + perl $(MAKEEXCEPTION) ConnectionException.txt + diff --git a/lib/server/Protocol.cpp b/lib/server/Protocol.cpp new file mode 100755 index 00000000..ca89bdf1 --- /dev/null +++ b/lib/server/Protocol.cpp @@ -0,0 +1,1120 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Protocol.cpp +// Purpose: Generic protocol support +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> + +#include <stdlib.h> +#include <string.h> + +#include <new> + +#include "Protocol.h" +#include "ProtocolWire.h" +#include "IOStream.h" +#include "ServerException.h" +#include "PartialReadStream.h" +#include "ProtocolUncertainStream.h" + +#include "MemLeakFindOn.h" + +#ifdef NDEBUG + #define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK 1024 +#else +// #define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK 1024 + #define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK 4 +#endif + +#define UNCERTAIN_STREAM_SIZE_BLOCK (64*1024) + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Protocol(IOStream &rStream) +// Purpose: Constructor +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +Protocol::Protocol(IOStream &rStream) + : mrStream(rStream), + mHandshakeDone(false), + mMaxObjectSize(PROTOCOL_DEFAULT_MAXOBJSIZE), + mTimeout(PROTOCOL_DEFAULT_TIMEOUT), + mpBuffer(0), + mBufferSize(0), + mReadOffset(-1), + mWriteOffset(-1), + mValidDataSize(-1), + mLastErrorType(NoError), + mLastErrorSubType(NoError) +{ + TRACE1("Send block allocation size is %d\n", PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::~Protocol() +// Purpose: Destructor +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +Protocol::~Protocol() +{ + // Free buffer? + if(mpBuffer != 0) + { + free(mpBuffer); + mpBuffer = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::GetLastError(int &, int &) +// Purpose: Returns true if there was an error, and type and subtype if there was. +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +bool Protocol::GetLastError(int &rTypeOut, int &rSubTypeOut) +{ + if(mLastErrorType == NoError) + { + // no error. + return false; + } + + // Return type and subtype in args + rTypeOut = mLastErrorType; + rSubTypeOut = mLastErrorSubType; + + // and unset them + mLastErrorType = NoError; + mLastErrorSubType = NoError; + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Handshake() +// Purpose: Handshake with peer (exchange ident strings) +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +void Protocol::Handshake() +{ + // Already done? + if(mHandshakeDone) + { + THROW_EXCEPTION(CommonException, Internal) + } + + // Make handshake block + PW_Handshake hsSend; + ::memset(&hsSend, 0, sizeof(hsSend)); + // Copy in ident string + ::strncpy(hsSend.mIdent, GetIdentString(), sizeof(hsSend.mIdent)); + + // Send it + mrStream.Write(&hsSend, sizeof(hsSend)); + mrStream.WriteAllBuffered(); + + // Receive a handshake from the peer + PW_Handshake hsReceive; + ::memset(&hsReceive, 0, sizeof(hsReceive)); + char *readInto = (char*)&hsReceive; + int bytesToRead = sizeof(hsReceive); + while(bytesToRead > 0) + { + // Get some data from the stream + int bytesRead = mrStream.Read(readInto, bytesToRead, mTimeout); + if(bytesRead == 0) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + } + readInto += bytesRead; + bytesToRead -= bytesRead; + } + ASSERT(bytesToRead == 0); + + // Are they the same? + if(::memcmp(&hsSend, &hsReceive, sizeof(hsSend)) != 0) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_HandshakeFailed) + } + + // Mark as done + mHandshakeDone = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::CheckAndReadHdr(void *) +// Purpose: Check read for recieve call and get object header from stream. +// Don't use type here to avoid dependency in .h file. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void Protocol::CheckAndReadHdr(void *hdr) +{ + // Check usage + if(mValidDataSize != -1 || mWriteOffset != -1 || mReadOffset != -1) + { + THROW_EXCEPTION(ServerException, Protocol_BadUsage) + } + + // Handshake done? + if(!mHandshakeDone) + { + Handshake(); + } + + // Get some data into this header + if(!mrStream.ReadFullBuffer(hdr, sizeof(PW_ObjectHeader), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Recieve() +// Purpose: Recieves an object from the stream, creating it from the factory object type +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> Protocol::Receive() +{ + // Get object header + PW_ObjectHeader objHeader; + CheckAndReadHdr(&objHeader); + + // Hope it's not a stream + if(ntohl(objHeader.mObjType) == SPECIAL_STREAM_OBJECT_TYPE) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_StreamWhenObjExpected) + } + + // Check the object size + u_int32_t objSize = ntohl(objHeader.mObjSize); + if(objSize < sizeof(objHeader) || objSize > mMaxObjectSize) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_ObjTooBig) + } + + // Create a blank object + std::auto_ptr<ProtocolObject> obj(MakeProtocolObject(ntohl(objHeader.mObjType))); + + // Make sure memory is allocated to read it into + EnsureBufferAllocated(objSize); + + // Read data + if(!mrStream.ReadFullBuffer(mpBuffer, objSize - sizeof(objHeader), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + } + + // Setup ready to read out data from the buffer + mValidDataSize = objSize - sizeof(objHeader); + mReadOffset = 0; + + // Get the object to read its properties from the data recieved + try + { + obj->SetPropertiesFromStreamData(*this); + } + catch(...) + { + // Make sure state is reset! + mValidDataSize = -1; + mReadOffset = -1; + throw; + } + + // Any data left over? + bool dataLeftOver = (mValidDataSize != mReadOffset); + + // Unset read state, so future read calls don't fail + mValidDataSize = -1; + mReadOffset = -1; + + // Exception if not all the data was consumed + if(dataLeftOver) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_BadCommandRecieved) + } + + return obj; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Send() +// Purpose: Send an object to the other side of the connection. +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Send(const ProtocolObject &rObject) +{ + // Check usage + if(mValidDataSize != -1 || mWriteOffset != -1 || mReadOffset != -1) + { + THROW_EXCEPTION(ServerException, Protocol_BadUsage) + } + + // Handshake done? + if(!mHandshakeDone) + { + Handshake(); + } + + // Make sure there's a little bit of space allocated + EnsureBufferAllocated(((sizeof(PW_ObjectHeader) + PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK - 1) / PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK) * PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK); + ASSERT(mBufferSize >= (int)sizeof(PW_ObjectHeader)); + + // Setup for write operation + mValidDataSize = 0; // Not used, but must not be -1 + mWriteOffset = sizeof(PW_ObjectHeader); + + try + { + rObject.WritePropertiesToStreamData(*this); + } + catch(...) + { + // Make sure state is reset! + mValidDataSize = -1; + mWriteOffset = -1; + throw; + } + + // How big? + int writtenSize = mWriteOffset; + + // Reset write state + mValidDataSize = -1; + mWriteOffset = -1; + + // Make header in the existing block + PW_ObjectHeader *pobjHeader = (PW_ObjectHeader*)(mpBuffer); + pobjHeader->mObjSize = htonl(writtenSize); + pobjHeader->mObjType = htonl(rObject.GetType()); + + // Write data + mrStream.Write(mpBuffer, writtenSize); + mrStream.WriteAllBuffered(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::EnsureBufferAllocated(int) +// Purpose: Private. Ensures the buffer is at least the size requested. +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::EnsureBufferAllocated(int Size) +{ + if(mpBuffer != 0 && mBufferSize >= Size) + { + // Nothing to do! + return; + } + + // Need to allocate, or reallocate, the block + if(mpBuffer != 0) + { + // Reallocate + void *b = realloc(mpBuffer, Size); + if(b == 0) + { + throw std::bad_alloc(); + } + mpBuffer = (char*)b; + mBufferSize = Size; + } + else + { + // Just allocate + mpBuffer = (char*)malloc(Size); + if(mpBuffer == 0) + { + throw std::bad_alloc(); + } + mBufferSize = Size; + } +} + + +#define READ_START_CHECK \ + if(mValidDataSize == -1 || mWriteOffset != -1 || mReadOffset == -1) \ + { \ + THROW_EXCEPTION(ServerException, Protocol_BadUsage) \ + } + +#define READ_CHECK_BYTES_AVAILABLE(bytesRequired) \ + if((mReadOffset + (int)(bytesRequired)) > mValidDataSize) \ + { \ + THROW_EXCEPTION(ConnectionException, Conn_Protocol_BadCommandRecieved) \ + } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(void *, int) +// Purpose: Read raw data from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(void *Buffer, int Size) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(Size) + + // Copy data out + ::memmove(Buffer, mpBuffer + mReadOffset, Size); + mReadOffset += Size; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(std::string &, int) +// Purpose: Read raw data from the stream (buffered), into a std::string +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void Protocol::Read(std::string &rOut, int Size) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(Size) + + rOut.assign(mpBuffer + mReadOffset, Size); + mReadOffset += Size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(int64_t &) +// Purpose: Read a value from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(int64_t &rOut) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(sizeof(int64_t)) + + rOut = ntoh64(*((int64_t*)(mpBuffer + mReadOffset))); + mReadOffset += sizeof(int64_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(int32_t &) +// Purpose: Read a value from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(int32_t &rOut) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(sizeof(int32_t)) + + rOut = ntohl(*((int32_t*)(mpBuffer + mReadOffset))); + mReadOffset += sizeof(int32_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(int16_t &) +// Purpose: Read a value from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(int16_t &rOut) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(sizeof(int16_t)) + + rOut = ntohs(*((int16_t*)(mpBuffer + mReadOffset))); + mReadOffset += sizeof(int16_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(int8_t &) +// Purpose: Read a value from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(int8_t &rOut) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(sizeof(int8_t)) + + rOut = *((int8_t*)(mpBuffer + mReadOffset)); + mReadOffset += sizeof(int8_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(std::string &) +// Purpose: Read a value from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(std::string &rOut) +{ + // READ_START_CHECK implied + int32_t size; + Read(size); + + READ_CHECK_BYTES_AVAILABLE(size) + + // initialise string + rOut.assign(mpBuffer + mReadOffset, size); + mReadOffset += size; +} + + + + +#define WRITE_START_CHECK \ + if(mValidDataSize == -1 || mWriteOffset == -1 || mReadOffset != -1) \ + { \ + THROW_EXCEPTION(ServerException, Protocol_BadUsage) \ + } + +#define WRITE_ENSURE_BYTES_AVAILABLE(bytesToWrite) \ + if(mWriteOffset + (int)(bytesToWrite) > mBufferSize) \ + { \ + EnsureBufferAllocated((((mWriteOffset + (int)(bytesToWrite)) + PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK - 1) / PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK) * PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK); \ + ASSERT(mWriteOffset + (int)(bytesToWrite) <= mBufferSize); \ + } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(const void *, int) +// Purpose: Writes the contents of a buffer to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(const void *Buffer, int Size) +{ + WRITE_START_CHECK + WRITE_ENSURE_BYTES_AVAILABLE(Size) + + ::memmove(mpBuffer + mWriteOffset, Buffer, Size); + mWriteOffset += Size; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(int64_t) +// Purpose: Writes a value to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(int64_t Value) +{ + WRITE_START_CHECK + WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int64_t)) + + *((int64_t*)(mpBuffer + mWriteOffset)) = hton64(Value); + mWriteOffset += sizeof(int64_t); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(int32_t) +// Purpose: Writes a value to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(int32_t Value) +{ + WRITE_START_CHECK + WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int32_t)) + + *((int32_t*)(mpBuffer + mWriteOffset)) = htonl(Value); + mWriteOffset += sizeof(int32_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(int16_t) +// Purpose: Writes a value to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(int16_t Value) +{ + WRITE_START_CHECK + WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int16_t)) + + *((int16_t*)(mpBuffer + mWriteOffset)) = htons(Value); + mWriteOffset += sizeof(int16_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(int8_t) +// Purpose: Writes a value to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(int8_t Value) +{ + WRITE_START_CHECK + WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int8_t)) + + *((int8_t*)(mpBuffer + mWriteOffset)) = Value; + mWriteOffset += sizeof(int8_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(const std::string &) +// Purpose: Writes a value to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(const std::string &rValue) +{ + // WRITE_START_CHECK implied + Write((int32_t)(rValue.size())); + + WRITE_ENSURE_BYTES_AVAILABLE(rValue.size()) + Write(rValue.c_str(), rValue.size()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::ReceieveStream() +// Purpose: Receive a stream from the remote side +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> Protocol::ReceiveStream() +{ + // Get object header + PW_ObjectHeader objHeader; + CheckAndReadHdr(&objHeader); + + // Hope it's not an object + if(ntohl(objHeader.mObjType) != SPECIAL_STREAM_OBJECT_TYPE) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_ObjWhenStreamExpected) + } + + // Get the stream size + u_int32_t streamSize = ntohl(objHeader.mObjSize); + + // Inform sub class + InformStreamReceiving(streamSize); + + // Return a stream object + return std::auto_ptr<IOStream>((streamSize == ProtocolStream_SizeUncertain)? + ((IOStream*)(new ProtocolUncertainStream(mrStream))) + :((IOStream*)(new PartialReadStream(mrStream, streamSize)))); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::SendStream(IOStream &) +// Purpose: Send a stream to the remote side +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void Protocol::SendStream(IOStream &rStream) +{ + // Check usage + if(mValidDataSize != -1 || mWriteOffset != -1 || mReadOffset != -1) + { + THROW_EXCEPTION(ServerException, Protocol_BadUsage) + } + + // Handshake done? + if(!mHandshakeDone) + { + Handshake(); + } + + // How should this be streamed? + bool uncertainSize = false; + IOStream::pos_type streamSize = rStream.BytesLeftToRead(); + if(streamSize == IOStream::SizeOfStreamUnknown + || streamSize > 0x7fffffff) + { + // Can't send this using the fixed size header + uncertainSize = true; + } + + // Inform sub class + InformStreamSending(streamSize); + + // Make header + PW_ObjectHeader objHeader; + objHeader.mObjSize = htonl(uncertainSize?(ProtocolStream_SizeUncertain):streamSize); + objHeader.mObjType = htonl(SPECIAL_STREAM_OBJECT_TYPE); + + // Write header + mrStream.Write(&objHeader, sizeof(objHeader)); + // Could be sent in one of two ways + if(uncertainSize) + { + // Don't know how big this is going to be -- so send it in chunks + + // Allocate memory + uint8_t *blockA = (uint8_t *)malloc(UNCERTAIN_STREAM_SIZE_BLOCK + sizeof(int)); + if(blockA == 0) + { + throw std::bad_alloc(); + } + uint8_t *block = blockA + sizeof(int); // so that everything is word aligned for reading, but can put the one byte header before it + + try + { + int bytesInBlock = 0; + while(rStream.StreamDataLeft()) + { + // Read some of it + bytesInBlock += rStream.Read(block + bytesInBlock, UNCERTAIN_STREAM_SIZE_BLOCK - bytesInBlock); + + // Send as much as we can out + bytesInBlock -= SendStreamSendBlock(block, bytesInBlock); + } + + // Everything recieved from stream, but need to send whatevers left in the block + while(bytesInBlock > 0) + { + bytesInBlock -= SendStreamSendBlock(block, bytesInBlock); + } + + // Send final byte to finish the stream + uint8_t endOfStream = ProtocolStreamHeader_EndOfStream; + mrStream.Write(&endOfStream, 1); + } + catch(...) + { + free(blockA); + throw; + } + + // Clean up + free(blockA); + } + else + { + // Fixed size stream, send it all in one go + if(!rStream.CopyStreamTo(mrStream, mTimeout, 4096 /* slightly larger buffer */)) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_TimeOutWhenSendingStream) + } + } + // Make sure everything is written + mrStream.WriteAllBuffered(); + +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::SendStreamSendBlock(uint8_t *, int) +// Purpose: Sends as much of the block as can be sent, moves the remainer down to the beginning, +// and returns the number of bytes sent. WARNING: Will write to Block[-1] +// Created: 5/12/03 +// +// -------------------------------------------------------------------------- +int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) +{ + // Quick sanity check + if(BytesInBlock == 0) + { + return 0; + } + + // Work out the header byte + uint8_t header = 0; + int writeSize = 0; + if(BytesInBlock >= (64*1024)) + { + header = ProtocolStreamHeader_SizeIs64k; + writeSize = (64*1024); + } + else + { + // Scan the table to find the most that can be written + for(int s = ProtocolStreamHeader_MaxEncodedSizeValue; s > 0; --s) + { + if(sProtocolStreamHeaderLengths[s] <= BytesInBlock) + { + header = s; + writeSize = sProtocolStreamHeaderLengths[s]; + break; + } + } + } + ASSERT(header > 0); + + // Store the header + Block[-1] = header; + + // Write everything out + mrStream.Write(Block - 1, writeSize + 1); + + // move the remainer to the beginning of the block for the next time round + if(writeSize != BytesInBlock) + { + ::memmove(Block, Block + writeSize, BytesInBlock - writeSize); + } + + return writeSize; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::InformStreamReceiving(u_int32_t) +// Purpose: Informs sub classes about streams being received +// Created: 2003/10/27 +// +// -------------------------------------------------------------------------- +void Protocol::InformStreamReceiving(u_int32_t Size) +{ + // Do nothing +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::InformStreamSending(u_int32_t) +// Purpose: Informs sub classes about streams being sent +// Created: 2003/10/27 +// +// -------------------------------------------------------------------------- +void Protocol::InformStreamSending(u_int32_t Size) +{ + // Do nothing +} + + +/* +perl code to generate the table below + +#!/usr/bin/perl +use strict; +open OUT,">protolengths.txt"; +my $len = 0; +for(0 .. 255) +{ + print OUT "\t$len,\t// $_\n"; + my $inc = 1; + $inc = 8 if $_ >= 64; + $inc = 16 if $_ >= 96; + $inc = 32 if $_ >= 112; + $inc = 64 if $_ >= 128; + $inc = 128 if $_ >= 135; + $inc = 256 if $_ >= 147; + $inc = 512 if $_ >= 159; + $inc = 1024 if $_ >= 231; + $len += $inc; +} +close OUT; + +*/ +const uint16_t Protocol::sProtocolStreamHeaderLengths[256] = +{ + 0, // 0 + 1, // 1 + 2, // 2 + 3, // 3 + 4, // 4 + 5, // 5 + 6, // 6 + 7, // 7 + 8, // 8 + 9, // 9 + 10, // 10 + 11, // 11 + 12, // 12 + 13, // 13 + 14, // 14 + 15, // 15 + 16, // 16 + 17, // 17 + 18, // 18 + 19, // 19 + 20, // 20 + 21, // 21 + 22, // 22 + 23, // 23 + 24, // 24 + 25, // 25 + 26, // 26 + 27, // 27 + 28, // 28 + 29, // 29 + 30, // 30 + 31, // 31 + 32, // 32 + 33, // 33 + 34, // 34 + 35, // 35 + 36, // 36 + 37, // 37 + 38, // 38 + 39, // 39 + 40, // 40 + 41, // 41 + 42, // 42 + 43, // 43 + 44, // 44 + 45, // 45 + 46, // 46 + 47, // 47 + 48, // 48 + 49, // 49 + 50, // 50 + 51, // 51 + 52, // 52 + 53, // 53 + 54, // 54 + 55, // 55 + 56, // 56 + 57, // 57 + 58, // 58 + 59, // 59 + 60, // 60 + 61, // 61 + 62, // 62 + 63, // 63 + 64, // 64 + 72, // 65 + 80, // 66 + 88, // 67 + 96, // 68 + 104, // 69 + 112, // 70 + 120, // 71 + 128, // 72 + 136, // 73 + 144, // 74 + 152, // 75 + 160, // 76 + 168, // 77 + 176, // 78 + 184, // 79 + 192, // 80 + 200, // 81 + 208, // 82 + 216, // 83 + 224, // 84 + 232, // 85 + 240, // 86 + 248, // 87 + 256, // 88 + 264, // 89 + 272, // 90 + 280, // 91 + 288, // 92 + 296, // 93 + 304, // 94 + 312, // 95 + 320, // 96 + 336, // 97 + 352, // 98 + 368, // 99 + 384, // 100 + 400, // 101 + 416, // 102 + 432, // 103 + 448, // 104 + 464, // 105 + 480, // 106 + 496, // 107 + 512, // 108 + 528, // 109 + 544, // 110 + 560, // 111 + 576, // 112 + 608, // 113 + 640, // 114 + 672, // 115 + 704, // 116 + 736, // 117 + 768, // 118 + 800, // 119 + 832, // 120 + 864, // 121 + 896, // 122 + 928, // 123 + 960, // 124 + 992, // 125 + 1024, // 126 + 1056, // 127 + 1088, // 128 + 1152, // 129 + 1216, // 130 + 1280, // 131 + 1344, // 132 + 1408, // 133 + 1472, // 134 + 1536, // 135 + 1664, // 136 + 1792, // 137 + 1920, // 138 + 2048, // 139 + 2176, // 140 + 2304, // 141 + 2432, // 142 + 2560, // 143 + 2688, // 144 + 2816, // 145 + 2944, // 146 + 3072, // 147 + 3328, // 148 + 3584, // 149 + 3840, // 150 + 4096, // 151 + 4352, // 152 + 4608, // 153 + 4864, // 154 + 5120, // 155 + 5376, // 156 + 5632, // 157 + 5888, // 158 + 6144, // 159 + 6656, // 160 + 7168, // 161 + 7680, // 162 + 8192, // 163 + 8704, // 164 + 9216, // 165 + 9728, // 166 + 10240, // 167 + 10752, // 168 + 11264, // 169 + 11776, // 170 + 12288, // 171 + 12800, // 172 + 13312, // 173 + 13824, // 174 + 14336, // 175 + 14848, // 176 + 15360, // 177 + 15872, // 178 + 16384, // 179 + 16896, // 180 + 17408, // 181 + 17920, // 182 + 18432, // 183 + 18944, // 184 + 19456, // 185 + 19968, // 186 + 20480, // 187 + 20992, // 188 + 21504, // 189 + 22016, // 190 + 22528, // 191 + 23040, // 192 + 23552, // 193 + 24064, // 194 + 24576, // 195 + 25088, // 196 + 25600, // 197 + 26112, // 198 + 26624, // 199 + 27136, // 200 + 27648, // 201 + 28160, // 202 + 28672, // 203 + 29184, // 204 + 29696, // 205 + 30208, // 206 + 30720, // 207 + 31232, // 208 + 31744, // 209 + 32256, // 210 + 32768, // 211 + 33280, // 212 + 33792, // 213 + 34304, // 214 + 34816, // 215 + 35328, // 216 + 35840, // 217 + 36352, // 218 + 36864, // 219 + 37376, // 220 + 37888, // 221 + 38400, // 222 + 38912, // 223 + 39424, // 224 + 39936, // 225 + 40448, // 226 + 40960, // 227 + 41472, // 228 + 41984, // 229 + 42496, // 230 + 43008, // 231 + 44032, // 232 + 45056, // 233 + 46080, // 234 + 47104, // 235 + 48128, // 236 + 49152, // 237 + 50176, // 238 + 51200, // 239 + 52224, // 240 + 53248, // 241 + 54272, // 242 + 55296, // 243 + 56320, // 244 + 57344, // 245 + 58368, // 246 + 59392, // 247 + 60416, // 248 + 61440, // 249 + 62464, // 250 + 63488, // 251 + 64512, // 252 + 0, // 253 = 65536 / 64k + 0, // 254 = special (reserved) + 0 // 255 = special (reserved) +}; + + + + diff --git a/lib/server/Protocol.h b/lib/server/Protocol.h new file mode 100755 index 00000000..e037e33c --- /dev/null +++ b/lib/server/Protocol.h @@ -0,0 +1,201 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Protocol.h +// Purpose: Generic protocol support +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- + +#ifndef PROTOCOL__H +#define PROTOCOL__H + +#include <sys/types.h> + +class IOStream; +#include "ProtocolObject.h" +#include <memory> +#include <vector> +#include <string> + +// default timeout is 15 minutes +#define PROTOCOL_DEFAULT_TIMEOUT (15*60*1000) +// 16 default maximum object size -- should be enough +#define PROTOCOL_DEFAULT_MAXOBJSIZE (16*1024) + +// -------------------------------------------------------------------------- +// +// Class +// Name: Protocol +// Purpose: Generic command / response protocol support +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +class Protocol +{ +public: + Protocol(IOStream &rStream); + virtual ~Protocol(); + +private: + Protocol(const Protocol &rToCopy); + +public: + void Handshake(); + std::auto_ptr<ProtocolObject> Receive(); + void Send(const ProtocolObject &rObject); + + std::auto_ptr<IOStream> ReceiveStream(); + void SendStream(IOStream &rStream); + + enum + { + NoError = -1, + UnknownError = 0 + }; + + bool GetLastError(int &rTypeOut, int &rSubTypeOut); + + // -------------------------------------------------------------------------- + // + // Function + // Name: Protocol::SetTimeout(int) + // Purpose: Sets the timeout for sending and reciving + // Created: 2003/08/19 + // + // -------------------------------------------------------------------------- + void SetTimeout(int NewTimeout) {mTimeout = NewTimeout;} + + + // -------------------------------------------------------------------------- + // + // Function + // Name: Protocol::GetTimeout() + // Purpose: Get current timeout for sending and receiving + // Created: 2003/09/06 + // + // -------------------------------------------------------------------------- + int GetTimeout() {return mTimeout;} + + // -------------------------------------------------------------------------- + // + // Function + // Name: Protocol::SetMaxObjectSize(int) + // Purpose: Sets the maximum size of an object which will be accepted + // Created: 2003/08/19 + // + // -------------------------------------------------------------------------- + void SetMaxObjectSize(unsigned int NewMaxObjSize) {mMaxObjectSize = NewMaxObjSize;} + + // For ProtocolObject derived classes + void Read(void *Buffer, int Size); + void Read(std::string &rOut, int Size); + void Read(int64_t &rOut); + void Read(int32_t &rOut); + void Read(int16_t &rOut); + void Read(int8_t &rOut); + void Read(bool &rOut) {int8_t read; Read(read); rOut = (read == true);} + void Read(std::string &rOut); + template<typename type> + void Read(type &rOut) + { + rOut.ReadFromProtocol(*this); + } + // -------------------------------------------------------------------------- + // + // Function + // Name: Protocol::ReadVector(std::vector<> &) + // Purpose: Reads a vector/list of items from the stream + // Created: 2003/08/19 + // + // -------------------------------------------------------------------------- + template<typename type> + void ReadVector(std::vector<type> &rOut) + { + rOut.clear(); + int16_t num = 0; + Read(num); + for(int16_t n = 0; n < num; ++n) + { + type v; + Read(v); + rOut.push_back(v); + } + } + + void Write(const void *Buffer, int Size); + void Write(int64_t Value); + void Write(int32_t Value); + void Write(int16_t Value); + void Write(int8_t Value); + void Write(bool Value) {int8_t write = Value; Write(write);} + void Write(const std::string &rValue); + template<typename type> + void Write(const type &rValue) + { + rValue.WriteToProtocol(*this); + } + template<typename type> + // -------------------------------------------------------------------------- + // + // Function + // Name: Protocol::WriteVector(const std::vector<> &) + // Purpose: Writes a vector/list of items from the stream + // Created: 2003/08/19 + // + // -------------------------------------------------------------------------- + void WriteVector(const std::vector<type> &rValue) + { + int16_t num = rValue.size(); + Write(num); + for(int16_t n = 0; n < num; ++n) + { + Write(rValue[n]); + } + } + +public: + static const uint16_t sProtocolStreamHeaderLengths[256]; + enum + { + ProtocolStreamHeader_EndOfStream = 0, + ProtocolStreamHeader_MaxEncodedSizeValue = 252, + ProtocolStreamHeader_SizeIs64k = 253, + ProtocolStreamHeader_Reserved1 = 254, + ProtocolStreamHeader_Reserved2 = 255 + }; + enum + { + ProtocolStream_SizeUncertain = 0xffffffff + }; + +protected: + virtual std::auto_ptr<ProtocolObject> MakeProtocolObject(int ObjType) = 0; + virtual const char *GetIdentString() = 0; + void SetError(int Type, int SubType) {mLastErrorType = Type; mLastErrorSubType = SubType;} + void CheckAndReadHdr(void *hdr); // don't use type here to avoid dependency + + // Will be used for logging + virtual void InformStreamReceiving(u_int32_t Size); + virtual void InformStreamSending(u_int32_t Size); + +private: + void EnsureBufferAllocated(int Size); + int SendStreamSendBlock(uint8_t *Block, int BytesInBlock); + +private: + IOStream &mrStream; + bool mHandshakeDone; + unsigned int mMaxObjectSize; + int mTimeout; + char *mpBuffer; + int mBufferSize; + int mReadOffset; + int mWriteOffset; + int mValidDataSize; + int mLastErrorType; + int mLastErrorSubType; +}; + +#endif // PROTOCOL__H + diff --git a/lib/server/ProtocolObject.cpp b/lib/server/ProtocolObject.cpp new file mode 100755 index 00000000..fb09f820 --- /dev/null +++ b/lib/server/ProtocolObject.cpp @@ -0,0 +1,125 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ProtocolObject.h +// Purpose: Protocol object base class +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "ProtocolObject.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::ProtocolObject() +// Purpose: Default constructor +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +ProtocolObject::ProtocolObject() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::ProtocolObject() +// Purpose: Destructor +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +ProtocolObject::~ProtocolObject() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::ProtocolObject() +// Purpose: Copy constructor +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +ProtocolObject::ProtocolObject(const ProtocolObject &rToCopy) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::IsError(int &, int &) +// Purpose: Does this represent an error, and if so, what is the type and subtype? +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +bool ProtocolObject::IsError(int &rTypeOut, int &rSubTypeOut) const +{ + return false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::IsConversationEnd() +// Purpose: Does this command end the conversation? +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +bool ProtocolObject::IsConversationEnd() const +{ + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::GetType() +// Purpose: Return type of the object +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +int ProtocolObject::GetType() const +{ + // This isn't implemented in the base class! + THROW_EXCEPTION(CommonException, Internal) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::SetPropertiesFromStreamData(Protocol &) +// Purpose: Set the properties of the object from the stream data ready in the Protocol object +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void ProtocolObject::SetPropertiesFromStreamData(Protocol &rProtocol) +{ + // This isn't implemented in the base class! + THROW_EXCEPTION(CommonException, Internal) +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::WritePropertiesToStreamData(Protocol &) +// Purpose: Write the properties of the object into the stream data in the Protocol object +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void ProtocolObject::WritePropertiesToStreamData(Protocol &rProtocol) const +{ + // This isn't implemented in the base class! + THROW_EXCEPTION(CommonException, Internal) +} + + + diff --git a/lib/server/ProtocolObject.h b/lib/server/ProtocolObject.h new file mode 100755 index 00000000..0a127ab5 --- /dev/null +++ b/lib/server/ProtocolObject.h @@ -0,0 +1,41 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ProtocolObject.h +// Purpose: Protocol object base class +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- + +#ifndef PROTOCOLOBJECT__H +#define PROTOCOLOBJECT__H + +class Protocol; + +// -------------------------------------------------------------------------- +// +// Class +// Name: ProtocolObject +// Purpose: Basic object representation of objects to pass through a Protocol session +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +class ProtocolObject +{ +public: + ProtocolObject(); + virtual ~ProtocolObject(); + ProtocolObject(const ProtocolObject &rToCopy); + + // Info about this object + virtual int GetType() const; + virtual bool IsError(int &rTypeOut, int &rSubTypeOut) const; + virtual bool IsConversationEnd() const; + + // reading and writing with Protocol objects + virtual void SetPropertiesFromStreamData(Protocol &rProtocol); + virtual void WritePropertiesToStreamData(Protocol &rProtocol) const; +}; + +#endif // PROTOCOLOBJECT__H + diff --git a/lib/server/ProtocolUncertainStream.cpp b/lib/server/ProtocolUncertainStream.cpp new file mode 100755 index 00000000..60c1fa1d --- /dev/null +++ b/lib/server/ProtocolUncertainStream.cpp @@ -0,0 +1,189 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ProtocolUncertainStream.h +// Purpose: Read part of another stream +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "ProtocolUncertainStream.h" +#include "ServerException.h" +#include "Protocol.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::ProtocolUncertainStream(IOStream &, int) +// Purpose: Constructor, taking another stream. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +ProtocolUncertainStream::ProtocolUncertainStream(IOStream &rSource) + : mrSource(rSource), + mBytesLeftInCurrentBlock(0), + mFinished(false) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::~ProtocolUncertainStream() +// Purpose: Destructor. Won't absorb any unread bytes. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +ProtocolUncertainStream::~ProtocolUncertainStream() +{ + if(!mFinished) + { + TRACE0("ProtocolUncertainStream::~ProtocolUncertainStream() destroyed when stream not complete\n"); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::Read(void *, int, int) +// Purpose: As interface. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Finished? + if(mFinished) + { + return 0; + } + + int read = 0; + while(read < NBytes) + { + // Anything we can get from the current block? + ASSERT(mBytesLeftInCurrentBlock >= 0); + if(mBytesLeftInCurrentBlock > 0) + { + // Yes, let's use some of these up + int toRead = (NBytes - read); + if(toRead > mBytesLeftInCurrentBlock) + { + // Adjust downwards to only read stuff out of the current block + toRead = mBytesLeftInCurrentBlock; + } + + // Read it + int r = mrSource.Read(((uint8_t*)pBuffer) + read, toRead, Timeout); + // Give up now if it didn't return anything + if(r == 0) + { + return read; + } + + // Adjust counts of bytes by the bytes recieved + read += r; + mBytesLeftInCurrentBlock -= r; + + // stop now if the stream returned less than we asked for -- avoid blocking + if(r != toRead) + { + return read; + } + } + else + { + // Read the header byte to find out how much there is in the next block + uint8_t header; + if(mrSource.Read(&header, 1, Timeout) == 0) + { + // Didn't get the byte, return now + return read; + } + + // Interpret the byte... + if(header == Protocol::ProtocolStreamHeader_EndOfStream) + { + // All done. + mFinished = true; + return read; + } + else if(header <= Protocol::ProtocolStreamHeader_MaxEncodedSizeValue) + { + // get size of the block from the Protocol's lovely list + mBytesLeftInCurrentBlock = Protocol::sProtocolStreamHeaderLengths[header]; + } + else if(header == Protocol::ProtocolStreamHeader_SizeIs64k) + { + // 64k + mBytesLeftInCurrentBlock = (64*1024); + } + else + { + // Bad. It used the reserved values. + THROW_EXCEPTION(ServerException, ProtocolUncertainStreamBadBlockHeader) + } + } + } + + // Return the number read + return read; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::BytesLeftToRead() +// Purpose: As interface. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ProtocolUncertainStream::BytesLeftToRead() +{ + // Only know how much is left if everything is finished + return mFinished?(0):(IOStream::SizeOfStreamUnknown); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::Write(const void *, int) +// Purpose: As interface. But will exception. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +void ProtocolUncertainStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(ServerException, CantWriteToProtocolUncertainStream) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::StreamDataLeft() +// Purpose: As interface. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +bool ProtocolUncertainStream::StreamDataLeft() +{ + return !mFinished; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::StreamClosed() +// Purpose: As interface. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +bool ProtocolUncertainStream::StreamClosed() +{ + // always closed + return true; +} + diff --git a/lib/server/ProtocolUncertainStream.h b/lib/server/ProtocolUncertainStream.h new file mode 100755 index 00000000..66397a39 --- /dev/null +++ b/lib/server/ProtocolUncertainStream.h @@ -0,0 +1,47 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: PartialReadStream.h +// Purpose: Read part of another stream +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- + +#ifndef PROTOCOLUNCERTAINSTREAM__H +#define PROTOCOLUNCERTAINSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: PartialReadStream +// Purpose: Read part of another stream +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +class ProtocolUncertainStream : public IOStream +{ +public: + ProtocolUncertainStream(IOStream &rSource); + ~ProtocolUncertainStream(); +private: + // no copying allowed + ProtocolUncertainStream(const IOStream &); + ProtocolUncertainStream(const ProtocolUncertainStream &); + +public: + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + IOStream &mrSource; + int mBytesLeftInCurrentBlock; + bool mFinished; +}; + +#endif // PROTOCOLUNCERTAINSTREAM__H + diff --git a/lib/server/ProtocolWire.h b/lib/server/ProtocolWire.h new file mode 100755 index 00000000..b6d3bd37 --- /dev/null +++ b/lib/server/ProtocolWire.h @@ -0,0 +1,43 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ProtocolWire.h +// Purpose: On the wire structures for Protocol +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- + +#ifndef PROTOCOLWIRE__H +#define PROTOCOLWIRE__H + +#include <sys/types.h> + +// set packing to one byte +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + char mIdent[32]; +} PW_Handshake; + +typedef struct +{ + u_int32_t mObjSize; + u_int32_t mObjType; +} PW_ObjectHeader; + +#define SPECIAL_STREAM_OBJECT_TYPE 0xffffffff + +// Use default packing +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +#endif // PROTOCOLWIRE__H + diff --git a/lib/server/SSLLib.cpp b/lib/server/SSLLib.cpp new file mode 100755 index 00000000..e9f3a59d --- /dev/null +++ b/lib/server/SSLLib.cpp @@ -0,0 +1,83 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SSLLib.cpp +// Purpose: Utility functions for dealing with the OpenSSL library +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#define TLS_CLASS_IMPLEMENTATION_CPP +#include <openssl/ssl.h> +#include <openssl/err.h> +#include <openssl/rand.h> + +#include <syslog.h> + +#include "SSLLib.h" +#include "ServerException.h" + +#include "MemLeakFindOn.h" + +#ifndef NDEBUG + bool SSLLib__TraceErrors = false; +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: SSLLib::Initialise() +// Purpose: Initialise SSL library +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SSLLib::Initialise() +{ + if(!::SSL_library_init()) + { + LogError("Initialisation"); + THROW_EXCEPTION(ServerException, SSLLibraryInitialisationError) + } + + // More helpful error messages + ::SSL_load_error_strings(); + + // Extra seeding over and above what's already done by the library +#ifndef PLATFORM_RANDOM_DEVICE_NONE + if(::RAND_load_file(PLATFORM_RANDOM_DEVICE, 1024) != 1024) + { + THROW_EXCEPTION(ServerException, SSLRandomInitFailed) + } +#else + ::fprintf(stderr, "No random device -- additional seeding of random number generator not performed.\n"); +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SSLLib::LogError(const char *) +// Purpose: Logs an error +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SSLLib::LogError(const char *ErrorDuringAction) +{ + unsigned long errcode; + char errname[256]; // SSL docs say at least 120 bytes + while((errcode = ERR_get_error()) != 0) + { + ::ERR_error_string_n(errcode, errname, sizeof(errname)); + #ifndef NDEBUG + if(SSLLib__TraceErrors) + { + TRACE2("SSL err during %s: %s\n", ErrorDuringAction, errname); + } + #endif + ::syslog(LOG_ERR, "SSL err during %s: %s", ErrorDuringAction, errname); + } +} + diff --git a/lib/server/SSLLib.h b/lib/server/SSLLib.h new file mode 100755 index 00000000..cdff4f04 --- /dev/null +++ b/lib/server/SSLLib.h @@ -0,0 +1,36 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SSLLib.h +// Purpose: Utility functions for dealing with the OpenSSL library +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#ifndef SSLLIB__H +#define SSLLIB__H + +#ifndef NDEBUG + extern bool SSLLib__TraceErrors; + #define SET_DEBUG_SSLLIB_TRACE_ERRORS {SSLLib__TraceErrors = true;} +#else + #define SET_DEBUG_SSLLIB_TRACE_ERRORS +#endif + + +// -------------------------------------------------------------------------- +// +// Namespace +// Name: SSLLib +// Purpose: Utility functions for dealing with the OpenSSL library +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +namespace SSLLib +{ + void Initialise(); + void LogError(const char *ErrorDuringAction); +}; + +#endif // SSLLIB__H + diff --git a/lib/server/ServerException.h b/lib/server/ServerException.h new file mode 100755 index 00000000..8851b90a --- /dev/null +++ b/lib/server/ServerException.h @@ -0,0 +1,46 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ServerException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef SERVEREXCEPTION__H +#define SERVEREXCEPTION__H + +// Compatibility header +#include "autogen_ServerException.h" +#include "autogen_ConnectionException.h" + +// Rename old connection exception names to new names without Conn_ prefix +// This is all because ConnectionException used to be derived from ServerException +// with some funky magic with subtypes. Perhaps a little unreliable, and the +// usefulness of it never really was used. +#define Conn_SocketWriteError SocketWriteError +#define Conn_SocketReadError SocketReadError +#define Conn_SocketNameLookupError SocketNameLookupError +#define Conn_SocketShutdownError SocketShutdownError +#define Conn_SocketConnectError SocketConnectError +#define Conn_TLSHandshakeFailed TLSHandshakeFailed +#define Conn_TLSShutdownFailed TLSShutdownFailed +#define Conn_TLSWriteFailed TLSWriteFailed +#define Conn_TLSReadFailed TLSReadFailed +#define Conn_TLSNoPeerCertificate TLSNoPeerCertificate +#define Conn_TLSPeerCertificateInvalid TLSPeerCertificateInvalid +#define Conn_TLSClosedWhenWriting TLSClosedWhenWriting +#define Conn_TLSHandshakeTimedOut TLSHandshakeTimedOut +#define Conn_Protocol_Timeout Protocol_Timeout +#define Conn_Protocol_ObjTooBig Protocol_ObjTooBig +#define Conn_Protocol_BadCommandRecieved Protocol_BadCommandRecieved +#define Conn_Protocol_UnknownCommandRecieved Protocol_UnknownCommandRecieved +#define Conn_Protocol_TriedToExecuteReplyCommand Protocol_TriedToExecuteReplyCommand +#define Conn_Protocol_UnexpectedReply Protocol_UnexpectedReply +#define Conn_Protocol_HandshakeFailed Protocol_HandshakeFailed +#define Conn_Protocol_StreamWhenObjExpected Protocol_StreamWhenObjExpected +#define Conn_Protocol_ObjWhenStreamExpected Protocol_ObjWhenStreamExpected +#define Conn_Protocol_TimeOutWhenSendingStream Protocol_TimeOutWhenSendingStream + +#endif // SERVEREXCEPTION__H + diff --git a/lib/server/ServerException.txt b/lib/server/ServerException.txt new file mode 100644 index 00000000..ed591b73 --- /dev/null +++ b/lib/server/ServerException.txt @@ -0,0 +1,39 @@ +EXCEPTION Server 3 + +# for historic reasons, some codes are not used + +Internal 0 +FailedToLoadConfiguration 1 +DaemoniseFailed 2 +AlreadyDaemonConstructed 3 +BadSocketHandle 4 +DupError 5 +SocketAlreadyOpen 8 +SocketOpenError 10 +SocketPollError 11 +SocketCloseError 13 +SocketNameUNIXPathTooLong 14 +SocketBindError 16 Check the ListenAddresses directive in your config file -- must refer to local IP addresses only +SocketAcceptError 17 +ServerStreamBadListenAddrs 18 +ServerForkError 19 +ServerWaitOnChildError 20 +TooManySocketsInMultiListen 21 There is a limit on how many addresses you can listen on simulatiously. +ServerStreamTooManyListenAddresses 22 +TLSContextNotInitialised 23 +TLSAllocationFailed 24 +TLSLoadCertificatesFailed 25 +TLSLoadPrivateKeyFailed 26 +TLSLoadTrustedCAsFailed 27 +TLSSetCiphersFailed 28 +SSLLibraryInitialisationError 29 +TLSNoSSLObject 31 +TLSAlreadyHandshaked 35 +SocketSetNonBlockingFailed 40 +Protocol_BadUsage 43 +Protocol_UnsuitableStreamTypeForSending 51 +CantWriteToProtocolUncertainStream 53 +ProtocolUncertainStreamBadBlockHeader 54 +SocketPairFailed 55 +CouldNotChangePIDFileOwner 56 +SSLRandomInitFailed 57 Read from /dev/*random device failed diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h new file mode 100755 index 00000000..d087a321 --- /dev/null +++ b/lib/server/ServerStream.h @@ -0,0 +1,340 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ServerStream.h +// Purpose: Stream based server daemons +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef SERVERSTREAM__H +#define SERVERSTREAM__H + +#include <syslog.h> +#include <stdlib.h> +#include <errno.h> +#include <sys/wait.h> + +#include "Daemon.h" +#include "SocketListen.h" +#include "Utils.h" +#include "Configuration.h" +#include "WaitForEvent.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: ServerStream +// Purpose: Stream based server daemon +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +template<typename StreamType, int Port, int ListenBacklog = 128, bool ForkToHandleRequests = true> +class ServerStream : public Daemon +{ +public: + ServerStream() + { + } + ~ServerStream() + { + DeleteSockets(); + } +private: + ServerStream(const ServerStream &rToCopy) + { + } +public: + + virtual const char *DaemonName() const + { + return "generic-stream-server"; + } + + virtual void Run() + { + // Set process title as appropraite + SetProcessTitle(ForkToHandleRequests?"server":"idle"); + + // Handle exceptions and child task quitting gracefully. + bool childExit = false; + try + { + Run2(childExit); + } + catch(BoxException &e) + { + if(childExit) + { + ::syslog(LOG_ERR, "in server child, exception %s (%d/%d) -- terminating child", e.what(), e.GetType(), e.GetSubType()); + _exit(1); + } + else throw; + } + catch(std::exception &e) + { + if(childExit) + { + ::syslog(LOG_ERR, "in server child, exception %s -- terminating child", e.what()); + _exit(1); + } + else throw; + } + catch(...) + { + if(childExit) + { + ::syslog(LOG_ERR, "in server child, unknown exception -- terminating child"); + _exit(1); + } + else throw; + } + + // if it's a child fork, exit the process now + if(childExit) + { + // Child task, dump leaks to trace, which we make sure is on + #ifdef BOX_MEMORY_LEAK_TESTING + #ifndef NDEBUG + TRACE_TO_SYSLOG(true); + TRACE_TO_STDOUT(true); + #endif + memleakfinder_traceblocksinsection(); + #endif + + // If this is a child quitting, exit now to stop bad things happening + _exit(0); + } + } + + virtual void Run2(bool &rChildExit) + { + try + { + // Wait object with a timeout of 10 seconds, which is a reasonable time to wait before + // cleaning up finished child processes. + WaitForEvent connectionWait(10000); + + // BLOCK + { + // Get the address we need to bind to + // this-> in next line required to build under some gcc versions + const Configuration &config(this->GetConfiguration()); + const Configuration &server(config.GetSubConfiguration("Server")); + std::string addrs = server.GetKeyValue("ListenAddresses"); + + // split up the list of addresses + std::vector<std::string> addrlist; + SplitString(addrs, ',', addrlist); + + for(unsigned int a = 0; a < addrlist.size(); ++a) + { + // split the address up into components + std::vector<std::string> c; + SplitString(addrlist[a], ':', c); + + // listen! + SocketListen<StreamType, ListenBacklog> *psocket = new SocketListen<StreamType, ListenBacklog>; + try + { + if(c[0] == "inet") + { + // Check arguments + if(c.size() != 2 && c.size() != 3) + { + THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) + } + + // Which port? + int port = Port; + + if(c.size() == 3) + { + // Convert to number + port = ::atol(c[2].c_str()); + if(port <= 0 || port > ((64*1024)-1)) + { + THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) + } + } + + // Listen + psocket->Listen(Socket::TypeINET, c[1].c_str(), port); + } + else if(c[0] == "unix") + { + // Check arguments size + if(c.size() != 2) + { + THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) + } + + // unlink anything there + ::unlink(c[1].c_str()); + + psocket->Listen(Socket::TypeUNIX, c[1].c_str()); + } + else + { + delete psocket; + THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) + } + + // Add to list of sockets + mSockets.push_back(psocket); + } + catch(...) + { + delete psocket; + throw; + } + + // Add to the list of things to wait on + connectionWait.Add(psocket); + } + } + + while(!StopRun()) + { + // Wait for a connection, or timeout + SocketListen<StreamType, ListenBacklog> *psocket + = (SocketListen<StreamType, ListenBacklog> *)connectionWait.Wait(); + + if(psocket) + { + // Get the incomming connection (with zero wait time) + std::string logMessage; + std::auto_ptr<StreamType> connection(psocket->Accept(0, &logMessage)); + + // Was there one (there should be...) + if(connection.get()) + { + // Since this is a template parameter, the if() will be optimised out by the compiler + if(ForkToHandleRequests) + { + pid_t pid = ::fork(); + switch(pid) + { + case -1: + // Error! + THROW_EXCEPTION(ServerException, ServerForkError) + break; + + case 0: + // Child process + rChildExit = true; + // Close listening sockets + DeleteSockets(); + + // Set up daemon + EnterChild(); + SetProcessTitle("transaction"); + + // Memory leak test the forked process + #ifdef BOX_MEMORY_LEAK_TESTING + memleakfinder_startsectionmonitor(); + #endif + + // The derived class does some server magic with the connection + HandleConnection(*connection); + // Since rChildExit == true, the forked process will call _exit() on return from this fn + return; + + default: + // parent daemon process + break; + } + + // Log it + ::syslog(LOG_INFO, "%s (handling in child %d)", logMessage.c_str(), pid); + } + else + { + // Just handle in this connection + SetProcessTitle("handling"); + HandleConnection(*connection); + SetProcessTitle("idle"); + } + } + } + + // Clean up child processes (if forking daemon) + if(ForkToHandleRequests) + { + int status = 0; + int p = 0; + do + { + if((p = ::waitpid(0 /* any child in process group */, &status, WNOHANG)) == -1 + && errno != ECHILD && errno != EINTR) + { + THROW_EXCEPTION(ServerException, ServerWaitOnChildError) + } + } while(p > 0); + } + } + } + catch(...) + { + DeleteSockets(); + throw; + } + + // Delete the sockets + DeleteSockets(); + } + + virtual void HandleConnection(StreamType &rStream) + { + Connection(rStream); + } + + virtual void Connection(StreamType &rStream) = 0; + +protected: + // For checking code in dervied classes -- use if you have an algorithm which + // depends on the forking model in case someone changes it later. + bool WillForkToHandleRequests() + { + return ForkToHandleRequests; + } + +private: + // -------------------------------------------------------------------------- + // + // Function + // Name: ServerStream::DeleteSockets() + // Purpose: Delete sockets + // Created: 9/3/04 + // + // -------------------------------------------------------------------------- + void DeleteSockets() + { + for(unsigned int l = 0; l < mSockets.size(); ++l) + { + if(mSockets[l]) + { + mSockets[l]->Close(); + delete mSockets[l]; + } + mSockets[l] = 0; + } + mSockets.clear(); + } + +private: + std::vector<SocketListen<StreamType, ListenBacklog> *> mSockets; +}; + +#define SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ + {"ListenAddresses", DEFAULT_ADDRESSES, 0, 0}, \ + DAEMON_VERIFY_SERVER_KEYS + +#include "MemLeakFindOff.h" + +#endif // SERVERSTREAM__H + + + diff --git a/lib/server/ServerTLS.h b/lib/server/ServerTLS.h new file mode 100755 index 00000000..71d35380 --- /dev/null +++ b/lib/server/ServerTLS.h @@ -0,0 +1,80 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ServerTLS.h +// Purpose: Implementation of a server using TLS streams +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#ifndef SERVERTLS__H +#define SERVERTLS__H + +#include "ServerStream.h" +#include "SocketStreamTLS.h" +#include "SSLLib.h" +#include "TLSContext.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: ServerTLS +// Purpose: Implementation of a server using TLS streams +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +template<int Port, int ListenBacklog = 128, bool ForkToHandleRequests = true> +class ServerTLS : public ServerStream<SocketStreamTLS, Port, ListenBacklog, ForkToHandleRequests> +{ +public: + ServerTLS() + { + // Safe to call this here, as the Daemon class makes sure there is only one instance every of a Daemon. + SSLLib::Initialise(); + } + + ~ServerTLS() + { + } +private: + ServerTLS(const ServerTLS &) + { + } +public: + + virtual void Run2(bool &rChildExit) + { + // First, set up the SSL context. + // Get parameters from the configuration + // this-> in next line required to build under some gcc versions + const Configuration &conf(this->GetConfiguration()); + const Configuration &serverconf(conf.GetSubConfiguration("Server")); + std::string certFile(serverconf.GetKeyValue("CertificateFile")); + std::string keyFile(serverconf.GetKeyValue("PrivateKeyFile")); + std::string caFile(serverconf.GetKeyValue("TrustedCAsFile")); + mContext.Initialise(true /* as server */, certFile.c_str(), keyFile.c_str(), caFile.c_str()); + + // Then do normal stream server stuff + ServerStream<SocketStreamTLS, Port, ListenBacklog>::Run2(rChildExit); + } + + virtual void HandleConnection(SocketStreamTLS &rStream) + { + rStream.Handshake(mContext, true /* is server */); + // this-> in next line required to build under some gcc versions + this->Connection(rStream); + } + +private: + TLSContext mContext; +}; + +#define SERVERTLS_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ + {"CertificateFile", 0, ConfigTest_Exists, 0}, \ + {"PrivateKeyFile", 0, ConfigTest_Exists, 0}, \ + {"TrustedCAsFile", 0, ConfigTest_Exists, 0}, \ + SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) + + +#endif // SERVERTLS__H + diff --git a/lib/server/Socket.cpp b/lib/server/Socket.cpp new file mode 100755 index 00000000..52eb79e3 --- /dev/null +++ b/lib/server/Socket.cpp @@ -0,0 +1,171 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Socket.cpp +// Purpose: Socket related stuff +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <unistd.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netdb.h> +#include <syslog.h> +#include <netinet/in.h> +#include <arpa/inet.h> + +#include <string.h> +#include <stdio.h> + +#include "Socket.h" +#include "ServerException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: Socket::NameLookupToSockAddr(SocketAllAddr &, int, char *, int) +// Purpose: Sets up a sockaddr structure given a name and type +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type, const char *Name, int Port, int &rSockAddrLenOut) +{ + int sockAddrLen = 0; + + switch(Type) + { + case TypeINET: + sockDomain = AF_INET; + { + // Lookup hostname + struct hostent *phost = ::gethostbyname(Name); + if(phost != NULL) + { + if(phost->h_addr_list[0] != 0) + { + sockAddrLen = sizeof(addr.sa_inet); +#ifndef PLATFORM_sockaddr_NO_len + addr.sa_inet.sin_len = sizeof(addr.sa_inet); +#endif + addr.sa_inet.sin_family = PF_INET; + addr.sa_inet.sin_port = htons(Port); + addr.sa_inet.sin_addr = *((in_addr*)phost->h_addr_list[0]); + for(unsigned int l = 0; l < sizeof(addr.sa_inet.sin_zero); ++l) + { + addr.sa_inet.sin_zero[l] = 0; + } + } + else + { + THROW_EXCEPTION(ConnectionException, Conn_SocketNameLookupError); + } + } + else + { + THROW_EXCEPTION(ConnectionException, Conn_SocketNameLookupError); + } + } + break; + + case TypeUNIX: + sockDomain = AF_UNIX; + { + // Check length of name is OK + unsigned int nameLen = ::strlen(Name); + if(nameLen >= (sizeof(addr.sa_unix.sun_path) - 1)) + { + THROW_EXCEPTION(ServerException, SocketNameUNIXPathTooLong); + } + sockAddrLen = nameLen + (((char*)(&(addr.sa_unix.sun_path[0]))) - ((char*)(&addr.sa_unix))); +#ifndef PLATFORM_sockaddr_NO_len + addr.sa_unix.sun_len = sockAddrLen; +#endif + addr.sa_unix.sun_family = PF_UNIX; + ::strcpy(addr.sa_unix.sun_path, Name); + } + break; + + default: + THROW_EXCEPTION(CommonException, BadArguments) + break; + } + + // Return size of structure to caller + rSockAddrLenOut = sockAddrLen; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Socket::LogIncomingConnection(const struct sockaddr *, socklen_t) +// Purpose: Writes a message logging the connection to syslog +// Created: 2003/08/01 +// +// -------------------------------------------------------------------------- +void Socket::LogIncomingConnection(const struct sockaddr *addr, socklen_t addrlen) +{ + if(addr == NULL) {THROW_EXCEPTION(CommonException, BadArguments)} + + switch(addr->sa_family) + { + case AF_UNIX: + ::syslog(LOG_INFO, "Incoming connection from local (UNIX socket)"); + break; + + case AF_INET: + { + sockaddr_in *a = (sockaddr_in*)addr; + ::syslog(LOG_INFO, "Incoming connection from %s port %d", inet_ntoa(a->sin_addr), ntohs(a->sin_port)); + } + break; + + default: + ::syslog(LOG_INFO, "Incoming connection of unknown type"); + break; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Socket::IncomingConnectionLogMessage(const struct sockaddr *, socklen_t) +// Purpose: Returns a string for use in log messages +// Created: 2003/08/01 +// +// -------------------------------------------------------------------------- +std::string Socket::IncomingConnectionLogMessage(const struct sockaddr *addr, socklen_t addrlen) +{ + if(addr == NULL) {THROW_EXCEPTION(CommonException, BadArguments)} + + switch(addr->sa_family) + { + case AF_UNIX: + return std::string("Incoming connection from local (UNIX socket)"); + break; + + case AF_INET: + { + char msg[256]; // more than enough + sockaddr_in *a = (sockaddr_in*)addr; + sprintf(msg, "Incoming connection from %s port %d", inet_ntoa(a->sin_addr), ntohs(a->sin_port)); + return std::string(msg); + } + break; + + default: + return std::string("Incoming connection of unknown type"); + break; + } + + // Dummy. + return std::string(); +} + diff --git a/lib/server/Socket.h b/lib/server/Socket.h new file mode 100755 index 00000000..86a06097 --- /dev/null +++ b/lib/server/Socket.h @@ -0,0 +1,47 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Socket.h +// Purpose: Socket related stuff +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef SOCKET__H +#define SOCKET__H + +#include <sys/socket.h> +#include <netinet/in.h> +#include <sys/un.h> + +#include <string> + +typedef union { + struct sockaddr sa_generic; + struct sockaddr_in sa_inet; + struct sockaddr_un sa_unix; +} SocketAllAddr; + +// -------------------------------------------------------------------------- +// +// Namespace +// Name: Socket +// Purpose: Socket utilities +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +namespace Socket +{ + enum + { + TypeINET = 1, + TypeUNIX = 2 + }; + + void NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type, const char *Name, int Port, int &rSockAddrLenOut); + void LogIncomingConnection(const struct sockaddr *addr, socklen_t addrlen); + std::string IncomingConnectionLogMessage(const struct sockaddr *addr, socklen_t addrlen); +}; + +#endif // SOCKET__H + diff --git a/lib/server/SocketListen.h b/lib/server/SocketListen.h new file mode 100755 index 00000000..f1f5e377 --- /dev/null +++ b/lib/server/SocketListen.h @@ -0,0 +1,265 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SocketListen.h +// Purpose: Stream based sockets for servers +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef SOCKETLISTEN__H +#define SOCKETLISTEN__H + +#include <errno.h> +#include <unistd.h> +#include <new> +#include <poll.h> +#include <memory> +#include <string> +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + #include <sys/event.h> + #include <sys/time.h> +#endif + +#include "Socket.h" +#include "ServerException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: _NoSocketLocking +// Purpose: Default locking class for SocketListen +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +class _NoSocketLocking +{ +public: + _NoSocketLocking(int sock) + { + } + + ~_NoSocketLocking() + { + } + + bool HaveLock() + { + return true; + } + +private: + _NoSocketLocking(const _NoSocketLocking &rToCopy) + { + } +}; + + +// -------------------------------------------------------------------------- +// +// Class +// Name: SocketListen +// Purpose: +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +template<typename SocketType, int ListenBacklog = 128, typename SocketLockingType = _NoSocketLocking, int MaxMultiListenSockets = 16> +class SocketListen +{ +public: + // Initialise + SocketListen() + : mSocketHandle(-1) + { + } + // Close socket nicely + ~SocketListen() + { + Close(); + } +private: + SocketListen(const SocketListen &rToCopy) + { + } +public: + + enum + { + MaxMultipleListenSockets = MaxMultiListenSockets + }; + + void Close() + { + if(mSocketHandle != -1) + { + if(::close(mSocketHandle) == -1) + { + THROW_EXCEPTION(ServerException, SocketCloseError) + } + } + mSocketHandle = -1; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: SocketListen::Listen(int, char*, int) + // Purpose: Initialises, starts the socket listening. + // Created: 2003/07/31 + // + // -------------------------------------------------------------------------- + void Listen(int Type, const char *Name, int Port = 0) + { + if(mSocketHandle != -1) {THROW_EXCEPTION(ServerException, SocketAlreadyOpen)} + + // Setup parameters based on type, looking up names if required + int sockDomain = 0; + SocketAllAddr addr; + int addrLen = 0; + Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, Port, addrLen); + + // Create the socket + mSocketHandle = ::socket(sockDomain, SOCK_STREAM, 0 /* let OS choose protocol */); + if(mSocketHandle == -1) + { + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + // Set an option to allow reuse (useful for -HUP situations!) + int option = true; + if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, &option, sizeof(option)) == -1) + { + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + // Bind it to the right port, and start listening + if(::bind(mSocketHandle, &addr.sa_generic, addrLen) == -1 + || ::listen(mSocketHandle, ListenBacklog) == -1) + { + // Dispose of the socket + ::close(mSocketHandle); + mSocketHandle = -1; + THROW_EXCEPTION(ServerException, SocketBindError) + } + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: SocketListen::Accept(int) + // Purpose: Accepts a connection, returning a pointer to a class of + // the specified type. May return a null pointer if a signal happens, + // or there's a timeout. Timeout specified in milliseconds, defaults to infinite time. + // Created: 2003/07/31 + // + // -------------------------------------------------------------------------- + std::auto_ptr<SocketType> Accept(int Timeout = INFTIM, std::string *pLogMsg = 0) + { + if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)} + + // Do the accept, using the supplied locking type + int sock; + struct sockaddr addr; + socklen_t addrlen = sizeof(addr); + // BLOCK + { + SocketLockingType socklock(mSocketHandle); + + if(!socklock.HaveLock()) + { + // Didn't get the lock for some reason. Wait a while, then + // return nothing. + ::sleep(1); + return std::auto_ptr<SocketType>(); + } + + // poll this socket + struct pollfd p; + p.fd = mSocketHandle; + p.events = POLLIN; + p.revents = 0; + switch(::poll(&p, 1, Timeout)) + { + case -1: + // signal? + if(errno == EINTR) + { + // return nothing + return std::auto_ptr<SocketType>(); + } + else + { + THROW_EXCEPTION(ServerException, SocketPollError) + } + break; + case 0: // timed out + return std::auto_ptr<SocketType>(); + break; + default: // got some thing... + // control flows on... + break; + } + + sock = ::accept(mSocketHandle, &addr, &addrlen); + } + // Got socket (or error), unlock (implcit in destruction) + if(sock == -1) + { + THROW_EXCEPTION(ServerException, SocketAcceptError) + } + + // Log it + if(pLogMsg) + { + *pLogMsg = Socket::IncomingConnectionLogMessage(&addr, addrlen); + } + else + { + // Do logging ourselves + Socket::LogIncomingConnection(&addr, addrlen); + } + + return std::auto_ptr<SocketType>(new SocketType(sock)); + } + + // Functions to allow adding to WaitForEvent class, for efficient waiting + // on multiple sockets. +#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED + // -------------------------------------------------------------------------- + // + // Function + // Name: SocketListen::FillInKEevent + // Purpose: Fills in a kevent structure for this socket + // Created: 9/3/04 + // + // -------------------------------------------------------------------------- + void FillInKEvent(struct kevent &rEvent, int Flags = 0) const + { + EV_SET(&rEvent, mSocketHandle, EVFILT_READ, 0, 0, 0, (void*)this); + } +#else + // -------------------------------------------------------------------------- + // + // Function + // Name: SocketListen::FillInPoll + // Purpose: Fills in the data necessary for a poll operation + // Created: 9/3/04 + // + // -------------------------------------------------------------------------- + void FillInPoll(int &fd, short &events, int Flags = 0) const + { + fd = mSocketHandle; + events = POLLIN; + } +#endif + +private: + int mSocketHandle; +}; + +#include "MemLeakFindOff.h" + +#endif // SOCKETLISTEN__H + diff --git a/lib/server/SocketStream.cpp b/lib/server/SocketStream.cpp new file mode 100755 index 00000000..3c8bf453 --- /dev/null +++ b/lib/server/SocketStream.cpp @@ -0,0 +1,405 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SocketStream.cpp +// Purpose: I/O stream interface for sockets +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <unistd.h> +#include <sys/types.h> +#include <poll.h> +#include <errno.h> + +#include "SocketStream.h" +#include "ServerException.h" +#include "CommonException.h" +#include "Socket.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::SocketStream() +// Purpose: Constructor (create stream ready for Open() call) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +SocketStream::SocketStream() + : mSocketHandle(-1), + mReadClosed(false), + mWriteClosed(false) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::SocketStream(int) +// Purpose: Create stream from existing socket handle +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +SocketStream::SocketStream(int socket) + : mSocketHandle(socket), + mReadClosed(false), + mWriteClosed(false) +{ + if(socket < 0) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::SocketStream(const SocketStream &) +// Purpose: Copy constructor (dup()s socket) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +SocketStream::SocketStream(const SocketStream &rToCopy) + : mSocketHandle(::dup(rToCopy.mSocketHandle)), + mReadClosed(rToCopy.mReadClosed), + mWriteClosed(rToCopy.mWriteClosed) + +{ + if(rToCopy.mSocketHandle < 0) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + if(mSocketHandle == -1) + { + THROW_EXCEPTION(ServerException, DupError); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::~SocketStream() +// Purpose: Destructor, closes stream if open +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +SocketStream::~SocketStream() +{ + if(mSocketHandle != -1) + { + Close(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Attach(int) +// Purpose: Attach a socket handle to this stream +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void SocketStream::Attach(int socket) +{ + if(mSocketHandle != -1) {THROW_EXCEPTION(ServerException, SocketAlreadyOpen)} + + mSocketHandle = socket; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Open(int, char *, int) +// Purpose: Opens a connection to a listening socket (INET or UNIX) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void SocketStream::Open(int Type, const char *Name, int Port) +{ + if(mSocketHandle != -1) {THROW_EXCEPTION(ServerException, SocketAlreadyOpen)} + + // Setup parameters based on type, looking up names if required + int sockDomain = 0; + SocketAllAddr addr; + int addrLen = 0; + Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, Port, addrLen); + + // Create the socket + mSocketHandle = ::socket(sockDomain, SOCK_STREAM, 0 /* let OS choose protocol */); + if(mSocketHandle == -1) + { + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + // Connect it + if(::connect(mSocketHandle, &addr.sa_generic, addrLen) == -1) + { + // Dispose of the socket + ::close(mSocketHandle); + mSocketHandle = -1; + THROW_EXCEPTION(ConnectionException, Conn_SocketConnectError) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Read(void *pBuffer, int NBytes) +// Purpose: Reads data from stream. Maybe returns less than asked for. +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)} + + if(Timeout != IOStream::TimeOutInfinite) + { + struct pollfd p; + p.fd = mSocketHandle; + p.events = POLLIN; + p.revents = 0; + switch(::poll(&p, 1, (Timeout == IOStream::TimeOutInfinite)?INFTIM:Timeout)) + { + case -1: + // error + if(errno == EINTR) + { + // Signal. Just return 0 bytes + return 0; + } + else + { + // Bad! + THROW_EXCEPTION(ServerException, SocketPollError) + } + break; + + case 0: + // no data + return 0; + break; + + default: + // good to go! + break; + } + } + + int r = ::read(mSocketHandle, pBuffer, NBytes); + if(r == -1) + { + if(errno == EINTR) + { + // Nothing could be read + return 0; + } + else + { + // Other error + THROW_EXCEPTION(ConnectionException, Conn_SocketReadError) + } + } + // Closed for reading? + if(r == 0) + { + mReadClosed = true; + } + + return r; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Write(void *pBuffer, int NBytes) +// Purpose: Writes data, blocking until it's all done. +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void SocketStream::Write(const void *pBuffer, int NBytes) +{ + if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)} + + // Buffer in byte sized type. + ASSERT(sizeof(char) == 1); + const char *buffer = (char *)pBuffer; + + // Bytes left to send + int bytesLeft = NBytes; + + while(bytesLeft > 0) + { + // Try to send. + int sent = ::write(mSocketHandle, buffer, bytesLeft); + if(sent == -1) + { + // Error. + mWriteClosed = true; // assume can't write again + THROW_EXCEPTION(ConnectionException, Conn_SocketWriteError) + } + + // Knock off bytes sent + bytesLeft -= sent; + // Move buffer pointer + buffer += sent; + + // Need to wait until it can send again? + if(bytesLeft > 0) + { + TRACE3("Waiting to send data on socket %d, (%d to send of %d)\n", mSocketHandle, bytesLeft, NBytes); + + // Wait for data to send. + struct pollfd p; + p.fd = mSocketHandle; + p.events = POLLOUT; + p.revents = 0; + + if(::poll(&p, 1, 16000 /* 16 seconds */) == -1) + { + // Don't exception if it's just a signal + if(errno != EINTR) + { + THROW_EXCEPTION(ServerException, SocketPollError) + } + } + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Close() +// Purpose: Closes connection to remote socket +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void SocketStream::Close() +{ + if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)} + + if(::close(mSocketHandle) == -1) + { + THROW_EXCEPTION(ServerException, SocketCloseError) + } + mSocketHandle = -1; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Shutdown(bool, bool) +// Purpose: Shuts down a socket for further reading and/or writing +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void SocketStream::Shutdown(bool Read, bool Write) +{ + if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)} + + // Do anything? + if(!Read && !Write) return; + + int how = SHUT_RDWR; + if(Read && !Write) how = SHUT_RD; + if(!Read && Write) how = SHUT_WR; + + // Shut it down! + if(::shutdown(mSocketHandle, how) == -1) + { + THROW_EXCEPTION(ConnectionException, Conn_SocketShutdownError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::StreamDataLeft() +// Purpose: Still capable of reading data? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool SocketStream::StreamDataLeft() +{ + return !mReadClosed; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::StreamClosed() +// Purpose: Connection been closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool SocketStream::StreamClosed() +{ + return mWriteClosed; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::GetSocketHandle() +// Purpose: Returns socket handle for this stream (derived classes only). +// Will exception if there's no valid socket. +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +int SocketStream::GetSocketHandle() +{ + if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)} + return mSocketHandle; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::GetPeerCredentials(uid_t &, gid_t &) +// Purpose: Returns true if the peer credientials are available. +// (will work on UNIX domain sockets only) +// Created: 19/2/04 +// +// -------------------------------------------------------------------------- +bool SocketStream::GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut) +{ +#ifdef PLATFORM_HAVE_getpeereid + uid_t remoteEUID = 0xffff; + gid_t remoteEGID = 0xffff; + + if(::getpeereid(mSocketHandle, &remoteEUID, &remoteEGID) == 0) + { + rUidOut = remoteEUID; + rGidOut = remoteEGID; + return true; + } +#endif // PLATFORM_HAVE_getpeereid + +#ifdef PLATFORM_HAVE_getsockopt_SO_PEERCRED + struct ucred cred; + socklen_t credLen = sizeof(cred); + + if(::getsockopt(mSocketHandle, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) + { + rUidOut = cred.uid; + rGidOut = cred.gid; + return true; + } +#endif // PLATFORM_HAVE_getsockopt_SO_PEERCRED + + // Not available + return false; +} + + + + diff --git a/lib/server/SocketStream.h b/lib/server/SocketStream.h new file mode 100755 index 00000000..9b16dbfc --- /dev/null +++ b/lib/server/SocketStream.h @@ -0,0 +1,56 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SocketStream.h +// Purpose: I/O stream interface for sockets +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef SOCKETSTREAM__H +#define SOCKETSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: SocketStream +// Purpose: Stream interface for sockets +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +class SocketStream : public IOStream +{ +public: + SocketStream(); + SocketStream(int socket); + SocketStream(const SocketStream &rToCopy); + ~SocketStream(); + + void Open(int Type, const char *Name, int Port = 0); + void Attach(int socket); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes); + virtual void Close(); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + virtual void Shutdown(bool Read = true, bool Write = true); + + virtual bool GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut); + +protected: + int GetSocketHandle(); + void MarkAsReadClosed() {mReadClosed = true;} + void MarkAsWriteClosed() {mWriteClosed = true;} + +private: + int mSocketHandle; + bool mReadClosed; + bool mWriteClosed; +}; + +#endif // SOCKETSTREAM__H + diff --git a/lib/server/SocketStreamTLS.cpp b/lib/server/SocketStreamTLS.cpp new file mode 100755 index 00000000..63ac7bb5 --- /dev/null +++ b/lib/server/SocketStreamTLS.cpp @@ -0,0 +1,457 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SocketStreamTLS.cpp +// Purpose: Socket stream encrpyted and authenticated by TLS +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#define TLS_CLASS_IMPLEMENTATION_CPP +#include <openssl/ssl.h> +#include <openssl/bio.h> +#include <poll.h> +#include <errno.h> +#include <sys/ioctl.h> + +#include "SocketStreamTLS.h" +#include "SSLLib.h" +#include "ServerException.h" +#include "TLSContext.h" + +#include "MemLeakFindOn.h" + +// Allow 5 minutes to handshake (in milliseconds) +#define TLS_HANDSHAKE_TIMEOUT (5*60*1000) + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::SocketStreamTLS() +// Purpose: Constructor +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +SocketStreamTLS::SocketStreamTLS() + : mpSSL(0), mpBIO(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::SocketStreamTLS(int) +// Purpose: Constructor, taking previously connected socket +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +SocketStreamTLS::SocketStreamTLS(int socket) + : SocketStream(socket), + mpSSL(0), mpBIO(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::~SocketStreamTLS() +// Purpose: Destructor +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +SocketStreamTLS::~SocketStreamTLS() +{ + if(mpSSL) + { + // Attempt to close to avoid problems + Close(); + + // And if that didn't work... + if(mpSSL) + { + ::SSL_free(mpSSL); + mpSSL = 0; + mpBIO = 0; // implicity freed by the SSL_free call + } + } + + // If we only got to creating that BIO. + if(mpBIO) + { + ::BIO_free(mpBIO); + mpBIO = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Open(const TLSContext &, int, const char *, int) +// Purpose: Open connection, and perform TLS handshake +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SocketStreamTLS::Open(const TLSContext &rContext, int Type, const char *Name, int Port) +{ + SocketStream::Open(Type, Name, Port); + Handshake(rContext); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Handshake(const TLSContext &, bool) +// Purpose: Perform TLS handshake +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) +{ + if(mpBIO || mpSSL) {THROW_EXCEPTION(ServerException, TLSAlreadyHandshaked)} + + // Create a BIO for this socket + mpBIO = ::BIO_new(::BIO_s_socket()); + if(mpBIO == 0) + { + SSLLib::LogError("Create socket bio"); + THROW_EXCEPTION(ServerException, TLSAllocationFailed) + } + int socket = GetSocketHandle(); + BIO_set_fd(mpBIO, socket, BIO_NOCLOSE); + + // Then the SSL object + mpSSL = ::SSL_new(rContext.GetRawContext()); + if(mpSSL == 0) + { + SSLLib::LogError("Create ssl"); + THROW_EXCEPTION(ServerException, TLSAllocationFailed) + } + + // Make the socket non-blocking so timeouts on Read work + int nonblocking = true; + if(::ioctl(socket, FIONBIO, &nonblocking) == -1) + { + THROW_EXCEPTION(ServerException, SocketSetNonBlockingFailed) + } + + // Set the two to know about each other + ::SSL_set_bio(mpSSL, mpBIO, mpBIO); + + bool waitingForHandshake = true; + while(waitingForHandshake) + { + // Attempt to do the handshake + int r = 0; + if(IsServer) + { + r = ::SSL_accept(mpSSL); + } + else + { + r = ::SSL_connect(mpSSL); + } + + // check return code + int se; + switch((se = ::SSL_get_error(mpSSL, r))) + { + case SSL_ERROR_NONE: + // No error, handshake succeeded + waitingForHandshake = false; + break; + + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + // wait for the requried data + if(WaitWhenRetryRequired(se, TLS_HANDSHAKE_TIMEOUT) == false) + { + // timed out + THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeTimedOut) + } + break; + + default: // (and SSL_ERROR_ZERO_RETURN) + // Error occured + if(IsServer) + { + SSLLib::LogError("Accept"); + THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) + } + else + { + SSLLib::LogError("Connect"); + THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) + } + } + } + + // And that's it +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WaitWhenRetryRequired(int, int) +// Purpose: Waits until the condition required by the TLS layer is met. +// Returns true if the condition is met, false if timed out. +// Created: 2003/08/15 +// +// -------------------------------------------------------------------------- +bool SocketStreamTLS::WaitWhenRetryRequired(int SSLErrorCode, int Timeout) +{ + struct pollfd p; + p.fd = GetSocketHandle(); + switch(SSLErrorCode) + { + case SSL_ERROR_WANT_READ: + p.events = POLLIN; + break; + + case SSL_ERROR_WANT_WRITE: + p.events = POLLOUT; + break; + + default: + // Not good! + THROW_EXCEPTION(ServerException, Internal) + break; + } + p.revents = 0; + switch(::poll(&p, 1, (Timeout == IOStream::TimeOutInfinite)?INFTIM:Timeout)) + { + case -1: + // error + if(errno == EINTR) + { + // Signal. Do "time out" + return false; + } + else + { + // Bad! + THROW_EXCEPTION(ServerException, SocketPollError) + } + break; + + case 0: + // Condition not met, timed out + return false; + break; + + default: + // good to go! + return true; + break; + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Read(void *, int, int Timeout) +// Purpose: See base class +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) +{ + if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} + + // Make sure zero byte reads work as expected + if(NBytes == 0) + { + return 0; + } + + while(true) + { + int r = ::SSL_read(mpSSL, pBuffer, NBytes); + + int se; + switch((se = ::SSL_get_error(mpSSL, r))) + { + case SSL_ERROR_NONE: + // No error, return number of bytes read + return r; + break; + + case SSL_ERROR_ZERO_RETURN: + // Connection closed + MarkAsReadClosed(); + return 0; + break; + + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + // wait for the requried data + // Will only get once around this loop, so don't need to calculate timeout values + if(WaitWhenRetryRequired(se, Timeout) == false) + { + // timed out + return 0; + } + break; + + default: + SSLLib::LogError("Read"); + THROW_EXCEPTION(ConnectionException, Conn_TLSReadFailed) + break; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Write(const void *, int) +// Purpose: See base class +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SocketStreamTLS::Write(const void *pBuffer, int NBytes) +{ + if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} + + // Make sure zero byte writes work as expected + if(NBytes == 0) + { + return; + } + + // from man SSL_write + // + // SSL_write() will only return with success, when the + // complete contents of buf of length num has been written. + // + // So no worries about partial writes and moving the buffer around + + while(true) + { + // try the write + int r = ::SSL_write(mpSSL, pBuffer, NBytes); + + int se; + switch((se = ::SSL_get_error(mpSSL, r))) + { + case SSL_ERROR_NONE: + // No error, data sent, return success + return; + break; + + case SSL_ERROR_ZERO_RETURN: + // Connection closed + MarkAsWriteClosed(); + THROW_EXCEPTION(ConnectionException, Conn_TLSClosedWhenWriting) + break; + + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + // wait for the requried data + { + #ifndef NDEBUG + bool conditionmet = + #endif + WaitWhenRetryRequired(se, IOStream::TimeOutInfinite); + ASSERT(conditionmet); + } + break; + + default: + SSLLib::LogError("Write"); + THROW_EXCEPTION(ConnectionException, Conn_TLSWriteFailed) + break; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Close() +// Purpose: See base class +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SocketStreamTLS::Close() +{ + if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} + + // Base class to close + SocketStream::Close(); + + // Free resources + ::SSL_free(mpSSL); + mpSSL = 0; + mpBIO = 0; // implicitly freed by SSL_free +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Shutdown() +// Purpose: See base class +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SocketStreamTLS::Shutdown(bool Read, bool Write) +{ + if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} + + if(::SSL_shutdown(mpSSL) < 0) + { + SSLLib::LogError("Shutdown"); + THROW_EXCEPTION(ConnectionException, Conn_TLSShutdownFailed) + } + + // Don't ask the base class to shutdown -- BIO does this, apparently. +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::GetPeerCommonName() +// Purpose: Returns the common name of the other end of the connection +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +std::string SocketStreamTLS::GetPeerCommonName() +{ + if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} + + // Get certificate + X509 *cert = ::SSL_get_peer_certificate(mpSSL); + if(cert == 0) + { + ::X509_free(cert); + THROW_EXCEPTION(ConnectionException, Conn_TLSNoPeerCertificate) + } + + // Subject details + X509_NAME *subject = ::X509_get_subject_name(cert); + if(subject == 0) + { + ::X509_free(cert); + THROW_EXCEPTION(ConnectionException, Conn_TLSPeerCertificateInvalid) + } + + // Common name + char commonName[256]; + if(::X509_NAME_get_text_by_NID(subject, NID_commonName, commonName, sizeof(commonName)) <= 0) + { + ::X509_free(cert); + THROW_EXCEPTION(ConnectionException, Conn_TLSPeerCertificateInvalid) + } + // Terminate just in case + commonName[sizeof(commonName)-1] = '\0'; + + // Done. + return std::string(commonName); +} + + diff --git a/lib/server/SocketStreamTLS.h b/lib/server/SocketStreamTLS.h new file mode 100755 index 00000000..64e52833 --- /dev/null +++ b/lib/server/SocketStreamTLS.h @@ -0,0 +1,60 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SocketStreamTLS.h +// Purpose: Socket stream encrpyted and authenticated by TLS +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#ifndef SOCKETSTREAMTLS__H +#define SOCKETSTREAMTLS__H + +#include <string> + +#include "SocketStream.h" + +class TLSContext; +#ifndef TLS_CLASS_IMPLEMENTATION_CPP + class SSL; + class BIO; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: SocketStreamTLS +// Purpose: Socket stream encrpyted and authenticated by TLS +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +class SocketStreamTLS : public SocketStream +{ +public: + SocketStreamTLS(); + SocketStreamTLS(int socket); + ~SocketStreamTLS(); +private: + SocketStreamTLS(const SocketStreamTLS &rToCopy); +public: + + void Open(const TLSContext &rContext, int Type, const char *Name, int Port = 0); + void Handshake(const TLSContext &rContext, bool IsServer = false); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes); + virtual void Close(); + virtual void Shutdown(bool Read = true, bool Write = true); + + std::string GetPeerCommonName(); + +private: + bool WaitWhenRetryRequired(int SSLErrorCode, int Timeout); + +private: + SSL *mpSSL; + BIO *mpBIO; +}; + +#endif // SOCKETSTREAMTLS__H + diff --git a/lib/server/TLSContext.cpp b/lib/server/TLSContext.cpp new file mode 100755 index 00000000..cc125d00 --- /dev/null +++ b/lib/server/TLSContext.cpp @@ -0,0 +1,120 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: TLSContext.h +// Purpose: TLS (SSL) context for connections +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#define TLS_CLASS_IMPLEMENTATION_CPP +#include <openssl/ssl.h> + +#include "TLSContext.h" +#include "ServerException.h" +#include "SSLLib.h" +#include "TLSContext.h" + +#include "MemLeakFindOn.h" + +#define MAX_VERIFICATION_DEPTH 2 +#define CIPHER_LIST "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH" + +// -------------------------------------------------------------------------- +// +// Function +// Name: TLSContext::TLSContext() +// Purpose: Constructor +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +TLSContext::TLSContext() + : mpContext(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: TLSContext::~TLSContext() +// Purpose: Destructor +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +TLSContext::~TLSContext() +{ + if(mpContext != 0) + { + ::SSL_CTX_free(mpContext); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: TLSContext::Initialise(bool, const char *, const char *, const char *) +// Purpose: Initialise the context, loading in the specified certificate and private key files +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const char *PrivateKeyFile, const char *TrustedCAsFile) +{ + mpContext = ::SSL_CTX_new(AsServer?TLSv1_server_method():TLSv1_client_method()); + if(mpContext == NULL) + { + THROW_EXCEPTION(ServerException, TLSAllocationFailed) + } + + // Setup our identity + if(::SSL_CTX_use_certificate_chain_file(mpContext, CertificatesFile) != 1) + { + SSLLib::LogError("Load certificates"); + THROW_EXCEPTION(ServerException, TLSLoadCertificatesFailed) + } + if(::SSL_CTX_use_PrivateKey_file(mpContext, PrivateKeyFile, SSL_FILETYPE_PEM) != 1) + { + SSLLib::LogError("Load private key"); + THROW_EXCEPTION(ServerException, TLSLoadPrivateKeyFailed) + } + + // Setup the identify of CAs we trust + if(::SSL_CTX_load_verify_locations(mpContext, TrustedCAsFile, NULL) != 1) + { + SSLLib::LogError("Load CA cert"); + THROW_EXCEPTION(ServerException, TLSLoadTrustedCAsFailed) + } + + // Setup options to require these certificates + ::SSL_CTX_set_verify(mpContext, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL); + // and a sensible maximum depth + ::SSL_CTX_set_verify_depth(mpContext, MAX_VERIFICATION_DEPTH); + + // Setup allowed ciphers + if(::SSL_CTX_set_cipher_list(mpContext, CIPHER_LIST) != 1) + { + SSLLib::LogError("Set cipher list"); + THROW_EXCEPTION(ServerException, TLSSetCiphersFailed) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: TLSContext::GetRawContext() +// Purpose: Get the raw context for OpenSSL API +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +SSL_CTX *TLSContext::GetRawContext() const +{ + if(mpContext == 0) + { + THROW_EXCEPTION(ServerException, TLSContextNotInitialised) + } + return mpContext; +} + + + diff --git a/lib/server/TLSContext.h b/lib/server/TLSContext.h new file mode 100755 index 00000000..f52f5457 --- /dev/null +++ b/lib/server/TLSContext.h @@ -0,0 +1,41 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: TLSContext.h +// Purpose: TLS (SSL) context for connections +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#ifndef TLSCONTEXT__H +#define TLSCONTEXT__H + +#ifndef TLS_CLASS_IMPLEMENTATION_CPP + class SSL_CTX; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: TLSContext +// Purpose: TLS (SSL) context for connections +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +class TLSContext +{ +public: + TLSContext(); + ~TLSContext(); +private: + TLSContext(const TLSContext &); +public: + void Initialise(bool AsServer, const char *CertificatesFile, const char *PrivateKeyFile, const char *TrustedCAsFile); + SSL_CTX *GetRawContext() const; + +private: + SSL_CTX *mpContext; +}; + +#endif // TLSCONTEXT__H + diff --git a/lib/server/makeprotocol.pl b/lib/server/makeprotocol.pl new file mode 100755 index 00000000..2a69c59c --- /dev/null +++ b/lib/server/makeprotocol.pl @@ -0,0 +1,993 @@ +#!/usr/bin/perl +use strict; + +# Make protocol C++ classes from a protocol description file + +# built in type info (values are is basic type, C++ typename) +# may get stuff added to it later if protocol uses extra types +my %translate_type_info = +( + 'int64' => [1, 'int64_t'], + 'int32' => [1, 'int32_t'], + 'int16' => [1, 'int16_t'], + 'int8' => [1, 'int8_t'], + 'bool' => [1, 'bool'], + 'string' => [0, 'std::string'] +); + +# built in instructions for logging various types +# may be added to +my %log_display_types = +( + 'int64' => ['0x%llx', 'VAR'], + 'int32' => ['0x%x', 'VAR'], + 'int16' => ['0x%x', 'VAR'], + 'int8' => ['0x%x', 'VAR'], + 'bool' => ['%s', '((VAR)?"true":"false")'], + 'string' => ['%s', 'VAR.c_str()'] +); + + + +my ($type, $file) = @ARGV; + +if($type ne 'Server' && $type ne 'Client') +{ + die "Neither Server or Client is specified on command line\n"; +} + +open IN, $file or die "Can't open input file $file\n"; + +print "Making $type protocol classes from $file...\n"; + +my @extra_header_files; + +my $implement_syslog = 0; +my $implement_filelog = 0; + +# read attributes +my %attr; +while(<IN>) +{ + # get and clean line + my $l = $_; $l =~ s/#.*\Z//; $l =~ s/\A\s+//; $l =~ s/\s+\Z//; next unless $l =~ m/\S/; + + last if $l eq 'BEGIN_OBJECTS'; + + my ($k,$v) = split /\s+/,$l,2; + + if($k eq 'ClientType') + { + add_type($v) if $type eq 'Client'; + } + elsif($k eq 'ServerType') + { + add_type($v) if $type eq 'Server'; + } + elsif($k eq 'ImplementLog') + { + my ($log_if_type,$log_type) = split /\s+/,$v; + if($type eq $log_if_type) + { + if($log_type eq 'syslog') + { + $implement_syslog = 1; + } + elsif($log_type eq 'file') + { + $implement_filelog = 1; + } + else + { + printf("ERROR: Unknown log type for implementation: $log_type\n"); + exit(1); + } + } + } + elsif($k eq 'LogTypeToText') + { + my ($log_if_type,$type_name,$printf_format,$arg_template) = split /\s+/,$v; + if($type eq $log_if_type) + { + $log_display_types{$type_name} = [$printf_format,$arg_template] + } + } + else + { + $attr{$k} = $v; + } +} + +sub add_type +{ + my ($protocol_name, $cpp_name, $header_file) = split /\s+/,$_[0]; + + $translate_type_info{$protocol_name} = [0, $cpp_name]; + push @extra_header_files, $header_file; +} + +# check attributes +for(qw/Name ServerContextClass IdentString/) +{ + if(!exists $attr{$_}) + { + die "Attribute $_ is required, but not specified\n"; + } +} + +my $protocol_name = $attr{'Name'}; +my ($context_class, $context_class_inc) = split /\s+/,$attr{'ServerContextClass'}; +my $ident_string = $attr{'IdentString'}; + +my $current_cmd = ''; +my %cmd_contents; +my %cmd_attributes; +my %cmd_constants; +my %cmd_id; +my @cmd_list; + +# read in the command definitions +while(<IN>) +{ + # get and clean line + my $l = $_; $l =~ s/#.*\Z//; $l =~ s/\s+\Z//; next unless $l =~ m/\S/; + + # definitions or new command thing? + if($l =~ m/\A\s+/) + { + die "No command defined yet" if $current_cmd eq ''; + + # definition of component + $l =~ s/\A\s+//; + + my ($type,$name,$value) = split /\s+/,$l; + if($type eq 'CONSTANT') + { + push @{$cmd_constants{$current_cmd}},"$name = $value" + } + else + { + push @{$cmd_contents{$current_cmd}},$type,$name; + } + } + else + { + # new command + my ($name,$id,@attributes) = split /\s+/,$l; + $cmd_attributes{$name} = [@attributes]; + $cmd_id{$name} = int($id); + $current_cmd = $name; + push @cmd_list,$name; + } +} + +close IN; + + + +# open files +my $h_filename = 'autogen_'.$protocol_name.'Protocol'.$type.'.h'; +open CPP,'>autogen_'.$protocol_name.'Protocol'.$type.'.cpp'; +open H,">$h_filename"; + +print CPP <<__E; + +// Auto-generated file -- do not edit + +#include "Box.h" +#include "$h_filename" +#include "IOStream.h" + +__E + +if($implement_syslog) +{ + print H qq~#include <syslog.h>\n~; +} + + +my $guardname = uc 'AUTOGEN_'.$protocol_name.'Protocol'.$type.'_H'; +print H <<__E; + +// Auto-generated file -- do not edit + +#ifndef $guardname +#define $guardname + +#include "Protocol.h" +#include "ProtocolObject.h" +#include "ServerException.h" + +class IOStream; + +__E + +if($implement_filelog) +{ + print H qq~#include <stdio.h>\n~; +} + +# extra headers +for(@extra_header_files) +{ + print H qq~#include "$_"\n~ +} +print H "\n"; + +if($type eq 'Server') +{ + # need utils file for the server + print H '#include "Utils.h"',"\n\n" +} + + +my $derive_objects_from = 'ProtocolObject'; +my $objects_extra_h = ''; +my $objects_extra_cpp = ''; +if($type eq 'Server') +{ + # define the context + print H "class $context_class;\n\n"; + print CPP "#include \"$context_class_inc\"\n\n"; + + # change class we derive the objects from + $derive_objects_from = $protocol_name.'ProtocolObject'; + + $objects_extra_h = <<__E; + virtual std::auto_ptr<ProtocolObject> DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext); +__E + $objects_extra_cpp = <<__E; +std::auto_ptr<ProtocolObject> ${derive_objects_from}::DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext) +{ + THROW_EXCEPTION(ConnectionException, Conn_Protocol_TriedToExecuteReplyCommand) +} +__E +} + +print CPP qq~#include "MemLeakFindOn.h"\n~; + +if($type eq 'Client' && ($implement_syslog || $implement_filelog)) +{ + # change class we derive the objects from + $derive_objects_from = $protocol_name.'ProtocolObjectCl'; +} +if($implement_syslog) +{ + $objects_extra_h .= <<__E; + virtual void LogSysLog(const char *Action) const = 0; +__E +} +if($implement_filelog) +{ + $objects_extra_h .= <<__E; + virtual void LogFile(const char *Action, FILE *file) const = 0; +__E +} + +if($derive_objects_from ne 'ProtocolObject') +{ + # output a definition for the protocol object derviced class + print H <<__E; +class ${protocol_name}ProtocolServer; + +class $derive_objects_from : public ProtocolObject +{ +public: + $derive_objects_from(); + virtual ~$derive_objects_from(); + $derive_objects_from(const $derive_objects_from &rToCopy); + +$objects_extra_h +}; +__E + + # and some cpp definitions + print CPP <<__E; +${derive_objects_from}::${derive_objects_from}() +{ +} +${derive_objects_from}::~${derive_objects_from}() +{ +} +${derive_objects_from}::${derive_objects_from}(const $derive_objects_from &rToCopy) +{ +} +$objects_extra_cpp +__E +} + + + +my $classname_base = $protocol_name.'Protocol'.$type; + +# output the classes +for my $cmd (@cmd_list) +{ + print H <<__E; +class $classname_base$cmd : public $derive_objects_from +{ +public: + $classname_base$cmd(); + $classname_base$cmd(const $classname_base$cmd &rToCopy); + ~$classname_base$cmd(); + int GetType() const; + enum + { + TypeID = $cmd_id{$cmd} + }; +__E + # constants + if(exists $cmd_constants{$cmd}) + { + print H "\tenum\n\t{\n\t\t"; + print H join(",\n\t\t",@{$cmd_constants{$cmd}}); + print H "\n\t};\n"; + } + # flags + if(obj_is_type($cmd,'EndsConversation')) + { + print H "\tbool IsConversationEnd() const;\n"; + } + if(obj_is_type($cmd,'IsError')) + { + print H "\tbool IsError(int &rTypeOut, int &rSubTypeOut) const;\n"; + } + if($type eq 'Server' && obj_is_type($cmd, 'Command')) + { + print H "\tstd::auto_ptr<ProtocolObject> DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext); // IMPLEMENT THIS\n" + } + + # want to be able to read from streams? + my $read_from_streams = (obj_is_type($cmd,'Command') && $type eq 'Server') || (obj_is_type($cmd,'Reply') && $type eq 'Client'); + my $write_to_streams = (obj_is_type($cmd,'Command') && $type eq 'Client') || (obj_is_type($cmd,'Reply') && $type eq 'Server'); + + if($read_from_streams) + { + print H "\tvoid SetPropertiesFromStreamData(Protocol &rProtocol);\n"; + + # write Get functions + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + print H "\t".translate_type_to_arg_type($ty)." Get$nm() {return m$nm;}\n"; + } + } + my $param_con_args = ''; + if($write_to_streams) + { + # extra constructor? + if($#{$cmd_contents{$cmd}} >= 0) + { + my @a; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + push @a,translate_type_to_arg_type($ty)." $nm"; + } + $param_con_args = join(', ',@a); + print H "\t$classname_base$cmd(".$param_con_args.");\n"; + } + print H "\tvoid WritePropertiesToStreamData(Protocol &rProtocol) const;\n"; + # set functions + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + print H "\tvoid Set$nm(".translate_type_to_arg_type($ty)." $nm) {m$nm = $nm;}\n"; + } + } + + if($implement_syslog) + { + print H "\tvirtual void LogSysLog(const char *Action) const;\n"; + } + if($implement_filelog) + { + print H "\tvirtual void LogFile(const char *Action, FILE *file) const;\n"; + } + + + # write member variables and setup for cpp file + my @def_constructor_list; + my @copy_constructor_list; + my @param_constructor_list; + + print H "private:\n"; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + print H "\t".translate_type_to_member_type($ty)." m$nm;\n"; + + my ($basic,$typename) = translate_type($ty); + if($basic) + { + push @def_constructor_list, "m$nm(0)"; + } + push @copy_constructor_list, "m$nm(rToCopy.m$nm)"; + push @param_constructor_list, "m$nm($nm)"; + } + + # finish off + print H "};\n\n"; + + # now the cpp file... + my $def_con_vars = join(",\n\t ",@def_constructor_list); + $def_con_vars = "\n\t: ".$def_con_vars if $def_con_vars ne ''; + my $copy_con_vars = join(",\n\t ",@copy_constructor_list); + $copy_con_vars = "\n\t: ".$copy_con_vars if $copy_con_vars ne ''; + my $param_con_vars = join(",\n\t ",@param_constructor_list); + $param_con_vars = "\n\t: ".$param_con_vars if $param_con_vars ne ''; + + my $class = "$classname_base$cmd".'::'; + print CPP <<__E; +$class$classname_base$cmd()$def_con_vars +{ +} +$class$classname_base$cmd(const $classname_base$cmd &rToCopy)$copy_con_vars +{ +} +$class~$classname_base$cmd() +{ +} +int ${class}GetType() const +{ + return $cmd_id{$cmd}; +} +__E + if($read_from_streams) + { + print CPP "void ${class}SetPropertiesFromStreamData(Protocol &rProtocol)\n{\n"; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + if($ty =~ m/\Avector/) + { + print CPP "\trProtocol.ReadVector(m$nm);\n"; + } + else + { + print CPP "\trProtocol.Read(m$nm);\n"; + } + } + print CPP "}\n"; + } + if($write_to_streams) + { + # implement extra constructor? + if($param_con_vars ne '') + { + print CPP "$class$classname_base$cmd($param_con_args)$param_con_vars\n{\n}\n"; + } + print CPP "void ${class}WritePropertiesToStreamData(Protocol &rProtocol) const\n{\n"; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + if($ty =~ m/\Avector/) + { + print CPP "\trProtocol.WriteVector(m$nm);\n"; + } + else + { + print CPP "\trProtocol.Write(m$nm);\n"; + } + } + print CPP "}\n"; + } + if(obj_is_type($cmd,'EndsConversation')) + { + print CPP "bool ${class}IsConversationEnd() const\n{\n\treturn true;\n}\n"; + } + if(obj_is_type($cmd,'IsError')) + { + # get parameters + my ($mem_type,$mem_subtype) = split /,/,obj_get_type_params($cmd,'IsError'); + print CPP <<__E; +bool ${class}IsError(int &rTypeOut, int &rSubTypeOut) const +{ + rTypeOut = m$mem_type; + rSubTypeOut = m$mem_subtype; + return true; +} +__E + } + + if($implement_syslog) + { + my ($format,$args) = make_log_strings($cmd); + print CPP <<__E; +void ${class}LogSysLog(const char *Action) const +{ + ::syslog(LOG_INFO,"%s $format",Action$args); +} +__E + } + if($implement_filelog) + { + my ($format,$args) = make_log_strings($cmd); + print CPP <<__E; +void ${class}LogFile(const char *Action, FILE *File) const +{ + ::fprintf(File,"%s $format\\n",Action$args); + ::fflush(File); +} +__E + } +} + +# finally, the protocol object itself +print H <<__E; +class $classname_base : public Protocol +{ +public: + $classname_base(IOStream &rStream); + virtual ~$classname_base(); + + std::auto_ptr<$derive_objects_from> Receive(); + void Send(const ${derive_objects_from} &rObject); +__E +if($implement_syslog) +{ + print H "\tvoid SetLogToSysLog(bool Log = false) {mLogToSysLog = Log;}\n"; +} +if($implement_filelog) +{ + print H "\tvoid SetLogToFile(FILE *File = 0) {mLogToFile = File;}\n"; +} +if($type eq 'Server') +{ + # need to put in the conversation function + print H "\tvoid DoServer($context_class &rContext);\n\n"; + # and the send vector thing + print H "\tvoid SendStreamAfterCommand(IOStream *pStream);\n\n"; +} +if($type eq 'Client') +{ + # add plain object taking query functions + my $with_params; + for my $cmd (@cmd_list) + { + if(obj_is_type($cmd,'Command')) + { + my $has_stream = obj_is_type($cmd,'StreamWithCommand'); + my $argextra = $has_stream?', IOStream &rStream':''; + my $queryextra = $has_stream?', rStream':''; + my $reply = obj_get_type_params($cmd,'Command'); + print H "\tstd::auto_ptr<$classname_base$reply> Query(const $classname_base$cmd &rQuery$argextra);\n"; + my @a; + my @na; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + push @a,translate_type_to_arg_type($ty)." $nm"; + push @na,"$nm"; + } + my $ar = join(', ',@a); + my $nar = join(', ',@na); + $nar = "($nar)" if $nar ne ''; + + $with_params .= "\tinline std::auto_ptr<$classname_base$reply> Query$cmd($ar$argextra)\n\t{\n"; + $with_params .= "\t\t$classname_base$cmd send$nar;\n"; + $with_params .= "\t\treturn Query(send$queryextra);\n"; + $with_params .= "\t}\n"; + } + } + # quick hack to correct bad argument lists for commands with zero paramters but with streams + $with_params =~ s/\(, /(/g; + print H "\n",$with_params,"\n"; +} +print H <<__E; +private: + $classname_base(const $classname_base &rToCopy); +__E +if($type eq 'Server') +{ + # need to put the streams to send vector + print H "\tstd::vector<IOStream*> mStreamsToSend;\n\tvoid DeleteStreamsToSend();\n"; +} + +if($implement_filelog || $implement_syslog) +{ + print H <<__E; + virtual void InformStreamReceiving(u_int32_t Size); + virtual void InformStreamSending(u_int32_t Size); +__E +} + +if($implement_syslog) +{ + print H "private:\n\tbool mLogToSysLog;\n"; +} +if($implement_filelog) +{ + print H "private:\n\tFILE *mLogToFile;\n"; +} +print H <<__E; + +protected: + virtual std::auto_ptr<ProtocolObject> MakeProtocolObject(int ObjType); + virtual const char *GetIdentString(); +}; + +__E + +my $construtor_extra = ''; +$construtor_extra .= ', mLogToSysLog(false)' if $implement_syslog; +$construtor_extra .= ', mLogToFile(0)' if $implement_filelog; + +my $destructor_extra = ($type eq 'Server')?"\n\tDeleteStreamsToSend();":''; + +my $prefix = $classname_base.'::'; +print CPP <<__E; +$prefix$classname_base(IOStream &rStream) + : Protocol(rStream)$construtor_extra +{ +} +$prefix~$classname_base() +{$destructor_extra +} +const char *${prefix}GetIdentString() +{ + return "$ident_string"; +} +std::auto_ptr<ProtocolObject> ${prefix}MakeProtocolObject(int ObjType) +{ + switch(ObjType) + { +__E + +# do objects within this +for my $cmd (@cmd_list) +{ + print CPP <<__E; + case $cmd_id{$cmd}: + return std::auto_ptr<ProtocolObject>(new $classname_base$cmd); + break; +__E +} + +print CPP <<__E; + default: + THROW_EXCEPTION(ConnectionException, Conn_Protocol_UnknownCommandRecieved) + } +} +__E +# write receieve and send functions +print CPP <<__E; +std::auto_ptr<$derive_objects_from> ${prefix}Receive() +{ + std::auto_ptr<${derive_objects_from}> preply((${derive_objects_from}*)(Protocol::Receive().release())); + +__E + if($implement_syslog) + { + print CPP <<__E; + if(mLogToSysLog) + { + preply->LogSysLog("Receive"); + } +__E + } + if($implement_filelog) + { + print CPP <<__E; + if(mLogToFile != 0) + { + preply->LogFile("Receive", mLogToFile); + } +__E + } +print CPP <<__E; + + return preply; +} + +void ${prefix}Send(const ${derive_objects_from} &rObject) +{ +__E + if($implement_syslog) + { + print CPP <<__E; + if(mLogToSysLog) + { + rObject.LogSysLog("Send"); + } +__E + } + if($implement_filelog) + { + print CPP <<__E; + if(mLogToFile != 0) + { + rObject.LogFile("Send", mLogToFile); + } +__E + } + +print CPP <<__E; + Protocol::Send(rObject); +} + +__E +# write server function? +if($type eq 'Server') +{ + print CPP <<__E; +void ${prefix}DoServer($context_class &rContext) +{ + // Handshake with client + Handshake(); + + // Command processing loop + bool inProgress = true; + while(inProgress) + { + // Get an object from the conversation + std::auto_ptr<${derive_objects_from}> pobj(Receive()); + +__E + if($implement_syslog) + { + print CPP <<__E; + if(mLogToSysLog) + { + pobj->LogSysLog("Receive"); + } +__E + } + if($implement_filelog) + { + print CPP <<__E; + if(mLogToFile != 0) + { + pobj->LogFile("Receive", mLogToFile); + } +__E + } + print CPP <<__E; + + // Run the command + std::auto_ptr<${derive_objects_from}> preply((${derive_objects_from}*)(pobj->DoCommand(*this, rContext).release())); + +__E + if($implement_syslog) + { + print CPP <<__E; + if(mLogToSysLog) + { + preply->LogSysLog("Send"); + } +__E + } + if($implement_filelog) + { + print CPP <<__E; + if(mLogToFile != 0) + { + preply->LogFile("Send", mLogToFile); + } +__E + } + print CPP <<__E; + + // Send the reply + Send(*(preply.get())); + + // Send any streams + for(unsigned int s = 0; s < mStreamsToSend.size(); s++) + { + // Send the streams + SendStream(*mStreamsToSend[s]); + } + // Delete these streams + DeleteStreamsToSend(); + + // Does this end the conversation? + if(pobj->IsConversationEnd()) + { + inProgress = false; + } + } +} + +void ${prefix}SendStreamAfterCommand(IOStream *pStream) +{ + ASSERT(pStream != NULL); + mStreamsToSend.push_back(pStream); +} + +void ${prefix}DeleteStreamsToSend() +{ + for(std::vector<IOStream*>::iterator i(mStreamsToSend.begin()); i != mStreamsToSend.end(); ++i) + { + delete (*i); + } + mStreamsToSend.clear(); +} + +__E +} + +# write logging functions? +if($implement_filelog || $implement_syslog) +{ + my ($fR,$fS); + + if($implement_syslog) + { + $fR .= qq~\tif(mLogToSysLog) { ::syslog(LOG_INFO, (Size==Protocol::ProtocolStream_SizeUncertain)?"Receiving stream, size uncertain":"Receiving stream, size %d", Size); }\n~; + $fS .= qq~\tif(mLogToSysLog) { ::syslog(LOG_INFO, (Size==Protocol::ProtocolStream_SizeUncertain)?"Sending stream, size uncertain":"Sending stream, size %d", Size); }\n~; + } + if($implement_filelog) + { + $fR .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Receiving stream, size uncertain":"Receiving stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~; + $fS .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Sending stream, size uncertain":"Sending stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~; + } + + print CPP <<__E; + +void ${prefix}InformStreamReceiving(u_int32_t Size) +{ +$fR} + +void ${prefix}InformStreamSending(u_int32_t Size) +{ +$fS} + +__E +} + + +# write client Query functions? +if($type eq 'Client') +{ + for my $cmd (@cmd_list) + { + if(obj_is_type($cmd,'Command')) + { + my $reply = obj_get_type_params($cmd,'Command'); + my $reply_id = $cmd_id{$reply}; + my $has_stream = obj_is_type($cmd,'StreamWithCommand'); + my $argextra = $has_stream?', IOStream &rStream':''; + my $send_stream_extra = ''; + if($has_stream) + { + $send_stream_extra = <<__E; + + // Send stream after the command + SendStream(rStream); +__E + } + print CPP <<__E; +std::auto_ptr<$classname_base$reply> ${classname_base}::Query(const $classname_base$cmd &rQuery$argextra) +{ + // Send query + Send(rQuery); + $send_stream_extra + // Wait for the reply + std::auto_ptr<${derive_objects_from}> preply(Receive().release()); + + if(preply->GetType() == $reply_id) + { + // Correct response + return std::auto_ptr<$classname_base$reply>(($classname_base$reply*)preply.release()); + } + else + { + // Set protocol error + int type, subType; + if(preply->IsError(type, subType)) + { + SetError(type, subType); + TRACE2("Protocol: Error received %d/%d\\n", type, subType); + } + else + { + SetError(Protocol::UnknownError, Protocol::UnknownError); + } + + // Throw an exception + THROW_EXCEPTION(ConnectionException, Conn_Protocol_UnexpectedReply) + } +} +__E + } + } +} + + + +print H <<__E; +#endif // $guardname + +__E + +# close files +close H; +close CPP; + + +sub obj_is_type +{ + my ($c,$ty) = @_; + for(@{$cmd_attributes{$c}}) + { + return 1 if $_ =~ m/\A$ty/; + } + + return 0; +} + +sub obj_get_type_params +{ + my ($c,$ty) = @_; + for(@{$cmd_attributes{$c}}) + { + return $1 if $_ =~ m/\A$ty\((.+?)\)\Z/; + } + die "Can't find attribute $ty\n" +} + +# returns (is basic type, typename) +sub translate_type +{ + my $ty = $_[0]; + + if($ty =~ m/\Avector\<(.+?)\>\Z/) + { + my $v_type = $1; + my (undef,$v_ty) = translate_type($v_type); + return (0, 'std::vector<'.$v_ty.'>') + } + else + { + if(!exists $translate_type_info{$ty}) + { + die "Don't know about type name $ty\n"; + } + return @{$translate_type_info{$ty}} + } +} + +sub translate_type_to_arg_type +{ + my ($basic,$typename) = translate_type(@_); + return $basic?$typename:'const '.$typename.'&' +} + +sub translate_type_to_member_type +{ + my ($basic,$typename) = translate_type(@_); + return $typename +} + +sub make_log_strings +{ + my ($cmd) = @_; + + my @str; + my @arg; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + if(exists $log_display_types{$ty}) + { + # need to translate it + my ($format,$arg) = @{$log_display_types{$ty}}; + push @str,$format; + $arg =~ s/VAR/m$nm/g; + push @arg,$arg; + } + else + { + # is opaque + push @str,'OPAQUE'; + } + } + return ($cmd.'('.join(',',@str).')', join(',','',@arg)); +} + + |