diff options
Diffstat (limited to 'lib')
235 files changed, 47655 insertions, 0 deletions
diff --git a/lib/backupclient/BackupClientCryptoKeys.cpp b/lib/backupclient/BackupClientCryptoKeys.cpp new file mode 100644 index 00000000..7a8da7ba --- /dev/null +++ b/lib/backupclient/BackupClientCryptoKeys.cpp @@ -0,0 +1,85 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientCryptoKeys.cpp +// Purpose: function for setting up all the backup client keys +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "BackupClientCryptoKeys.h" +#include "FileStream.h" +#include "BackupStoreFilenameClear.h" +#include "BackupStoreException.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreFile.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientCryptoKeys_Setup(const char *) +// Purpose: Read in the key material file, and set keys to all the backup elements required. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientCryptoKeys_Setup(const std::string& rKeyMaterialFilename) +{ + // Read in the key material + unsigned char KeyMaterial[BACKUPCRYPTOKEYS_FILE_SIZE]; + + // Open the file + FileStream file(rKeyMaterialFilename); + + // Read in data + if(!file.ReadFullBuffer(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE, 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntLoadClientKeyMaterial) + } + + // Setup keys and encoding method for filename encryption + BackupStoreFilenameClear::SetBlowfishKey( + KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, + BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH, + KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, + BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH); + BackupStoreFilenameClear::SetEncodingMethod( + BackupStoreFilename::Encoding_Blowfish); + + // Setup key for attributes encryption + BackupClientFileAttributes::SetBlowfishKey( + KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH); + + // Setup secret for attribute hashing + BackupClientFileAttributes::SetAttributeHashSecret( + KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH); + + // Setup keys for file data encryption + BackupStoreFile::SetBlowfishKeys( + KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH, + KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH); + +#ifndef HAVE_OLD_SSL + // Use AES where available + BackupStoreFile::SetAESKey( + KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, + BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH); +#endif + + // Wipe the key material from memory + #ifdef _MSC_VER // not defined on MinGW + SecureZeroMemory(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE); + #else + ::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE); + #endif +} + diff --git a/lib/backupclient/BackupClientCryptoKeys.h b/lib/backupclient/BackupClientCryptoKeys.h new file mode 100644 index 00000000..f40e2e03 --- /dev/null +++ b/lib/backupclient/BackupClientCryptoKeys.h @@ -0,0 +1,55 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientCryptoKeys.h +// Purpose: Format of crypto keys file, and function for setting everything up +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTCRYTOKEYS__H +#define BACKUPCLIENTCRYTOKEYS__H + + +// All keys are the maximum size that Blowfish supports. Since only the +// setup time is affected by key length (encryption same speed whatever) +// there is no disadvantage to using long keys as they are never +// transmitted and are static over long periods of time. + + +// All sizes in bytes. Some gaps deliberately left in the used material. + +// How long the key material file is expected to be +#define BACKUPCRYPTOKEYS_FILE_SIZE 1024 + +// key for encrypting filenames (448 bits) +#define BACKUPCRYPTOKEYS_FILENAME_KEY_START 0 +#define BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH 56 +#define BACKUPCRYPTOKEYS_FILENAME_IV_START (0 + BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH) +#define BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH 8 + +// key for encrypting attributes (448 bits) +#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START (BACKUPCRYPTOKEYS_FILENAME_KEY_START+64) +#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH 56 + +// Blowfish key for encrypting file data (448 bits (max blowfish key length)) +#define BACKUPCRYPTOKEYS_FILE_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START+64) +#define BACKUPCRYPTOKEYS_FILE_KEY_LENGTH 56 + +// key for encrypting file block index entries +#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START (BACKUPCRYPTOKEYS_FILE_KEY_START+64) +#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH 56 + +// Secret for hashing attributes +#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START (BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START+64) +#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH 128 + +// AES key for encrypting file data (256 bits (max AES key length)) +#define BACKUPCRYPTOKEYS_FILE_AES_KEY_START (BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START+128) +#define BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH 32 + + +void BackupClientCryptoKeys_Setup(const std::string& rKeyMaterialFilename); + +#endif // BACKUPCLIENTCRYTOKEYS__H + diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp new file mode 100644 index 00000000..b25ed9c7 --- /dev/null +++ b/lib/backupclient/BackupClientFileAttributes.cpp @@ -0,0 +1,1186 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientFileAttributes.cpp +// Purpose: Storage of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> +#include <limits.h> + +#include <algorithm> +#include <cstring> +#include <new> +#include <vector> + +#ifdef HAVE_SYS_XATTR_H +#include <cerrno> +#include <sys/xattr.h> +#endif + +#include <cstring> + +#include "BackupClientFileAttributes.h" +#include "CommonException.h" +#include "FileModificationTime.h" +#include "BoxTimeToUnix.h" +#include "BackupStoreException.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "MD5Digest.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +#define ATTRIBUTETYPE_GENERIC_UNIX 1 + +#define ATTRIBUTE_ENCODING_BLOWFISH 2 + +typedef struct +{ + int32_t AttributeType; + u_int32_t UID; + u_int32_t GID; + u_int64_t ModificationTime; + u_int64_t AttrModificationTime; + u_int32_t UserDefinedFlags; + u_int32_t FileGenerationNumber; + u_int16_t Mode; + // Symbolic link filename may follow + // Extended attribute (xattr) information may follow, format is: + // u_int32_t Size of extended attribute block (excluding this word) + // For each of NumberOfAttributes (sorted by AttributeName): + // u_int16_t AttributeNameLength + // char AttributeName[AttributeNameLength] + // u_int32_t AttributeValueLength + // unsigned char AttributeValue[AttributeValueLength] + // AttributeName is 0 terminated, AttributeValue is not (and may be binary data) +} attr_StreamFormat; + +// This has wire packing so it's compatible across platforms +// Use wider than necessary sizes, just to be careful. +typedef struct +{ + int32_t uid, gid, mode; + #ifdef WIN32 + int64_t fileCreationTime; + #endif +} attributeHashData; + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + + +#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256 + +// Hide private static variables from the rest of the world +// -- don't put them as static class variables to avoid openssl/evp.h being +// included all over the project. +namespace +{ + CipherContext sBlowfishEncrypt; + CipherContext sBlowfishDecrypt; + uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH]; + int sAttributeHashSecretLength = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::BackupClientFileAttributes() +// Purpose: Default constructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::BackupClientFileAttributes() + : mpClearAttributes(0) +{ + ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &) +// Purpose: Copy constructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy) + : StreamableMemBlock(rToCopy), // base class does the hard work + mpClearAttributes(0) +{ +} +BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy) + : StreamableMemBlock(rToCopy), // base class does the hard work + mpClearAttributes(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::~BackupClientFileAttributes() +// Purpose: Destructor +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes::~BackupClientFileAttributes() +{ + if(mpClearAttributes) + { + delete mpClearAttributes; + mpClearAttributes = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &) +// Purpose: Assignment operator +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr) +{ + StreamableMemBlock::Set(rAttr); + RemoveClear(); // make sure no decrypted version held + return *this; +} +// Assume users play nice +BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr) +{ + StreamableMemBlock::Set(rAttr); + RemoveClear(); // make sure no decrypted version held + return *this; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &) +// Purpose: Comparison operator +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const +{ + EnsureClearAvailable(); + rAttr.EnsureClearAvailable(); + + return mpClearAttributes->operator==(*rAttr.mpClearAttributes); +} +// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data. +/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const +{ + StreamableMemBlock *pDecoded = 0; + + try + { + EnsureClearAvailable(); + StreamableMemBlock *pDecoded = MakeClear(rAttr); + + // Compare using clear version + bool compared = mpClearAttributes->operator==(rAttr); + + // Delete temporary + delete pDecoded; + + return compared; + } + catch(...) + { + delete pDecoded; + throw; + } +}*/ + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool) +// Purpose: Compare, optionally ignoring the attribute +// modification time and/or modification time, and some +// data which is irrelevant in practise (eg file +// generation number) +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, + bool IgnoreAttrModTime, bool IgnoreModTime) const +{ + EnsureClearAvailable(); + rAttr.EnsureClearAvailable(); + + // Check sizes are the same, as a first check + if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize()) + { + BOX_TRACE("Attribute Compare: Attributes objects are " + "different sizes, cannot compare them: local " << + mpClearAttributes->GetSize() << " bytes, remote " << + rAttr.mpClearAttributes->GetSize() << " bytes"); + return false; + } + + // Then check the elements of the two things + // Bytes are checked in network order, but this doesn't matter as we're only checking for equality. + attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer(); + + #define COMPARE(attribute, message) \ + if (a1->attribute != a2->attribute) \ + { \ + BOX_TRACE("Attribute Compare: " << message << " differ: " \ + "local " << ntoh(a1->attribute) << ", " \ + "remote " << ntoh(a2->attribute)); \ + return false; \ + } + COMPARE(AttributeType, "Attribute types"); + COMPARE(UID, "UIDs"); + COMPARE(GID, "GIDs"); + COMPARE(UserDefinedFlags, "User-defined flags"); + COMPARE(Mode, "Modes"); + + if(!IgnoreModTime) + { + uint64_t t1 = box_ntoh64(a1->ModificationTime); + uint64_t t2 = box_ntoh64(a2->ModificationTime); + time_t s1 = BoxTimeToSeconds(t1); + time_t s2 = BoxTimeToSeconds(t2); + if(s1 != s2) + { + BOX_TRACE("Attribute Compare: File modification " + "times differ: local " << + FormatTime(t1, true) << " (" << s1 << "), " + "remote " << + FormatTime(t2, true) << " (" << s2 << ")"); + return false; + } + } + + if(!IgnoreAttrModTime) + { + uint64_t t1 = box_ntoh64(a1->AttrModificationTime); + uint64_t t2 = box_ntoh64(a2->AttrModificationTime); + time_t s1 = BoxTimeToSeconds(t1); + time_t s2 = BoxTimeToSeconds(t2); + if(s1 != s2) + { + BOX_TRACE("Attribute Compare: Attribute modification " + "times differ: local " << + FormatTime(t1, true) << " (" << s1 << "), " + "remote " << + FormatTime(t2, true) << " (" << s2 << ")"); + return false; + } + } + + // Check symlink string? + unsigned int size = mpClearAttributes->GetSize(); + if(size > sizeof(attr_StreamFormat)) + { + // Symlink strings don't match. This also compares xattrs + int datalen = size - sizeof(attr_StreamFormat); + + if(::memcmp(a1 + 1, a2 + 1, datalen) != 0) + { + std::string s1((char *)(a1 + 1), datalen); + std::string s2((char *)(a2 + 1), datalen); + BOX_TRACE("Attribute Compare: Symbolic link target " + "or extended attributes differ: " + "local " << PrintEscapedBinaryData(s1) << ", " + "remote " << PrintEscapedBinaryData(s2)); + return false; + } + } + + // Passes all test, must be OK + return true; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadAttributes( +// const char *Filename, bool ZeroModificationTimes, +// box_time_t *pModTime, box_time_t *pAttrModTime, +// int64_t *pFileSize, InodeRefType *pInodeNumber, +// bool *pHasMultipleLinks) +// Purpose: Read the attributes of the file, and store them +// ready for streaming. Optionally retrieve the +// modification time and attribute modification time. +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::ReadAttributes(const char *Filename, + bool ZeroModificationTimes, box_time_t *pModTime, + box_time_t *pAttrModTime, int64_t *pFileSize, + InodeRefType *pInodeNumber, bool *pHasMultipleLinks) +{ + StreamableMemBlock *pnewAttr = 0; + try + { + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) != 0) + { + BOX_LOG_SYS_ERROR("Failed to stat file: '" << + Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Modification times etc + if(pModTime) {*pModTime = FileModificationTime(st);} + if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);} + if(pFileSize) {*pFileSize = st.st_size;} + if(pInodeNumber) {*pInodeNumber = st.st_ino;} + if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);} + + pnewAttr = new StreamableMemBlock; + + FillAttributes(*pnewAttr, Filename, st, ZeroModificationTimes); + +#ifndef WIN32 + // Is it a link? + if((st.st_mode & S_IFMT) == S_IFLNK) + { + FillAttributesLink(*pnewAttr, Filename, st); + } +#endif + + FillExtendedAttr(*pnewAttr, Filename); + +#ifdef WIN32 + //this is to catch those problems with invalid time stamps stored... + //need to find out the reason why - but also a catch as well. + + attr_StreamFormat *pattr = + (attr_StreamFormat*)pnewAttr->GetBuffer(); + ASSERT(pattr != 0); + + // __time64_t winTime = BoxTimeToSeconds( + // pnewAttr->ModificationTime); + + u_int64_t modTime = box_ntoh64(pattr->ModificationTime); + box_time_t modSecs = BoxTimeToSeconds(modTime); + __time64_t winTime = modSecs; + + // _MAX__TIME64_T doesn't seem to be defined, but the code below + // will throw an assertion failure if we exceed it :-) + // Microsoft says dates up to the year 3000 are valid, which + // is a bit more than 15 * 2^32. Even that doesn't seem + // to be true (still aborts), but it can at least hold 2^32. + if (winTime >= 0x100000000LL || _gmtime64(&winTime) == 0) + { + BOX_ERROR("Invalid Modification Time caught for " + "file: '" << Filename << "'"); + pattr->ModificationTime = 0; + } + + modTime = box_ntoh64(pattr->AttrModificationTime); + modSecs = BoxTimeToSeconds(modTime); + winTime = modSecs; + + if (winTime > 0x100000000LL || _gmtime64(&winTime) == 0) + { + BOX_ERROR("Invalid Attribute Modification Time " + "caught for file: '" << Filename << "'"); + pattr->AttrModificationTime = 0; + } +#endif + + // Attributes ready. Encrypt into this block + EncryptAttr(*pnewAttr); + + // Store the new attributes + RemoveClear(); + mpClearAttributes = pnewAttr; + pnewAttr = 0; + } + catch(...) + { + // clean up + delete pnewAttr; + pnewAttr = 0; + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadAttributesLink() +// Purpose: Private function, handles standard attributes for all objects +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, EMU_STRUCT_STAT &st, bool ZeroModificationTimes) +{ + outputBlock.ResizeBlock(sizeof(attr_StreamFormat)); + attr_StreamFormat *pattr = (attr_StreamFormat*)outputBlock.GetBuffer(); + ASSERT(pattr != 0); + + // Fill in the entries + pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX); + pattr->UID = htonl(st.st_uid); + pattr->GID = htonl(st.st_gid); + if(ZeroModificationTimes) + { + pattr->ModificationTime = 0; + pattr->AttrModificationTime = 0; + } + else + { + pattr->ModificationTime = box_hton64(FileModificationTime(st)); + pattr->AttrModificationTime = box_hton64(FileAttrModificationTime(st)); + } + pattr->Mode = htons(st.st_mode); + +#ifndef HAVE_STRUCT_STAT_ST_FLAGS + pattr->UserDefinedFlags = 0; + pattr->FileGenerationNumber = 0; +#else + pattr->UserDefinedFlags = htonl(st.st_flags); + pattr->FileGenerationNumber = htonl(st.st_gen); +#endif +} +#ifndef WIN32 +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadAttributesLink() +// Purpose: Private function, handles the case where a symbolic link is needed +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st) +{ + // Make sure we're only called for symbolic links + ASSERT((st.st_mode & S_IFMT) == S_IFLNK); + + // Get the filename the link is linked to + char linkedTo[PATH_MAX+4]; + int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX); + if(linkedToSize == -1) + { + BOX_LOG_SYS_ERROR("Failed to readlink '" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError); + } + + int oldSize = outputBlock.GetSize(); + outputBlock.ResizeBlock(oldSize+linkedToSize+1); + char* buffer = static_cast<char*>(outputBlock.GetBuffer()); + + // Add the path name for the symbolic link, and add 0 termination + std::memcpy(buffer+oldSize, linkedTo, linkedToSize); + buffer[oldSize+linkedToSize] = '\0'; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::ReadExtendedAttr(const char *, unsigned char**) +// Purpose: Private function, read the extended attributes of the file into the block +// Created: 2005/06/12 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename) +{ +#ifdef HAVE_SYS_XATTR_H + int listBufferSize = 10000; + char* list = new char[listBufferSize]; + + try + { + // This returns an unordered list of attribute names, each 0 terminated, + // concatenated together + int listSize = ::llistxattr(Filename, list, listBufferSize); + + if(listSize>listBufferSize) + { + delete[] list, list = NULL; + list = new char[listSize]; + listSize = ::llistxattr(Filename, list, listSize); + } + + if(listSize>0) + { + // Extract list of attribute names so we can sort them + std::vector<std::string> attrKeys; + for(int i = 0; i<listSize; ++i) + { + std::string attrKey(list+i); + i += attrKey.size(); + attrKeys.push_back(attrKey); + } + sort(attrKeys.begin(), attrKeys.end()); + + // Make initial space in block + int xattrSize = outputBlock.GetSize(); + int xattrBufferSize = (xattrSize+listSize)>500 ? (xattrSize+listSize)*2 : 1000; + outputBlock.ResizeBlock(xattrBufferSize); + unsigned char* buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); + + // Leave space for attr block size later + int xattrBlockSizeOffset = xattrSize; + xattrSize += sizeof(u_int32_t); + + // Loop for each attribute + for(std::vector<std::string>::const_iterator attrKeyI = attrKeys.begin(); attrKeyI!=attrKeys.end(); ++attrKeyI) + { + std::string attrKey(*attrKeyI); + + if(xattrSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t)>static_cast<unsigned int>(xattrBufferSize)) + { + xattrBufferSize = (xattrBufferSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t))*2; + outputBlock.ResizeBlock(xattrBufferSize); + buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); + } + + // Store length and text for attibute name + u_int16_t keyLength = htons(attrKey.size()+1); + std::memcpy(buffer+xattrSize, &keyLength, sizeof(u_int16_t)); + xattrSize += sizeof(u_int16_t); + std::memcpy(buffer+xattrSize, attrKey.c_str(), attrKey.size()+1); + xattrSize += attrKey.size()+1; + + // Leave space for value size + int valueSizeOffset = xattrSize; + xattrSize += sizeof(u_int32_t); + + // Find size of attribute (must call with buffer and length 0 on some platforms, + // as -1 is returned if the data doesn't fit.) + int valueSize = ::lgetxattr(Filename, attrKey.c_str(), 0, 0); + if(valueSize<0) + { + BOX_LOG_SYS_ERROR("Failed to get " + "extended attributes size " + "for '" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError); + } + + // Resize block, if needed + if(xattrSize+valueSize>xattrBufferSize) + { + xattrBufferSize = (xattrBufferSize+valueSize)*2; + outputBlock.ResizeBlock(xattrBufferSize); + buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); + } + + // This gets the attribute value (may be text or binary), no termination + valueSize = ::lgetxattr(Filename, attrKey.c_str(), buffer+xattrSize, xattrBufferSize-xattrSize); + if(valueSize<0) + { + BOX_LOG_SYS_ERROR("Failed to get " + "extended attributes for " + "'" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError); + } + xattrSize += valueSize; + + // Fill in value size + u_int32_t valueLength = htonl(valueSize); + std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(u_int32_t)); + } + + // Fill in attribute block size + u_int32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(u_int32_t)); + std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(u_int32_t)); + + outputBlock.ResizeBlock(xattrSize); + } + else if(listSize<0) + { + if(errno == EOPNOTSUPP || errno == EACCES) + { + // fail silently + } + else if(errno == ERANGE) + { + BOX_ERROR("Failed to list extended " + "attributes of '" << Filename << "': " + "buffer too small, not backed up"); + } + else + { + BOX_LOG_SYS_ERROR("Failed to list extended " + "attributes of '" << Filename << "', " + "not backed up"); + THROW_EXCEPTION(CommonException, OSFileError); + } + } + } + catch(...) + { + delete[] list; + throw; + } + delete[] list; +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::GetModificationTimes() +// Purpose: Returns the modification time embedded in the +// attributes. +// Created: 2010/02/24 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::GetModificationTimes( + box_time_t *pModificationTime, + box_time_t *pAttrModificationTime) const +{ + // Got something loaded + if(GetSize() <= 0) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Make sure there are clear attributes to use + EnsureClearAvailable(); + ASSERT(mpClearAttributes != 0); + + // Check if the decrypted attributes are small enough, and the type of attributes stored + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) + { + // Don't know what to do with these + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + + // Check there is enough space for an attributes block + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get pointer to structure + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + + if(pModificationTime) + { + *pModificationTime = box_ntoh64(pattr->ModificationTime); + } + + if(pAttrModificationTime) + { + *pAttrModificationTime = box_ntoh64(pattr->AttrModificationTime); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::WriteAttributes(const char *) +// Purpose: Apply the stored attributes to the file +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::WriteAttributes(const char *Filename, + bool MakeUserWritable) const +{ + // Got something loaded + if(GetSize() <= 0) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Make sure there are clear attributes to use + EnsureClearAvailable(); + ASSERT(mpClearAttributes != 0); + + // Check if the decrypted attributes are small enough, and the type of attributes stored + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) + { + // Don't know what to do with these + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + + // Check there is enough space for an attributes block + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get pointer to structure + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + int xattrOffset = sizeof(attr_StreamFormat); + + // is it a symlink? + int16_t mode = ntohs(pattr->Mode); + if((mode & S_IFMT) == S_IFLNK) + { + // Check things are sensible + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + +#ifdef WIN32 + BOX_WARNING("Cannot create symbolic links on Windows: '" << + Filename << "'"); +#else + // Make a symlink, first deleting anything in the way + ::unlink(Filename); + if(::symlink((char*)(pattr + 1), Filename) != 0) + { + BOX_LOG_SYS_ERROR("Failed to symlink '" << Filename << + "' to '" << (char*)(pattr + 1) << "'"); + THROW_EXCEPTION(CommonException, OSFileError) + } +#endif + + xattrOffset += std::strlen(reinterpret_cast<char*>(pattr+1))+1; + } + + // If working as root, set user IDs + if(::geteuid() == 0) + { + #ifndef HAVE_LCHOWN + // only if not a link, can't set their owner on this platform + if((mode & S_IFMT) != S_IFLNK) + { + // Not a link, use normal chown + if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) + { + BOX_LOG_SYS_ERROR("Failed to change " + "owner of file " + "'" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError) + } + } + #else + // use the version which sets things on symlinks + if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) + { + BOX_LOG_SYS_ERROR("Failed to change owner of " + "symbolic link '" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError) + } + #endif + } + + if(static_cast<int>(xattrOffset+sizeof(u_int32_t))<=mpClearAttributes->GetSize()) + { + WriteExtendedAttr(Filename, xattrOffset); + } + + // Stop now if symlink, because otherwise it'll just be applied to the target + if((mode & S_IFMT) == S_IFLNK) + { + return; + } + + // Set modification time? + box_time_t modtime = box_ntoh64(pattr->ModificationTime); + if(modtime != 0) + { + // Work out times as timevals + struct timeval times[2]; + + #ifdef WIN32 + BoxTimeToTimeval(box_ntoh64(pattr->ModificationTime), + times[1]); + BoxTimeToTimeval(box_ntoh64(pattr->AttrModificationTime), + times[0]); + // Because stat() returns the creation time in the ctime + // field under Windows, and this gets saved in the + // AttrModificationTime field of the serialised attributes, + // we subvert the first parameter of emu_utimes() to allow + // it to be reset to the right value on the restored file. + #else + BoxTimeToTimeval(modtime, times[1]); + // Copy access time as well, why not, got to set it to something + times[0] = times[1]; + // Attr modification time will be changed anyway, + // nothing that can be done about it + #endif + + // Try to apply + if(::utimes(Filename, times) != 0) + { + BOX_LOG_SYS_WARNING("Failed to change times of " + "file '" << Filename << "' to ctime=" << + BOX_FORMAT_TIMESPEC(times[0]) << ", mtime=" << + BOX_FORMAT_TIMESPEC(times[1])); + } + } + + if (MakeUserWritable) + { + mode |= S_IRWXU; + } + + // Apply everything else... (allowable mode flags only) + // Mode must be done last (think setuid) + if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID + | S_ISGID | S_ISVTX)) != 0) + { + BOX_LOG_SYS_ERROR("Failed to change permissions of file " + "'" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::IsSymLink() +// Purpose: Do these attributes represent a symbolic link? +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +bool BackupClientFileAttributes::IsSymLink() const +{ + EnsureClearAvailable(); + + // Got the right kind of thing? + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get the type of attributes stored + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat)) + { + // Check link + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK; + } + + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::RemoveClear() +// Purpose: Private. Deletes any clear version of the attributes that may be held +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::RemoveClear() const +{ + if(mpClearAttributes) + { + delete mpClearAttributes; + } + mpClearAttributes = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::EnsureClearAvailable() +// Purpose: Private. Makes sure the clear version is available +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::EnsureClearAvailable() const +{ + if(mpClearAttributes == 0) + { + mpClearAttributes = MakeClear(*this); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) +// Purpose: Private function, apply the stored extended attributes to the file +// Created: 2005/06/13 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) const +{ +#ifdef HAVE_SYS_XATTR_H + const char* buffer = static_cast<char*>(mpClearAttributes->GetBuffer()); + + u_int32_t xattrBlockLength = 0; + std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(u_int32_t)); + int xattrBlockSize = ntohl(xattrBlockLength); + xattrOffset += sizeof(u_int32_t); + + int xattrEnd = xattrOffset+xattrBlockSize; + if(xattrEnd>mpClearAttributes->GetSize()) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + while(xattrOffset<xattrEnd) + { + u_int16_t keyLength = 0; + std::memcpy(&keyLength, buffer+xattrOffset, sizeof(u_int16_t)); + int keySize = ntohs(keyLength); + xattrOffset += sizeof(u_int16_t); + + const char* key = buffer+xattrOffset; + xattrOffset += keySize; + + u_int32_t valueLength = 0; + std::memcpy(&valueLength, buffer+xattrOffset, sizeof(u_int32_t)); + int valueSize = ntohl(valueLength); + xattrOffset += sizeof(u_int32_t); + + // FIXME: Warn on EOPNOTSUPP + if(::lsetxattr(Filename, key, buffer+xattrOffset, valueSize, 0)!=0 && errno!=EOPNOTSUPP) + { + BOX_LOG_SYS_ERROR("Failed to set extended attributes " + "on file '" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileError); + } + + xattrOffset += valueSize; + } + + ASSERT(xattrOffset==xattrEnd); +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &) +// Purpose: Static. Decrypts stored attributes. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted) +{ + // New block + StreamableMemBlock *pdecrypted = 0; + + try + { + // Check the block is big enough for IV and header + int ivSize = sBlowfishEncrypt.GetIVLength(); + if(rEncrypted.GetSize() <= (ivSize + 1)) + { + THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes); + } + + // How much space is needed for the output? + int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize); + + // Allocate it + pdecrypted = new StreamableMemBlock(maxDecryptedSize); + + // ptr to block + uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer(); + + // Check that the header has right type + if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH) + { + THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding); + } + + // Set IV + sBlowfishDecrypt.SetIV(encBlock + 1); + + // Decrypt + int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1)); + + // Resize block to fit + pdecrypted->ResizeBlock(decryptedSize); + } + catch(...) + { + delete pdecrypted; + pdecrypted = 0; + } + + return pdecrypted; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int) +// Purpose: Static. Sets the key to use for encryption and decryption. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength) +{ + // IVs set later + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &) +// Purpose: Private. Encrypt the given attributes into this block. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt) +{ + // Free any existing block + FreeBlock(); + + // Work out the maximum amount of space we need + int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize()); + // And the size of the IV + int ivSize = sBlowfishEncrypt.GetIVLength(); + + // Allocate this space + AllocateBlock(maxEncryptedSize + ivSize + 1); + + // Store the encoding byte + uint8_t *block = (uint8_t*)GetBuffer(); + block[0] = ATTRIBUTE_ENCODING_BLOWFISH; + + // Generate and store an IV for this attribute block + int ivSize2 = 0; + const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2); + ASSERT(ivSize == ivSize2); + + // Copy into the encrypted block + ::memcpy(block + 1, iv, ivSize); + + // Do the transform + int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize()); + + // Resize this block + ResizeBlock(encrytedSize + ivSize + 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int) +// Purpose: Set the secret for the filename attribute hash +// Created: 25/4/04 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength) +{ + if(SecretLength > (int)sizeof(sAttributeHashSecret)) + { + SecretLength = sizeof(sAttributeHashSecret); + } + if(SecretLength < 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Copy + ::memcpy(sAttributeHashSecret, pSecret, SecretLength); + sAttributeHashSecretLength = SecretLength; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::GenerateAttributeHash( +// struct stat &, const std::string &, +// const std::string &) +// Purpose: Generate a 64 bit hash from the attributes, used to +// detect changes. Include filename in the hash, so +// that it changes from one file to another, so don't +// reveal identical attributes. +// Created: 25/4/04 +// +// -------------------------------------------------------------------------- +uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, + const std::string &filename, const std::string &leafname) +{ + if(sAttributeHashSecretLength == 0) + { + THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet) + } + + // Assemble stuff we're interested in + attributeHashData hashData; + memset(&hashData, 0, sizeof(hashData)); + // Use network byte order and large sizes to be cross platform + hashData.uid = htonl(st.st_uid); + hashData.gid = htonl(st.st_gid); + hashData.mode = htonl(st.st_mode); + + #ifdef WIN32 + // On Windows, the "file attribute modification time" is the + // file creation time, and we want to back this up, restore + // it and compare it. + // + // On other platforms, it's not very important and can't + // reliably be set to anything other than the current time. + hashData.fileCreationTime = box_hton64(st.st_ctime); + #endif + + StreamableMemBlock xattr; + FillExtendedAttr(xattr, filename.c_str()); + + // Create a MD5 hash of the data, filename, and secret + MD5Digest digest; + digest.Add(&hashData, sizeof(hashData)); + digest.Add(xattr.GetBuffer(), xattr.GetSize()); + digest.Add(leafname.c_str(), leafname.size()); + digest.Add(sAttributeHashSecret, sAttributeHashSecretLength); + digest.Finish(); + + // Return the first 64 bits of the hash + uint64_t result = *((uint64_t *)(digest.DigestAsData())); + return result; +} diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h new file mode 100644 index 00000000..f9a0d883 --- /dev/null +++ b/lib/backupclient/BackupClientFileAttributes.h @@ -0,0 +1,78 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientFileAttributes.h +// Purpose: Storage of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTFILEATTRIBUTES__H +#define BACKUPCLIENTFILEATTRIBUTES__H + +#include <string> + +#include "StreamableMemBlock.h" +#include "BoxTime.h" + +EMU_STRUCT_STAT; // declaration + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientFileAttributes +// Purpose: Storage, streaming and application of file attributes +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- +class BackupClientFileAttributes : public StreamableMemBlock +{ +public: + BackupClientFileAttributes(); + BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy); + BackupClientFileAttributes(const StreamableMemBlock &rToCopy); + ~BackupClientFileAttributes(); + BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr); + BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr); + bool operator==(const BackupClientFileAttributes &rAttr) const; +// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous? + + bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const; + + // Prevent access to base class members accidently + void Set(); + + void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false, + box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0, + InodeRefType *pInodeNumber = 0, bool *pHasMultipleLinks = 0); + void WriteAttributes(const char *Filename, + bool MakeUserWritable = false) const; + void GetModificationTimes(box_time_t *pModificationTime, + box_time_t *pAttrModificationTime) const; + + bool IsSymLink() const; + + static void SetBlowfishKey(const void *pKey, int KeyLength); + static void SetAttributeHashSecret(const void *pSecret, int SecretLength); + + static uint64_t GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname); + static void FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename); + +private: + static void FillAttributes(StreamableMemBlock &outputBlock, + const char *Filename, EMU_STRUCT_STAT &st, + bool ZeroModificationTimes); + static void FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st); + void WriteExtendedAttr(const char *Filename, int xattrOffset) const; + + void RemoveClear() const; + void EnsureClearAvailable() const; + static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted); + void EncryptAttr(const StreamableMemBlock &rToEncrypt); + +private: + mutable StreamableMemBlock *mpClearAttributes; +}; + +#endif // BACKUPCLIENTFILEATTRIBUTES__H + diff --git a/lib/backupclient/BackupClientMakeExcludeList.cpp b/lib/backupclient/BackupClientMakeExcludeList.cpp new file mode 100644 index 00000000..0615faaa --- /dev/null +++ b/lib/backupclient/BackupClientMakeExcludeList.cpp @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientMakeExcludeList.cpp +// Purpose: Makes exclude lists from bbbackupd config location entries +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "BackupClientMakeExcludeList.h" +#include "Configuration.h" +#include "ExcludeList.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientMakeExcludeList(const Configuration &, const char *, const char *) +// Purpose: Given a Configuration object corresponding to a bbackupd Location, and the +// two names of the keys for definite and regex entries, return a ExcludeList. +// Or 0 if it isn't required. +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName, + const char *AlwaysIncludeDefiniteName, const char *AlwaysIncludeRegexName) +{ + // Check that at least one of the entries exists + if(!rConfig.KeyExists(DefiniteName) && !rConfig.KeyExists(RegexName)) + { + // Neither exists -- return 0 as an Exclude list isn't required. + return 0; + } + + // Create the exclude list + ExcludeList *pexclude = new ExcludeList; + + try + { + // Definite names to add? + if(rConfig.KeyExists(DefiniteName)) + { + pexclude->AddDefiniteEntries(rConfig.GetKeyValue(DefiniteName)); + } + // Regular expressions to add? + if(rConfig.KeyExists(RegexName)) + { + pexclude->AddRegexEntries(rConfig.GetKeyValue(RegexName)); + } + + // Add a "always include" list? + if(AlwaysIncludeDefiniteName != 0 && AlwaysIncludeRegexName != 0) + { + // This will accept NULL as a valid argument, so safe to do this. + pexclude->SetAlwaysIncludeList( + BackupClientMakeExcludeList(rConfig, AlwaysIncludeDefiniteName, AlwaysIncludeRegexName) + ); + } + } + catch(...) + { + // Clean up + delete pexclude; + throw; + } + + return pexclude; +} + + + diff --git a/lib/backupclient/BackupClientMakeExcludeList.h b/lib/backupclient/BackupClientMakeExcludeList.h new file mode 100644 index 00000000..d5dd8af4 --- /dev/null +++ b/lib/backupclient/BackupClientMakeExcludeList.h @@ -0,0 +1,48 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientMakeExcludeList.h +// Purpose: Makes exclude lists from bbbackupd config location entries +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTMAKEEXCLUDELIST__H +#define BACKUPCLIENTMAKEEXCLUDELIST__H + +class ExcludeList; +class Configuration; + +ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName, + const char *AlwaysIncludeDefiniteName = 0, const char *AlwaysIncludeRegexName = 0); + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientMakeExcludeList_Files(const Configuration &) +// Purpose: Create a exclude list from config file entries for files. May return 0. +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +inline ExcludeList *BackupClientMakeExcludeList_Files(const Configuration &rConfig) +{ + return BackupClientMakeExcludeList(rConfig, "ExcludeFile", "ExcludeFilesRegex", "AlwaysIncludeFile", "AlwaysIncludeFilesRegex"); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientMakeExcludeList_Dirs(const Configuration &) +// Purpose: Create a exclude list from config file entries for directories. May return 0. +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +inline ExcludeList *BackupClientMakeExcludeList_Dirs(const Configuration &rConfig) +{ + return BackupClientMakeExcludeList(rConfig, "ExcludeDir", "ExcludeDirsRegex", "AlwaysIncludeDir", "AlwaysIncludeDirsRegex"); +} + + +#endif // BACKUPCLIENTMAKEEXCLUDELIST__H + diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp new file mode 100644 index 00000000..fa61bb59 --- /dev/null +++ b/lib/backupclient/BackupClientRestore.cpp @@ -0,0 +1,918 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientRestore.cpp +// Purpose: +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <sys/types.h> +#include <sys/stat.h> +#include <string> +#include <set> +#include <limits.h> +#include <stdio.h> +#include <errno.h> + +#include "BackupClientRestore.h" +#include "autogen_BackupProtocolClient.h" +#include "CommonException.h" +#include "BackupClientFileAttributes.h" +#include "IOStream.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreFile.h" +#include "CollectInBufferStream.h" +#include "FileStream.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +#define MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES (128*1024) + +class RestoreResumeInfo +{ +public: + // constructor + RestoreResumeInfo() + : mNextLevelID(0), + mpNextLevel(0) + { + } + + // destructor + ~RestoreResumeInfo() + { + delete mpNextLevel; + mpNextLevel = 0; + } + + // Get a next level object + RestoreResumeInfo &AddLevel(int64_t ID, const std::string &rLocalName) + { + ASSERT(mpNextLevel == 0 && mNextLevelID == 0); + mpNextLevel = new RestoreResumeInfo; + mNextLevelID = ID; + mNextLevelLocalName = rLocalName; + return *mpNextLevel; + } + + // Remove the next level info + void RemoveLevel() + { + ASSERT(mpNextLevel != 0 && mNextLevelID != 0); + delete mpNextLevel; + mpNextLevel = 0; + mNextLevelID = 0; + mNextLevelLocalName.erase(); + } + + void Save(const std::string &rFilename) const + { + // TODO: use proper buffered streams when they're done + // Build info in memory buffer + CollectInBufferStream write; + + // Save this level + SaveLevel(write); + + // Store in file + write.SetForReading(); + FileStream file(rFilename.c_str(), O_WRONLY | O_CREAT); + write.CopyStreamTo(file, IOStream::TimeOutInfinite, 8*1024 /* large buffer */); + } + + void SaveLevel(IOStream &rWrite) const + { + // Write the restored objects + int64_t numObjects = mRestoredObjects.size(); + rWrite.Write(&numObjects, sizeof(numObjects)); + for(std::set<int64_t>::const_iterator i(mRestoredObjects.begin()); i != mRestoredObjects.end(); ++i) + { + int64_t id = *i; + rWrite.Write(&id, sizeof(id)); + } + + // Next level? + if(mpNextLevel != 0) + { + // ID + rWrite.Write(&mNextLevelID, sizeof(mNextLevelID)); + // Name string + int32_t nsize = mNextLevelLocalName.size(); + rWrite.Write(&nsize, sizeof(nsize)); + rWrite.Write(mNextLevelLocalName.c_str(), nsize); + // And then the level itself + mpNextLevel->SaveLevel(rWrite); + } + else + { + // Just write a zero + int64_t zero = 0; + rWrite.Write(&zero, sizeof(zero)); + } + } + + // Not written to be efficient -- shouldn't be called very often. + bool Load(const std::string &rFilename) + { + // Delete and reset if necessary + if(mpNextLevel != 0) + { + RemoveLevel(); + } + + // Open file + FileStream file(rFilename.c_str()); + + // Load this level + return LoadLevel(file); + } + + #define CHECKED_READ(x, s) if(!rRead.ReadFullBuffer(x, s, 0)) {return false;} + bool LoadLevel(IOStream &rRead) + { + // Load the restored objects list + mRestoredObjects.clear(); + int64_t numObjects = 0; + CHECKED_READ(&numObjects, sizeof(numObjects)); + for(int64_t o = 0; o < numObjects; ++o) + { + int64_t id; + CHECKED_READ(&id, sizeof(id)); + mRestoredObjects.insert(id); + } + + // ID of next level? + int64_t nextID = 0; + CHECKED_READ(&nextID, sizeof(nextID)); + if(nextID != 0) + { + // Load the next level! + std::string name; + int32_t nsize = 0; + CHECKED_READ(&nsize, sizeof(nsize)); + char n[PATH_MAX]; + if(nsize > PATH_MAX) return false; + CHECKED_READ(n, nsize); + name.assign(n, nsize); + + // Create a new level + mpNextLevel = new RestoreResumeInfo; + mNextLevelID = nextID; + mNextLevelLocalName = name; + + // And ask it to read itself in + if(!mpNextLevel->LoadLevel(rRead)) + { + return false; + } + } + + return true; + } + + // List of objects at this level which have been done already + std::set<int64_t> mRestoredObjects; + // Next level ID + int64_t mNextLevelID; + // Pointer to next level + RestoreResumeInfo *mpNextLevel; + // Local filename of next level + std::string mNextLevelLocalName; +}; + +// parameters structure +typedef struct +{ + bool PrintDots; + bool RestoreDeleted; + bool ContinueAfterErrors; + bool ContinuedAfterError; + std::string mRestoreResumeInfoFilename; + RestoreResumeInfo mResumeInfo; +} RestoreParams; + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientRestoreDir(BackupProtocolClient &, +// int64_t, const char *, bool) +// Purpose: Restore a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +static int BackupClientRestoreDir(BackupProtocolClient &rConnection, + int64_t DirectoryID, const std::string &rRemoteDirectoryName, + const std::string &rLocalDirectoryName, + RestoreParams &Params, RestoreResumeInfo &rLevel) +{ + // If we're resuming... check that we haven't got a next level to + // look at + if(rLevel.mpNextLevel != 0) + { + // Recurse immediately + std::string localDirname(rLocalDirectoryName + + DIRECTORY_SEPARATOR_ASCHAR + + rLevel.mNextLevelLocalName); + BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, + rRemoteDirectoryName + '/' + + rLevel.mNextLevelLocalName, localDirname, + Params, *rLevel.mpNextLevel); + + // Add it to the list of done itmes + rLevel.mRestoredObjects.insert(rLevel.mNextLevelID); + + // Remove the level for the recursed directory + rLevel.RemoveLevel(); + } + + // Create the local directory, if not already done. + // Path and owner set later, just use restrictive owner mode. + + int exists; + + try + { + exists = ObjectExists(rLocalDirectoryName.c_str()); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to check existence for " << + rLocalDirectoryName << ": " << e.what()); + return Restore_UnknownError; + } + catch(std::exception &e) + { + BOX_ERROR("Failed to check existence for " << + rLocalDirectoryName << ": " << e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to check existence for " << + rLocalDirectoryName << ": unknown error"); + return Restore_UnknownError; + } + + switch(exists) + { + case ObjectExists_Dir: + // Do nothing + break; + case ObjectExists_File: + { + // File exists with this name, which is fun. + // Get rid of it. + BOX_WARNING("File present with name '" << + rLocalDirectoryName << "', removing " + "out of the way of restored directory. " + "Use specific restore with ID to " + "restore this object."); + if(::unlink(rLocalDirectoryName.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("Failed to delete " + "file '" << + rLocalDirectoryName << "'"); + return Restore_UnknownError; + } + BOX_TRACE("In restore, directory name " + "collision with file '" << + rLocalDirectoryName << "'"); + } + break; + case ObjectExists_NoObject: + // we'll create it in a second, after checking + // whether the parent directory exists + break; + default: + ASSERT(false); + break; + } + + std::string parentDirectoryName(rLocalDirectoryName); + if(parentDirectoryName[parentDirectoryName.size() - 1] == + DIRECTORY_SEPARATOR_ASCHAR) + { + parentDirectoryName.resize(parentDirectoryName.size() - 1); + } + + size_t lastSlash = parentDirectoryName.rfind(DIRECTORY_SEPARATOR_ASCHAR); + + if(lastSlash == std::string::npos) + { + // might be a forward slash separator, + // especially in the unit tests! + lastSlash = parentDirectoryName.rfind('/'); + } + + if(lastSlash != std::string::npos) + { + // the target directory is a deep path, remove the last + // directory name and check that the resulting parent + // exists, otherwise the restore should fail. + parentDirectoryName.resize(lastSlash); + + #ifdef WIN32 + // if the path is a drive letter, then we need to + // add a a backslash to query the root directory. + if (lastSlash == 2 && parentDirectoryName[1] == ':') + { + parentDirectoryName += '\\'; + } + else if (lastSlash == 0) + { + parentDirectoryName += '\\'; + } + #endif + + int parentExists; + + try + { + parentExists = ObjectExists(parentDirectoryName.c_str()); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to check existence for " << + parentDirectoryName << ": " << e.what()); + return Restore_UnknownError; + } + catch(std::exception &e) + { + BOX_ERROR("Failed to check existence for " << + parentDirectoryName << ": " << e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to check existence for " << + parentDirectoryName << ": unknown error"); + return Restore_UnknownError; + } + + switch(parentExists) + { + case ObjectExists_Dir: + // this is fine, do nothing + break; + + case ObjectExists_File: + BOX_ERROR("Failed to restore: '" << + parentDirectoryName << "' " + "is a file, but should be a " + "directory."); + return Restore_TargetPathNotFound; + + case ObjectExists_NoObject: + BOX_ERROR("Failed to restore: parent '" << + parentDirectoryName << "' of target " + "directory does not exist."); + return Restore_TargetPathNotFound; + + default: + BOX_ERROR("Failed to restore: unknown " + "result from ObjectExists('" << + parentDirectoryName << "')"); + return Restore_UnknownError; + } + } + + if((exists == ObjectExists_NoObject || + exists == ObjectExists_File) && + ::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0) + { + BOX_LOG_SYS_ERROR("Failed to create directory '" << + rLocalDirectoryName << "'"); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + + // Save the restore info, in case it's needed later + try + { + Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << "': " << + e.what()); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + catch(...) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << + "': unknown error"); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + + // Fetch the directory listing from the server -- getting a + // list of files which is appropriate to the restore type + rConnection.QueryListDirectory( + DirectoryID, + Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING), + BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)), + true /* want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(rConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, rConnection.GetTimeout()); + + // Apply attributes to the directory + const StreamableMemBlock &dirAttrBlock(dir.GetAttributes()); + BackupClientFileAttributes dirAttr(dirAttrBlock); + + try + { + dirAttr.WriteAttributes(rLocalDirectoryName.c_str(), true); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to restore attributes for '" << + rLocalDirectoryName << "': " << e.what()); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + catch(...) + { + BOX_ERROR("Failed to restore attributes for '" << + rLocalDirectoryName << "': unknown error"); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + + int64_t bytesWrittenSinceLastRestoreInfoSave = 0; + + // Process files + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) + != 0) + { + // Check ID hasn't already been done + if(rLevel.mRestoredObjects.find(en->GetObjectID()) + == rLevel.mRestoredObjects.end()) + { + // Local name + BackupStoreFilenameClear nm(en->GetName()); + std::string localFilename(rLocalDirectoryName + + DIRECTORY_SEPARATOR_ASCHAR + + nm.GetClearFilename()); + + // Unlink anything which already exists: + // For resuming restores, we can't overwrite + // files already there. + if(ObjectExists(localFilename) + != ObjectExists_NoObject && + ::unlink(localFilename.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("Failed to delete " + "file '" << localFilename << + "'"); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + + BOX_TRACE("Restoring file: " << + rRemoteDirectoryName + '/' + + nm.GetClearFilename() << " (" << + en->GetSizeInBlocks() << " blocks)"); + + // Request it from the store + rConnection.QueryGetFile(DirectoryID, + en->GetObjectID()); + + // Stream containing encoded file + std::auto_ptr<IOStream> objectStream( + rConnection.ReceiveStream()); + + // Decode the file -- need to do different + // things depending on whether the directory + // entry has additional attributes + try + { + if(en->HasAttributes()) + { + // Use these attributes + const StreamableMemBlock &storeAttr(en->GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout(), &attr); + } + else + { + // Use attributes stored in file + BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout()); + } + } + catch(std::exception &e) + { + BOX_ERROR("Failed to restore file '" << + localFilename << "': " << + e.what()); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + catch(...) + { + BOX_ERROR("Failed to restore file '" << + localFilename << + "': unknown error"); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + + // Progress display? + if(Params.PrintDots) + { + printf("."); + fflush(stdout); + } + + // Add it to the list of done itmes + rLevel.mRestoredObjects.insert(en->GetObjectID()); + + // Save restore info? + int64_t fileSize; + bool exists = false; + + try + { + exists = FileExists( + localFilename.c_str(), + &fileSize, + true /* treat links as not + existing */); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to determine " + "whether file exists: '" << + localFilename << "': " << + e.what()); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + catch(...) + { + BOX_ERROR("Failed to determine " + "whether file exists: '" << + localFilename << "': " + "unknown error"); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + + if(exists) + { + // File exists... + bytesWrittenSinceLastRestoreInfoSave + += fileSize; + + if(bytesWrittenSinceLastRestoreInfoSave > MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES) + { + // Save the restore info, in + // case it's needed later + try + { + Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << + "': " << e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << + "': unknown error"); + return Restore_UnknownError; + } + + bytesWrittenSinceLastRestoreInfoSave = 0; + } + } + } + } + } + + // Make sure the restore info has been saved + if(bytesWrittenSinceLastRestoreInfoSave != 0) + { + // Save the restore info, in case it's needed later + try + { + Params.mResumeInfo.Save( + Params.mRestoreResumeInfoFilename); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << "': " << + e.what()); + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + catch(...) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << + "': unknown error"); + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + + bytesWrittenSinceLastRestoreInfoSave = 0; + } + + + // Recurse to directories + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) + != 0) + { + // Check ID hasn't already been done + if(rLevel.mRestoredObjects.find(en->GetObjectID()) + == rLevel.mRestoredObjects.end()) + { + // Local name + BackupStoreFilenameClear nm(en->GetName()); + std::string localDirname(rLocalDirectoryName + + DIRECTORY_SEPARATOR_ASCHAR + + nm.GetClearFilename()); + + // Add the level for the next entry + RestoreResumeInfo &rnextLevel( + rLevel.AddLevel(en->GetObjectID(), + nm.GetClearFilename())); + + // Recurse + + BOX_TRACE("Entering directory: " << + rRemoteDirectoryName + '/' + + nm.GetClearFilename()); + + int result = BackupClientRestoreDir( + rConnection, en->GetObjectID(), + rRemoteDirectoryName + '/' + + nm.GetClearFilename(), localDirname, + Params, rnextLevel); + + if (result != Restore_Complete) + { + return result; + } + + // Remove the level for the above call + rLevel.RemoveLevel(); + + // Add it to the list of done itmes + rLevel.mRestoredObjects.insert(en->GetObjectID()); + } + } + } + + // now remove the user writable flag, if we added it earlier + try + { + dirAttr.WriteAttributes(rLocalDirectoryName.c_str(), false); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to restore attributes for '" << + rLocalDirectoryName << "': " << e.what()); + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + catch(...) + { + BOX_ERROR("Failed to restore attributes for '" << + rLocalDirectoryName << "': unknown error"); + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } + } + + return Restore_Complete; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientRestore(BackupProtocolClient &, int64_t, +// const char *, bool, bool, bool, bool, bool) +// Purpose: Restore a directory on the server to a local +// directory on the disc. The local directory must not +// already exist. +// +// If a restore is aborted for any reason, then it may +// be resumed if Resume == true. If Resume == false +// and resumption is possible, then +// Restore_ResumePossible is returned. +// +// Set RestoreDeleted to restore a deleted directory. +// This may not give the directory structure when it +// was deleted, because files may have been deleted +// within it before it was deleted. +// +// Returns Restore_TargetExists if the target +// directory exists, but there is no restore possible. +// (Won't attempt to overwrite things.) +// +// Returns Restore_Complete on success. (Exceptions +// on error, unless ContinueAfterError is true and +// the error is recoverable, in which case it returns +// Restore_CompleteWithErrors) +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +int BackupClientRestore(BackupProtocolClient &rConnection, + int64_t DirectoryID, const char *RemoteDirectoryName, + const char *LocalDirectoryName, bool PrintDots, bool RestoreDeleted, + bool UndeleteAfterRestoreDeleted, bool Resume, + bool ContinueAfterErrors) +{ + // Parameter block + RestoreParams params; + params.PrintDots = PrintDots; + params.RestoreDeleted = RestoreDeleted; + params.ContinueAfterErrors = ContinueAfterErrors; + params.ContinuedAfterError = false; + params.mRestoreResumeInfoFilename = LocalDirectoryName; + params.mRestoreResumeInfoFilename += ".boxbackupresume"; + + // Target exists? + int targetExistance = ObjectExists(LocalDirectoryName); + + // Does any resumption information exist? + bool doingResume = false; + if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && + targetExistance == ObjectExists_Dir) + { + if(!Resume) + { + // Caller didn't specify that resume should be done, + // so refuse to do it but say why. + return Restore_ResumePossible; + } + + // Attempt to load the resume info file + if(!params.mResumeInfo.Load(params.mRestoreResumeInfoFilename)) + { + // failed -- bad file, so things have gone a bit wrong + return Restore_TargetExists; + } + + // Flag as doing resume so next check isn't actually performed + doingResume = true; + } + + // Does the directory already exist? + if(targetExistance != ObjectExists_NoObject && !doingResume) + { + // Don't do anything in this case! + return Restore_TargetExists; + } + + // Restore the directory + int result = BackupClientRestoreDir(rConnection, DirectoryID, + RemoteDirectoryName, LocalDirectoryName, params, + params.mResumeInfo); + if (result != Restore_Complete) + { + return result; + } + + // Undelete the directory on the server? + if(RestoreDeleted && UndeleteAfterRestoreDeleted) + { + // Send the command + rConnection.QueryUndeleteDirectory(DirectoryID); + } + + // Finish progress display? + if(PrintDots) + { + printf("\n"); + fflush(stdout); + } + + // Delete the resume information file + ::unlink(params.mRestoreResumeInfoFilename.c_str()); + + return params.ContinuedAfterError ? Restore_CompleteWithErrors + : Restore_Complete; +} + diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h new file mode 100644 index 00000000..311a15bd --- /dev/null +++ b/lib/backupclient/BackupClientRestore.h @@ -0,0 +1,36 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientRestore.h +// Purpose: Functions to restore files from a backup store +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSCLIENTRESTORE_H +#define BACKUPSCLIENTRESTORE__H + +class BackupProtocolClient; + +enum +{ + Restore_Complete = 0, + Restore_ResumePossible, + Restore_TargetExists, + Restore_TargetPathNotFound, + Restore_UnknownError, + Restore_CompleteWithErrors, +}; + +int BackupClientRestore(BackupProtocolClient &rConnection, + int64_t DirectoryID, + const char *RemoteDirectoryName, + const char *LocalDirectoryName, + bool PrintDots = false, + bool RestoreDeleted = false, + bool UndeleteAfterRestoreDeleted = false, + bool Resume = false, + bool ContinueAfterErrors = false); + +#endif // BACKUPSCLIENTRESTORE__H + diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp new file mode 100644 index 00000000..dfef5b03 --- /dev/null +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -0,0 +1,132 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemonConfigVerify.cpp +// Purpose: Configuration file definition for bbackupd +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupDaemonConfigVerify.h" +#include "Daemon.h" +#include "BoxPortsAndFiles.h" + +#include "MemLeakFindOn.h" + + +static const ConfigurationVerifyKey backuplocationkeys[] = +{ + ConfigurationVerifyKey("ExcludeFile", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("ExcludeFilesRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("ExcludeDir", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("ExcludeDirsRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeFile", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeFilesRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeDir", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeDirsRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("Path", ConfigTest_Exists | ConfigTest_LastEntry) +}; + +static const ConfigurationVerify backuplocations[] = +{ + { + "*", + 0, + backuplocationkeys, + ConfigTest_LastEntry, + 0 + } +}; + +static const ConfigurationVerifyKey verifyserverkeys[] = +{ + DAEMON_VERIFY_SERVER_KEYS +}; + +static const ConfigurationVerify verifyserver[] = +{ + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists, + 0 + }, + { + "BackupLocations", + backuplocations, + 0, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } +}; + +static const ConfigurationVerifyKey verifyrootkeys[] = +{ + ConfigurationVerifyKey("AccountNumber", + ConfigTest_Exists | ConfigTest_IsUint32), + ConfigurationVerifyKey("UpdateStoreInterval", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("MinimumFileAge", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("MaxUploadWait", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("MaxFileTimeInFuture", ConfigTest_IsInt, 172800), + // file is uploaded if the file is this much in the future + // (2 days default) + ConfigurationVerifyKey("AutomaticBackup", ConfigTest_IsBool, true), + + ConfigurationVerifyKey("SyncAllowScript", 0), + // script that returns "now" if backup is allowed now, or a number + // of seconds to wait before trying again if not + + ConfigurationVerifyKey("MaximumDiffingTime", ConfigTest_IsInt), + ConfigurationVerifyKey("DeleteRedundantLocationsAfter", + ConfigTest_IsInt, 172800), + + ConfigurationVerifyKey("FileTrackingSizeThreshold", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("DiffingUploadSizeThreshold", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("StoreHostname", ConfigTest_Exists), + ConfigurationVerifyKey("StorePort", ConfigTest_IsInt, + BOX_PORT_BBSTORED), + ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), + // extended log to syslog + ConfigurationVerifyKey("ExtendedLogFile", 0), + // extended log to a file + ConfigurationVerifyKey("LogAllFileAccess", ConfigTest_IsBool, false), + // enable logging reasons why each file is backed up or not + ConfigurationVerifyKey("LogFile", 0), + // enable logging to a file + ConfigurationVerifyKey("LogFileLevel", 0), + // set the level of verbosity of file logging + ConfigurationVerifyKey("CommandSocket", 0), + // not compulsory to have this + ConfigurationVerifyKey("KeepAliveTime", ConfigTest_IsInt), + ConfigurationVerifyKey("StoreObjectInfoFile", 0), + // optional + + ConfigurationVerifyKey("NotifyScript", 0), + // optional script to run when backup needs attention, eg store full + + ConfigurationVerifyKey("NotifyAlways", ConfigTest_IsBool, false), + // option to disable the suppression of duplicate notifications + + ConfigurationVerifyKey("CertificateFile", ConfigTest_Exists), + ConfigurationVerifyKey("PrivateKeyFile", ConfigTest_Exists), + ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_Exists), + ConfigurationVerifyKey("KeysFile", ConfigTest_Exists), + ConfigurationVerifyKey("DataDirectory", + ConfigTest_Exists | ConfigTest_LastEntry), +}; + +const ConfigurationVerify BackupDaemonConfigVerify = +{ + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 +}; diff --git a/lib/backupclient/BackupDaemonConfigVerify.h b/lib/backupclient/BackupDaemonConfigVerify.h new file mode 100644 index 00000000..fefa703c --- /dev/null +++ b/lib/backupclient/BackupDaemonConfigVerify.h @@ -0,0 +1,18 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemonConfigVerify.h +// Purpose: Configuration file definition for bbackupd +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPDAEMONCONFIGVERIFY__H +#define BACKUPDAEMONCONFIGVERIFY__H + +#include "Configuration.h" + +extern const ConfigurationVerify BackupDaemonConfigVerify; + +#endif // BACKUPDAEMONCONFIGVERIFY__H + diff --git a/lib/backupclient/BackupStoreConstants.h b/lib/backupclient/BackupStoreConstants.h new file mode 100644 index 00000000..2c33fd8f --- /dev/null +++ b/lib/backupclient/BackupStoreConstants.h @@ -0,0 +1,44 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreContants.h +// Purpose: constants for the backup system +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTORECONSTANTS__H +#define BACKUPSTORECONSTANTS__H + +#define BACKUPSTORE_ROOT_DIRECTORY_ID 1 + +#define BACKUP_STORE_SERVER_VERSION 1 + +// Minimum size for a chunk to be compressed +#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256 + +// min and max sizes for blocks +#define BACKUP_FILE_MIN_BLOCK_SIZE 4096 +#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024) + +// Increase the block size if there are more than this number of blocks +#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096 + +// Avoid creating blocks smaller than this +#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128 + +// Maximum number of sizes to do an rsync-like scan for +#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 64 + +// When doing rsync scans, do not scan for blocks smaller than +#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 128 + +// A limit to stop diffing running out of control: If more than this +// times the number of blocks in the original index are found, stop +// looking. This stops really bad cases of diffing files containing +// all the same byte using huge amounts of memory and processor time. +// This is a multiple of the number of blocks in the diff from file. +#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096 + +#endif // BACKUPSTORECONSTANTS__H + diff --git a/lib/backupclient/BackupStoreDirectory.cpp b/lib/backupclient/BackupStoreDirectory.cpp new file mode 100644 index 00000000..0d06da34 --- /dev/null +++ b/lib/backupclient/BackupStoreDirectory.cpp @@ -0,0 +1,568 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory.h +// Purpose: Representation of a backup directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> + +#include "BackupStoreDirectory.h" +#include "IOStream.h" +#include "BackupStoreException.h" +#include "BackupStoreObjectMagic.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mNumEntries; + int64_t mObjectID; // this object ID + int64_t mContainerID; // ID of container + uint64_t mAttributesModTime; + int32_t mOptionsPresent; // bit mask of optional sections / features present + // Then a StreamableMemBlock for attributes +} dir_StreamFormat; + +typedef enum +{ + Option_DependencyInfoPresent = 1 +} dir_StreamFormatOptions; + +typedef struct +{ + uint64_t mModificationTime; + int64_t mObjectID; + int64_t mSizeInBlocks; + uint64_t mAttributesHash; + int16_t mFlags; // order smaller items after bigger ones (for alignment) + // Then a BackupStoreFilename + // Then a StreamableMemBlock for attributes +} en_StreamFormat; + +typedef struct +{ + int64_t mDependsNewer; + int64_t mDependsOlder; +} en_StreamFormatDepends; + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::BackupStoreDirectory() +// Purpose: Constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::BackupStoreDirectory() + : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0) +{ + ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); +} + + +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t) +// Purpose: Constructor giving object and container IDs +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID) + : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::~BackupStoreDirectory() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::~BackupStoreDirectory() +{ + for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) + { + delete (*i); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int) +// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Get the header + dir_StreamFormat hdr; + if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic value... + if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue)) + { + THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat) + } + + // Get data + mObjectID = box_ntoh64(hdr.mObjectID); + mContainerID = box_ntoh64(hdr.mContainerID); + mAttributesModTime = box_ntoh64(hdr.mAttributesModTime); + + // Options + int32_t options = ntohl(hdr.mOptionsPresent); + + // Get attributes + mAttributes.ReadFromStream(rStream, Timeout); + + // Decode count + int count = ntohl(hdr.mNumEntries); + + // Clear existing list + for(std::vector<Entry*>::iterator i = mEntries.begin(); + i != mEntries.end(); i++) + { + delete (*i); + } + mEntries.clear(); + + // Read them in! + for(int c = 0; c < count; ++c) + { + Entry *pen = new Entry; + try + { + // Read from stream + pen->ReadFromStream(rStream, Timeout); + + // Add to list + mEntries.push_back(pen); + } + catch(...) + { + delete pen; + throw; + } + } + + // Read in dependency info? + if(options & Option_DependencyInfoPresent) + { + // Read in extra dependency data + for(int c = 0; c < count; ++c) + { + mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool) +// Purpose: Writes a selection of entries to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const +{ + // Get count of entries + int32_t count = mEntries.size(); + if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING) + { + // Need to count the entries + count = 0; + Iterator i(*this); + while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0) + { + count++; + } + } + + // Check that sensible IDs have been set + ASSERT(mObjectID != 0); + ASSERT(mContainerID != 0); + + // Need dependency info? + bool dependencyInfoRequired = false; + if(StreamDependencyInfo) + { + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + if(pen->HasDependencies()) + { + dependencyInfoRequired = true; + } + } + } + + // Options + int32_t options = 0; + if(dependencyInfoRequired) options |= Option_DependencyInfoPresent; + + // Build header + dir_StreamFormat hdr; + hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE); + hdr.mNumEntries = htonl(count); + hdr.mObjectID = box_hton64(mObjectID); + hdr.mContainerID = box_hton64(mContainerID); + hdr.mAttributesModTime = box_hton64(mAttributesModTime); + hdr.mOptionsPresent = htonl(options); + + // Write header + rStream.Write(&hdr, sizeof(hdr)); + + // Write the attributes? + if(StreamAttributes) + { + mAttributes.WriteToStream(rStream); + } + else + { + // Write a blank header instead + StreamableMemBlock::WriteEmptyBlockToStream(rStream); + } + + // Then write all the entries + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + pen->WriteToStream(rStream); + } + + // Write dependency info? + if(dependencyInfoRequired) + { + Iterator i(*this); + Entry *pen = 0; + while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) + { + pen->WriteToStreamDependencyInfo(rStream); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddEntry(const Entry &) +// Purpose: Adds entry to directory (no checking) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy) +{ + Entry *pnew = new Entry(rEntryToCopy); + try + { + mEntries.push_back(pnew); + } + catch(...) + { + delete pnew; + throw; + } + + return pnew; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t) +// Purpose: Adds entry to directory (no checking) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime) +{ + Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime); + try + { + mEntries.push_back(pnew); + } + catch(...) + { + delete pnew; + throw; + } + + return pnew; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::DeleteEntry(int64_t) +// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient) +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) +{ + for(std::vector<Entry*>::iterator i(mEntries.begin()); + i != mEntries.end(); ++i) + { + if((*i)->mObjectID == ObjectID) + { + // Delete + delete (*i); + // Remove from list + mEntries.erase(i); + // Done + return; + } + } + + // Not found + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::FindEntryByID(int64_t) +// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist. +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const +{ + for(std::vector<Entry*>::const_iterator i(mEntries.begin()); + i != mEntries.end(); ++i) + { + if((*i)->mObjectID == ObjectID) + { + // Found + return (*i); + } + } + + // Not found + return 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry() +// Purpose: Constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry() + : mModificationTime(0), + mObjectID(0), + mSizeInBlocks(0), + mFlags(0), + mAttributesHash(0), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::~Entry() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::~Entry() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry(const Entry &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) + : mName(rToCopy.mName), + mModificationTime(rToCopy.mModificationTime), + mObjectID(rToCopy.mObjectID), + mSizeInBlocks(rToCopy.mSizeInBlocks), + mFlags(rToCopy.mFlags), + mAttributesHash(rToCopy.mAttributesHash), + mAttributes(rToCopy.mAttributes), + mMinMarkNumber(rToCopy.mMinMarkNumber), + mMarkNumber(rToCopy.mMarkNumber), + mDependsNewer(rToCopy.mDependsNewer), + mDependsOlder(rToCopy.mDependsOlder) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t) +// Purpose: Constructor from values +// Created: 2003/08/27 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) + : mName(rName), + mModificationTime(ModificationTime), + mObjectID(ObjectID), + mSizeInBlocks(SizeInBlocks), + mFlags(Flags), + mAttributesHash(AttributesHash), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int) +// Purpose: Read an entry from a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Grab the raw bytes from the stream which compose the header + en_StreamFormat entry; + if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Do reading first before modifying the variables, to be more exception safe + + // Get the filename + BackupStoreFilename name; + name.ReadFromStream(rStream, Timeout); + + // Get the attributes + mAttributes.ReadFromStream(rStream, Timeout); + + // Store the rest of the bits + mModificationTime = box_ntoh64(entry.mModificationTime); + mObjectID = box_ntoh64(entry.mObjectID); + mSizeInBlocks = box_ntoh64(entry.mSizeInBlocks); + mAttributesHash = box_ntoh64(entry.mAttributesHash); + mFlags = ntohs(entry.mFlags); + mName = name; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &) +// Purpose: Writes the entry to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const +{ + // Build a structure + en_StreamFormat entry; + entry.mModificationTime = box_hton64(mModificationTime); + entry.mObjectID = box_hton64(mObjectID); + entry.mSizeInBlocks = box_hton64(mSizeInBlocks); + entry.mAttributesHash = box_hton64(mAttributesHash); + entry.mFlags = htons(mFlags); + + // Write it + rStream.Write(&entry, sizeof(entry)); + + // Write the filename + mName.WriteToStream(rStream); + + // Write any attributes + mAttributes.WriteToStream(rStream); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int) +// Purpose: Read the optional dependency info from a stream +// Created: 13/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout) +{ + // Grab the raw bytes from the stream which compose the header + en_StreamFormatDepends depends; + if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Store the data + mDependsNewer = box_ntoh64(depends.mDependsNewer); + mDependsOlder = box_ntoh64(depends.mDependsOlder); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &) +// Purpose: Write the optional dependency info to a stream +// Created: 13/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const +{ + // Build structure + en_StreamFormatDepends depends; + depends.mDependsNewer = box_hton64(mDependsNewer); + depends.mDependsOlder = box_hton64(mDependsOlder); + // Write + rStream.Write(&depends, sizeof(depends)); +} + + + diff --git a/lib/backupclient/BackupStoreDirectory.h b/lib/backupclient/BackupStoreDirectory.h new file mode 100644 index 00000000..958eee81 --- /dev/null +++ b/lib/backupclient/BackupStoreDirectory.h @@ -0,0 +1,268 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDirectory.h +// Purpose: Representation of a backup directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREDIRECTORY__H +#define BACKUPSTOREDIRECTORY__H + +#include <string> +#include <vector> + +#include "BackupStoreFilenameClear.h" +#include "StreamableMemBlock.h" +#include "BoxTime.h" + +class IOStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreDirectory +// Purpose: In memory representation of a directory +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreDirectory +{ +public: + BackupStoreDirectory(); + BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID); +private: + // Copying not allowed + BackupStoreDirectory(const BackupStoreDirectory &rToCopy); +public: + ~BackupStoreDirectory(); + + class Entry + { + public: + friend class BackupStoreDirectory; + + Entry(); + ~Entry(); + Entry(const Entry &rToCopy); + Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash); + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream) const; + + const BackupStoreFilename &GetName() const {return mName;} + box_time_t GetModificationTime() const {return mModificationTime;} + int64_t GetObjectID() const {return mObjectID;} + int64_t GetSizeInBlocks() const {return mSizeInBlocks;} + int16_t GetFlags() const {return mFlags;} + void AddFlags(int16_t Flags) {mFlags |= Flags;} + void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;} + + // Some things can be changed + void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;} + void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;} + + // Attributes + bool HasAttributes() const {return !mAttributes.IsEmpty();} + void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;} + const StreamableMemBlock &GetAttributes() const {return mAttributes;} + uint64_t GetAttributesHash() const {return mAttributesHash;} + + // Marks + // The lowest mark number a version of a file of this name has ever had + uint32_t GetMinMarkNumber() const {return mMinMarkNumber;} + // The mark number on this file + uint32_t GetMarkNumber() const {return mMarkNumber;} + + // Make sure these flags are synced with those in backupprocotol.txt + // ListDirectory command + enum + { + Flags_INCLUDE_EVERYTHING = -1, + Flags_EXCLUDE_NOTHING = 0, + Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below! + Flags_File = 1, + Flags_Dir = 2, + Flags_Deleted = 4, + Flags_OldVersion = 8, + Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion + }; + // characters for textual listing of files -- see bbackupquery/BackupQueries + #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR" + + bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) + { + return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet)) + && ((mFlags & FlagsNotToBeSet) == 0); + }; + + // Get dependency info + // new version this depends on + int64_t GetDependsNewer() const {return mDependsNewer;} + void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;} + // older version which depends on this + int64_t GetDependsOlder() const {return mDependsOlder;} + void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;} + + // Dependency info saving + bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;} + void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout); + void WriteToStreamDependencyInfo(IOStream &rStream) const; + + private: + BackupStoreFilename mName; + box_time_t mModificationTime; + int64_t mObjectID; + int64_t mSizeInBlocks; + int16_t mFlags; + uint64_t mAttributesHash; + StreamableMemBlock mAttributes; + uint32_t mMinMarkNumber; + uint32_t mMarkNumber; + + uint64_t mDependsNewer; // new version this depends on + uint64_t mDependsOlder; // older version which depends on this + }; + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream, + int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, + int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING, + bool StreamAttributes = true, bool StreamDependencyInfo = true) const; + + Entry *AddEntry(const Entry &rEntryToCopy); + Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime); + void DeleteEntry(int64_t ObjectID); + Entry *FindEntryByID(int64_t ObjectID) const; + + int64_t GetObjectID() const {return mObjectID;} + int64_t GetContainerID() const {return mContainerID;} + + // Need to be able to update the container ID when moving objects + void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;} + + // Purely for use of server -- not serialised into streams + int64_t GetRevisionID() const {return mRevisionID;} + void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;} + + unsigned int GetNumberOfEntries() const {return mEntries.size();} + + // User info -- not serialised into streams + int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;} + void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;} + + // Attributes + bool HasAttributes() const {return !mAttributes.IsEmpty();} + void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;} + const StreamableMemBlock &GetAttributes() const {return mAttributes;} + box_time_t GetAttributesModTime() const {return mAttributesModTime;} + + class Iterator + { + public: + Iterator(const BackupStoreDirectory &rDir) + : mrDir(rDir), i(rDir.mEntries.begin()) + { + } + + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags + while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.end()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + // WARNING: This function is really very inefficient. + // Only use when you want to look up ONE filename, not in a loop looking up lots. + // In a looping situation, cache the decrypted filenames in another memory structure. + BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags or filename + while( (i != mrDir.mEntries.end()) + && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) ) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.end()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + private: + const BackupStoreDirectory &mrDir; + std::vector<Entry*>::const_iterator i; + }; + + friend class Iterator; + + class ReverseIterator + { + public: + ReverseIterator(const BackupStoreDirectory &rDir) + : mrDir(rDir), i(rDir.mEntries.rbegin()) + { + } + + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) + { + // Skip over things which don't match the required flags + while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) + { + ++i; + } + // Not the last one? + if(i == mrDir.mEntries.rend()) + { + return 0; + } + // Return entry, and increment + return (*(i++)); + } + + private: + const BackupStoreDirectory &mrDir; + std::vector<Entry*>::const_reverse_iterator i; + }; + + friend class ReverseIterator; + + // For recovery of the store + // Implemented in BackupStoreCheck2.cpp + bool CheckAndFix(); + void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags); + bool NameInUse(const BackupStoreFilename &rName); + // Don't use these functions in normal code! + + // For testing + void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;} + + // Debug and diagonistics + void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere + +private: + int64_t mRevisionID; + int64_t mObjectID; + int64_t mContainerID; + std::vector<Entry*> mEntries; + box_time_t mAttributesModTime; + StreamableMemBlock mAttributes; + int64_t mUserInfo1; +}; + +#endif // BACKUPSTOREDIRECTORY__H + diff --git a/lib/backupclient/BackupStoreException.h b/lib/backupclient/BackupStoreException.h new file mode 100644 index 00000000..981dfa60 --- /dev/null +++ b/lib/backupclient/BackupStoreException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREEXCEPTION__H +#define BACKUPSTOREEXCEPTION__H + +// Compatibility +#include "autogen_BackupStoreException.h" + +#endif // BACKUPSTOREEXCEPTION__H + diff --git a/lib/backupclient/BackupStoreException.txt b/lib/backupclient/BackupStoreException.txt new file mode 100644 index 00000000..528a8c94 --- /dev/null +++ b/lib/backupclient/BackupStoreException.txt @@ -0,0 +1,71 @@ +EXCEPTION BackupStore 4 + +Internal 0 +BadAccountDatabaseFile 1 +AccountDatabaseNoSuchEntry 2 +InvalidBackupStoreFilename 3 +UnknownFilenameEncoding 4 +CouldntReadEntireStructureFromStream 5 +BadDirectoryFormat 6 +CouldNotFindEntryInDirectory 7 +OutputFileAlreadyExists 8 +OSFileError 9 +StreamDoesntHaveRequiredFeatures 10 +BadBackupStoreFile 11 +CouldNotLoadStoreInfo 12 +BadStoreInfoOnLoad 13 +StoreInfoIsReadOnly 14 +StoreInfoDirNotInList 15 +StoreInfoBlockDeltaMakesValueNegative 16 +DirectoryHasBeenDeleted 17 +StoreInfoNotInitialised 18 +StoreInfoAlreadyLoaded 19 +StoreInfoNotLoaded 20 +ReadFileFromStreamTimedOut 21 +FileWrongSizeAfterBeingStored 22 +AddedFileDoesNotVerify 23 +StoreInfoForWrongAccount 24 +ContextIsReadOnly 25 +AttributesNotLoaded 26 +AttributesNotUnderstood 27 +WrongServerVersion 28 # client side +ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account. +NameAlreadyExistsInDirectory 30 +BerkelyDBFailure 31 # client side +InodeMapIsReadOnly 32 # client side +InodeMapNotOpen 33 # client side +FilenameEncryptionKeyNotKnown 34 +FilenameEncryptionNoKeyForSpecifiedMethod 35 +FilenameEncryptionNotSetup 36 +CouldntLoadClientKeyMaterial 37 +BadEncryptedAttributes 38 +EncryptedAttributesHaveUnknownEncoding 39 +OutputSizeTooSmallForChunk 40 +BadEncodedChunk 41 +NotEnoughSpaceToDecodeChunk 42 +ChunkHasUnknownEncoding 43 +ChunkContainsBadCompressedData 44 +CantWriteToEncodedFileStream 45 +Temp_FileEncodeStreamDidntReadBuffer 46 +CantWriteToDecodedFileStream 47 +WhenDecodingExpectedToReadButCouldnt 48 +BackupStoreFileFailedIntegrityCheck 49 +ThereIsNoDataInASymLink 50 +IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51 +BlockEntryEncodingDidntGiveExpectedLength 52 +CouldNotFindUnusedIDDuringAllocation 53 +AddedFileExceedsStorageLimit 54 +CannotDiffAnIncompleteStoreFile 55 +CannotDecodeDiffedFilesWithoutCombining 56 +FailedToReadBlockOnCombine 57 +OnCombineFromFileIsIncomplete 58 +BadNotifySysadminEventCode 59 +InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60 +CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server? +AttributeHashSecretNotSet 62 +AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES +SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection. +IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related +DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory +PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. +UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp new file mode 100644 index 00000000..17e145a3 --- /dev/null +++ b/lib/backupclient/BackupStoreFile.cpp @@ -0,0 +1,1556 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFile.cpp +// Purpose: Utils for manipulating files +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <sys/stat.h> +#include <string.h> +#include <new> +#include <string.h> + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + #include <stdio.h> +#endif + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFilename.h" +#include "BackupStoreException.h" +#include "IOStream.h" +#include "Guards.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreObjectMagic.h" +#include "Compress.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "CipherAES.h" +#include "BackupStoreConstants.h" +#include "CollectInBufferStream.h" +#include "RollingChecksum.h" +#include "MD5Digest.h" +#include "ReadGatherStream.h" +#include "Random.h" +#include "BackupStoreFileEncodeStream.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +using namespace BackupStoreFileCryptVar; + +// How big a buffer to use for copying files +#define COPY_BUFFER_SIZE (8*1024) + +// Statistics +BackupStoreFileStats BackupStoreFile::msStats = {0,0,0}; + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool sWarnedAboutBackwardsCompatiblity = false; +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &) +// Purpose: Encode a file into something for storing on file server. +// Requires a real filename so full info can be stored. +// +// Returns a stream. Most of the work is done by the stream +// when data is actually requested -- the file will be held +// open until the stream is deleted or the file finished. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(const char *Filename, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) +{ + // Create the stream + std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); + + // Do the initial setup + ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, + 0 /* no recipe, just encode */, + ContainerID, rStoreFilename, pModificationTime, pLogger, + pRunStatusProvider); + + // Return the stream for the caller + return stream; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &) +// Purpose: Verify that an encoded file meets the format requirements. +// Doesn't verify that the data is intact and can be decoded. +// Optionally returns the ID of the file which it is diffed from, +// and the (original) container ID. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut) +{ + // Get the size of the file + int64_t fileSize = rFile.BytesLeftToRead(); + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Get the header... + file_StreamFormat hdr; + if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read header + return false; + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + return false; + } + + // Get a filename, see if it loads OK + try + { + BackupStoreFilename fn; + fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); + } + catch(...) + { + // an error occured while reading it, so that's not good + return false; + } + + // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not + try + { + int32_t size_s; + if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + int size = ntohl(size_s); + // Skip forward the size + rFile.Seek(size, IOStream::SeekType_Relative); + } + catch(...) + { + // an error occured while reading it, so that's not good + return false; + } + + // Get current position in file -- the end of the header + int64_t headerEnd = rFile.GetPosition(); + + // Get number of blocks + int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); + + // Calculate where the block index will be, check it's reasonable + int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + if(blockIndexLoc < headerEnd) + { + // Not enough space left for the block index, let alone the blocks themselves + return false; + } + + // Load the block index header + rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute); + file_BlockIndexHeader blkhdr; + if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read block index header -- assume bad file + return false; + } + + // Check header + if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != numBlocks) + { + // Bad header -- either magic value or number of blocks is wrong + return false; + } + + // Flag for recording whether a block is referenced from another file + bool blockFromOtherFileReferenced = false; + + // Read the index, checking that the length values all make sense + int64_t currentBlockStart = headerEnd; + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read block entry + file_BlockIndexEntry blk; + if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */)) + { + // Couldn't read block index entry -- assume bad file + return false; + } + + // Check size and location + int64_t blkSize = box_ntoh64(blk.mEncodedSize); + if(blkSize <= 0) + { + // Mark that this file references another file + blockFromOtherFileReferenced = true; + } + else + { + // This block is actually in this file + if((currentBlockStart + blkSize) > blockIndexLoc) + { + // Encoded size makes the block run over the index + return false; + } + + // Move the current block start ot the end of this block + currentBlockStart += blkSize; + } + } + + // Check that there's no empty space + if(currentBlockStart != blockIndexLoc) + { + return false; + } + + // Check that if another block is references, then the ID is there, and if one isn't there is no ID. + int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); + if((otherID != 0 && blockFromOtherFileReferenced == false) + || (otherID == 0 && blockFromOtherFileReferenced == true)) + { + // Doesn't look good! + return false; + } + + // Does the caller want the other ID? + if(pDiffFromObjectIDOut) + { + *pDiffFromObjectIDOut = otherID; + } + + // Does the caller want the container ID? + if(pContainerIDOut) + { + *pContainerIDOut = box_ntoh64(hdr.mContainerID); + } + + // Passes all tests + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeFile(IOStream &, const char *) +// Purpose: Decode a file. Will set file attributes. File must not exist. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) +{ + // Does file exist? + EMU_STRUCT_STAT st; + if(EMU_STAT(DecodedFilename, &st) == 0) + { + THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) + } + + // Try, delete output file if error + try + { + // Make a stream for outputting this file + FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL); + + // Get the decoding stream + std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr)); + + // Is it a symlink? + if(!stream->IsSymLink()) + { + // Copy it out to the file + stream->CopyStreamTo(out); + } + + out.Close(); + + // The stream might have uncertain size, in which case + // we need to drain it to get the + // Protocol::ProtocolStreamHeader_EndOfStream byte + // out of our connection stream. + char buffer[1]; + int drained = rEncodedFile.Read(buffer, 1); + + // The Read will return 0 if we are actually at the end + // of the stream, but some tests decode files directly, + // in which case we are actually positioned at the start + // of the block index. I hope that reading an extra byte + // doesn't hurt! + // ASSERT(drained == 0); + + // Write the attributes + try + { + stream->GetAttributes().WriteAttributes(DecodedFilename); + } + catch (std::exception& e) + { + BOX_WARNING("Failed to restore attributes on " << + DecodedFilename << ": " << e.what()); + } + } + catch(...) + { + ::unlink(DecodedFilename); + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *) +// Purpose: Return a stream which will decode the encrypted file data on the fly. +// Accepts streams in block index first, or main header first, order. In the latter case, +// the stream must be Seek()able. +// +// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow +// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) +{ + // Create stream + std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout)); + + // Get it ready + stream->Setup(pAlterativeAttr); + + // Return to caller + return stream; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int) +// Purpose: Constructor +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout) + : mrEncodedFile(rEncodedFile), + mTimeout(Timeout), + mNumBlocks(0), + mpBlockIndex(0), + mpEncodedData(0), + mpClearData(0), + mClearDataSize(0), + mCurrentBlock(-1), + mCurrentBlockClearSize(0), + mPositionInCurrentBlock(0), + mEntryIVBase(42) // different to default value in the encoded stream! +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + , mIsOldVersion(false) +#endif +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::~DecodedStream() +// Purpose: Desctructor +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFile::DecodedStream::~DecodedStream() +{ + // Free any allocated memory + if(mpBlockIndex) + { + ::free(mpBlockIndex); + } + if(mpEncodedData) + { + BackupStoreFile::CodingChunkFree(mpEncodedData); + } + if(mpClearData) + { + ::free(mpClearData); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *) +// Purpose: Get the stream ready to decode -- reads in headers +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr) +{ + // Get the size of the file + int64_t fileSize = mrEncodedFile.BytesLeftToRead(); + + // Get the magic number to work out which order the stream is in + int32_t magic; + if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read magic value + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + bool inFileOrder = true; + switch(ntohl(magic)) + { +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_MAGIC_VALUE_V0: + mIsOldVersion = true; + // control flows on +#endif + case OBJECTMAGIC_FILE_MAGIC_VALUE_V1: + inFileOrder = true; + break; + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0: + mIsOldVersion = true; + // control flows on +#endif + case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1: + inFileOrder = false; + break; + + default: + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // If not in file order, then the index list must be read now + if(!inFileOrder) + { + ReadBlockIndex(true /* have already read and verified the magic number */); + } + + // Get header + file_StreamFormat hdr; + if(inFileOrder) + { + // Read the header, without the magic number + if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic), + 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + // Put in magic number + hdr.mMagicValue = magic; + } + else + { + // Not in file order, so need to read the full header + if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Get the filename + mFilename.ReadFromStream(mrEncodedFile, mTimeout); + + // Get the attributes (either from stream, or supplied attributes) + if(pAlterativeAttr != 0) + { + // Read dummy attributes + BackupClientFileAttributes attr; + attr.ReadFromStream(mrEncodedFile, mTimeout); + + // Set to supplied attributes + mAttributes = *pAlterativeAttr; + } + else + { + // Read the attributes from the stream + mAttributes.ReadFromStream(mrEncodedFile, mTimeout); + } + + // If it is in file order, go and read the file attributes + // Requires that the stream can seek + if(inFileOrder) + { + // Make sure the file size is known + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Store current location (beginning of encoded blocks) + int64_t endOfHeaderPos = mrEncodedFile.GetPosition(); + + // Work out where the index is + int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); + int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + + // Seek to that position + mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute); + + // Read the block index + ReadBlockIndex(false /* magic number still to be read */); + + // Seek back to the end of header position, ready for reading the chunks + mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute); + } + + // Check view of blocks from block header and file header match + if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Need to allocate some memory for the two blocks for reading encoded data, and clear data + if(mNumBlocks > 0) + { + // Find the maximum encoded data size + int32_t maxEncodedDataSize = 0; + const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; + ASSERT(entry != 0); + for(int64_t e = 0; e < mNumBlocks; e++) + { + // Get the clear and encoded size + int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize); + ASSERT(encodedSize > 0); + + // Larger? + if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize; + } + + // Allocate those blocks! + mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32); + + // Allocate the block for the clear data, using the hint from the header. + // If this is wrong, things will exception neatly later on, so it can't be used + // to do anything more than cause an error on downloading. + mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32; + mpClearData = (uint8_t*)::malloc(mClearDataSize); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool) +// Purpose: Read the block index from the stream, and store in internal buffer (minus header) +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) +{ + // Header + file_BlockIndexHeader blkhdr; + + // Read it in -- way depends on how whether the magic number has already been read + if(MagicAlreadyRead) + { + // Read the header, without the magic number + if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue), + 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } + else + { + // Magic not already read, so need to read the full header + if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Check magic value + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + } + + // Get the number of blocks out of the header + mNumBlocks = box_ntoh64(blkhdr.mNumBlocks); + + // Read the IV base + mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase); + + // Load the block entries in? + if(mNumBlocks > 0) + { + // How big is the index? + int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks; + + // Allocate some memory + mpBlockIndex = ::malloc(indexSize); + if(mpBlockIndex == 0) + { + throw std::bad_alloc(); + } + + // Read it in + if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Read(void *, int, int) +// Purpose: As interface. Reads decrpyted data. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Symlinks don't have data. So can't read it. Not even zero bytes. + if(IsSymLink()) + { + // Don't allow reading in this case + THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink); + } + + // Already finished? + if(mCurrentBlock >= mNumBlocks) + { + // At end of stream, nothing to do + return 0; + } + + int bytesToRead = NBytes; + uint8_t *output = (uint8_t*)pBuffer; + + while(bytesToRead > 0 && mCurrentBlock < mNumBlocks) + { + // Anything left in the current block? + if(mPositionInCurrentBlock < mCurrentBlockClearSize) + { + // Copy data out of this buffer + int s = mCurrentBlockClearSize - mPositionInCurrentBlock; + if(s > bytesToRead) s = bytesToRead; // limit to requested data + + // Copy + ::memcpy(output, mpClearData + mPositionInCurrentBlock, s); + + // Update positions + output += s; + mPositionInCurrentBlock += s; + bytesToRead -= s; + } + + // Need to get some more data? + if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize) + { + // Number of next block + ++mCurrentBlock; + if(mCurrentBlock >= mNumBlocks) + { + // Stop now! + break; + } + + // Get the size from the block index + const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; + int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize); + if(encodedSize <= 0) + { + // The caller is attempting to decode a file which is the direct result of a diff + // operation, and so does not contain all the data. + // It needs to be combined with the previous version first. + THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining) + } + + // Load in next block + if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Decode the data + mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize); + + // Calculate IV for this entry + uint64_t iv = mEntryIVBase; + iv += mCurrentBlock; + // Convert to network byte order before encrypting with it, so that restores work on + // platforms with different endiannesses. + iv = box_hton64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Make sure this is the right size + if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) + { +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(!mIsOldVersion) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the + // IV for the encrypted section. Try again, with the thing the other way round + iv = box_swap64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + else + { + // Warn and log this issue + if(!sWarnedAboutBackwardsCompatiblity) + { + BOX_WARNING("WARNING: Decoded one or more files using backwards compatibility mode for block index."); + sWarnedAboutBackwardsCompatiblity = true; + } + } +#else + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) +#endif + } + + // Check the digest + MD5Digest md5; + md5.Add(mpClearData, mCurrentBlockClearSize); + md5.Finish(); + if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum)) + { + THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck) + } + + // Set vars to say what's happening + mPositionInCurrentBlock = 0; + } + } + + ASSERT(bytesToRead >= 0); + ASSERT(bytesToRead <= NBytes); + + return NBytes - bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::IsSymLink() +// Purpose: Is the unencoded file actually a symlink? +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::IsSymLink() +{ + // First, check in with the attributes + if(!mAttributes.IsSymLink()) + { + return false; + } + + // So the attributes think it is a symlink. + // Consistency check... + if(mNumBlocks != 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::Write(const void *, int) +// Purpose: As interface. Throws exception, as you can't write to this stream. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::StreamDataLeft() +// Purpose: As interface. Any data left? +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::StreamDataLeft() +{ + return mCurrentBlock < mNumBlocks; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodedStream::StreamClosed() +// Purpose: As interface. Always returns true, no writing allowed. +// Created: 9/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::DecodedStream::StreamClosed() +{ + // Can't write to this stream! + return true; +} + + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::SetBlowfishKey(const void *, int) +// Purpose: Static. Sets the key to use for encryption and decryption. +// Created: 7/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength) +{ + // IVs set later + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + + sBlowfishEncryptBlockEntry.Reset(); + sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); + sBlowfishEncryptBlockEntry.UsePadding(false); + sBlowfishDecryptBlockEntry.Reset(); + sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); + sBlowfishDecryptBlockEntry.UsePadding(false); +} + + +#ifndef HAVE_OLD_SSL +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::SetAESKey(const void *, int) +// Purpose: Sets the AES key to use for file data encryption. Will select AES as +// the cipher to use when encrypting. +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength) +{ + // Setup context + sAESEncrypt.Reset(); + sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); + sAESDecrypt.Reset(); + sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); + + // Set encryption to use this key, instead of the "default" blowfish key + spEncrypt = &sAESEncrypt; + sEncryptCipherType = HEADER_AES_ENCODING; +} +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int) +// Purpose: The maximum output size of a block, given the chunk size +// Created: 7/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize) +{ + // Calculate... the maximum size of output by first the largest it could be after compression, + // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck + // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing + // an problem where a rather non-compressable file didn't fit in a block buffer.) + return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1 + + sBlowfishEncrypt.GetIVLength() + 128; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &) +// Purpose: Encodes a chunk (encryption, possible compressed beforehand) +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput) +{ + ASSERT(spEncrypt != 0); + + // Check there's some space in the output block + if(rOutput.mBufferSize < 256) + { + rOutput.Reallocate(256); + } + + // Check alignment of the block + ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + + // Want to compress it? + bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE); + + // Build header + uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT; + if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED; + + // Store header + rOutput.mpBuffer[0] = header; + int outOffset = 1; + + // Setup cipher, and store the IV + int ivLen = 0; + const void *iv = spEncrypt->SetRandomIV(ivLen); + ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen); + outOffset += ivLen; + + // Start encryption process + spEncrypt->Begin(); + + #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \ + { \ + if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \ + { \ + rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \ + } \ + } + + // Encode the chunk + if(compressChunk) + { + // buffer to compress into + uint8_t buffer[2048]; + + // Set compressor with all the chunk as an input + Compress<true> compress; + compress.Input(Chunk, ChunkSize); + compress.FinishInput(); + + // Get and encrypt output + while(!compress.OutputHasFinished()) + { + int s = compress.Output(buffer, sizeof(buffer)); + if(s > 0) + { + ENCODECHUNK_CHECK_SPACE(s) + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); + } + else + { + // Should never happen, as we put all the input in in one go. + // So if this happens, it means there's a logical problem somewhere + THROW_EXCEPTION(BackupStoreException, Internal) + } + } + ENCODECHUNK_CHECK_SPACE(16) + outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); + } + else + { + // Straight encryption + ENCODECHUNK_CHECK_SPACE(ChunkSize) + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize); + ENCODECHUNK_CHECK_SPACE(16) + outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); + } + + ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check + + return outOffset; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int) +// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find +// the extra output buffer size needed before calling. +// See notes in EncodeChunk() for notes re alignment of the +// encoded data. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize) +{ + // Check alignment of the encoded block + ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + + // First check + if(EncodedSize < 1) + { + THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) + } + + const uint8_t *input = (uint8_t*)Encoded; + + // Get header, make checks, etc + uint8_t header = input[0]; + bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED; + uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT); + if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING) + { + THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding) + } + +#ifndef HAVE_OLD_SSL + // Choose cipher + CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt); +#else + // AES not supported with this version of OpenSSL + if(encodingType == HEADER_AES_ENCODING) + { + THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL) + } + CipherContext &cipher(sBlowfishDecrypt); +#endif + + // Check enough space for header, an IV and one byte of input + int ivLen = cipher.GetIVLength(); + if(EncodedSize < (1 + ivLen + 1)) + { + THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) + } + + // Set IV in decrypt context, and start + cipher.SetIV(input + 1); + cipher.Begin(); + + // Setup vars for code + int inOffset = 1 + ivLen; + uint8_t *output = (uint8_t*)Output; + int outOffset = 0; + + // Do action + if(chunkCompressed) + { + // Do things in chunks + uint8_t buffer[2048]; + int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer)); + + // Decompressor + Compress<false> decompress; + + while(inOffset < EncodedSize) + { + // Decrypt a block + int bl = inputBlockLen; + if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long + int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl); + inOffset += bl; + + // Decompress the decrypted data + if(s > 0) + { + decompress.Input(buffer, s); + int os = 0; + do + { + os = decompress.Output(output + outOffset, OutputSize - outOffset); + outOffset += os; + } while(os > 0); + + // Check that there's space left in the output buffer -- there always should be + if(outOffset >= OutputSize) + { + THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) + } + } + } + + // Get any compressed data remaining in the cipher context and compression + int s = cipher.Final(buffer, sizeof(buffer)); + decompress.Input(buffer, s); + decompress.FinishInput(); + while(!decompress.OutputHasFinished()) + { + int os = decompress.Output(output + outOffset, OutputSize - outOffset); + outOffset += os; + + // Check that there's space left in the output buffer -- there always should be + if(outOffset >= OutputSize) + { + THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) + } + } + } + else + { + // Easy decryption + outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset); + outOffset += cipher.Final(output + outOffset, OutputSize - outOffset); + } + + return outOffset; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool) +// Purpose: Returns a stream which gives a Stream order version of the encoded file. +// If TakeOwnership == true, then the input stream will be deleted when the +// returned stream is deleted. +// The input stream must be seekable. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership) +{ + ASSERT(pStream != 0); + + // Get the size of the file + int64_t fileSize = pStream->BytesLeftToRead(); + if(fileSize == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) + } + + // Read the header + int bytesRead = 0; + file_StreamFormat hdr; + bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead); + + // Seek backwards to put the file pointer back where it was before we started this + pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative); + + // Check we got a block + if(!readBlock) + { + // Couldn't read header -- assume file bad + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Get number of blocks + int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); + + // Calculate where the block index will be, check it's reasonable + int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + int64_t blockIndexLoc = fileSize - blockIndexSize; + if(blockIndexLoc < 0) + { + // Doesn't look good! + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Build a reordered stream + std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership)); + + // Set it up... + ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get())); + int component = rreordered.AddComponent(pStream); + // Send out the block index + rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc); + // And then the rest of the file + rreordered.AddBlock(component, blockIndexLoc, true, 0); + + return reordered; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ResetStats() +// Purpose: Reset the gathered statistics +// Created: 20/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::ResetStats() +{ + msStats.mBytesInEncodedFiles = 0; + msStats.mBytesAlreadyOnServer = 0; + msStats.mTotalFileStreamSize = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &) +// Purpose: Compares the contents of a file against the checksums contained in the +// block index. Returns true if the checksums match, meaning the file is +// extremely likely to match the original. Will always consume the entire index. +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout) +{ + // is it a symlink? + bool sourceIsSymlink = false; + { + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + if((st.st_mode & S_IFMT) == S_IFLNK) + { + sourceIsSymlink = true; + } + } + + // Open file, if it's not a symlink + std::auto_ptr<FileStream> in; + if(!sourceIsSymlink) + { + in.reset(new FileStream(Filename)); + } + + // Read header + file_BlockIndexHeader hdr; + if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic + if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0) +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0); +#endif + + // Get basic information + int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); + uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase); + + //TODO: Verify that these sizes look reasonable + + // setup + void *data = 0; + int32_t dataSize = -1; + bool matches = true; + int64_t totalSizeInBlockIndex = 0; + + try + { + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read an entry from the stream + file_BlockIndexEntry entry; + if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read entry + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Calculate IV for this entry + uint64_t iv = entryIVBase; + iv += b; + iv = box_hton64(iv); +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(isOldVersion) + { + // Reverse the IV for compatibility + iv = box_swap64(iv); + } +#endif + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry.mEnEnc, sizeof(entry.mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Size of block + int32_t blockClearSize = ntohl(entryEnc.mSize); + if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + totalSizeInBlockIndex += blockClearSize; + + // Make sure there's enough memory allocated to load the block in + if(dataSize < blockClearSize) + { + // Too small, free the block if it's already allocated + if(data != 0) + { + ::free(data); + data = 0; + } + // Allocate a block + data = ::malloc(blockClearSize + 128); + if(data == 0) + { + throw std::bad_alloc(); + } + dataSize = blockClearSize + 128; + } + + // Load in the block from the file, if it's not a symlink + if(!sourceIsSymlink) + { + if(in->Read(data, blockClearSize) != blockClearSize) + { + // Not enough data left in the file, can't possibly match + matches = false; + } + else + { + // Check the checksum + MD5Digest md5; + md5.Add(data, blockClearSize); + md5.Finish(); + if(!md5.DigestMatches(entryEnc.mStrongChecksum)) + { + // Checksum didn't match + matches = false; + } + } + } + + // Keep on going regardless, to make sure the entire block index stream is read + // -- must always be consistent about what happens with the stream. + } + } + catch(...) + { + // clean up in case of errors + if(data != 0) + { + ::free(data); + data = 0; + } + throw; + } + + // free block + if(data != 0) + { + ::free(data); + data = 0; + } + + // Check for data left over if it's not a symlink + if(!sourceIsSymlink) + { + // Anything left to read in the file? + if(in->BytesLeftToRead() != 0) + { + // File has extra data at the end + matches = false; + } + } + + // Symlinks must have zero size on server + if(sourceIsSymlink) + { + matches = (totalSizeInBlockIndex == 0); + } + + return matches; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer() +// Purpose: Constructor +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +BackupStoreFile::EncodingBuffer::EncodingBuffer() + : mpBuffer(0), + mBufferSize(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer() +// Purpose: Destructor +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +BackupStoreFile::EncodingBuffer::~EncodingBuffer() +{ + if(mpBuffer != 0) + { + BackupStoreFile::CodingChunkFree(mpBuffer); + mpBuffer = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::Allocate(int) +// Purpose: Do initial allocation of block +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::EncodingBuffer::Allocate(int Size) +{ + ASSERT(mpBuffer == 0); + uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size); + if(buffer == 0) + { + throw std::bad_alloc(); + } + mpBuffer = buffer; + mBufferSize = Size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodingBuffer::Reallocate(int) +// Purpose: Reallocate the block. Try not to call this, it has to copy +// the entire contents as the block can't be reallocated straight. +// Created: 25/11/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) +{ + BOX_TRACE("Reallocating EncodingBuffer from " << mBufferSize << + " to " << NewSize); + ASSERT(mpBuffer != 0); + uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + // Copy data + ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize); + + // Free old + BackupStoreFile::CodingChunkFree(mpBuffer); + + // Store new buffer + mpBuffer = buffer; + mBufferSize = NewSize; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: DiffTimer::DiffTimer(); +// Purpose: Constructor +// Created: 2005/02/01 +// +// -------------------------------------------------------------------------- +DiffTimer::DiffTimer() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: DiffTimer::DiffTimer(); +// Purpose: Destructor +// Created: 2005/02/01 +// +// -------------------------------------------------------------------------- +DiffTimer::~DiffTimer() +{ +} diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h new file mode 100644 index 00000000..f4c60919 --- /dev/null +++ b/lib/backupclient/BackupStoreFile.h @@ -0,0 +1,228 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFile.h +// Purpose: Utils for manipulating files +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILE__H +#define BACKUPSTOREFILE__H + +#include <cstdlib> +#include <memory> +#include <cstdlib> + +#include "BackupClientFileAttributes.h" +#include "BackupStoreFilename.h" +#include "IOStream.h" +#include "ReadLoggingStream.h" +#include "RunStatusProvider.h" + +typedef struct +{ + int64_t mBytesInEncodedFiles; + int64_t mBytesAlreadyOnServer; + int64_t mTotalFileStreamSize; +} BackupStoreFileStats; + +// Uncomment to disable backwards compatibility +//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + + +// Output buffer to EncodeChunk and input data to DecodeChunk must +// have specific alignment, see function comments. +#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16 +#define BACKUPSTOREFILE_CODING_OFFSET 15 + +// Have some memory allocation commands, note closing "Off" at end of file. +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: DiffTimer +// Purpose: Interface for classes that can keep track of diffing time, +// and send SSL keepalive messages +// Created: 2006/01/19 +// +// -------------------------------------------------------------------------- +class DiffTimer +{ +public: + DiffTimer(); + virtual ~DiffTimer(); +public: + virtual void DoKeepAlive() = 0; + virtual int GetMaximumDiffingTime() = 0; + virtual bool IsManaged() = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFile +// Purpose: Class to hold together utils for manipulating files. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +class BackupStoreFile +{ +public: + class DecodedStream : public IOStream + { + friend class BackupStoreFile; + private: + DecodedStream(IOStream &rEncodedFile, int Timeout); + DecodedStream(const DecodedStream &); // not allowed + DecodedStream &operator=(const DecodedStream &); // not allowed + public: + ~DecodedStream(); + + // Stream functions + virtual int Read(void *pBuffer, int NBytes, int Timeout); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + // Accessor functions + const BackupClientFileAttributes &GetAttributes() {return mAttributes;} + const BackupStoreFilename &GetFilename() {return mFilename;} + int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests + + bool IsSymLink(); + + private: + void Setup(const BackupClientFileAttributes *pAlterativeAttr); + void ReadBlockIndex(bool MagicAlreadyRead); + + private: + IOStream &mrEncodedFile; + int mTimeout; + BackupClientFileAttributes mAttributes; + BackupStoreFilename mFilename; + int64_t mNumBlocks; + void *mpBlockIndex; + uint8_t *mpEncodedData; + uint8_t *mpClearData; + int mClearDataSize; + int mCurrentBlock; + int mCurrentBlockClearSize; + int mPositionInCurrentBlock; + uint64_t mEntryIVBase; +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + bool mIsOldVersion; +#endif + }; + + + // Main interface + static std::auto_ptr<IOStream> EncodeFile(const char *Filename, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime = 0, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL); + static std::auto_ptr<IOStream> EncodeFileDiff + ( + const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, + int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, + int Timeout, + DiffTimer *pDiffTimer, + int64_t *pModificationTime = 0, + bool *pIsCompletelyDifferent = 0 + ); + static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); + static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); + static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); + static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0); + static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); + static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); + static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout); + static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false); + + // Stream manipulation + static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership); + static void MoveStreamPositionToBlockIndex(IOStream &rStream); + + // Crypto setup + static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength); +#ifndef HAVE_OLD_SSL + static void SetAESKey(const void *pKey, int KeyLength); +#endif + + // Allocation of properly aligning chunks for decoding and encoding chunks + inline static void *CodingChunkAlloc(int Size) + { + uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3)); + if(a == 0) return 0; + // Align to main block size + ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size + uint8_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE + - (uint8_t)(((unsigned long)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE); + uint8_t *b = (a + adjustment); + // Store adjustment + *b = adjustment; + // Return offset + return b + BACKUPSTOREFILE_CODING_OFFSET; + } + inline static void CodingChunkFree(void *Block) + { + // Check alignment is as expected + ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size + ASSERT((uint8_t)(((unsigned long)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + uint8_t *a = (uint8_t*)Block; + a -= BACKUPSTOREFILE_CODING_OFFSET; + // Adjust downwards... + a -= *a; + free(a); + } + + static void DiffTimerExpired(); + + // Building blocks + class EncodingBuffer + { + public: + EncodingBuffer(); + ~EncodingBuffer(); + private: + // No copying + EncodingBuffer(const EncodingBuffer &); + EncodingBuffer &operator=(const EncodingBuffer &); + public: + void Allocate(int Size); + void Reallocate(int NewSize); + + uint8_t *mpBuffer; + int mBufferSize; + }; + static int MaxBlockSizeForChunkSize(int ChunkSize); + static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput); + + // Caller should know how big the output size is, but also allocate a bit more memory to cover various + // overheads allowed for in checks + static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize) + { + // Plenty big enough + return KnownChunkSize + 256; + } + static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize); + + // Statisitics, not designed to be completely reliable + static void ResetStats(); + static BackupStoreFileStats msStats; + + // For debug +#ifndef BOX_RELEASE_BUILD + static bool TraceDetailsOfDiffProcess; +#endif + + // For decoding encoded files + static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile); +}; + +#include "MemLeakFindOff.h" + +#endif // BACKUPSTOREFILE__H diff --git a/lib/backupclient/BackupStoreFileCmbDiff.cpp b/lib/backupclient/BackupStoreFileCmbDiff.cpp new file mode 100644 index 00000000..1a88fa3f --- /dev/null +++ b/lib/backupclient/BackupStoreFileCmbDiff.cpp @@ -0,0 +1,326 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCmbDiff.cpp +// Purpose: Combine two diffs together +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <stdlib.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CombineDiffs(IOStream &, IOStream &, IOStream &rOut) +// Purpose: Given two diffs, combine them into a single diff, to produce a diff +// which, combined with the original file, creates the result of applying +// rDiff, then rDiff2. Two opens of rDiff2 are required +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut) +{ + // Skip header of first diff, record where the data starts, and skip to the index + int64_t diff1DataStarts = 0; + { + // Read the header for the From file + file_StreamFormat diff1Hdr; + if(!rDiff1.ReadFullBuffer(&diff1Hdr, sizeof(diff1Hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diff1Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Skip over the filename and attributes of the From file + // BLOCK + { + BackupStoreFilename filename2; + filename2.ReadFromStream(rDiff1, IOStream::TimeOutInfinite); + int32_t size_s; + if(!rDiff1.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + int size = ntohl(size_s); + // Skip forward the size + rDiff1.Seek(size, IOStream::SeekType_Relative); + } + // Record position + diff1DataStarts = rDiff1.GetPosition(); + // Skip to index + rDiff1.Seek(0 - (((box_ntoh64(diff1Hdr.mNumBlocks)) * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + } + + // Read the index of the first diff + // Header first + file_BlockIndexHeader diff1IdxHdr; + if(!rDiff1.ReadFullBuffer(&diff1IdxHdr, sizeof(diff1IdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(diff1IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + int64_t diff1NumBlocks = box_ntoh64(diff1IdxHdr.mNumBlocks); + // Allocate some memory + int64_t *diff1BlockStartPositions = (int64_t*)::malloc((diff1NumBlocks + 1) * sizeof(int64_t)); + if(diff1BlockStartPositions == 0) + { + throw std::bad_alloc(); + } + + // Buffer data + void *buffer = 0; + int bufferSize = 0; + + try + { + // Then the entries: + // For each entry, want to know if it's in the file, and if so, how big it is. + // We'll store this as an array of file positions in the file, with an additioal + // entry on the end so that we can work out the length of the last block. + // If an entry isn't in the file, then store 0 - (position in other file). + int64_t diff1Position = diff1DataStarts; + for(int64_t b = 0; b < diff1NumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff1.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Where's the block? + int64_t blockEn = box_ntoh64(e.mEncodedSize); + if(blockEn <= 0) + { + // Just store the negated block number + diff1BlockStartPositions[b] = blockEn; + } + else + { + // Block is present in this file + diff1BlockStartPositions[b] = diff1Position; + diff1Position += blockEn; + } + } + + // Finish off the list, so the last entry can have it's size calcuated. + diff1BlockStartPositions[diff1NumBlocks] = diff1Position; + + // Now read the second diff's header, copying it to the out file + file_StreamFormat diff2Hdr; + if(!rDiff2.ReadFullBuffer(&diff2Hdr, sizeof(diff2Hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diff2Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Copy + rOut.Write(&diff2Hdr, sizeof(diff2Hdr)); + // Copy over filename and attributes + // BLOCK + { + BackupStoreFilename filename; + filename.ReadFromStream(rDiff2, IOStream::TimeOutInfinite); + filename.WriteToStream(rOut); + StreamableMemBlock attr; + attr.ReadFromStream(rDiff2, IOStream::TimeOutInfinite); + attr.WriteToStream(rOut); + } + + // Get to the index of rDiff2b, and read the header + MoveStreamPositionToBlockIndex(rDiff2b); + file_BlockIndexHeader diff2IdxHdr; + if(!rDiff2b.ReadFullBuffer(&diff2IdxHdr, sizeof(diff2IdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(diff2IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + int64_t diff2NumBlocks = box_ntoh64(diff2IdxHdr.mNumBlocks); + int64_t diff2IndexEntriesStart = rDiff2b.GetPosition(); + + // Then read all the entries + int64_t diff2FilePosition = rDiff2.GetPosition(); + for(int64_t b = 0; b < diff2NumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // What do to next about copying data + bool copyBlock = false; + int copySize = 0; + int64_t copyFrom = 0; + bool fromFileDiff1 = false; + + // Where's the block? + int64_t blockEn = box_ntoh64(e.mEncodedSize); + if(blockEn > 0) + { + // Block is present in this file -- copy to out + copyBlock = true; + copyFrom = diff2FilePosition; + copySize = (int)blockEn; + + // Move pointer onwards + diff2FilePosition += blockEn; + } + else + { + // Block isn't present here -- is it present in the old one? + int64_t blockIndex = 0 - blockEn; + if(blockIndex < 0 || blockIndex > diff1NumBlocks) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + if(diff1BlockStartPositions[blockIndex] > 0) + { + // Block is in the old diff file, copy it across + copyBlock = true; + copyFrom = diff1BlockStartPositions[blockIndex]; + int nb = blockIndex + 1; + while(diff1BlockStartPositions[nb] <= 0) + { + // This is safe, because the last entry will terminate it properly! + ++nb; + ASSERT(nb <= diff1NumBlocks); + } + copySize = diff1BlockStartPositions[nb] - copyFrom; + fromFileDiff1 = true; + } + } + //TRACE4("%d %d %lld %d\n", copyBlock, copySize, copyFrom, fromFileDiff1); + + // Copy data to the output file? + if(copyBlock) + { + // Allocate enough space + if(bufferSize < copySize || buffer == 0) + { + // Free old block + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + bufferSize = 0; + } + // Allocate new block + buffer = ::malloc(copySize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + bufferSize = copySize; + } + ASSERT(bufferSize >= copySize); + + // Load in the data + if(fromFileDiff1) + { + rDiff1.Seek(copyFrom, IOStream::SeekType_Absolute); + if(!rDiff1.ReadFullBuffer(buffer, copySize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + } + else + { + rDiff2.Seek(copyFrom, IOStream::SeekType_Absolute); + if(!rDiff2.ReadFullBuffer(buffer, copySize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + } + // Write out data + rOut.Write(buffer, copySize); + } + } + + // Write the modified header + diff2IdxHdr.mOtherFileID = diff1IdxHdr.mOtherFileID; + rOut.Write(&diff2IdxHdr, sizeof(diff2IdxHdr)); + + // Then we'll write out the index, reading the data again + rDiff2b.Seek(diff2IndexEntriesStart, IOStream::SeekType_Absolute); + for(int64_t b = 0; b < diff2NumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Where's the block? + int64_t blockEn = box_ntoh64(e.mEncodedSize); + + // If it's not in this file, it needs modification... + if(blockEn <= 0) + { + int64_t blockIndex = 0 - blockEn; + // In another file. Need to translate this against the other diff + if(diff1BlockStartPositions[blockIndex] > 0) + { + // Block is in the first diff file, stick in size + int nb = blockIndex + 1; + while(diff1BlockStartPositions[nb] <= 0) + { + // This is safe, because the last entry will terminate it properly! + ++nb; + ASSERT(nb <= diff1NumBlocks); + } + int64_t size = diff1BlockStartPositions[nb] - diff1BlockStartPositions[blockIndex]; + e.mEncodedSize = box_hton64(size); + } + else + { + // Block in the original file, use translated value + e.mEncodedSize = box_hton64(diff1BlockStartPositions[blockIndex]); + } + } + + // Write entry + rOut.Write(&e, sizeof(e)); + } + } + catch(...) + { + // clean up + ::free(diff1BlockStartPositions); + if(buffer != 0) + { + ::free(buffer); + } + throw; + } + + // Clean up allocated memory + ::free(diff1BlockStartPositions); + if(buffer != 0) + { + ::free(buffer); + } +} + diff --git a/lib/backupclient/BackupStoreFileCmbIdx.cpp b/lib/backupclient/BackupStoreFileCmbIdx.cpp new file mode 100644 index 00000000..c8bcc3b9 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCmbIdx.cpp @@ -0,0 +1,324 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCmbIdx.cpp +// Purpose: Combine indicies of a delta file and the file it's a diff from. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <string.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" + +#include "MemLeakFindOn.h" + +// Hide from outside world +namespace +{ + +class BSFCombinedIndexStream : public IOStream +{ +public: + BSFCombinedIndexStream(IOStream *pDiff); + ~BSFCombinedIndexStream(); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + virtual void Initialise(IOStream &rFrom); + +private: + IOStream *mpDiff; + bool mIsInitialised; + bool mHeaderWritten; + file_BlockIndexHeader mHeader; + int64_t mNumEntriesToGo; + int64_t mNumEntriesInFromFile; + int64_t *mFromBlockSizes; // NOTE: Entries in network byte order +}; + +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CombineFileIndices(IOStream &, IOStream &, bool) +// Purpose: Given a diff file and the file it's a diff from, return a stream from which +// can be read the index of the combined file, without actually combining them. +// The stream of the diff must have a lifetime greater than or equal to the +// lifetime of the returned stream object. The full "from" file stream +// only needs to exist during the actual function call. +// If you pass in dodgy files which aren't related, then you will either +// get an error or bad results. So don't do that. +// If DiffIsIndexOnly is true, then rDiff is assumed to be a stream positioned +// at the beginning of the block index. Similarly for FromIsIndexOnly. +// WARNING: Reads of the returned streams with buffer sizes less than 64 bytes +// will not return any data. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly, bool FromIsIndexOnly) +{ + // Reposition file pointers? + if(!DiffIsIndexOnly) + { + MoveStreamPositionToBlockIndex(rDiff); + } + if(!FromIsIndexOnly) + { + MoveStreamPositionToBlockIndex(rFrom); + } + + // Create object + std::auto_ptr<IOStream> stream(new BSFCombinedIndexStream(&rDiff)); + + // Initialise it + ((BSFCombinedIndexStream *)stream.get())->Initialise(rFrom); + + // And return the stream + return stream; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::BSFCombinedIndexStream() +// Purpose: Private class. Constructor. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +BSFCombinedIndexStream::BSFCombinedIndexStream(IOStream *pDiff) + : mpDiff(pDiff), + mIsInitialised(false), + mHeaderWritten(false), + mNumEntriesToGo(0), + mNumEntriesInFromFile(0), + mFromBlockSizes(0) +{ + ASSERT(mpDiff != 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::~BSFCombinedIndexStream() +// Purpose: Private class. Destructor. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +BSFCombinedIndexStream::~BSFCombinedIndexStream() +{ + if(mFromBlockSizes != 0) + { + ::free(mFromBlockSizes); + mFromBlockSizes = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::Initialise(IOStream &) +// Purpose: Private class. Initalise from the streams (diff passed in constructor). +// Both streams must have file pointer positioned at the block index. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +void BSFCombinedIndexStream::Initialise(IOStream &rFrom) +{ + // Paranoia is good. + if(mIsInitialised) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Look at the diff file: Read in the header + if(!mpDiff->ReadFullBuffer(&mHeader, sizeof(mHeader), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(mHeader.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Read relevant data. + mNumEntriesToGo = box_ntoh64(mHeader.mNumBlocks); + + // Adjust a bit to reflect the fact it's no longer a diff + mHeader.mOtherFileID = box_hton64(0); + + // Now look at the from file: Read header + file_BlockIndexHeader fromHdr; + if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Then... allocate memory for the list of sizes + mNumEntriesInFromFile = box_ntoh64(fromHdr.mNumBlocks); + mFromBlockSizes = (int64_t*)::malloc(mNumEntriesInFromFile * sizeof(int64_t)); + if(mFromBlockSizes == 0) + { + throw std::bad_alloc(); + } + + // And read them all in! + for(int64_t b = 0; b < mNumEntriesInFromFile; ++b) + { + file_BlockIndexEntry e; + if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check that the from file isn't a delta in itself + if(box_ntoh64(e.mEncodedSize) <= 0) + { + THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) + } + + // Store size (in network byte order) + mFromBlockSizes[b] = e.mEncodedSize; + } + + // Flag as initialised + mIsInitialised = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::Read(void *, int, int) +// Purpose: Private class. As interface. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Paranoia is good. + if(!mIsInitialised || mFromBlockSizes == 0 || mpDiff == 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + int written = 0; + + // Header output yet? + if(!mHeaderWritten) + { + // Enough space? + if(NBytes < (int)sizeof(mHeader)) return 0; + + // Copy in + ::memcpy(pBuffer, &mHeader, sizeof(mHeader)); + NBytes -= sizeof(mHeader); + written += sizeof(mHeader); + + // Flag it's done + mHeaderWritten = true; + } + + // How many entries can be written? + int entriesToWrite = NBytes / sizeof(file_BlockIndexEntry); + if(entriesToWrite > mNumEntriesToGo) + { + entriesToWrite = mNumEntriesToGo; + } + + // Setup ready to go + file_BlockIndexEntry *poutput = (file_BlockIndexEntry*)(((uint8_t*)pBuffer) + written); + + // Write entries + for(int b = 0; b < entriesToWrite; ++b) + { + if(!mpDiff->ReadFullBuffer(&(poutput[b]), sizeof(file_BlockIndexEntry), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Does this need adjusting? + int s = box_ntoh64(poutput[b].mEncodedSize); + if(s <= 0) + { + // A reference to a block in the from file + int block = 0 - s; + ASSERT(block >= 0); + if(block >= mNumEntriesInFromFile) + { + // That's not good, the block doesn't exist + THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) + } + + // Adjust the entry in the buffer + poutput[b].mEncodedSize = mFromBlockSizes[block]; // stored in network byte order, no translation necessary + } + } + + // Update written count + written += entriesToWrite * sizeof(file_BlockIndexEntry); + mNumEntriesToGo -= entriesToWrite; + + return written; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::Write(const void *, int) +// Purpose: Private class. As interface. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::StreamDataLeft() +// Purpose: Private class. As interface +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +bool BSFCombinedIndexStream::StreamDataLeft() +{ + return (!mHeaderWritten) || (mNumEntriesToGo > 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BSFCombinedIndexStream::StreamClosed() +// Purpose: Private class. As interface. +// Created: 8/7/04 +// +// -------------------------------------------------------------------------- +bool BSFCombinedIndexStream::StreamClosed() +{ + return true; // doesn't do writing +} + diff --git a/lib/backupclient/BackupStoreFileCombine.cpp b/lib/backupclient/BackupStoreFileCombine.cpp new file mode 100644 index 00000000..baa331f0 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCombine.cpp @@ -0,0 +1,410 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCombine.cpp +// Purpose: File combining for BackupStoreFile +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" +#include "FileStream.h" + +#include "MemLeakFindOn.h" + +typedef struct +{ + int64_t mFilePosition; +} FromIndexEntry; + +static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries); +static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut); +static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut); + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::CombineFile(IOStream &, IOStream &, IOStream &) +// Purpose: Where rDiff is a store file which is incomplete as a result of a +// diffing operation, rFrom is the file it is diffed from, and +// rOut is the stream in which to place the result, the old file +// and new file are combined into a file containing all the data. +// rDiff2 is the same file as rDiff, opened again to get two +// independent streams to the same file. +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut) +{ + // Read and copy the header. + file_StreamFormat hdr; + if(!rDiff.ReadFullBuffer(&hdr, sizeof(hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Copy + rOut.Write(&hdr, sizeof(hdr)); + // Copy over filename and attributes + // BLOCK + { + BackupStoreFilename filename; + filename.ReadFromStream(rDiff, IOStream::TimeOutInfinite); + filename.WriteToStream(rOut); + StreamableMemBlock attr; + attr.ReadFromStream(rDiff, IOStream::TimeOutInfinite); + attr.WriteToStream(rOut); + } + + // Read the header for the From file + file_StreamFormat fromHdr; + if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Skip over the filename and attributes of the From file + // BLOCK + { + BackupStoreFilename filename2; + filename2.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + int32_t size_s; + if(!rFrom.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + int size = ntohl(size_s); + // Skip forward the size + rFrom.Seek(size, IOStream::SeekType_Relative); + } + + // Allocate memory for the block index of the From file + int64_t fromNumBlocks = box_ntoh64(fromHdr.mNumBlocks); + // NOTE: An extra entry is required so that the length of the last block can be calculated + FromIndexEntry *pFromIndex = (FromIndexEntry*)::malloc((fromNumBlocks+1) * sizeof(FromIndexEntry)); + if(pFromIndex == 0) + { + throw std::bad_alloc(); + } + + try + { + // Load the index from the From file, calculating the offsets in the + // file as we go along, and enforce that everything should be present. + LoadFromIndex(rFrom, pFromIndex, fromNumBlocks); + + // Read in the block index of the Diff file in small chunks, and output data + // for each block, either from this file, or the other file. + int64_t diffNumBlocks = box_ntoh64(hdr.mNumBlocks); + CopyData(rDiff /* positioned at start of data */, rDiff2, diffNumBlocks, rFrom, pFromIndex, fromNumBlocks, rOut); + + // Read in the block index again, and output the new block index, simply + // filling in the sizes of blocks from the old file. + WriteNewIndex(rDiff, diffNumBlocks, pFromIndex, fromNumBlocks, rOut); + + // Free buffers + ::free(pFromIndex); + pFromIndex = 0; + } + catch(...) + { + // Clean up + if(pFromIndex != 0) + { + ::free(pFromIndex); + pFromIndex = 0; + } + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static LoadFromIndex(IOStream &, FromIndexEntry *, int64_t) +// Purpose: Static. Load the index from the From file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries) +{ + ASSERT(pIndex != 0); + ASSERT(NumEntries >= 0); + + // Get the starting point in the file + int64_t filePos = rFrom.GetPosition(); + + // Jump to the end of the file to read the index + rFrom.Seek(0 - ((NumEntries * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + + // Read block index header + file_BlockIndexHeader blkhdr; + if(!rFrom.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != NumEntries) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // And then the block entries + for(int64_t b = 0; b < NumEntries; ++b) + { + // Read + file_BlockIndexEntry en; + if(!rFrom.ReadFullBuffer(&en, sizeof(en), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // Add to list + pIndex[b].mFilePosition = filePos; + + // Encoded size? + int64_t encodedSize = box_ntoh64(en.mEncodedSize); + // Check that the block is actually there + if(encodedSize <= 0) + { + THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) + } + + // Move file pointer on + filePos += encodedSize; + } + + // Store the position in the very last entry, so the size of the last entry can be calculated + pIndex[NumEntries].mFilePosition = filePos; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static CopyData(IOStream &, IOStream &, int64_t, IOStream &, FromIndexEntry *, int64_t, IOStream &) +// Purpose: Static. Copy data from the Diff and From file to the out file. +// rDiffData is at beginning of data. +// rDiffIndex at any position. +// rFrom is at any position. +// rOut is after the header, ready for data +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, + IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut) +{ + // Jump to the end of the diff file to read the index + rDiffIndex.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + + // Read block index header + file_BlockIndexHeader diffBlkhdr; + if(!rDiffIndex.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || (int64_t)box_ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Record where the From file is + int64_t fromPos = rFrom.GetPosition(); + + // Buffer data + void *buffer = 0; + int bufferSize = 0; + + try + { + // Read the blocks in! + for(int64_t b = 0; b < DiffNumBlocks; ++b) + { + // Read + file_BlockIndexEntry en; + if(!rDiffIndex.ReadFullBuffer(&en, sizeof(en), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // What's the size value stored in the entry + int64_t encodedSize = box_ntoh64(en.mEncodedSize); + + // How much data will be read? + int32_t blockSize = 0; + if(encodedSize > 0) + { + // The block is actually in the diff file + blockSize = encodedSize; + } + else + { + // It's in the from file. First, check to see if it's valid + int64_t blockIdx = (0 - encodedSize); + if(blockIdx > FromNumBlocks) + { + // References a block which doesn't actually exist + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Calculate size. This operation is safe because of the extra entry at the end + blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition; + } + ASSERT(blockSize > 0); + + // Make sure there's memory available to copy this + if(bufferSize < blockSize || buffer == 0) + { + // Free old block + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + bufferSize = 0; + } + // Allocate new block + buffer = ::malloc(blockSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + bufferSize = blockSize; + } + ASSERT(bufferSize >= blockSize); + + // Load in data from one of the files + if(encodedSize > 0) + { + // Load from diff file + if(!rDiffData.ReadFullBuffer(buffer, blockSize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + } + else + { + // Locate and read the data from the from file + int64_t blockIdx = (0 - encodedSize); + // Seek if necessary + if(fromPos != pFromIndex[blockIdx].mFilePosition) + { + rFrom.Seek(pFromIndex[blockIdx].mFilePosition, IOStream::SeekType_Absolute); + fromPos = pFromIndex[blockIdx].mFilePosition; + } + // Read + if(!rFrom.ReadFullBuffer(buffer, blockSize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // Update fromPos to current position + fromPos += blockSize; + } + + // Write data to out file + rOut.Write(buffer, blockSize); + } + + // Free buffer, if allocated + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + } + } + catch(...) + { + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + } + throw; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static WriteNewIndex(IOStream &, int64_t, FromIndexEntry *, int64_t, IOStream &) +// Purpose: Write the index to the out file, just copying from the diff file and +// adjusting the entries. +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut) +{ + // Jump to the end of the diff file to read the index + rDiff.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); + + // Read block index header + file_BlockIndexHeader diffBlkhdr; + if(!rDiff.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || (int64_t)box_ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Write it out with a blanked out other file ID + diffBlkhdr.mOtherFileID = box_hton64(0); + rOut.Write(&diffBlkhdr, sizeof(diffBlkhdr)); + + // Rewrite the index + for(int64_t b = 0; b < DiffNumBlocks; ++b) + { + file_BlockIndexEntry en; + if(!rDiff.ReadFullBuffer(&en, sizeof(en), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + + // What's the size value stored in the entry + int64_t encodedSize = box_ntoh64(en.mEncodedSize); + + // Need to adjust it? + if(encodedSize <= 0) + { + // This actually refers to a block in the from file. So rewrite this. + int64_t blockIdx = (0 - encodedSize); + if(blockIdx > FromNumBlocks) + { + // References a block which doesn't actually exist + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Calculate size. This operation is safe because of the extra entry at the end + int32_t blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition; + // Then replace entry + en.mEncodedSize = box_hton64(((uint64_t)blockSize)); + } + + // Write entry + rOut.Write(&en, sizeof(en)); + } +} + + + + + diff --git a/lib/backupclient/BackupStoreFileCryptVar.cpp b/lib/backupclient/BackupStoreFileCryptVar.cpp new file mode 100644 index 00000000..e826de4e --- /dev/null +++ b/lib/backupclient/BackupStoreFileCryptVar.cpp @@ -0,0 +1,31 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCryptVar.cpp +// Purpose: Cryptographic keys for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileWire.h" + +#include "MemLeakFindOn.h" + +CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt; +CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt; + +#ifndef HAVE_OLD_SSL + CipherContext BackupStoreFileCryptVar::sAESEncrypt; + CipherContext BackupStoreFileCryptVar::sAESDecrypt; +#endif + +// Default to blowfish +CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt; +uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING; + +CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry; +CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry; + diff --git a/lib/backupclient/BackupStoreFileCryptVar.h b/lib/backupclient/BackupStoreFileCryptVar.h new file mode 100644 index 00000000..566813c8 --- /dev/null +++ b/lib/backupclient/BackupStoreFileCryptVar.h @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileCryptVar.h +// Purpose: Cryptographic keys for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILECRYPTVAR__H +#define BACKUPSTOREFILECRYPTVAR__H + +#include "CipherContext.h" + +// Hide private static variables from the rest of the world by putting them +// as static variables in a namespace. +// -- don't put them as static class variables to avoid openssl/evp.h being +// included all over the project. +namespace BackupStoreFileCryptVar +{ + // Keys for the main file data + extern CipherContext sBlowfishEncrypt; + extern CipherContext sBlowfishDecrypt; + // Use AES when available +#ifndef HAVE_OLD_SSL + extern CipherContext sAESEncrypt; + extern CipherContext sAESDecrypt; +#endif + // How encoding will be done + extern CipherContext *spEncrypt; + extern uint8_t sEncryptCipherType; + + // Keys for the block indicies + extern CipherContext sBlowfishEncryptBlockEntry; + extern CipherContext sBlowfishDecryptBlockEntry; +} + +#endif // BACKUPSTOREFILECRYPTVAR__H + diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp new file mode 100644 index 00000000..5705c3aa --- /dev/null +++ b/lib/backupclient/BackupStoreFileDiff.cpp @@ -0,0 +1,1046 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileDiff.cpp +// Purpose: Functions relating to diffing BackupStoreFiles +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include <new> +#include <map> + +#ifdef HAVE_TIME_H + #include <time.h> +#elif HAVE_SYS_TIME_H + #include <sys/time.h> +#endif + +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "CommonException.h" +#include "FileStream.h" +#include "MD5Digest.h" +#include "RollingChecksum.h" +#include "Timer.h" + +#include "MemLeakFindOn.h" + +#include <cstring> + +using namespace BackupStoreFileCryptVar; +using namespace BackupStoreFileCreation; + +// By default, don't trace out details of the diff as we go along -- would fill up logs significantly. +// But it's useful for the test. +#ifndef BOX_RELEASE_BUILD + bool BackupStoreFile::TraceDetailsOfDiffProcess = false; +#endif + +static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis); +static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]); +static void SearchForMatchingBlocks(IOStream &rFile, + std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, + int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], + DiffTimer *pDiffTimer); +static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable); +static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum &fastSum, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber, +BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks); +static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile); + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &) +// Purpose: Move the file pointer in this stream to just before the block index. +// Assumes that the stream is at the beginning, seekable, and +// reading from the stream is OK. +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream) +{ + // Size of file + int64_t fileSize = rStream.BytesLeftToRead(); + + // Get header + file_StreamFormat hdr; + + // Read the header + if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Work out where the index is + int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); + int64_t blockHeaderPosFromEnd = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); + + // Sanity check + if(blockHeaderPosFromEnd > static_cast<int64_t>(fileSize - sizeof(file_StreamFormat))) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Seek to that position + rStream.Seek(0 - blockHeaderPosFromEnd, IOStream::SeekType_End); + + // Done. Stream now in right position (as long as the file is formatted correctly) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *) +// Purpose: Similar to EncodeFile, but takes the object ID of the file it's +// diffing from, and the index of the blocks in a stream. It'll then +// calculate which blocks can be reused from that old file. +// The timeout is the timeout value for reading the diff block index. +// If pIsCompletelyDifferent != 0, it will be set to true if the +// the two files are completely different (do not share any block), false otherwise. +// +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff +( + const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, + IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, + int64_t *pModificationTime, bool *pIsCompletelyDifferent) +{ + // Is it a symlink? + { + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + if((st.st_mode & S_IFLNK) == S_IFLNK) + { + // Don't do diffs for symlinks + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = true; + } + return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + } + } + + // Load in the blocks + BlocksAvailableEntry *pindex = 0; + int64_t blocksInIndex = 0; + bool canDiffFromThis = false; + LoadIndex(rDiffFromBlockIndex, DiffFromObjectID, &pindex, blocksInIndex, Timeout, canDiffFromThis); + // BOX_TRACE("Diff: Blocks in index: " << blocksInIndex); + + if(!canDiffFromThis) + { + // Don't do diffing... + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = true; + } + return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + } + + // Pointer to recipe we're going to create + BackupStoreFileEncodeStream::Recipe *precipe = 0; + + try + { + // Find which sizes should be scanned + int32_t sizesToScan[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]; + FindMostUsedSizes(pindex, blocksInIndex, sizesToScan); + + // Flag for reporting to the user + bool completelyDifferent; + + // BLOCK + { + // Search the file to find matching blocks + std::map<int64_t, int64_t> foundBlocks; // map of offset in file to index in block index + int64_t sizeOfInputFile = 0; + // BLOCK + { + FileStream file(Filename); + // Get size of file + sizeOfInputFile = file.BytesLeftToRead(); + // Find all those lovely matching blocks + SearchForMatchingBlocks(file, foundBlocks, pindex, + blocksInIndex, sizesToScan, pDiffTimer); + + // Is it completely different? + completelyDifferent = (foundBlocks.size() == 0); + } + + // Create a recipe -- if the two files are completely different, don't put the from file ID in the recipe. + precipe = new BackupStoreFileEncodeStream::Recipe(pindex, blocksInIndex, completelyDifferent?(0):(DiffFromObjectID)); + BlocksAvailableEntry *pindexKeptRef = pindex; // we need this later, but must set pindex == 0 now, because of exceptions + pindex = 0; // Recipe now has ownership + + // Fill it in + GenerateRecipe(*precipe, pindexKeptRef, blocksInIndex, foundBlocks, sizeOfInputFile); + } + // foundBlocks no longer required + + // Create the stream + std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); + + // Do the initial setup + ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime); + precipe = 0; // Stream has taken ownership of this + + // Tell user about completely different status? + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = completelyDifferent; + } + + // Return the stream for the caller + return stream; + } + catch(...) + { + // cleanup + if(pindex != 0) + { + ::free(pindex); + pindex = 0; + } + if(precipe != 0) + { + delete precipe; + precipe = 0; + } + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static LoadIndex(IOStream &, int64_t, BlocksAvailableEntry **, int64_t, bool &) +// Purpose: Read in an index, and decrypt, and store in the in memory block format. +// rCanDiffFromThis is set to false if the version of the from file is too old. +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis) +{ + // Reset + rNumBlocksOut = 0; + rCanDiffFromThis = false; + + // Read header + file_BlockIndexHeader hdr; + if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + // Check against backwards comptaibility stuff + if(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)) + { + // Won't diff against old version + + // Absorb rest of stream + char buffer[2048]; + while(rBlockIndex.StreamDataLeft()) + { + rBlockIndex.Read(buffer, sizeof(buffer), 1000 /* 1 sec timeout */); + } + + // Tell caller + rCanDiffFromThis = false; + return; + } +#endif + + // Check magic + if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Check that we're not trying to diff against a file which references blocks from another file + if(((int64_t)box_ntoh64(hdr.mOtherFileID)) != 0) + { + THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile) + } + + // Mark as an acceptable diff. + rCanDiffFromThis = true; + + // Get basic information + int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); + uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase); + + //TODO: Verify that these sizes look reasonable + + // Allocate space for the index + BlocksAvailableEntry *pindex = (BlocksAvailableEntry*)::malloc(sizeof(BlocksAvailableEntry) * numBlocks); + if(pindex == 0) + { + throw std::bad_alloc(); + } + + try + { + for(int64_t b = 0; b < numBlocks; ++b) + { + // Read an entry from the stream + file_BlockIndexEntry entry; + if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + { + // Couldn't read entry + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Calculate IV for this entry + uint64_t iv = entryIVBase; + iv += b; + // Network byte order + iv = box_hton64(iv); + sBlowfishDecryptBlockEntry.SetIV(&iv); + + // Decrypt the encrypted section + file_BlockIndexEntryEnc entryEnc; + int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), + entry.mEnEnc, sizeof(entry.mEnEnc)); + if(sectionSize != sizeof(entryEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Check that we're not trying to diff against a file which references blocks from another file + if(((int64_t)box_ntoh64(entry.mEncodedSize)) <= 0) + { + THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile) + } + + // Store all the required information + pindex[b].mpNextInHashList = 0; // hash list not set up yet + pindex[b].mSize = ntohl(entryEnc.mSize); + pindex[b].mWeakChecksum = ntohl(entryEnc.mWeakChecksum); + ::memcpy(pindex[b].mStrongChecksum, entryEnc.mStrongChecksum, sizeof(pindex[b].mStrongChecksum)); + } + + // Store index pointer for called + ASSERT(ppIndex != 0); + *ppIndex = pindex; + + // Store number of blocks for caller + rNumBlocksOut = numBlocks; + + } + catch(...) + { + // clean up and send the exception along its way + ::free(pindex); + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static FindMostUsedSizes(BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +// Purpose: Finds the most commonly used block sizes in the index +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +{ + // Array for lengths + int64_t sizeCounts[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]; + + // Set arrays to lots of zeros (= unused entries) + for(int l = 0; l < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++l) + { + Sizes[l] = 0; + sizeCounts[l] = 0; + } + + // Array for collecting sizes + std::map<int32_t, int64_t> foundSizes; + + // Run through blocks and make a count of the entries + for(int64_t b = 0; b < NumBlocks; ++b) + { + // Only if the block size is bigger than the minimum size we'll scan for + if(pIndex[b].mSize > BACKUP_FILE_DIFF_MIN_BLOCK_SIZE) + { + // Find entry? + std::map<int32_t, int64_t>::const_iterator f(foundSizes.find(pIndex[b].mSize)); + if(f != foundSizes.end()) + { + // Increment existing entry + foundSizes[pIndex[b].mSize] = foundSizes[pIndex[b].mSize] + 1; + } + else + { + // New entry + foundSizes[pIndex[b].mSize] = 1; + } + } + } + + // Make the block sizes + for(std::map<int32_t, int64_t>::const_iterator i(foundSizes.begin()); i != foundSizes.end(); ++i) + { + // Find the position of the size in the array + for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t) + { + // Instead of sorting on the raw count of blocks, + // take the file area covered by this block size. + if(i->second * i->first > sizeCounts[t] * Sizes[t]) + { + // Then this size belong before this entry -- shuffle them up + for(int s = (BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1); s >= t; --s) + { + Sizes[s] = Sizes[s-1]; + sizeCounts[s] = sizeCounts[s-1]; + } + + // Insert this size + Sizes[t] = i->first; + sizeCounts[t] = i->second; + + // Shouldn't do any more searching + break; + } + } + } + + // trace the size table in debug builds +#ifndef BOX_RELEASE_BUILD + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t) + { + BOX_TRACE("Diff block size " << t << ": " << + Sizes[t] << " (count = " << + sizeCounts[t] << ")"); + } + } +#endif +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) +// Purpose: Find the matching blocks within the file. +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- +static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, + BlocksAvailableEntry *pIndex, int64_t NumBlocks, + int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer) +{ + Timer maximumDiffingTime(0, "MaximumDiffingTime"); + + if(pDiffTimer && pDiffTimer->IsManaged()) + { + maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime(), + "MaximumDiffingTime"); + } + + std::map<int64_t, int32_t> goodnessOfFit; + + // Allocate the hash lookup table + BlocksAvailableEntry **phashTable = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024)); + + // Choose a size for the buffer, just a little bit more than the maximum block size + int32_t bufSize = Sizes[0]; + for(int z = 1; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z) + { + if(Sizes[z] > bufSize) bufSize = Sizes[z]; + } + bufSize += 4; + ASSERT(bufSize > Sizes[0]); + ASSERT(bufSize > 0); + if(bufSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024)) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // TODO: Because we read in the file a scanned block size at a time, + // it is likely to be inefficient. Probably will be much better to + // calculate checksums for all block sizes in a single pass. + + // Allocate the buffers. + uint8_t *pbuffer0 = (uint8_t *)::malloc(bufSize); + uint8_t *pbuffer1 = (uint8_t *)::malloc(bufSize); + try + { + // Check buffer allocation + if(pbuffer0 == 0 || pbuffer1 == 0 || phashTable == 0) + { + // If a buffer got allocated, it will be cleaned up in the catch block + throw std::bad_alloc(); + } + + // Flag to abort the run, if too many blocks are found -- avoid using + // huge amounts of processor time when files contain many similar blocks. + bool abortSearch = false; + + // Search for each block size in turn + // NOTE: Do the smallest size first, so that the scheme for adding + // entries in the found list works as expected and replaces smallers block + // with larger blocks when it finds matches at the same offset in the file. + for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s) + { + ASSERT(Sizes[s] <= bufSize); + BOX_TRACE("Diff pass " << s << ", for block size " << + Sizes[s]); + + // Check we haven't finished + if(Sizes[s] == 0) + { + // empty entry, try next size + continue; + } + + // Set up the hash table entries + SetupHashTable(pIndex, NumBlocks, Sizes[s], phashTable); + + // Shift file position to beginning + rFile.Seek(0, IOStream::SeekType_Absolute); + + // Read first block + if(rFile.Read(pbuffer0, Sizes[s]) != Sizes[s]) + { + // Size of file too short to match -- do next size + continue; + } + + // Setup block pointers + uint8_t *beginnings = pbuffer0; + uint8_t *endings = pbuffer1; + int offset = 0; + + // Calculate the first checksum, ready for rolling + RollingChecksum rolling(beginnings, Sizes[s]); + + // Then roll, until the file is exhausted + int64_t fileBlockNumber = 0; + int64_t fileOffset = 0; + int rollOverInitialBytes = 0; + while(true) + { + if(maximumDiffingTime.HasExpired()) + { + ASSERT(pDiffTimer != NULL); + BOX_INFO("MaximumDiffingTime reached - " + "suspending file diff"); + abortSearch = true; + break; + } + + if(pDiffTimer) + { + pDiffTimer->DoKeepAlive(); + } + + // Load in another block of data, and record how big it is + int bytesInEndings = rFile.Read(endings, Sizes[s]); + int tmp; + + // Skip any bytes from a previous matched block + if(rollOverInitialBytes > 0 && offset < bytesInEndings) + { + int spaceLeft = bytesInEndings - offset; + int thisRoll = (rollOverInitialBytes > spaceLeft) ? spaceLeft : rollOverInitialBytes; + + rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll); + + offset += thisRoll; + fileOffset += thisRoll; + rollOverInitialBytes -= thisRoll; + + if(rollOverInitialBytes) + { + goto refresh; + } + } + + if(goodnessOfFit.count(fileOffset)) + { + tmp = goodnessOfFit[fileOffset]; + } + else + { + tmp = 0; + } + + if(tmp >= Sizes[s]) + { + // Skip over bigger ready-matched blocks completely + rollOverInitialBytes = tmp; + int spaceLeft = bytesInEndings - offset; + int thisRoll = (rollOverInitialBytes > spaceLeft) ? spaceLeft : rollOverInitialBytes; + + rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll); + + offset += thisRoll; + fileOffset += thisRoll; + rollOverInitialBytes -= thisRoll; + + if(rollOverInitialBytes) + { + goto refresh; + } + } + + while(offset < bytesInEndings) + { + // Is current checksum in hash list? + uint16_t hash = rolling.GetComponentForHashing(); + if(phashTable[hash] != 0 && (goodnessOfFit.count(fileOffset) == 0 || goodnessOfFit[fileOffset] < Sizes[s])) + { + if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks)) + { + BOX_TRACE("Found block match for " << hash << " of " << Sizes[s] << " bytes at offset " << fileOffset); + goodnessOfFit[fileOffset] = Sizes[s]; + + // Block matched, roll the checksum forward to the next block without doing + // any more comparisons, because these are pointless (as any more matches will be ignored when + // the recipe is generated) and just take up valuable processor time. Edge cases are + // especially nasty, using huge amounts of time and memory. + int skip = Sizes[s]; + if(offset < bytesInEndings && skip > 0) + { + int spaceLeft = bytesInEndings - offset; + int thisRoll = (skip > spaceLeft) ? spaceLeft : skip; + + rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll); + + offset += thisRoll; + fileOffset += thisRoll; + skip -= thisRoll; + } + // Not all the bytes necessary will have been skipped, so get them + // skipped after the next block is loaded. + rollOverInitialBytes = skip; + + // End this loop, so the final byte isn't used again + break; + } + else + { + BOX_TRACE("False alarm match for " << hash << " of " << Sizes[s] << " bytes at offset " << fileOffset); + } + + int64_t NumBlocksFound = static_cast<int64_t>( + rFoundBlocks.size()); + int64_t MaxBlocksFound = NumBlocks * + BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE; + + if(NumBlocksFound > MaxBlocksFound) + { + abortSearch = true; + break; + } + } + + // Roll checksum forward + rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]); + + // Increment offsets + ++offset; + ++fileOffset; + } + + if(abortSearch) break; + + refresh: + // Finished? + if(bytesInEndings != Sizes[s]) + { + // No more data in file -- check the final block + // (Do a copy and paste of 5 lines of code instead of introducing a comparison for + // each byte of the file) + uint16_t hash = rolling.GetComponentForHashing(); + if(phashTable[hash] != 0 && (goodnessOfFit.count(fileOffset) == 0 || goodnessOfFit[fileOffset] < Sizes[s])) + { + if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks)) + { + goodnessOfFit[fileOffset] = Sizes[s]; + } + } + + // finish + break; + } + + // Switch buffers, reset offset + beginnings = endings; + endings = (beginnings == pbuffer0)?(pbuffer1):(pbuffer0); // ie the other buffer + offset = 0; + + // And count the blocks which have been done + ++fileBlockNumber; + } + + if(abortSearch) break; + } + + // Free buffers and hash table + ::free(pbuffer1); + pbuffer1 = 0; + ::free(pbuffer0); + pbuffer0 = 0; + ::free(phashTable); + phashTable = 0; + } + catch(...) + { + // Cleanup and throw + if(pbuffer1 != 0) ::free(pbuffer1); + if(pbuffer0 != 0) ::free(pbuffer0); + if(phashTable != 0) ::free(phashTable); + throw; + } + +#ifndef BOX_RELEASE_BUILD + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + // Trace out the found blocks in debug mode + BOX_TRACE("Diff: list of found blocks"); + BOX_TRACE("======== ======== ======== ========"); + BOX_TRACE(" Offset BlkIdx Size Movement"); + for(std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); i != rFoundBlocks.end(); ++i) + { + int64_t orgLoc = 0; + for(int64_t b = 0; b < i->second; ++b) + { + orgLoc += pIndex[b].mSize; + } + BOX_TRACE(std::setw(8) << i->first << " " << + std::setw(8) << i->second << " " << + std::setw(8) << pIndex[i->second].mSize << + " " << + std::setw(8) << (i->first - orgLoc)); + } + BOX_TRACE("======== ======== ======== ========"); + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static SetupHashTable(BlocksAvailableEntry *, int64_t, in32_t, BlocksAvailableEntry **) +// Purpose: Set up the hash table ready for a scan +// Created: 14/1/04 +// +// -------------------------------------------------------------------------- +static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable) +{ + // Set all entries in the hash table to zero + ::memset(pHashTable, 0, (sizeof(BlocksAvailableEntry *) * (64*1024))); + + // Scan through the blocks, building the hash table + for(int64_t b = 0; b < NumBlocks; ++b) + { + // Only look at the required block size + if(pIndex[b].mSize == BlockSize) + { + // Get the value under which to hash this entry + uint16_t hash = RollingChecksum::ExtractHashingComponent(pIndex[b].mWeakChecksum); + + // Already present in table? + if(pHashTable[hash] != 0) + { + //BOX_TRACE("Another hash entry for " << hash << " found"); + // Yes -- need to set the pointer in this entry to the current entry to build the linked list + pIndex[b].mpNextInHashList = pHashTable[hash]; + } + + // Put a pointer to this entry in the hash table + pHashTable[hash] = pIndex + b; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static bool SecondStageMatch(xxx) +// Purpose: When a match in the hash table is found, scan for second stage match using strong checksum. +// Created: 14/1/04 +// +// -------------------------------------------------------------------------- +static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum &fastSum, uint8_t *pBeginnings, uint8_t *pEndings, + int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks) +{ + // Check parameters + ASSERT(pBeginnings != 0); + ASSERT(pEndings != 0); + ASSERT(Offset >= 0); + ASSERT(BlockSize > 0); + ASSERT(pFirstInHashList != 0); + ASSERT(pIndex != 0); + +#ifndef BOX_RELEASE_BUILD + uint16_t DEBUG_Hash = fastSum.GetComponentForHashing(); +#endif + uint32_t Checksum = fastSum.GetChecksum(); + + // Before we go to the expense of the MD5, make sure it's a darn good match on the checksum we already know. + BlocksAvailableEntry *scan = pFirstInHashList; + bool found=false; + while(scan != 0) + { + if(scan->mWeakChecksum == Checksum) + { + found = true; + break; + } + scan = scan->mpNextInHashList; + } + if(!found) + { + return false; + } + + // Calculate the strong MD5 digest for this block + MD5Digest strong; + // Add the data from the beginnings + strong.Add(pBeginnings + Offset, BlockSize - Offset); + // Add any data from the endings + if(Offset > 0) + { + strong.Add(pEndings, Offset); + } + strong.Finish(); + + // Then go through the entries in the hash list, comparing with the strong digest calculated + scan = pFirstInHashList; + //BOX_TRACE("second stage match"); + while(scan != 0) + { + //BOX_TRACE("scan size " << scan->mSize << + // ", block size " << BlockSize << + // ", hash " << Hash); + ASSERT(scan->mSize == BlockSize); + ASSERT(RollingChecksum::ExtractHashingComponent(scan->mWeakChecksum) == DEBUG_Hash); + + // Compare? + if(strong.DigestMatches(scan->mStrongChecksum)) + { + //BOX_TRACE("Match!\n"); + // Found! Add to list of found blocks... + int64_t fileOffset = (FileBlockNumber * BlockSize) + Offset; + int64_t blockIndex = (scan - pIndex); // pointer arthmitic is frowned upon. But most efficient way of doing it here -- alternative is to use more memory + + // We do NOT search for smallest blocks first, as this code originally assumed. + // To prevent this from potentially overwriting a better match, the caller must determine + // the relative "goodness" of any existing match and this one, and avoid the call if it + // could be detrimental. + rFoundBlocks[fileOffset] = blockIndex; + + // No point in searching further, report success + return true; + } + + // Next + scan = scan->mpNextInHashList; + } + + // Not matched + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static GenerateRecipe(BackupStoreFileEncodeStream::Recipe &, BlocksAvailableEntry *, int64_t, std::map<int64_t, int64_t> &) +// Purpose: Fills in the recipe from the found block list +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, + int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile) +{ + // NOTE: This function could be a lot more sophisiticated. For example, if + // a small block overlaps a big block like this + // **** + // ******************************* + // then the small block will be used, not the big one. But it'd be better to + // just ignore the small block and keep the big one. However, some stats should + // be gathered about real world files before writing complex code which might + // go wrong. + + // Initialise a blank instruction + BackupStoreFileEncodeStream::RecipeInstruction instruction; + #define RESET_INSTRUCTION \ + instruction.mSpaceBefore = 0; \ + instruction.mBlocks = 0; \ + instruction.mpStartBlock = 0; + RESET_INSTRUCTION + + // First, a special case for when there are no found blocks + if(rFoundBlocks.size() == 0) + { + // No blocks, just a load of space + instruction.mSpaceBefore = SizeOfInputFile; + rRecipe.push_back(instruction); + + #ifndef BOX_RELEASE_BUILD + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + BOX_TRACE("Diff: Default recipe generated, " << + SizeOfInputFile << " bytes of file"); + } + #endif + + // Don't do anything + return; + } + + // Current location + int64_t loc = 0; + + // Then iterate through the list, generating the recipe + std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); + ASSERT(i != rFoundBlocks.end()); // check logic + + // Counting for debug tracing +#ifndef BOX_RELEASE_BUILD + int64_t debug_NewBytesFound = 0; + int64_t debug_OldBlocksUsed = 0; +#endif + + for(; i != rFoundBlocks.end(); ++i) + { + // Remember... map is (position in file) -> (index of block in pIndex) + + if(i->first < loc) + { + // This block overlaps the last one + continue; + } + else if(i->first > loc) + { + // There's a gap between the end of the last thing and this block. + // If there's an instruction waiting, push it onto the list + if(instruction.mSpaceBefore != 0 || instruction.mpStartBlock != 0) + { + rRecipe.push_back(instruction); + } + // Start a new instruction, with the gap ready + RESET_INSTRUCTION + instruction.mSpaceBefore = i->first - loc; + // Move location forward to match + loc += instruction.mSpaceBefore; +#ifndef BOX_RELEASE_BUILD + debug_NewBytesFound += instruction.mSpaceBefore; +#endif + } + + // First, does the current instruction need pushing back, because this block is not + // sequential to the last one? + if(instruction.mpStartBlock != 0 && (pIndex + i->second) != (instruction.mpStartBlock + instruction.mBlocks)) + { + rRecipe.push_back(instruction); + RESET_INSTRUCTION + } + + // Add in this block + if(instruction.mpStartBlock == 0) + { + // This block starts a new instruction + instruction.mpStartBlock = pIndex + i->second; + instruction.mBlocks = 1; + } + else + { + // It continues the previous section of blocks + instruction.mBlocks += 1; + } + +#ifndef BOX_RELEASE_BUILD + debug_OldBlocksUsed++; +#endif + + // Move location forward + loc += pIndex[i->second].mSize; + } + + // Push the last instruction generated + rRecipe.push_back(instruction); + + // Is there any space left at the end which needs sending? + if(loc != SizeOfInputFile) + { + RESET_INSTRUCTION + instruction.mSpaceBefore = SizeOfInputFile - loc; +#ifndef BOX_RELEASE_BUILD + debug_NewBytesFound += instruction.mSpaceBefore; +#endif + rRecipe.push_back(instruction); + } + + // dump out the recipe +#ifndef BOX_RELEASE_BUILD + BOX_TRACE("Diff: " << + debug_NewBytesFound << " new bytes found, " << + debug_OldBlocksUsed << " old blocks used"); + if(BackupStoreFile::TraceDetailsOfDiffProcess) + { + BOX_TRACE("Diff: Recipe generated (size " << rRecipe.size()); + BOX_TRACE("======== ========= ========"); + BOX_TRACE("Space b4 FirstBlk NumBlks"); + { + for(unsigned int e = 0; e < rRecipe.size(); ++e) + { + char b[64]; +#ifdef WIN32 + sprintf(b, "%8I64d", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); +#else + sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); +#endif + BOX_TRACE(std::setw(8) << + rRecipe[e].mSpaceBefore << + " " << + ((rRecipe[e].mpStartBlock == 0)?" -":b) << + " " << std::setw(8) << + rRecipe[e].mBlocks); + } + } + BOX_TRACE("======== ========= ========"); + } +#endif +} diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp new file mode 100644 index 00000000..54c2463d --- /dev/null +++ b/lib/backupclient/BackupStoreFileEncodeStream.cpp @@ -0,0 +1,715 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileEncodeStream.cpp +// Purpose: Implement stream-based file encoding for the backup store +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "BackupClientFileAttributes.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BoxTime.h" +#include "FileStream.h" +#include "Random.h" +#include "RollingChecksum.h" + +#include "MemLeakFindOn.h" + +#include <cstring> + +using namespace BackupStoreFileCryptVar; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream +// Purpose: Constructor (opens file) +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() + : mpRecipe(0), + mpFile(0), + mpLogging(0), + mpRunStatusProvider(NULL), + mStatus(Status_Header), + mSendData(true), + mTotalBlocks(0), + mAbsoluteBlockNumber(-1), + mInstructionNumber(-1), + mNumBlocks(0), + mCurrentBlock(-1), + mCurrentBlockEncodedSize(0), + mPositionInCurrentBlock(0), + mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), + mLastBlockSize(0), + mpRawBuffer(0), + mAllocatedBufferSize(0), + mEntryIVBase(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() +// Purpose: Destructor +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() +{ + // Free buffers + if(mpRawBuffer) + { + ::free(mpRawBuffer); + mpRawBuffer = 0; + } + + // Close the file, which we might have open + if(mpFile) + { + delete mpFile; + mpFile = 0; + } + + // Clear up logging stream + if(mpLogging) + { + delete mpLogging; + mpLogging = 0; + } + + // Free the recipe + if(mpRecipe != 0) + { + delete mpRecipe; + mpRecipe = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *) +// Purpose: Reads file information, and builds file header reading for sending. +// Takes ownership of the Recipe. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::Setup(const char *Filename, + BackupStoreFileEncodeStream::Recipe *pRecipe, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) +{ + // Pointer to a blank recipe which we might create + BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; + + try + { + // Get file attributes + box_time_t modTime = 0; + int64_t fileSize = 0; + BackupClientFileAttributes attr; + attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime, + 0 /* not interested in attr mod time */, &fileSize); + + // Might need to create a blank recipe... + if(pRecipe == 0) + { + pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); + + BackupStoreFileEncodeStream::RecipeInstruction instruction; + instruction.mSpaceBefore = fileSize; // whole file + instruction.mBlocks = 0; // no blocks + instruction.mpStartBlock = 0; // no block + pblankRecipe->push_back(instruction); + + pRecipe = pblankRecipe; + } + + // Tell caller? + if(pModificationTime != 0) + { + *pModificationTime = modTime; + } + + // Go through each instruction in the recipe and work out how many blocks + // it will add, and the max clear size of these blocks + int maxBlockClearSize = 0; + for(uint64_t inst = 0; inst < pRecipe->size(); ++inst) + { + if((*pRecipe)[inst].mSpaceBefore > 0) + { + // Calculate the number of blocks the space before requires + int64_t numBlocks; + int32_t blockSize, lastBlockSize; + CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize); + // Add to accumlated total + mTotalBlocks += numBlocks; + // Update maximum clear size + if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize; + if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize; + } + + // Add number of blocks copied from the previous file + mTotalBlocks += (*pRecipe)[inst].mBlocks; + + // Check for bad things + if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Run through blocks to get the max clear size + for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b) + { + if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize; + } + } + + // Send data? (symlinks don't have any data in them) + mSendData = !attr.IsSymLink(); + + // If not data is being sent, then the max clear block size is zero + if(!mSendData) + { + maxBlockClearSize = 0; + } + + // Header + file_StreamFormat hdr; + hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1); + hdr.mNumBlocks = (mSendData)?(box_hton64(mTotalBlocks)):(0); + hdr.mContainerID = box_hton64(ContainerID); + hdr.mModificationTime = box_hton64(modTime); + // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size + hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128); + hdr.mOptions = 0; // no options defined yet + + // Write header to stream + mData.Write(&hdr, sizeof(hdr)); + + // Write filename to stream + rStoreFilename.WriteToStream(mData); + + // Write attributes to stream + attr.WriteToStream(mData); + + // Allocate some buffers for writing data + if(mSendData) + { + // Open the file + mpFile = new FileStream(Filename); + + if (pLogger) + { + // Create logging stream + mpLogging = new ReadLoggingStream(*mpFile, + *pLogger); + } + else + { + // re-use FileStream instead + mpLogging = mpFile; + mpFile = NULL; + } + + // Work out the largest possible block required for the encoded data + mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); + + // Then allocate two blocks of this size + mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize); + if(mpRawBuffer == 0) + { + throw std::bad_alloc(); + } +#ifndef BOX_RELEASE_BUILD + // In debug builds, make sure that the reallocation code is exercised. + mEncodedBuffer.Allocate(mAllocatedBufferSize / 4); +#else + mEncodedBuffer.Allocate(mAllocatedBufferSize); +#endif + } + else + { + // Write an empty block index for the symlink + file_BlockIndexHeader blkhdr; + blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); + blkhdr.mOtherFileID = box_hton64(0); // not other file ID + blkhdr.mEntryIVBase = box_hton64(0); + blkhdr.mNumBlocks = box_hton64(0); + mData.Write(&blkhdr, sizeof(blkhdr)); + } + + // Ready for reading + mData.SetForReading(); + + // Update stats + BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize; + + // Finally, store the pointer to the recipe, when we know exceptions won't occur + mpRecipe = pRecipe; + } + catch(...) + { + // Clean up any blank recipe + if(pblankRecipe != 0) + { + delete pblankRecipe; + pblankRecipe = 0; + } + throw; + } + + mpRunStatusProvider = pRunStatusProvider; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &) +// Purpose: Calculates the sizes of blocks in a section of the file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut) +{ + // How many blocks, and how big? + rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2; + do + { + rBlockSizeOut *= 2; + + rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut; + + } while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); + + // Last block size + rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut); + + // Avoid small blocks? + if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN + && rNumBlocksOut > 1) + { + // Add the small bit of data to the last block + --rNumBlocksOut; + rLastBlockSizeOut += rBlockSizeOut; + } + + // checks! + ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize); + //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Read(void *, int, int) +// Purpose: As interface -- generates encoded file data on the fly from the raw file +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Check there's something to do. + if(mStatus == Status_Finished) + { + return 0; + } + + if(mpRunStatusProvider && mpRunStatusProvider->StopRun()) + { + THROW_EXCEPTION(BackupStoreException, SignalReceived); + } + + int bytesToRead = NBytes; + uint8_t *buffer = (uint8_t*)pBuffer; + + while(bytesToRead > 0 && mStatus != Status_Finished) + { + if(mStatus == Status_Header || mStatus == Status_BlockListing) + { + // Header or block listing phase -- send from the buffered stream + + // Send bytes from the data buffer + int b = mData.Read(buffer, bytesToRead, Timeout); + bytesToRead -= b; + buffer += b; + + // Check to see if all the data has been used from this stream + if(!mData.StreamDataLeft()) + { + // Yes, move on to next phase (or finish, if there's no file data) + if(!mSendData) + { + mStatus = Status_Finished; + } + else + { + // Reset the buffer so it can be used for the next phase + mData.Reset(); + + // Get buffer ready for index? + if(mStatus == Status_Header) + { + // Just finished doing the stream header, create the block index header + file_BlockIndexHeader blkhdr; + blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); + ASSERT(mpRecipe != 0); + blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID()); + blkhdr.mNumBlocks = box_hton64(mTotalBlocks); + + // Generate the IV base + Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase)); + blkhdr.mEntryIVBase = box_hton64(mEntryIVBase); + + mData.Write(&blkhdr, sizeof(blkhdr)); + } + + ++mStatus; + } + } + } + else if(mStatus == Status_Blocks) + { + // Block sending phase + + if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize) + { + // Next block! + ++mCurrentBlock; + ++mAbsoluteBlockNumber; + if(mCurrentBlock >= mNumBlocks) + { + // Output extra blocks for this instruction and move forward in file + if(mInstructionNumber >= 0) + { + SkipPreviousBlocksInInstruction(); + } + + // Is there another instruction to go? + ++mInstructionNumber; + + // Skip instructions which don't contain any data + while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size()) + && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0) + { + SkipPreviousBlocksInInstruction(); + ++mInstructionNumber; + } + + if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size())) + { + // End of blocks, go to next phase + ++mStatus; + + // Set the data to reading so the index can be written + mData.SetForReading(); + } + else + { + // Get ready for this instruction + SetForInstruction(); + } + } + + // Can't use 'else' here as SetForInstruction() will change this + if(mCurrentBlock < mNumBlocks) + { + EncodeCurrentBlock(); + } + } + + // Send data from the current block (if there's data to send) + if(mPositionInCurrentBlock < mCurrentBlockEncodedSize) + { + // How much data to put in the buffer? + int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock; + if(s > bytesToRead) s = bytesToRead; + + // Copy it in + ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s); + + // Update variables + bytesToRead -= s; + buffer += s; + mPositionInCurrentBlock += s; + } + } + else + { + // Should never get here, as it'd be an invalid status + ASSERT(false); + } + } + + // Add encoded size to stats + BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead); + + // Return size of data to caller + return NBytes - bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction() +// Purpose: Private. Stores the blocks of the old file referenced in the current +// instruction into the index and skips over the data in the file +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction() +{ + // Check something is necessary + if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0) + { + return; + } + + // Index of the first block in old file (being diffed from) + int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock); + + int64_t sizeToSkip = 0; + + for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b) + { + // Update stats + BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; + + // Store the entry + StoreBlockIndexEntry(0 - (firstIndex + b), + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize, + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum, + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); + + // Increment the absolute block number -- kept encryption IV in sync + ++mAbsoluteBlockNumber; + + // Add the size of this block to the size to skip + sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; + } + + // Move forward in the stream + mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::SetForInstruction() +// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::SetForInstruction() +{ + // Calculate block sizes + CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize); + + // Set variables + mCurrentBlock = 0; + mCurrentBlockEncodedSize = 0; + mPositionInCurrentBlock = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock() +// Purpose: Private. Encodes the current block, and writes the block data to the index +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::EncodeCurrentBlock() +{ + // How big is the block, raw? + int blockRawSize = mBlockSize; + if(mCurrentBlock == (mNumBlocks - 1)) + { + blockRawSize = mLastBlockSize; + } + ASSERT(blockRawSize < mAllocatedBufferSize); + + // Check file open + if(mpLogging == 0) + { + // File should be open, but isn't. So logical error. + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Read the data in + if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, + 0 /* not interested in size if failure */)) + { + // TODO: Do something more intelligent, and abort + // this upload because the file has changed. + THROW_EXCEPTION(BackupStoreException, + Temp_FileEncodeStreamDidntReadBuffer) + } + + // Encode it + mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, + blockRawSize, mEncodedBuffer); + + //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); + + // Create block listing data -- generate checksums + RollingChecksum weakChecksum(mpRawBuffer, blockRawSize); + MD5Digest strongChecksum; + strongChecksum.Add(mpRawBuffer, blockRawSize); + strongChecksum.Finish(); + + // Add entry to the index + StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, + weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); + + // Set vars to reading this block + mPositionInCurrentBlock = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *) +// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream. +// Created: 16/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum) +{ + // First, the encrypted section + file_BlockIndexEntryEnc entryEnc; + entryEnc.mSize = htonl(ClearSize); + entryEnc.mWeakChecksum = htonl(WeakChecksum); + ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum)); + + // Then the clear section + file_BlockIndexEntry entry; + entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex)); + + // Then encrypt the encryted section + // Generate the IV from the block number + if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase)) + { + THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements) + } + uint64_t iv = mEntryIVBase; + iv += mAbsoluteBlockNumber; + // Convert to network byte order before encrypting with it, so that restores work on + // platforms with different endiannesses. + iv = box_hton64(iv); + sBlowfishEncryptBlockEntry.SetIV(&iv); + + // Encode the data + int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc)); + if(encodedSize != sizeof(entry.mEnEnc)) + { + THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) + } + + // Save to data block for sending at the end of the stream + mData.Write(&entry, sizeof(entry)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Write(const void *, int) +// Purpose: As interface. Exceptions. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StreamDataLeft() +// Purpose: As interface -- end of stream reached? +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFileEncodeStream::StreamDataLeft() +{ + return (mStatus != Status_Finished); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::StreamClosed() +// Purpose: As interface +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFileEncodeStream::StreamClosed() +{ + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t) +// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, + int64_t NumBlocksInIndex, int64_t OtherFileID) + : mpBlockIndex(pBlockIndex), + mNumBlocksInIndex(NumBlocksInIndex), + mOtherFileID(OtherFileID) +{ + ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFileEncodeStream::Recipe::~Recipe() +// Purpose: Destructor +// Created: 15/1/04 +// +// -------------------------------------------------------------------------- +BackupStoreFileEncodeStream::Recipe::~Recipe() +{ + // Free the block index, if there is one + if(mpBlockIndex != 0) + { + ::free(mpBlockIndex); + } +} + + + + diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h new file mode 100644 index 00000000..c5fa780a --- /dev/null +++ b/lib/backupclient/BackupStoreFileEncodeStream.h @@ -0,0 +1,135 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileEncodeStream.h +// Purpose: Implement stream-based file encoding for the backup store +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILEENCODESTREAM__H +#define BACKUPSTOREFILEENCODESTREAM__H + +#include <vector> + +#include "IOStream.h" +#include "BackupStoreFilename.h" +#include "CollectInBufferStream.h" +#include "MD5Digest.h" +#include "BackupStoreFile.h" +#include "ReadLoggingStream.h" +#include "RunStatusProvider.h" + +namespace BackupStoreFileCreation +{ + // Diffing and creation of files share some implementation details. + typedef struct _BlocksAvailableEntry + { + struct _BlocksAvailableEntry *mpNextInHashList; + int32_t mSize; // size in clear + uint32_t mWeakChecksum; // weak, rolling checksum + uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum + } BlocksAvailableEntry; + +} + + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFileEncodeStream +// Purpose: Encode a file into a stream +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +class BackupStoreFileEncodeStream : public IOStream +{ +public: + BackupStoreFileEncodeStream(); + ~BackupStoreFileEncodeStream(); + + typedef struct + { + int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go + int32_t mBlocks; // number of block to reuse, starting at this one + BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null + } RecipeInstruction; + + class Recipe : public std::vector<RecipeInstruction> + { + // NOTE: This class is rather tied in with the implementation of diffing. + public: + Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex, + int64_t OtherFileID = 0); + ~Recipe(); + + int64_t GetOtherFileID() {return mOtherFileID;} + int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock) + { + return pBlock - mpBlockIndex; + } + + private: + BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex; + int64_t mNumBlocksInIndex; + int64_t mOtherFileID; + }; + + void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL); + + virtual int Read(void *pBuffer, int NBytes, int Timeout); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + enum + { + Status_Header = 0, + Status_Blocks = 1, + Status_BlockListing = 2, + Status_Finished = 3 + }; + +private: + void EncodeCurrentBlock(); + void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); + void SkipPreviousBlocksInInstruction(); + void SetForInstruction(); + void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum); + +private: + Recipe *mpRecipe; + IOStream *mpFile; // source file + CollectInBufferStream mData; // buffer for header and index entries + IOStream *mpLogging; + RunStatusProvider* mpRunStatusProvider; + int mStatus; + bool mSendData; // true if there's file data to send (ie not a symlink) + int64_t mTotalBlocks; // Total number of blocks in the file + int64_t mAbsoluteBlockNumber; // The absolute block number currently being output + // Instruction number + int64_t mInstructionNumber; + // All the below are within the current instruction + int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases + int64_t mCurrentBlock; + int32_t mCurrentBlockEncodedSize; + int32_t mPositionInCurrentBlock; // for reading out + int32_t mBlockSize; // Basic block size of most of the blocks in the file + int32_t mLastBlockSize; // the size (unencoded) of the last block in the file + // Buffers + uint8_t *mpRawBuffer; // buffer for raw data + BackupStoreFile::EncodingBuffer mEncodedBuffer; + // buffer for encoded data + int32_t mAllocatedBufferSize; // size of above two allocated blocks + uint64_t mEntryIVBase; // base for block entry IV +}; + + + +#endif // BACKUPSTOREFILEENCODESTREAM__H + diff --git a/lib/backupclient/BackupStoreFileRevDiff.cpp b/lib/backupclient/BackupStoreFileRevDiff.cpp new file mode 100644 index 00000000..509eef61 --- /dev/null +++ b/lib/backupclient/BackupStoreFileRevDiff.cpp @@ -0,0 +1,258 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileRevDiff.cpp +// Purpose: Reverse a patch, to build a new patch from new to old files +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <stdlib.h> + +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreException.h" +#include "BackupStoreConstants.h" +#include "BackupStoreFilename.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t) +// Purpose: Reverse a patch, to build a new patch from new to old files. Takes +// two independent copies to the From file, for efficiency. +// Created: 12/7/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent) +{ + // Read and copy the header from the from file to the out file -- beginnings of the patch + file_StreamFormat hdr; + if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + // Copy + rOut.Write(&hdr, sizeof(hdr)); + // Copy over filename and attributes + // BLOCK + { + BackupStoreFilename filename; + filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + filename.WriteToStream(rOut); + StreamableMemBlock attr; + attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite); + attr.WriteToStream(rOut); + } + + // Build an index of common blocks. + // For each block in the from file, we want to know it's index in the + // diff file. Allocate memory for this information. + int64_t fromNumBlocks = box_ntoh64(hdr.mNumBlocks); + int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t)); + if(pfromIndexInfo == 0) + { + throw std::bad_alloc(); + } + + // Buffer data + void *buffer = 0; + int bufferSize = 0; + + // flag + bool isCompletelyDifferent = true; + + try + { + // Initialise the index to be all 0, ie not filled in yet + for(int64_t i = 0; i < fromNumBlocks; ++i) + { + pfromIndexInfo[i] = 0; + } + + // Within the from file, skip to the index + MoveStreamPositionToBlockIndex(rDiff); + + // Read in header of index + file_BlockIndexHeader diffIdxHdr; + if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // And then read in each entry + int64_t diffNumBlocks = box_ntoh64(diffIdxHdr.mNumBlocks); + for(int64_t b = 0; b < diffNumBlocks; ++b) + { + file_BlockIndexEntry e; + if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Where's the block? + int64_t blockEn = box_ntoh64(e.mEncodedSize); + if(blockEn > 0) + { + // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file + } + else + { + // Block is in the original file, store which block it is in this file + int64_t fromIndex = 0 - blockEn; + if(fromIndex < 0 || fromIndex >= fromNumBlocks) + { + THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles) + } + + // Store information about where it is in the new file + // NOTE: This is slight different to how it'll be stored in the final index. + pfromIndexInfo[fromIndex] = -1 - b; + } + } + + // Open the index for the second copy of the from file + MoveStreamPositionToBlockIndex(rFrom2); + + // Read in header of index + file_BlockIndexHeader fromIdxHdr; + if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 + || box_ntoh64(fromIdxHdr.mOtherFileID) != 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // So, we can now start building the data in the file + int64_t filePosition = rFrom.GetPosition(); + for(int64_t b = 0; b < fromNumBlocks; ++b) + { + // Read entry from from index + file_BlockIndexEntry e; + if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Get size + int64_t blockSize = box_hton64(e.mEncodedSize); + if(blockSize < 0) + { + THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) + } + + // Copy this block? + if(pfromIndexInfo[b] == 0) + { + // Copy it, first move to file location + rFrom.Seek(filePosition, IOStream::SeekType_Absolute); + + // Make sure there's memory available to copy this + if(bufferSize < blockSize || buffer == 0) + { + // Free old block + if(buffer != 0) + { + ::free(buffer); + buffer = 0; + bufferSize = 0; + } + // Allocate new block + buffer = ::malloc(blockSize); + if(buffer == 0) + { + throw std::bad_alloc(); + } + bufferSize = blockSize; + } + ASSERT(bufferSize >= blockSize); + + // Copy the block + if(!rFrom.ReadFullBuffer(buffer, blockSize, 0)) + { + THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) + } + rOut.Write(buffer, blockSize); + + // Store the size + pfromIndexInfo[b] = blockSize; + } + else + { + // Block isn't needed, so it's not completely different + isCompletelyDifferent = false; + } + filePosition += blockSize; + } + + // Then write the index, modified header first + fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(box_hton64(ObjectIDOfFrom)); + rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr)); + + // Move to start of index entries + rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute); + + // Then copy modified entries + for(int64_t b = 0; b < fromNumBlocks; ++b) + { + // Read entry from from index + file_BlockIndexEntry e; + if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // Modify... + int64_t s = pfromIndexInfo[b]; + // Adjust to reflect real block index (remember 0 has a different meaning here) + if(s < 0) ++s; + // Insert + e.mEncodedSize = box_hton64(s); + // Write + rOut.Write(&e, sizeof(e)); + } + } + catch(...) + { + ::free(pfromIndexInfo); + if(buffer != 0) + { + ::free(buffer); + } + throw; + } + + // Free memory used (oh for finally {} blocks) + ::free(pfromIndexInfo); + if(buffer != 0) + { + ::free(buffer); + } + + // return completely different flag + if(pIsCompletelyDifferent != 0) + { + *pIsCompletelyDifferent = isCompletelyDifferent; + } +} + + + diff --git a/lib/backupclient/BackupStoreFileWire.h b/lib/backupclient/BackupStoreFileWire.h new file mode 100644 index 00000000..49e94aa5 --- /dev/null +++ b/lib/backupclient/BackupStoreFileWire.h @@ -0,0 +1,74 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFileWire.h +// Purpose: On the wire / disc formats for backup store files +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILEWIRE__H +#define BACKUPSTOREFILEWIRE__H + +#include "MD5Digest.h" + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + int32_t mMagicValue; // also the version number + int64_t mNumBlocks; // number of blocks contained in the file + int64_t mContainerID; + int64_t mModificationTime; + int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block + int32_t mOptions; // bitmask of options used + // Then a BackupStoreFilename + // Then a BackupClientFileAttributes +} file_StreamFormat; + +typedef struct +{ + int32_t mMagicValue; // different magic value + int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index + uint64_t mEntryIVBase; // base value for block IV + int64_t mNumBlocks; // repeat of value in file header +} file_BlockIndexHeader; + +typedef struct +{ + int32_t mSize; // size in clear + uint32_t mWeakChecksum; // weak, rolling checksum + uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum +} file_BlockIndexEntryEnc; + +typedef struct +{ + union + { + int64_t mEncodedSize; // size encoded, if > 0 + int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0 + }; + uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section +} file_BlockIndexEntry; + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +// header for blocks of compressed data in files +#define HEADER_CHUNK_IS_COMPRESSED 1 // bit +#define HEADER_ENCODING_SHIFT 1 // shift value +#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7 +#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7 + + +#endif // BACKUPSTOREFILEWIRE__H + diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp new file mode 100644 index 00000000..72cd1acd --- /dev/null +++ b/lib/backupclient/BackupStoreFilename.cpp @@ -0,0 +1,281 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilename.cpp +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreFilename.h" +#include "Protocol.h" +#include "BackupStoreException.h" +#include "IOStream.h" +#include "Guards.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::BackupStoreFilename() +// Purpose: Default constructor -- creates an invalid filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::BackupStoreFilename() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy) + : mEncryptedName(rToCopy.mEncryptedName) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::~BackupStoreFilename() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilename::~BackupStoreFilename() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::CheckValid(bool) +// Purpose: Checks the encoded filename for validity +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const +{ + bool ok = true; + + if(mEncryptedName.size() < 2) + { + // Isn't long enough to have a header + ok = false; + } + else + { + // Check size is consistent + unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(this->mEncryptedName); + if(dsize != mEncryptedName.size()) + { + ok = false; + } + + // And encoding is an accepted value + unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName); + if(encoding < Encoding_Min || encoding > Encoding_Max) + { + ok = false; + } + } + + // Exception? + if(!ok && ExceptionIfInvalid) + { + THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename) + } + + return ok; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::ReadFromProtocol(Protocol &) +// Purpose: Reads the filename from the protocol object +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol) +{ + // Read the header + char hdr[2]; + rProtocol.Read(hdr, 2); + + // How big is it? + int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); + + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + std::string data; + rProtocol.Read(data, dsize - 2); + + // assign to this string, storing the header and the extra data + mEncryptedName.assign(hdr, 2); + mEncryptedName.append(data.c_str(), data.size()); + + // Check it + CheckValid(); + + // Alert derived classes + EncodedFilenameChanged(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::WriteToProtocol(Protocol &) +// Purpose: Writes the filename to the protocol object +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const +{ + CheckValid(); + + rProtocol.Write(mEncryptedName.c_str(), mEncryptedName.size()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::ReadFromStream(IOStream &) +// Purpose: Reads the filename from a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Read the header + char hdr[2]; + if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + + // How big is it? + unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); + + // Assume most filenames are small + char buf[256]; + if(dsize < sizeof(buf)) + { + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + // Copy in header + buf[0] = hdr[0]; buf[1] = hdr[1]; + + // assign to this string, storing the header and the extra data + mEncryptedName.assign(buf, dsize); + } + else + { + // Block of memory to hold it + MemoryBlockGuard<char*> dataB(dsize+2); + char *data = dataB; + + // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us + if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) + { + THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) + } + // Copy in header + data[0] = hdr[0]; data[1] = hdr[1]; + + // assign to this string, storing the header and the extra data + mEncryptedName.assign(data, dsize); + } + + // Check it + CheckValid(); + + // Alert derived classes + EncodedFilenameChanged(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::WriteToStream(IOStream &) +// Purpose: Writes the filename to a stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::WriteToStream(IOStream &rStream) const +{ + CheckValid(); + + rStream.Write(mEncryptedName.c_str(), mEncryptedName.size()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::EncodedFilenameChanged() +// Purpose: The encoded filename stored has changed +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::EncodedFilenameChanged() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::IsEncrypted() +// Purpose: Returns true if the filename is stored using an encrypting encoding +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreFilename::IsEncrypted() const +{ + return BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName) != + Encoding_Clear; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilename::SetAsClearFilename(const char *) +// Purpose: Sets this object to be a valid filename, but with a +// filename in the clear. Used on the server to create +// filenames when there's no way of encrypting it. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFilename::SetAsClearFilename(const char *Clear) +{ + // Make std::string from the clear name + std::string toEncode(Clear); + + // Make an encoded string + char hdr[2]; + BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear); + std::string encoded(hdr, 2); + encoded += toEncode; + ASSERT(encoded.size() == toEncode.size() + 2); + + // Store the encoded string + mEncryptedName.assign(encoded); + + // Stuff which must be done + EncodedFilenameChanged(); + CheckValid(false); +} + + + diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h new file mode 100644 index 00000000..80db9516 --- /dev/null +++ b/lib/backupclient/BackupStoreFilename.h @@ -0,0 +1,107 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilename.h +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILENAME__H +#define BACKUPSTOREFILENAME__H + +#include <string> + +class Protocol; +class IOStream; + +// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE +// don't define this -- the problem of memory usage still appears without this. +// It's just that this class really showed up the problem. Instead, malloc allocation +// is globally defined in BoxPlatform.h, for troublesome libraries. + +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE + // Use a malloc_allocated string, because the STL default allocators really screw up with + // memory allocation, particularly with this class. + // Makes a few things a bit messy and inefficient with conversions. + // Given up using this, and use global malloc allocation instead, but thought it + // worth leaving this code in just in case it's useful for the future. + typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base; + // If this is changed, change GetClearFilename() back to returning a reference. +#else + typedef std::string BackupStoreFilename_base; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFilename +// Purpose: Filename for the backup store +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreFilename /* : public BackupStoreFilename_base */ +{ +private: + std::string mEncryptedName; + +public: + BackupStoreFilename(); + BackupStoreFilename(const BackupStoreFilename &rToCopy); + virtual ~BackupStoreFilename(); + + bool CheckValid(bool ExceptionIfInvalid = true) const; + + void ReadFromProtocol(Protocol &rProtocol); + void WriteToProtocol(Protocol &rProtocol) const; + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream) const; + + void SetAsClearFilename(const char *Clear); + + // Check that it's encrypted + bool IsEncrypted() const; + + // These enumerated types belong in the base class so + // the CheckValid() function can make sure that the encoding + // is a valid encoding + enum + { + Encoding_Min = 1, + Encoding_Clear = 1, + Encoding_Blowfish = 2, + Encoding_Max = 2 + }; + + const std::string& GetEncodedFilename() const + { + return mEncryptedName; + } + + bool operator==(const BackupStoreFilename& rOther) const + { + return mEncryptedName == rOther.mEncryptedName; + } + + bool operator!=(const BackupStoreFilename& rOther) const + { + return mEncryptedName != rOther.mEncryptedName; + } + +protected: + virtual void EncodedFilenameChanged(); + void SetEncodedFilename(const std::string &rEncoded) + { + mEncryptedName = rEncoded; + } +}; + +// On the wire utilities for class and derived class +#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2) +#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3) + +#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;} + +#endif // BACKUPSTOREFILENAME__H + diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp new file mode 100644 index 00000000..e529d8d3 --- /dev/null +++ b/lib/backupclient/BackupStoreFilenameClear.cpp @@ -0,0 +1,335 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilenameClear.cpp +// Purpose: BackupStoreFilenames in the clear +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreFilenameClear.h" +#include "BackupStoreException.h" +#include "CipherContext.h" +#include "CipherBlowfish.h" +#include "Guards.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +// Hide private variables from the rest of the world +namespace +{ + int sEncodeMethod = BackupStoreFilename::Encoding_Clear; + CipherContext sBlowfishEncrypt; + CipherContext sBlowfishDecrypt; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear() +// Purpose: Default constructor, creates an invalid filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &) +// Purpose: Creates a filename, encoding from the given string +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode) +{ + SetClearFilename(rToEncode); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &) +// Purpose: Copy constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy) + : BackupStoreFilename(rToCopy), + mClearFilename(rToCopy.mClearFilename) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) +// Purpose: Copy from base class +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) + : BackupStoreFilename(rToCopy) +{ + // Will get a clear filename when it's required +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +BackupStoreFilenameClear::~BackupStoreFilenameClear() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::GetClearFilename() +// Purpose: Get the unencoded filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE +const std::string BackupStoreFilenameClear::GetClearFilename() const +{ + MakeClearAvailable(); + // When modifying, remember to change back to reference return if at all possible + // -- returns an object rather than a reference to allow easy use with other code. + return std::string(mClearFilename.c_str(), mClearFilename.size()); +} +#else +const std::string &BackupStoreFilenameClear::GetClearFilename() const +{ + MakeClearAvailable(); + return mClearFilename; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &) +// Purpose: Encode and make available the clear filename +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode) +{ + // Only allow Blowfish encodings + if(sEncodeMethod != Encoding_Blowfish) + { + THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup) + } + + // Make an encoded string with blowfish encryption + EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish); + + // Store the clear filename + mClearFilename.assign(rToEncode.c_str(), rToEncode.size()); + + // Make sure we did the right thing + if(!CheckValid(false)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::MakeClearAvailable() +// Purpose: Private. Make sure the clear filename is available +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::MakeClearAvailable() const +{ + if(!mClearFilename.empty()) + return; // nothing to do + + // Check valid + CheckValid(); + + // Decode the header + int size = BACKUPSTOREFILENAME_GET_SIZE(GetEncodedFilename()); + int encoding = BACKUPSTOREFILENAME_GET_ENCODING(GetEncodedFilename()); + + // Decode based on encoding given in the header + switch(encoding) + { + case Encoding_Clear: + BOX_TRACE("**** BackupStoreFilename encoded with " + "Clear encoding ****"); + mClearFilename.assign(GetEncodedFilename().c_str() + 2, + size - 2); + break; + + case Encoding_Blowfish: + DecryptEncoded(sBlowfishDecrypt); + break; + + default: + THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding) + break; + } +} + + +// Buffer for encoding and decoding -- do this all in one single buffer to +// avoid lots of string allocation, which stuffs up memory usage. +// These static memory vars are, of course, not thread safe, but we don't use threads. +static int sEncDecBufferSize = 0; +static MemoryBlockGuard<uint8_t *> *spEncDecBuffer = 0; + +static void EnsureEncDecBufferSize(int BufSize) +{ + if(spEncDecBuffer == 0) + { +#ifndef WIN32 + BOX_TRACE("Allocating filename encoding/decoding buffer " + "with size " << BufSize); +#endif + spEncDecBuffer = new MemoryBlockGuard<uint8_t *>(BufSize); + MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer); + MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer); + sEncDecBufferSize = BufSize; + } + else + { + if(sEncDecBufferSize < BufSize) + { + BOX_TRACE("Reallocating filename encoding/decoding " + "buffer from " << sEncDecBufferSize << + " to " << BufSize); + spEncDecBuffer->Resize(BufSize); + sEncDecBufferSize = BufSize; + MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int) +// Purpose: Private. Assigns the encoded filename string, encrypting. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding) +{ + // Work out max size + int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4; + + // Make sure encode/decode buffer has enough space + EnsureEncDecBufferSize(maxOutSize); + + // Pointer to buffer + uint8_t *buffer = *spEncDecBuffer; + + // Encode -- do entire block in one go + int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size()); + // and add in header size + encSize += 2; + + // Adjust header + BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding); + + // Store the encoded string + SetEncodedFilename(std::string((char*)buffer, encSize)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &) +// Purpose: Decrypt the encoded filename using the cipher context +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const +{ + const std::string& rEncoded = GetEncodedFilename(); + + // Work out max size + int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rEncoded.size()) + 4; + + // Make sure encode/decode buffer has enough space + EnsureEncDecBufferSize(maxOutSize); + + // Pointer to buffer + uint8_t *buffer = *spEncDecBuffer; + + // Decrypt + const char *str = rEncoded.c_str() + 2; + int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, rEncoded.size() - 2); + + // Assign to this + mClearFilename.assign((char*)buffer, sizeOut); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::EncodedFilenameChanged() +// Purpose: The encoded filename stored has changed +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::EncodedFilenameChanged() +{ + BackupStoreFilename::EncodedFilenameChanged(); + + // Delete stored filename in clear + mClearFilename.erase(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int) +// Purpose: Set the key used for Blowfish encryption of filenames +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength) +{ + // Initialisation vector not used. Can't use a different vector for each filename as + // that would stop comparisions on the server working. + sBlowfishEncrypt.Reset(); + sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength); + sBlowfishEncrypt.SetIV(pIV); + sBlowfishDecrypt.Reset(); + sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); + ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength); + sBlowfishDecrypt.SetIV(pIV); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFilenameClear::SetEncodingMethod(int) +// Purpose: Set the encoding method used for filenames +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreFilenameClear::SetEncodingMethod(int Method) +{ + sEncodeMethod = Method; +} + + + diff --git a/lib/backupclient/BackupStoreFilenameClear.h b/lib/backupclient/BackupStoreFilenameClear.h new file mode 100644 index 00000000..d4c45701 --- /dev/null +++ b/lib/backupclient/BackupStoreFilenameClear.h @@ -0,0 +1,60 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreFilenameClear.h +// Purpose: BackupStoreFilenames in the clear +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREFILENAMECLEAR__H +#define BACKUPSTOREFILENAMECLEAR__H + +#include "BackupStoreFilename.h" + +class CipherContext; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreFilenameClear +// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class BackupStoreFilenameClear : public BackupStoreFilename +{ +public: + BackupStoreFilenameClear(); + BackupStoreFilenameClear(const std::string &rToEncode); + BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy); + BackupStoreFilenameClear(const BackupStoreFilename &rToCopy); + virtual ~BackupStoreFilenameClear(); + + // Because we need to use a different allocator for this class to avoid + // nasty things happening, can't return this as a reference. Which is a + // pity. But probably not too bad. +#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE + const std::string GetClearFilename() const; +#else + const std::string &GetClearFilename() const; +#endif + void SetClearFilename(const std::string &rToEncode); + + // Setup for encryption of filenames + static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength); + static void SetEncodingMethod(int Method); + +protected: + void MakeClearAvailable() const; + virtual void EncodedFilenameChanged(); + void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding); + void DecryptEncoded(CipherContext &rCipherContext) const; + +private: + mutable BackupStoreFilename_base mClearFilename; +}; + +#endif // BACKUPSTOREFILENAMECLEAR__H + + diff --git a/lib/backupclient/BackupStoreObjectDump.cpp b/lib/backupclient/BackupStoreObjectDump.cpp new file mode 100644 index 00000000..654317c1 --- /dev/null +++ b/lib/backupclient/BackupStoreObjectDump.cpp @@ -0,0 +1,227 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreObjectDump.cpp +// Purpose: Implementations of dumping objects to stdout/TRACE +// Created: 3/5/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <stdarg.h> +#include <map> + +#include "BackupStoreDirectory.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreFilename.h" +#include "BackupClientFileAttributes.h" +#include "BackupStoreObjectMagic.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void OutputLine(FILE *, bool, const char *, ...) +// Purpose: Output a line for the object dumping, to file and/or trace... +// Created: 3/5/04 +// +// -------------------------------------------------------------------------- +static void OutputLine(FILE *file, bool ToTrace, const char *format, ...) +{ + char text[512]; + int r = 0; + va_list ap; + va_start(ap, format); + r = vsnprintf(text, sizeof(text), format, ap); + va_end(ap); + + if(file != 0) + { + ::fprintf(file, "%s", text); + } + if(ToTrace) + { + BOX_TRACE(text); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) +// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere) +// Dump the contents to a file, or trace. +// Created: 3/5/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) +{ + FILE *file = (FILE*)clibFileHandle; + + OutputLine(file, ToTrace, "Directory object.\nObject ID: %llx\nContainer ID: %llx\nNumber entries: %d\n"\ + "Attributes mod time: %llx\nAttributes size: %d\n", mObjectID, mContainerID, mEntries.size(), + mAttributesModTime, mAttributes.GetSize()); + + // So repeated filenames can be illustrated, even though they can't be decoded + std::map<std::string, int> nameNum; + int nameNumI = 0; + + // Dump items + OutputLine(file, ToTrace, "Items:\nID Size AttrHash AtSz NSz NIdx Flags\n"); + for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i) + { + // Choose file name index number for this file + std::map<std::string, int>::iterator nn(nameNum.find((*i)->GetName().GetEncodedFilename())); + int ni = nameNumI; + if(nn != nameNum.end()) + { + ni = nn->second; + } + else + { + nameNum[(*i)->GetName().GetEncodedFilename()] = nameNumI; + ++nameNumI; + } + + // Do dependencies + char depends[128]; + depends[0] = '\0'; + int depends_l = 0; + if((*i)->GetDependsNewer() != 0) + { +#ifdef _MSC_VER + depends_l += ::sprintf(depends + depends_l, " depNew(%I64x)", (*i)->GetDependsNewer()); +#else + depends_l += ::sprintf(depends + depends_l, " depNew(%llx)", (long long)((*i)->GetDependsNewer())); +#endif + } + if((*i)->GetDependsOlder() != 0) + { +#ifdef _MSC_VER + depends_l += ::sprintf(depends + depends_l, " depOld(%I64x)", (*i)->GetDependsOlder()); +#else + depends_l += ::sprintf(depends + depends_l, " depOld(%llx)", (long long)((*i)->GetDependsOlder())); +#endif + } + + // Output item + int16_t f = (*i)->GetFlags(); +#ifdef WIN32 + OutputLine(file, ToTrace, + "%06I64x %4I64d %016I64x %4d %3d %4d%s%s%s%s%s%s\n", +#else + OutputLine(file, ToTrace, + "%06llx %4lld %016llx %4d %3d %4d%s%s%s%s%s%s\n", +#endif + (*i)->GetObjectID(), + (*i)->GetSizeInBlocks(), + (*i)->GetAttributesHash(), + (*i)->GetAttributes().GetSize(), + (*i)->GetName().GetEncodedFilename().size(), + ni, + ((f & BackupStoreDirectory::Entry::Flags_File)?" file":""), + ((f & BackupStoreDirectory::Entry::Flags_Dir)?" dir":""), + ((f & BackupStoreDirectory::Entry::Flags_Deleted)?" del":""), + ((f & BackupStoreDirectory::Entry::Flags_OldVersion)?" old":""), + ((f & BackupStoreDirectory::Entry::Flags_RemoveASAP)?" removeASAP":""), + depends); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::DumpFile(void *, bool, IOStream &) +// Purpose: (first arg is FILE *, but avoid including stdio.h everywhere) +// Dump the contents to a file, or trace. +// Created: 4/5/04 +// +// -------------------------------------------------------------------------- +void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile) +{ + FILE *file = (FILE*)clibFileHandle; + + // Read header + file_StreamFormat hdr; + if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), + 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite)) + { + // Couldn't read header + THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) + } + + // Check and output header info + if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1) + && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V0)) + { + OutputLine(file, ToTrace, "File header doesn't have the correct magic, aborting dump\n"); + return; + } + + OutputLine(file, ToTrace, "File object.\nContainer ID: %llx\nModification time: %llx\n"\ + "Max block clear size: %d\nOptions: %08x\nNum blocks: %d\n", box_ntoh64(hdr.mContainerID), + box_ntoh64(hdr.mModificationTime), ntohl(hdr.mMaxBlockClearSize), ntohl(hdr.mOptions), + box_ntoh64(hdr.mNumBlocks)); + + // Read the next two objects + BackupStoreFilename fn; + fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); + OutputLine(file, ToTrace, "Filename size: %d\n", + fn.GetEncodedFilename().size()); + + BackupClientFileAttributes attr; + attr.ReadFromStream(rFile, IOStream::TimeOutInfinite); + OutputLine(file, ToTrace, "Attributes size: %d\n", attr.GetSize()); + + // Dump the blocks + rFile.Seek(0, IOStream::SeekType_Absolute); + BackupStoreFile::MoveStreamPositionToBlockIndex(rFile); + + // Read in header + file_BlockIndexHeader bhdr; + rFile.ReadFullBuffer(&bhdr, sizeof(bhdr), 0); + if(bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) + && bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)) + { + OutputLine(file, ToTrace, "WARNING: Block header doesn't have the correct magic\n"); + } + // number of blocks + int64_t nblocks = box_ntoh64(bhdr.mNumBlocks); + OutputLine(file, ToTrace, "Other file ID (for block refs): %llx\nNum blocks (in blk hdr): %lld\n", + box_ntoh64(bhdr.mOtherFileID), nblocks); + + // Dump info about each block + OutputLine(file, ToTrace, "======== ===== ==========\n Index Where EncSz/Idx\n"); + int64_t nnew = 0, nold = 0; + for(int64_t b = 0; b < nblocks; ++b) + { + file_BlockIndexEntry en; + if(!rFile.ReadFullBuffer(&en, sizeof(en), 0)) + { + OutputLine(file, ToTrace, "Didn't manage to read block %lld from file\n", b); + continue; + } + int64_t s = box_ntoh64(en.mEncodedSize); + if(s > 0) + { + nnew++; + BOX_TRACE(std::setw(8) << b << " this s=" << + std::setw(8) << s); + } + else + { + nold++; + BOX_TRACE(std::setw(8) << b << " other i=" << + std::setw(8) << 0 - s); + } + } + BOX_TRACE("======== ===== =========="); +} + diff --git a/lib/backupclient/BackupStoreObjectMagic.h b/lib/backupclient/BackupStoreObjectMagic.h new file mode 100644 index 00000000..7ee600a2 --- /dev/null +++ b/lib/backupclient/BackupStoreObjectMagic.h @@ -0,0 +1,31 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreObjectMagic.h +// Purpose: Magic values for the start of objects in the backup store +// Created: 19/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREOBJECTMAGIC__H +#define BACKUPSTOREOBJECTMAGIC__H + +// Each of these values is the first 4 bytes of the object file. +// Remember to swap from network to host byte order. + +// Magic value for file streams +#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65 +// Do not use v0 in any new code! +#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45 + +// Magic for the block index at the file stream -- used to +// ensure streams are reordered as expected +#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478 +// Do not use v0 in any new code! +#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B + +// Magic value for directory streams +#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F + +#endif // BACKUPSTOREOBJECTMAGIC__H + diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra new file mode 100644 index 00000000..df3319df --- /dev/null +++ b/lib/backupclient/Makefile.extra @@ -0,0 +1,16 @@ + +MAKEPROTOCOL = ../../lib/server/makeprotocol.pl + +GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt + +# AUTOGEN SEEDING +autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt + $(_PERL) $(GEN_CMD_SRV) + + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt + $(_PERL) $(MAKEEXCEPTION) BackupStoreException.txt + diff --git a/lib/backupclient/RunStatusProvider.h b/lib/backupclient/RunStatusProvider.h new file mode 100644 index 00000000..89f361ca --- /dev/null +++ b/lib/backupclient/RunStatusProvider.h @@ -0,0 +1,29 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RunStatusProvider.h +// Purpose: Declares the RunStatusProvider interface. +// Created: 2008/08/14 +// +// -------------------------------------------------------------------------- + +#ifndef RUNSTATUSPROVIDER__H +#define RUNSTATUSPROVIDER__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: RunStatusProvider +// Purpose: Provides a StopRun() method which returns true if +// the current backup should be halted. +// Created: 2005/11/15 +// +// -------------------------------------------------------------------------- +class RunStatusProvider +{ + public: + virtual ~RunStatusProvider() { } + virtual bool StopRun() = 0; +}; + +#endif // RUNSTATUSPROVIDER__H diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp new file mode 100644 index 00000000..72a813d5 --- /dev/null +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -0,0 +1,373 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccountDatabase.cpp +// Purpose: Database of accounts for the backup store +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> +#include <string> +#include <map> +#include <stdio.h> +#include <sys/stat.h> + +#include "BackupStoreAccountDatabase.h" +#include "Guards.h" +#include "FdGetLine.h" +#include "BackupStoreException.h" +#include "CommonException.h" +#include "FileModificationTime.h" + +#include "MemLeakFindOn.h" + +class _BackupStoreAccountDatabase +{ +public: + std::string mFilename; + std::map<int32_t, BackupStoreAccountDatabase::Entry> mDatabase; + box_time_t mModificationTime; +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *) +// Purpose: Constructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *Filename) + : pImpl(new _BackupStoreAccountDatabase) +{ + pImpl->mFilename = Filename; + pImpl->mModificationTime = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::~BackupStoreAccountDatabase() +// Purpose: Destructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::~BackupStoreAccountDatabase() +{ + delete pImpl; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::Entry() +// Purpose: Default constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::Entry() + : mID(-1), + mDiscSet(-1) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::Entry(int32_t, int) +// Purpose: Constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::Entry(int32_t ID, int DiscSet) + : mID(ID), + mDiscSet(DiscSet) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::Entry(const Entry &) +// Purpose: Copy constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::Entry(const Entry &rEntry) + : mID(rEntry.mID), + mDiscSet(rEntry.mDiscSet) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::~Entry() +// Purpose: Destructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::~Entry() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Read(const char *) +// Purpose: Read in a database from disc +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreAccountDatabase> BackupStoreAccountDatabase::Read(const char *Filename) +{ + // Database object to use + std::auto_ptr<BackupStoreAccountDatabase> db(new BackupStoreAccountDatabase(Filename)); + + // Read in the file + db->ReadFile(); + + // Return to called + return db; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::ReadFile() +// Purpose: Read the file off disc +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::ReadFile() const +{ + // Open file + FileHandleGuard<> file(pImpl->mFilename.c_str()); + + // Clear existing entries + pImpl->mDatabase.clear(); + + // Read in lines + FdGetLine getLine(file); + + while(!getLine.IsEOF()) + { + // Read and split up line + std::string l(getLine.GetLine(true)); + + if(!l.empty()) + { + // Check... + int32_t id; + int discSet; + if(::sscanf(l.c_str(), "%x:%d", &id, &discSet) != 2) + { + THROW_EXCEPTION(BackupStoreException, BadAccountDatabaseFile) + } + + // Make a new entry + pImpl->mDatabase[id] = Entry(id, discSet); + } + } + + // Store the modification time of the file + pImpl->mModificationTime = GetDBFileModificationTime(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::CheckUpToDate() +// Purpose: Private. Ensure that the in memory database matches the one on disc +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::CheckUpToDate() const +{ + if(pImpl->mModificationTime != GetDBFileModificationTime()) + { + // File has changed -- load it in again + ReadFile(); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::GetDBFileModificationTime() +// Purpose: Get the current modification time of the database +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +box_time_t BackupStoreAccountDatabase::GetDBFileModificationTime() const +{ + EMU_STRUCT_STAT st; + if(EMU_STAT(pImpl->mFilename.c_str(), &st) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + return FileModificationTime(st); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Write() +// Purpose: Write the database back to disc after modifying it +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::Write() +{ + // Open file for writing + // Would use this... + // FileHandleGuard<O_WRONLY | O_TRUNC> file(pImpl->mFilename.c_str()); + // but gcc fails randomly on it on some platforms. Weird. + + int file = ::open(pImpl->mFilename.c_str(), O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + if(file == -1) + { + THROW_EXCEPTION(CommonException, OSFileOpenError) + } + + try + { + // Then write each entry + for(std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.begin()); + i != pImpl->mDatabase.end(); ++i) + { + // Write out the entry + char line[256]; // more than enough for a couple of integers in string form + int s = ::sprintf(line, "%x:%d\n", i->second.GetID(), i->second.GetDiscSet()); + if(::write(file, line, s) != s) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + + ::close(file); + } + catch(...) + { + ::close(file); + throw; + } + + // Done. +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::EntryExists(int32_t) +// Purpose: Does an entry exist in the database? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreAccountDatabase::EntryExists(int32_t ID) const +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + return pImpl->mDatabase.find(ID) != pImpl->mDatabase.end(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::GetEntry(int32_t) +// Purpose: Retrieve an entry +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry BackupStoreAccountDatabase::GetEntry( + int32_t ID) const +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.find(ID)); + if(i == pImpl->mDatabase.end()) + { + THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry) + } + + return i->second; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::AddEntry(int32_t, int) +// Purpose: Add a new entry to the database +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry BackupStoreAccountDatabase::AddEntry( + int32_t ID, int DiscSet) +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + pImpl->mDatabase[ID] = Entry(ID, DiscSet); + return pImpl->mDatabase[ID]; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::DeleteEntry(int32_t) +// Purpose: Delete an entry from the database +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::DeleteEntry(int32_t ID) +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.find(ID)); + if(i == pImpl->mDatabase.end()) + { + THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry) + } + + pImpl->mDatabase.erase(i); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t>) +// Purpose: +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t> &rIDsOut) +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + // Delete everything in the output list + rIDsOut.clear(); + + std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.begin()); + for(; i != pImpl->mDatabase.end(); ++i) + { + rIDsOut.push_back(i->first); + } +} + + + + diff --git a/lib/backupstore/BackupStoreAccountDatabase.h b/lib/backupstore/BackupStoreAccountDatabase.h new file mode 100644 index 00000000..79573242 --- /dev/null +++ b/lib/backupstore/BackupStoreAccountDatabase.h @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccountDatabase.h +// Purpose: Database of accounts for the backup store +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREACCOUNTDATABASE__H +#define BACKUPSTOREACCOUNTDATABASE__H + +#include <memory> +#include <vector> + +#include "BoxTime.h" + +class _BackupStoreAccountDatabase; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreAccountDatabase +// Purpose: Database of accounts for the backup store +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +class BackupStoreAccountDatabase +{ +public: + friend class _BackupStoreAccountDatabase; // to stop compiler warnings + ~BackupStoreAccountDatabase(); +private: + BackupStoreAccountDatabase(const char *Filename); + BackupStoreAccountDatabase(const BackupStoreAccountDatabase &); +public: + + static std::auto_ptr<BackupStoreAccountDatabase> Read(const char *Filename); + void Write(); + + class Entry + { + public: + Entry(); + Entry(int32_t ID, int DiscSet); + Entry(const Entry &rEntry); + ~Entry(); + + int32_t GetID() const {return mID;} + int GetDiscSet() const {return mDiscSet;} + + private: + int32_t mID; + int mDiscSet; + }; + + bool EntryExists(int32_t ID) const; + Entry GetEntry(int32_t ID) const; + Entry AddEntry(int32_t ID, int DiscSet); + void DeleteEntry(int32_t ID); + + // This interface should change in the future. But for now it'll do. + void GetAllAccountIDs(std::vector<int32_t> &rIDsOut); + +private: + void ReadFile() const; // const in concept only + void CheckUpToDate() const; // const in concept only + box_time_t GetDBFileModificationTime() const; + +private: + mutable _BackupStoreAccountDatabase *pImpl; +}; + +#endif // BACKUPSTOREACCOUNTDATABASE__H + diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp new file mode 100644 index 00000000..5c7e4d38 --- /dev/null +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -0,0 +1,170 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccounts.cpp +// Purpose: Account management for backup store server +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> + +#include "BoxPortsAndFiles.h" +#include "BackupStoreAccounts.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreRefCountDatabase.h" +#include "RaidFileWrite.h" +#include "BackupStoreInfo.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" +#include "UnixUser.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &) +// Purpose: Constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase) + : mrDatabase(rDatabase) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::~BackupStoreAccounts() +// Purpose: Destructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccounts::~BackupStoreAccounts() +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::Create(int32_t, int, int64_t, int64_t, const std::string &) +// Purpose: Create a new account on the specified disc set. +// If rAsUsername is not empty, then the account information will be written under the +// username specified. +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername) +{ + // Create the entry in the database + BackupStoreAccountDatabase::Entry Entry(mrDatabase.AddEntry(ID, + DiscSet)); + + { + // Become the user specified in the config file? + std::auto_ptr<UnixUser> user; + if(!rAsUsername.empty()) + { + // Username specified, change... + user.reset(new UnixUser(rAsUsername.c_str())); + user->ChangeProcessUser(true /* temporary */); + // Change will be undone at the end of this function + } + + // Get directory name + std::string dirName(MakeAccountRootDir(ID, DiscSet)); + + // Create a directory on disc + RaidFileWrite::CreateDirectory(DiscSet, dirName, true /* recursive */); + + // Create an info file + BackupStoreInfo::CreateNew(ID, dirName, DiscSet, SizeSoftLimit, SizeHardLimit); + + // And an empty directory + BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); + int64_t rootDirSize = 0; + // Write it, knowing the directory scheme + { + RaidFileWrite rf(DiscSet, dirName + "o01"); + rf.Open(); + rootDir.WriteToStream(rf); + rootDirSize = rf.GetDiscUsageInBlocks(); + rf.Commit(true); + } + + // Update the store info to reflect the size of the root directory + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, dirName, DiscSet, false /* ReadWrite */)); + info->ChangeBlocksUsed(rootDirSize); + info->ChangeBlocksInDirectories(rootDirSize); + + // Save it back + info->Save(); + + // Create the refcount database + BackupStoreRefCountDatabase::CreateNew(Entry); + std::auto_ptr<BackupStoreRefCountDatabase> refcount( + BackupStoreRefCountDatabase::Load(Entry, false)); + refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID); + } + + // As the original user... + // Write the database back + mrDatabase.Write(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::GetAccountRoot(int32_t, std::string &, int &) +// Purpose: Gets the root of an account, returning the info via references +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const +{ + // Find the account + const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID)); + + rRootDirOut = MakeAccountRootDir(ID, en.GetDiscSet()); + rDiscSetOut = en.GetDiscSet(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::MakeAccountRootDir(int32_t, int) +// Purpose: Private. Generates a root directory name for the account +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) +{ + char accid[64]; // big enough! + ::sprintf(accid, "%08x" DIRECTORY_SEPARATOR, ID); + return std::string(std::string(BOX_RAIDFILE_ROOT_BBSTORED + DIRECTORY_SEPARATOR) + accid); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::AccountExists(int32_t) +// Purpose: Does an account exist? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreAccounts::AccountExists(int32_t ID) +{ + return mrDatabase.EntryExists(ID); +} + + diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h new file mode 100644 index 00000000..224d7353 --- /dev/null +++ b/lib/backupstore/BackupStoreAccounts.h @@ -0,0 +1,52 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccounts.h +// Purpose: Account management for backup store server +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREACCOUNTS__H +#define BACKUPSTOREACCOUNTS__H + +#include <string> + +#include "BackupStoreAccountDatabase.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreAccounts +// Purpose: Account management for backup store server +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +class BackupStoreAccounts +{ +public: + BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase); + ~BackupStoreAccounts(); +private: + BackupStoreAccounts(const BackupStoreAccounts &rToCopy); + +public: + void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername); + + bool AccountExists(int32_t ID); + void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const; + static std::string GetAccountRoot(const + BackupStoreAccountDatabase::Entry &rEntry) + { + return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet()); + } + +private: + static std::string MakeAccountRootDir(int32_t ID, int DiscSet); + +private: + BackupStoreAccountDatabase &mrDatabase; +}; + +#endif // BACKUPSTOREACCOUNTS__H + diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp new file mode 100644 index 00000000..7598094e --- /dev/null +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -0,0 +1,776 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheck.cpp +// Purpose: Check a store for consistency +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <string.h> +#include <unistd.h> + +#include "BackupStoreCheck.h" +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreFile.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::BackupStoreCheck(const std::string &, int, int32_t, bool, bool) +// Purpose: Constructor +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet) + : mStoreRoot(rStoreRoot), + mDiscSetNumber(DiscSetNumber), + mAccountID(AccountID), + mFixErrors(FixErrors), + mQuiet(Quiet), + mNumberErrorsFound(0), + mLastIDInInfo(0), + mpInfoLastBlock(0), + mInfoLastBlockEntries(0), + mLostDirNameSerial(0), + mLostAndFoundDirectoryID(0), + mBlocksUsed(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::~BackupStoreCheck() +// Purpose: Destructor +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +BackupStoreCheck::~BackupStoreCheck() +{ + // Clean up + FreeInfo(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::Check() +// Purpose: Perform the check on the given account +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::Check() +{ + // Lock the account + { + std::string writeLockFilename; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); + + bool gotLock = false; + int triesLeft = 8; + do + { + gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */); + + if(!gotLock) + { + --triesLeft; + ::sleep(1); + } + } while(!gotLock && triesLeft > 0); + + if(!gotLock) + { + // Couldn't lock the account -- just stop now + if(!mQuiet) + { + BOX_ERROR("Failed to lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server."); + } + THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount) + } + } + + if(!mQuiet && mFixErrors) + { + BOX_NOTICE("Will fix errors encountered during checking."); + } + + // Phase 1, check objects + if(!mQuiet) + { + BOX_INFO("Checking store account ID " << + BOX_FORMAT_ACCOUNT(mAccountID) << "..."); + BOX_INFO("Phase 1, check objects..."); + } + CheckObjects(); + + // Phase 2, check directories + if(!mQuiet) + { + BOX_INFO("Phase 2, check directories..."); + } + CheckDirectories(); + + // Phase 3, check root + if(!mQuiet) + { + BOX_INFO("Phase 3, check root..."); + } + CheckRoot(); + + // Phase 4, check unattached objects + if(!mQuiet) + { + BOX_INFO("Phase 4, fix unattached objects..."); + } + CheckUnattachedObjects(); + + // Phase 5, fix bad info + if(!mQuiet) + { + BOX_INFO("Phase 5, fix unrecovered inconsistencies..."); + } + FixDirsWithWrongContainerID(); + FixDirsWithLostDirs(); + + // Phase 6, regenerate store info + if(!mQuiet) + { + BOX_INFO("Phase 6, regenerate store info..."); + } + WriteNewStoreInfo(); + +// DUMP_OBJECT_INFO + + if(mNumberErrorsFound > 0) + { + BOX_WARNING("Finished checking store account ID " << + BOX_FORMAT_ACCOUNT(mAccountID) << ": " << + mNumberErrorsFound << " errors found"); + if(!mFixErrors) + { + BOX_WARNING("No changes to the store account " + "have been made."); + } + if(!mFixErrors && mNumberErrorsFound > 0) + { + BOX_WARNING("Run again with fix option to " + "fix these errors"); + } + if(mFixErrors && mNumberErrorsFound > 0) + { + BOX_WARNING("You should now use bbackupquery " + "on the client machine to examine the store."); + if(mLostAndFoundDirectoryID != 0) + { + BOX_WARNING("A lost+found directory was " + "created in the account root.\n" + "This contains files and directories " + "which could not be matched to " + "existing directories.\n"\ + "bbackupd will delete this directory " + "in a few days time."); + } + } + } + else + { + BOX_NOTICE("Finished checking store account ID " << + BOX_FORMAT_ACCOUNT(mAccountID) << ": " + "no errors found"); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static TwoDigitHexToInt(const char *, int &) +// Purpose: Convert a two digit hex string to an int, returning whether it's valid or not +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +static inline bool TwoDigitHexToInt(const char *String, int &rNumberOut) +{ + int n = 0; + // Char 0 + if(String[0] >= '0' && String[0] <= '9') + { + n = (String[0] - '0') << 4; + } + else if(String[0] >= 'a' && String[0] <= 'f') + { + n = ((String[0] - 'a') + 0xa) << 4; + } + else + { + return false; + } + // Char 1 + if(String[1] >= '0' && String[1] <= '9') + { + n |= String[1] - '0'; + } + else if(String[1] >= 'a' && String[1] <= 'f') + { + n |= (String[1] - 'a') + 0xa; + } + else + { + return false; + } + + // Return a valid number + rNumberOut = n; + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckObjects() +// Purpose: Read in the contents of the directory, recurse to other levels, +// checking objects for sanity and readability +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckObjects() +{ + // Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything + int64_t maxDir = 0; + + // Find the maximum directory starting ID + { + // Make sure the starting root dir doesn't end with '/'. + std::string start(mStoreRoot); + if(start.size() > 0 && start[start.size() - 1] == '/') + { + start.resize(start.size() - 1); + } + + maxDir = CheckObjectsScanDir(0, 1, mStoreRoot); + BOX_TRACE("Max dir starting ID is " << + BOX_FORMAT_OBJECTID(maxDir)); + } + + // Then go through and scan all the objects within those directories + for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH)) + { + CheckObjectsDir(d); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckObjectsScanDir(int64_t, int, int, const std::string &) +// Purpose: Read in the contents of the directory, recurse to other levels, +// return the maximum starting ID of any directory found. +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName) +{ + //TRACE2("Scan directory for max dir starting ID %s, StartID %lld\n", rDirName.c_str(), StartID); + + int64_t maxID = StartID; + + // Read in all the directories, and recurse downwards + { + std::vector<std::string> dirs; + RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName, + RaidFileRead::DirReadType_DirsOnly, dirs); + + for(std::vector<std::string>::const_iterator i(dirs.begin()); i != dirs.end(); ++i) + { + // Check to see if it's the right name + int n = 0; + if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n) + && n < (1<<STORE_ID_SEGMENT_LENGTH)) + { + // Next level down + int64_t mi = CheckObjectsScanDir(StartID | (n << (Level * STORE_ID_SEGMENT_LENGTH)), Level + 1, + rDirName + DIRECTORY_SEPARATOR + *i); + // Found a greater starting ID? + if(mi > maxID) + { + maxID = mi; + } + } + else + { + BOX_WARNING("Spurious or invalid directory " << + rDirName << DIRECTORY_SEPARATOR << + (*i) << " found, " << + (mFixErrors?"deleting":"delete manually")); + ++mNumberErrorsFound; + } + } + } + + return maxID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckObjectsDir(int64_t) +// Purpose: Check all the files within this directory which has the given starting ID. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckObjectsDir(int64_t StartID) +{ + // Make directory name -- first generate the filename of an entry in it + std::string dirName; + StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */); + // Check expectations + ASSERT(dirName.size() > 4 && + dirName[dirName.size() - 4] == DIRECTORY_SEPARATOR_ASCHAR); + // Remove the filename from it + dirName.resize(dirName.size() - 4); // four chars for "/o00" + + // Check directory exists + if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName)) + { + BOX_WARNING("RaidFile dir " << dirName << " does not exist"); + return; + } + + // Read directory contents + std::vector<std::string> files; + RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName, + RaidFileRead::DirReadType_FilesOnly, files); + + // Array of things present + bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)]; + for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l) + { + idsPresent[l] = false; + } + + // Parse each entry, building up a list of object IDs which are present in the dir. + // This is done so that whatever order is retured from the directory, objects are scanned + // in order. + // Filename must begin with a 'o' and be three characters long, otherwise it gets deleted. + for(std::vector<std::string>::const_iterator i(files.begin()); i != files.end(); ++i) + { + bool fileOK = true; + int n = 0; + if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n) + && n < (1<<STORE_ID_SEGMENT_LENGTH)) + { + // Filename is valid, mark as existing + idsPresent[n] = true; + } + else + { + // info file in root dir is OK! + if(StartID != 0 || ::strcmp("info", (*i).c_str()) != 0) + { + fileOK = false; + } + } + + if(!fileOK) + { + // Unexpected or bad file, delete it + BOX_WARNING("Spurious file " << dirName << + DIRECTORY_SEPARATOR << (*i) << " found" << + (mFixErrors?", deleting":"")); + ++mNumberErrorsFound; + if(mFixErrors) + { + RaidFileWrite del(mDiscSetNumber, dirName + DIRECTORY_SEPARATOR + *i); + del.Delete(); + } + } + } + + // Check all the objects found in this directory + for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i) + { + if(idsPresent[i]) + { + // Check the object is OK, and add entry + char leaf[8]; + ::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i); + if(!CheckAndAddObject(StartID | i, dirName + leaf)) + { + // File was bad, delete it + BOX_WARNING("Corrupted file " << dirName << + leaf << " found" << + (mFixErrors?", deleting":"")); + ++mNumberErrorsFound; + if(mFixErrors) + { + RaidFileWrite del(mDiscSetNumber, dirName + leaf); + del.Delete(); + } + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckAndAddObject(int64_t, const std::string &) +// Purpose: Check a specific object and add it to the list if it's OK -- if +// there are any errors with the reading, return false and it'll be deleted. +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename) +{ + // Info on object... + bool isFile = true; + int64_t containerID = -1; + int64_t size = -1; + + try + { + // Open file + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, rFilename)); + size = file->GetDiscUsageInBlocks(); + + // Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile + uint32_t signature; + if(file->Read(&signature, sizeof(signature)) != sizeof(signature)) + { + // Too short, can't read signature from it + return false; + } + // Seek back to beginning + file->Seek(0, IOStream::SeekType_Absolute); + + // Then... check depending on the type + switch(ntohl(signature)) + { + case OBJECTMAGIC_FILE_MAGIC_VALUE_V1: +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_MAGIC_VALUE_V0: +#endif + // File... check + containerID = CheckFile(ObjectID, *file); + break; + + case OBJECTMAGIC_DIR_MAGIC_VALUE: + isFile = false; + containerID = CheckDirInitial(ObjectID, *file); + break; + + default: + // Unknown signature. Bad file. Very bad file. + return false; + break; + } + + // Add to usage counts + mBlocksUsed += size; + if(!isFile) + { + mBlocksInDirectories += size; + } + } + catch(...) + { + // Error caught, not a good file then, let it be deleted + return false; + } + + // Got a container ID? (ie check was successful) + if(containerID == -1) + { + return false; + } + + // Add to list of IDs known about + AddID(ObjectID, containerID, size, isFile); + + // Report success + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckFile(int64_t, IOStream &) +// Purpose: Do check on file, return original container ID if OK, or -1 on error +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) +{ + // Check that it's not the root directory ID. Having a file as the root directory would be bad. + if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID) + { + // Get that dodgy thing deleted! + BOX_ERROR("Have file as root directory. This is bad."); + return -1; + } + + // Check the format of the file, and obtain the container ID + int64_t originalContainerID = -1; + if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */, + &originalContainerID)) + { + // Didn't verify + return -1; + } + + return originalContainerID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckDirInitial(int64_t, IOStream &) +// Purpose: Do initial check on directory, return container ID if OK, or -1 on error +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream) +{ + // Simply attempt to read in the directory + BackupStoreDirectory dir; + dir.ReadFromStream(rStream, IOStream::TimeOutInfinite); + + // Check object ID + if(dir.GetObjectID() != ObjectID) + { + // Wrong object ID + return -1; + } + + // Return container ID + return dir.GetContainerID(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckDirectories() +// Purpose: Check the directories +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckDirectories() +{ + // Phase 1 did this: + // Checked that all the directories are readable + // Built a list of all directories and files which exist on the store + // + // This phase will check all the files in the directories, make + // a note of all directories which are missing, and do initial fixing. + + // Scan all objects + for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + IDBlock *pblock = i->second; + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + + for(int e = 0; e < bentries; ++e) + { + uint8_t flags = GetFlags(pblock, e); + if(flags & Flags_IsDir) + { + // Found a directory. Read it in. + std::string filename; + StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */); + BackupStoreDirectory dir; + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Flag for modifications + bool isModified = false; + + // Check for validity + if(dir.CheckAndFix()) + { + // Wasn't quite right, and has been modified + BOX_WARNING("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " has bad structure"); + ++mNumberErrorsFound; + isModified = true; + } + + // Go through, and check that everything in that directory exists and is valid + std::vector<int64_t> toDelete; + + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + // Lookup the item + int32_t iIndex; + IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); + bool badEntry = false; + if(piBlock != 0) + { + // Found. Get flags + uint8_t iflags = GetFlags(piBlock, iIndex); + + // Is the type the same? + if(((iflags & Flags_IsDir) == Flags_IsDir) + != ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir)) + { + // Entry is of wrong type + BOX_WARNING("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " references object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " which has a different type than expected."); + badEntry = true; + } + else + { + // Check that the entry is not already contained. + if(iflags & Flags_IsContained) + { + BOX_WARNING("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " references object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " which is already contained."); + badEntry = true; + } + else + { + // Not already contained -- mark as contained + SetFlags(piBlock, iIndex, iflags | Flags_IsContained); + + // Check that the container ID of the object is correct + if(piBlock->mContainer[iIndex] != pblock->mID[e]) + { + // Needs fixing... + if(iflags & Flags_IsDir) + { + // Add to will fix later list + BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " has wrong container ID."); + mDirsWithWrongContainerID.push_back(en->GetObjectID()); + } + else + { + // This is OK for files, they might move + BOX_WARNING("File ID " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " has different container ID, probably moved"); + } + + // Fix entry for now + piBlock->mContainer[iIndex] = pblock->mID[e]; + } + } + } + + // Check the object size, if it's OK and a file + if(!badEntry && !((iflags & Flags_IsDir) == Flags_IsDir)) + { + if(en->GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[iIndex]) + { + // Correct + en->SetSizeInBlocks(piBlock->mObjectSizeInBlocks[iIndex]); + // Mark as changed + isModified = true; + // Tell user + BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " has wrong size for object " << BOX_FORMAT_OBJECTID(en->GetObjectID())); + } + } + } + else + { + // Item can't be found. Is it a directory? + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) + { + // Store the directory for later attention + mDirsWhichContainLostDirs[en->GetObjectID()] = pblock->mID[e]; + } + else + { + // Just remove the entry + badEntry = true; + BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " references object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " which does not exist."); + } + } + + // Is this entry worth keeping? + if(badEntry) + { + toDelete.push_back(en->GetObjectID()); + } + else + { + // Add to sizes? + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) + { + mBlocksInOldFiles += en->GetSizeInBlocks(); + } + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) + { + mBlocksInDeletedFiles += en->GetSizeInBlocks(); + } + } + } + + if(toDelete.size() > 0) + { + // Delete entries from directory + for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) + { + dir.DeleteEntry(*d); + } + + // Mark as modified + isModified = true; + + // Check the directory again, now that entries have been removed + dir.CheckAndFix(); + + // Errors found + ++mNumberErrorsFound; + } + + if(isModified && mFixErrors) + { + BOX_WARNING("Fixing directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e])); + + // Save back to disc + RaidFileWrite fixed(mDiscSetNumber, filename); + fixed.Open(true /* allow overwriting */); + dir.WriteToStream(fixed); + // Commit it + fixed.Commit(true /* convert to raid representation now */); + } + } + } + } + +} + + diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h new file mode 100644 index 00000000..1d5c1b1e --- /dev/null +++ b/lib/backupstore/BackupStoreCheck.h @@ -0,0 +1,199 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheck.h +// Purpose: Check a store for consistency +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTORECHECK__H +#define BACKUPSTORECHECK__H + +#include <string> +#include <map> +#include <vector> +#include <set> + +#include "NamedLock.h" +class IOStream; +class BackupStoreFilename; + +/* + +The following problems can be fixed: + + * Spurious files deleted + * Corrupted files deleted + * Root ID as file, deleted + * Dirs with wrong object id inside, deleted + * Direcetory entries pointing to non-existant files, deleted + * Doubly references files have second reference deleted + * Wrong directory container IDs fixed + * Missing root recreated + * Reattach files which exist, but aren't referenced + - files go into directory original directory, if it still exists + - missing directories are inferred, and recreated + - or if all else fails, go into lost+found + - file dir entries take the original name and mod time + - directories go into lost+found + * Container IDs on directories corrected + * Inside directories, + - only one object per name has old version clear + - IDs aren't duplicated + * Bad store info files regenerated + * Bad sizes of files in directories fixed + +*/ + + +// Size of blocks in the list of IDs +#ifdef BOX_RELEASE_BUILD + #define BACKUPSTORECHECK_BLOCK_SIZE (64*1024) +#else + #define BACKUPSTORECHECK_BLOCK_SIZE 8 +#endif + +// The object ID type -- can redefine to uint32_t to produce a lower memory version for smaller stores +typedef int64_t BackupStoreCheck_ID_t; +// Can redefine the size type for lower memory usage too +typedef int64_t BackupStoreCheck_Size_t; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreCheck +// Purpose: Check a store for consistency +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +class BackupStoreCheck +{ +public: + BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet); + ~BackupStoreCheck(); +private: + // no copying + BackupStoreCheck(const BackupStoreCheck &); + BackupStoreCheck &operator=(const BackupStoreCheck &); +public: + + // Do the exciting things + void Check(); + + bool ErrorsFound() {return mNumberErrorsFound > 0;} + +private: + enum + { + // Bit mask + Flags_IsDir = 1, + Flags_IsContained = 2, + // Mask + Flags__MASK = 3, + // Number of bits + Flags__NumFlags = 2, + // Items per uint8_t + Flags__NumItemsPerEntry = 4 // ie 8 / 2 + }; + + typedef struct + { + // Note use arrays within the block, rather than the more obvious array of + // objects, to be more memory efficient -- think alignment of the byte values. + uint8_t mFlags[BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry]; + BackupStoreCheck_ID_t mID[BACKUPSTORECHECK_BLOCK_SIZE]; + BackupStoreCheck_ID_t mContainer[BACKUPSTORECHECK_BLOCK_SIZE]; + BackupStoreCheck_Size_t mObjectSizeInBlocks[BACKUPSTORECHECK_BLOCK_SIZE]; + } IDBlock; + + // Phases of the check + void CheckObjects(); + void CheckDirectories(); + void CheckRoot(); + void CheckUnattachedObjects(); + void FixDirsWithWrongContainerID(); + void FixDirsWithLostDirs(); + void WriteNewStoreInfo(); + + // Checking functions + int64_t CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName); + void CheckObjectsDir(int64_t StartID); + bool CheckAndAddObject(int64_t ObjectID, const std::string &rFilename); + int64_t CheckFile(int64_t ObjectID, IOStream &rStream); + int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); + + // Fixing functions + bool TryToRecreateDirectory(int64_t MissingDirectoryID); + void InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory); + int64_t GetLostAndFoundDirID(); + void CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID); + + // Data handling + void FreeInfo(); + void AddID(BackupStoreCheck_ID_t ID, BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile); + IDBlock *LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut); + inline void SetFlags(IDBlock *pBlock, int32_t Index, uint8_t Flags) + { + ASSERT(pBlock != 0); + ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE); + ASSERT(Flags < (1 << Flags__NumFlags)); + + pBlock->mFlags[Index / Flags__NumItemsPerEntry] + |= (Flags << ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)); + } + inline uint8_t GetFlags(IDBlock *pBlock, int32_t Index) + { + ASSERT(pBlock != 0); + ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE); + + return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK; + } + +#ifndef BOX_RELEASE_BUILD + void DumpObjectInfo(); + #define DUMP_OBJECT_INFO DumpObjectInfo(); +#else + #define DUMP_OBJECT_INFO +#endif + +private: + std::string mStoreRoot; + int mDiscSetNumber; + int32_t mAccountID; + bool mFixErrors; + bool mQuiet; + + int64_t mNumberErrorsFound; + + // Lock for the store account + NamedLock mAccountLock; + + // Storage for ID data + typedef std::map<BackupStoreCheck_ID_t, IDBlock*> Info_t; + Info_t mInfo; + BackupStoreCheck_ID_t mLastIDInInfo; + IDBlock *mpInfoLastBlock; + int32_t mInfoLastBlockEntries; + + // List of stuff to fix + std::vector<BackupStoreCheck_ID_t> mDirsWithWrongContainerID; + // This is a map of lost dir ID -> existing dir ID + std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t> mDirsWhichContainLostDirs; + + // Set of extra directories added + std::set<BackupStoreCheck_ID_t> mDirsAdded; + + // Misc stuff + int32_t mLostDirNameSerial; + int64_t mLostAndFoundDirectoryID; + + // Usage + int64_t mBlocksUsed; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; +}; + +#endif // BACKUPSTORECHECK__H + diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp new file mode 100644 index 00000000..bcb5c5e9 --- /dev/null +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -0,0 +1,916 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheck2.cpp +// Purpose: More backup store checking +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <string.h> + +#include "BackupStoreCheck.h" +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" +#include "BackupStoreInfo.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckRoot() +// Purpose: Check the root directory exists. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckRoot() +{ + int32_t index = 0; + IDBlock *pblock = LookupID(BACKUPSTORE_ROOT_DIRECTORY_ID, index); + + if(pblock != 0) + { + // Found it. Which is lucky. Mark it as contained. + SetFlags(pblock, index, Flags_IsContained); + } + else + { + BOX_WARNING("Root directory doesn't exist"); + + ++mNumberErrorsFound; + + if(mFixErrors) + { + // Create a new root directory + CreateBlankDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CreateBlankDirectory(int64_t, int64_t) +// Purpose: Creates a blank directory +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID) +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + BackupStoreDirectory dir(DirectoryID, ContainingDirID); + + // Serialise to disc + std::string filename; + StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); + RaidFileWrite obj(mDiscSetNumber, filename); + obj.Open(false /* don't allow overwriting */); + dir.WriteToStream(obj); + int64_t size = obj.GetDiscUsageInBlocks(); + obj.Commit(true /* convert to raid now */); + + // Record the fact we've done this + mDirsAdded.insert(DirectoryID); + + // Add to sizes + mBlocksUsed += size; + mBlocksInDirectories += size; +} + +class BackupStoreDirectoryFixer +{ + private: + BackupStoreDirectory mDirectory; + std::string mFilename; + std::string mStoreRoot; + int mDiscSetNumber; + + public: + BackupStoreDirectoryFixer(std::string storeRoot, int discSetNumber, + int64_t ID); + void InsertObject(int64_t ObjectID, bool IsDirectory, + int32_t lostDirNameSerial); + ~BackupStoreDirectoryFixer(); +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckUnattachedObjects() +// Purpose: Check for objects which aren't attached to anything +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckUnattachedObjects() +{ + typedef std::map<int64_t, BackupStoreDirectoryFixer*> fixers_t; + typedef std::pair<int64_t, BackupStoreDirectoryFixer*> fixer_pair_t; + fixers_t fixers; + + // Scan all objects, finding ones which have no container + for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + IDBlock *pblock = i->second; + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + + for(int e = 0; e < bentries; ++e) + { + uint8_t flags = GetFlags(pblock, e); + if((flags & Flags_IsContained) == 0) + { + // Unattached object... + BOX_WARNING("Object " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " is unattached."); + ++mNumberErrorsFound; + + // What's to be done? + int64_t putIntoDirectoryID = 0; + + if((flags & Flags_IsDir) == Flags_IsDir) + { + // Directory. Just put into lost and found. + putIntoDirectoryID = GetLostAndFoundDirID(); + } + else + { + // File. Only attempt to attach it somewhere if it isn't a patch + { + int64_t diffFromObjectID = 0; + std::string filename; + StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */); + // The easiest way to do this is to verify it again. Not such a bad penalty, because + // this really shouldn't be done very often. + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + BackupStoreFile::VerifyEncodedFileFormat(*file, &diffFromObjectID); + } + + // If not zero, then it depends on another file, which may or may not be available. + // Just delete it to be safe. + if(diffFromObjectID != 0) + { + BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached, and is a patch. Deleting, cannot reliably recover."); + + // Delete this object instead + if(mFixErrors) + { + RaidFileWrite del(mDiscSetNumber, filename); + del.Delete(); + } + + // Move on to next item + continue; + } + } + + // Files contain their original filename, so perhaps the orginal directory still exists, + // or we can infer the existance of a directory? + // Look for a matching entry in the mDirsWhichContainLostDirs map. + // Can't do this with a directory, because the name just wouldn't be known, which is + // pretty useless as bbackupd would just delete it. So better to put it in lost+found + // where the admin can do something about it. + int32_t dirindex; + IDBlock *pdirblock = LookupID(pblock->mContainer[e], dirindex); + if(pdirblock != 0) + { + // Something with that ID has been found. Is it a directory? + if(GetFlags(pdirblock, dirindex) & Flags_IsDir) + { + // Directory exists, add to that one + putIntoDirectoryID = pblock->mContainer[e]; + } + else + { + // Not a directory. Use lost and found dir + putIntoDirectoryID = GetLostAndFoundDirID(); + } + } + else if(mDirsAdded.find(pblock->mContainer[e]) != mDirsAdded.end() + || TryToRecreateDirectory(pblock->mContainer[e])) + { + // The directory reappeared, or was created somehow elsewhere + putIntoDirectoryID = pblock->mContainer[e]; + } + else + { + putIntoDirectoryID = GetLostAndFoundDirID(); + } + } + ASSERT(putIntoDirectoryID != 0); + + if (!mFixErrors) + { + continue; + } + + BackupStoreDirectoryFixer* pFixer; + fixers_t::iterator fi = + fixers.find(putIntoDirectoryID); + if (fi == fixers.end()) + { + // no match, create a new one + pFixer = new BackupStoreDirectoryFixer( + mStoreRoot, mDiscSetNumber, + putIntoDirectoryID); + fixers.insert(fixer_pair_t( + putIntoDirectoryID, pFixer)); + } + else + { + pFixer = fi->second; + } + + int32_t lostDirNameSerial = 0; + + if(flags & Flags_IsDir) + { + lostDirNameSerial = mLostDirNameSerial++; + } + + // Add it to the directory + pFixer->InsertObject(pblock->mID[e], + ((flags & Flags_IsDir) == Flags_IsDir), + lostDirNameSerial); + } + } + } + + // clean up all the fixers. Deleting them commits them automatically. + for (fixers_t::iterator i = fixers.begin(); i != fixers.end(); i++) + { + BackupStoreDirectoryFixer* pFixer = i->second; + delete pFixer; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::TryToRecreateDirectory(int64_t) +// Purpose: Recreate a missing directory +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) +{ + // During the directory checking phase, a map of "missing directory" to + // containing directory was built. If we can find it here, then it's + // something which can be recreated! + std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator missing( + mDirsWhichContainLostDirs.find(MissingDirectoryID)); + if(missing == mDirsWhichContainLostDirs.end()) + { + // Not a missing directory, can't recreate. + return false; + } + + // Can recreate this! Wooo! + if(!mFixErrors) + { + BOX_WARNING("Missing directory " << + BOX_FORMAT_OBJECTID(MissingDirectoryID) << + " could be recreated."); + mDirsAdded.insert(MissingDirectoryID); + return true; + } + + BOX_WARNING("Recreating missing directory " << + BOX_FORMAT_OBJECTID(MissingDirectoryID)); + + // Create a blank directory + BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */); + // Note that this directory already contains a directory entry pointing to + // this dir, so it doesn't have to be added. + + // Serialise to disc + std::string filename; + StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(false /* don't allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + + // Record the fact we've done this + mDirsAdded.insert(MissingDirectoryID); + + // Remove the entry from the map, so this doesn't happen again + mDirsWhichContainLostDirs.erase(missing); + + return true; +} + +BackupStoreDirectoryFixer::BackupStoreDirectoryFixer(std::string storeRoot, + int discSetNumber, int64_t ID) +: mStoreRoot(storeRoot), + mDiscSetNumber(discSetNumber) +{ + // Generate filename + StoreStructure::MakeObjectFilename(ID, mStoreRoot, mDiscSetNumber, + mFilename, false /* don't make sure the dir exists */); + + // Read it in + std::auto_ptr<RaidFileRead> file( + RaidFileRead::Open(mDiscSetNumber, mFilename)); + mDirectory.ReadFromStream(*file, IOStream::TimeOutInfinite); +} + +void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, + int32_t lostDirNameSerial) +{ + // Data for the object + BackupStoreFilename objectStoreFilename; + int64_t modTime = 100; // something which isn't zero or a special time + int32_t sizeInBlocks = 0; // suitable for directories + + if(IsDirectory) + { + // Directory -- simply generate a name for it. + char name[32]; + ::sprintf(name, "dir%08x", lostDirNameSerial); + objectStoreFilename.SetAsClearFilename(name); + } + else + { + // Files require a little more work... + // Open file + std::string fileFilename; + StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, + mDiscSetNumber, fileFilename, + false /* don't make sure the dir exists */); + std::auto_ptr<RaidFileRead> file( + RaidFileRead::Open(mDiscSetNumber, fileFilename)); + + // Fill in size information + sizeInBlocks = file->GetDiscUsageInBlocks(); + + // Read in header + file_StreamFormat hdr; + if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || + (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + )) + { + // This should never happen, everything has been + // checked before. + THROW_EXCEPTION(BackupStoreException, Internal) + } + // This tells us nice things + modTime = box_ntoh64(hdr.mModificationTime); + // And the filename comes next + objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Add a new entry in an appropriate place + mDirectory.AddUnattactedObject(objectStoreFilename, modTime, + ObjectID, sizeInBlocks, + IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); +} + +BackupStoreDirectoryFixer::~BackupStoreDirectoryFixer() +{ + // Fix any flags which have been broken, which there's a good chance of doing + mDirectory.CheckAndFix(); + + // Write it out + RaidFileWrite root(mDiscSetNumber, mFilename); + root.Open(true /* allow overwriting */); + mDirectory.WriteToStream(root); + root.Commit(true /* convert to raid now */); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::GetLostAndFoundDirID() +// Purpose: Returns the ID of the lost and found directory, creating it if necessary +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::GetLostAndFoundDirID() +{ + // Already allocated it? + if(mLostAndFoundDirectoryID != 0) + { + return mLostAndFoundDirectoryID; + } + + if(!mFixErrors) + { + // The result will never be used anyway if errors aren't being fixed + return 1; + } + + // Load up the root directory + BackupStoreDirectory dir; + std::string filename; + StoreStructure::MakeObjectFilename(BACKUPSTORE_ROOT_DIRECTORY_ID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Find a suitable name + BackupStoreFilename lostAndFound; + int n = 0; + while(true) + { + char name[32]; + ::sprintf(name, "lost+found%d", n++); + lostAndFound.SetAsClearFilename(name); + if(!dir.NameInUse(lostAndFound)) + { + // Found a name which can be used + BOX_WARNING("Lost and found dir has name " << name); + break; + } + } + + // Allocate an ID + int64_t id = mLastIDInInfo + 1; + + // Create a blank directory + CreateBlankDirectory(id, BACKUPSTORE_ROOT_DIRECTORY_ID); + + // Add an entry for it + dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0); + + // Write out root dir + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + + // Store + mLostAndFoundDirectoryID = id; + + // Tell caller + return mLostAndFoundDirectoryID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::FixDirsWithWrongContainerID() +// Purpose: Rewrites container IDs where required +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::FixDirsWithWrongContainerID() +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + // Run through things which need fixing + for(std::vector<BackupStoreCheck_ID_t>::iterator i(mDirsWithWrongContainerID.begin()); + i != mDirsWithWrongContainerID.end(); ++i) + { + int32_t index = 0; + IDBlock *pblock = LookupID(*i, index); + if(pblock == 0) continue; + + // Load in + BackupStoreDirectory dir; + std::string filename; + StoreStructure::MakeObjectFilename(*i, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Adjust container ID + dir.SetContainerID(pblock->mContainer[index]); + + // Write it out + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::FixDirsWithLostDirs() +// Purpose: Fix directories +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::FixDirsWithLostDirs() +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + // Run through things which need fixing + for(std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator i(mDirsWhichContainLostDirs.begin()); + i != mDirsWhichContainLostDirs.end(); ++i) + { + int32_t index = 0; + IDBlock *pblock = LookupID(i->second, index); + if(pblock == 0) continue; + + // Load in + BackupStoreDirectory dir; + std::string filename; + StoreStructure::MakeObjectFilename(i->second, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + { + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Delete the dodgy entry + dir.DeleteEntry(i->first); + + // Fix it up + dir.CheckAndFix(); + + // Write it out + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::WriteNewStoreInfo() +// Purpose: Regenerate store info +// Created: 23/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::WriteNewStoreInfo() +{ + // Attempt to load the existing store info file + std::auto_ptr<BackupStoreInfo> poldInfo; + try + { + poldInfo.reset(BackupStoreInfo::Load(mAccountID, mStoreRoot, mDiscSetNumber, true /* read only */).release()); + } + catch(...) + { + BOX_WARNING("Load of existing store info failed, regenerating."); + ++mNumberErrorsFound; + } + + // Minimum soft and hard limits + int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024; + int64_t minHard = ((minSoft * 11) / 10) + 1024; + + // Need to do anything? + if(poldInfo.get() != 0 && mNumberErrorsFound == 0 && poldInfo->GetAccountID() == mAccountID) + { + // Leave the store info as it is, no need to alter it because nothing really changed, + // and the only essential thing was that the account ID was correct, which is was. + return; + } + + // NOTE: We will always build a new store info, so the client store marker gets changed. + + // Work out the new limits + int64_t softLimit = minSoft; + int64_t hardLimit = minHard; + if(poldInfo.get() != 0 && poldInfo->GetBlocksSoftLimit() > minSoft) + { + softLimit = poldInfo->GetBlocksSoftLimit(); + } + else + { + BOX_WARNING("Soft limit for account changed to ensure housekeeping doesn't delete files on next run."); + } + if(poldInfo.get() != 0 && poldInfo->GetBlocksHardLimit() > minHard) + { + hardLimit = poldInfo->GetBlocksHardLimit(); + } + else + { + BOX_WARNING("Hard limit for account changed to ensure housekeeping doesn't delete files on next run."); + } + + // Object ID + int64_t lastObjID = mLastIDInInfo; + if(mLostAndFoundDirectoryID != 0) + { + mLastIDInInfo++; + } + + // Build a new store info + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::CreateForRegeneration( + mAccountID, + mStoreRoot, + mDiscSetNumber, + lastObjID, + mBlocksUsed, + mBlocksInOldFiles, + mBlocksInDeletedFiles, + mBlocksInDirectories, + softLimit, + hardLimit)); + + // Save to disc? + if(mFixErrors) + { + info->Save(); + BOX_NOTICE("New store info file written successfully."); + } +} + +#define FMT_OID(x) BOX_FORMAT_OBJECTID(x) +#define FMT_i BOX_FORMAT_OBJECTID((*i)->GetObjectID()) + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::CheckAndFix() +// Purpose: Check the directory for obvious logical problems, and fix them. +// Return true if the directory was changed. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreDirectory::CheckAndFix() +{ + bool changed = false; + + // Check that if a file depends on a new version, that version is in this directory + { + std::vector<Entry*>::iterator i(mEntries.begin()); + for(; i != mEntries.end(); ++i) + { + int64_t dependsNewer = (*i)->GetDependsNewer(); + if(dependsNewer != 0) + { + BackupStoreDirectory::Entry *newerEn = FindEntryByID(dependsNewer); + if(newerEn == 0) + { + // Depends on something, but it isn't there. + BOX_TRACE("Entry id " << FMT_i << + " removed because depends " + "on newer version " << + FMT_OID(dependsNewer) << + " which doesn't exist"); + + // Remove + delete *i; + mEntries.erase(i); + + // Start again at the beginning of the vector, the iterator is now invalid + i = mEntries.begin(); + + // Mark as changed + changed = true; + } + else + { + // Check that newerEn has it marked + if(newerEn->GetDependsOlder() != (*i)->GetObjectID()) + { + // Wrong entry + BOX_TRACE("Entry id " << + FMT_OID(dependsNewer) << + ", correcting DependsOlder to " << + FMT_i << + ", was " << + FMT_OID(newerEn->GetDependsOlder())); + newerEn->SetDependsOlder((*i)->GetObjectID()); + // Mark as changed + changed = true; + } + } + } + } + } + + // Check that if a file has a dependency marked, it exists, and remove it if it doesn't + { + std::vector<Entry*>::iterator i(mEntries.begin()); + for(; i != mEntries.end(); ++i) + { + int64_t dependsOlder = (*i)->GetDependsOlder(); + if(dependsOlder != 0 && FindEntryByID(dependsOlder) == 0) + { + // Has an older version marked, but this doesn't exist. Remove this mark + BOX_TRACE("Entry id " << FMT_i << + " was marked as depended on by " << + FMT_OID(dependsOlder) << ", " + "which doesn't exist, dependency " + "info cleared"); + + (*i)->SetDependsOlder(0); + + // Mark as changed + changed = true; + } + } + } + + bool ch = false; + do + { + // Reset change marker + ch = false; + + // Search backwards -- so see newer versions first + std::vector<Entry*>::iterator i(mEntries.end()); + if(i == mEntries.begin()) + { + // Directory is empty, stop now + return changed; // changed flag + } + + // Records of things seen + std::set<int64_t> idsEncountered; + std::set<std::string> filenamesEncountered; + + do + { + // Look at previous + --i; + + bool removeEntry = false; + if((*i) == 0) + { + BOX_TRACE("Remove because null pointer found"); + removeEntry = true; + } + else + { + bool isDir = (((*i)->GetFlags() & Entry::Flags_Dir) == Entry::Flags_Dir); + + // Check mutually exclusive flags + if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File)) + { + // Bad! Unset the file flag + BOX_TRACE("Entry " << FMT_i << + ": File flag and dir flag both set"); + (*i)->RemoveFlags(Entry::Flags_File); + changed = true; + } + + // Check... + if(idsEncountered.find((*i)->GetObjectID()) != idsEncountered.end()) + { + // ID already seen, or type doesn't match + BOX_TRACE("Entry " << FMT_i << + ": Remove because ID already seen"); + removeEntry = true; + } + else + { + // Haven't already seen this ID, remember it + idsEncountered.insert((*i)->GetObjectID()); + + // Check to see if the name has already been encountered -- if not, then it + // needs to have the old version flag set + if(filenamesEncountered.find((*i)->GetName().GetEncodedFilename()) != filenamesEncountered.end()) + { + // Seen before -- check old version flag set + if(((*i)->GetFlags() & Entry::Flags_OldVersion) != Entry::Flags_OldVersion + && ((*i)->GetFlags() & Entry::Flags_Deleted) == 0) + { + // Not set, set it + BOX_TRACE("Entry " << FMT_i << + ": Set old flag"); + (*i)->AddFlags(Entry::Flags_OldVersion); + changed = true; + } + } + else + { + // Check old version flag NOT set + if(((*i)->GetFlags() & Entry::Flags_OldVersion) == Entry::Flags_OldVersion) + { + // Set, unset it + BOX_TRACE("Entry " << FMT_i << + ": Old flag unset"); + (*i)->RemoveFlags(Entry::Flags_OldVersion); + changed = true; + } + + // Remember filename + filenamesEncountered.insert((*i)->GetName().GetEncodedFilename()); + } + } + } + + if(removeEntry) + { + // Mark something as changed, in loop + ch = true; + + // Mark something as globally changed + changed = true; + + // erase the thing from the list + Entry *pentry = (*i); + mEntries.erase(i); + + // And delete the entry object + delete pentry; + + // Stop going around this loop, as the iterator is now invalid + break; + } + } while(i != mEntries.begin()); + + } while(ch != false); + + return changed; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddUnattactedObject(...) +// Purpose: Adds an object which is currently unattached. Assume that CheckAndFix() will be called afterwards. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::AddUnattactedObject(const BackupStoreFilename &rName, + box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags) +{ + Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, + ModificationTime /* use as attr mod time too */); + try + { + // Want to order this just before the first object which has a higher ID, + // which is the place it's most likely to be correct. + std::vector<Entry*>::iterator i(mEntries.begin()); + for(; i != mEntries.end(); ++i) + { + if((*i)->GetObjectID() > ObjectID) + { + // Found a good place to insert it + break; + } + } + if(i == mEntries.end()) + { + mEntries.push_back(pnew); + } + else + { + mEntries.insert(i, 1 /* just the one copy */, pnew); + } + } + catch(...) + { + delete pnew; + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::NameInUse(const BackupStoreFilename &) +// Purpose: Returns true if the name is currently in use in the directory +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreDirectory::NameInUse(const BackupStoreFilename &rName) +{ + for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) + { + if((*i)->GetName() == rName) + { + return true; + } + } + + return false; +} + + diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp new file mode 100644 index 00000000..fed0c3f1 --- /dev/null +++ b/lib/backupstore/BackupStoreCheckData.cpp @@ -0,0 +1,208 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheckData.cpp +// Purpose: Data handling for store checking +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> +#include <memory> + +#include "BackupStoreCheck.h" +#include "autogen_BackupStoreException.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::FreeInfo() +// Purpose: Free all the data stored +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::FreeInfo() +{ + // Free all the blocks + for(Info_t::iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + ::free(i->second); + } + + // Clear the contents of the map + mInfo.clear(); + + // Reset the last ID, just in case + mpInfoLastBlock = 0; + mInfoLastBlockEntries = 0; + mLastIDInInfo = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::AddID(BackupStoreCheck_ID_t, BackupStoreCheck_ID_t, bool) +// Purpose: Add an ID to the list +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::AddID(BackupStoreCheck_ID_t ID, + BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile) +{ + // Check ID is OK. + if(ID <= mLastIDInInfo) + { + THROW_EXCEPTION(BackupStoreException, InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing) + } + + // Can this go in the current block? + if(mpInfoLastBlock == 0 || mInfoLastBlockEntries >= BACKUPSTORECHECK_BLOCK_SIZE) + { + // No. Allocate a new one + IDBlock *pblk = (IDBlock*)::malloc(sizeof(IDBlock)); + if(pblk == 0) + { + throw std::bad_alloc(); + } + // Zero all the flags entries + for(int z = 0; z < (BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry); ++z) + { + pblk->mFlags[z] = 0; + } + // Store in map + mInfo[ID] = pblk; + // Allocated and stored OK, setup for use + mpInfoLastBlock = pblk; + mInfoLastBlockEntries = 0; + } + ASSERT(mpInfoLastBlock != 0 && mInfoLastBlockEntries < BACKUPSTORECHECK_BLOCK_SIZE); + + // Add to block + mpInfoLastBlock->mID[mInfoLastBlockEntries] = ID; + mpInfoLastBlock->mContainer[mInfoLastBlockEntries] = Container; + mpInfoLastBlock->mObjectSizeInBlocks[mInfoLastBlockEntries] = ObjectSize; + SetFlags(mpInfoLastBlock, mInfoLastBlockEntries, IsFile?(0):(Flags_IsDir)); + + // Increment size + ++mInfoLastBlockEntries; + + // Store last ID + mLastIDInInfo = ID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::LookupID(BackupStoreCheck_ID_t, int32_t +// Purpose: Look up an ID. Return the block it's in, or zero if not found, and the +// index within that block if the thing is found. +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut) +{ + IDBlock *pblock = 0; + + // Find the lower matching block who's first entry is not less than ID + Info_t::const_iterator ib(mInfo.lower_bound(ID)); + + // Was there a block + if(ib == mInfo.end()) + { + // Block wasn't found... could be in last block + pblock = mpInfoLastBlock; + } + else + { + // Found it as first entry? + if(ib->first == ID) + { + rIndexOut = 0; + return ib->second; + } + + // Go back one block as it's not the first entry in this one + if(ib == mInfo.begin()) + { + // Was first block, can't go back + return 0; + } + // Go back... + --ib; + + // So, the ID will be in this block, if it's in anything + pblock = ib->second; + } + + ASSERT(pblock != 0); + if(pblock == 0) return 0; + + // How many entries are there in the block + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + + // Do binary search within block + int high = bentries; + int low = -1; + while(high - low > 1) + { + int i = (high + low) / 2; + if(ID <= pblock->mID[i]) + { + high = i; + } + else + { + low = i; + } + } + if(ID == pblock->mID[high]) + { + // Found + rIndexOut = high; + return pblock; + } + + // Not found + return 0; +} + + +#ifndef BOX_RELEASE_BUILD +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::DumpObjectInfo() +// Purpose: Debug only. Trace out all object info. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::DumpObjectInfo() +{ + for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + IDBlock *pblock = i->second; + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + BOX_TRACE("BLOCK @ " << BOX_FORMAT_HEX32(pblock) << + ", " << bentries << " entries"); + + for(int e = 0; e < bentries; ++e) + { + uint8_t flags = GetFlags(pblock, e); + BOX_TRACE(std::hex << + "id " << pblock->mID[e] << + ", c " << pblock->mContainer[e] << + ", " << ((flags & Flags_IsDir)?"dir":"file") << + ", " << ((flags & Flags_IsContained) ? + "contained":"unattached")); + } + } +} +#endif + diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp new file mode 100644 index 00000000..cc6efcf5 --- /dev/null +++ b/lib/backupstore/BackupStoreConfigVerify.cpp @@ -0,0 +1,57 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreConfigVerify.h +// Purpose: Configuration definition for the backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreConfigVerify.h" +#include "ServerTLS.h" +#include "BoxPortsAndFiles.h" + +#include "MemLeakFindOn.h" + +static const ConfigurationVerifyKey verifyserverkeys[] = +{ + SERVERTLS_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) + // no default listen addresses +}; + +static const ConfigurationVerify verifyserver[] = +{ + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } +}; + +static const ConfigurationVerifyKey verifyrootkeys[] = +{ + ConfigurationVerifyKey("AccountDatabase", ConfigTest_Exists), + ConfigurationVerifyKey("TimeBetweenHousekeeping", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), + // make value "yes" to enable in config file + + #ifdef WIN32 + ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry) + #else + ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry, + BOX_FILE_RAIDFILE_DEFAULT_CONFIG) + #endif +}; + +const ConfigurationVerify BackupConfigFileVerify = +{ + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 +}; diff --git a/lib/backupstore/BackupStoreConfigVerify.h b/lib/backupstore/BackupStoreConfigVerify.h new file mode 100644 index 00000000..815cfaed --- /dev/null +++ b/lib/backupstore/BackupStoreConfigVerify.h @@ -0,0 +1,18 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreConfigVerify.h +// Purpose: Configuration definition for the backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTORECONFIGVERIFY__H +#define BACKUPSTORECONFIGVERIFY__H + +#include "Configuration.h" + +extern const ConfigurationVerify BackupConfigFileVerify; + +#endif // BACKUPSTORECONFIGVERIFY__H + diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp new file mode 100644 index 00000000..1d55fdf0 --- /dev/null +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -0,0 +1,593 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreInfo.cpp +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <algorithm> + +#include "BackupStoreInfo.h" +#include "BackupStoreException.h" +#include "RaidFileWrite.h" +#include "RaidFileRead.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +// ****************** +// make sure the defaults in CreateNew are modified! +// ****************** +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mAccountID; + int64_t mClientStoreMarker; + int64_t mLastObjectIDUsed; + int64_t mBlocksUsed; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; + int64_t mBlocksSoftLimit; + int64_t mBlocksHardLimit; + uint32_t mCurrentMarkNumber; + uint32_t mOptionsPresent; // bit mask of optional elements present + int64_t mNumberDeletedDirectories; + // Then loads of int64_t IDs for the deleted directories +} info_StreamFormat; + +#define INFO_MAGIC_VALUE 0x34832476 + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +#ifdef BOX_RELEASE_BUILD + #define NUM_DELETED_DIRS_BLOCK 256 +#else + #define NUM_DELETED_DIRS_BLOCK 2 +#endif + +#define INFO_FILENAME "info" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::BackupStoreInfo() +// Purpose: Default constructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreInfo::BackupStoreInfo() + : mAccountID(-1), + mDiscSet(-1), + mReadOnly(true), + mIsModified(false), + mClientStoreMarker(0), + mLastObjectIDUsed(-1), + mBlocksUsed(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::~BackupStoreInfo +// Purpose: Destructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreInfo::~BackupStoreInfo() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::CreateNew(int32_t, const std::string &, int) +// Purpose: Create a new info file on disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit) +{ + // Initial header (is entire file) + info_StreamFormat hdr = { + htonl(INFO_MAGIC_VALUE), // mMagicValue + htonl(AccountID), // mAccountID + 0, // mClientStoreMarker + box_hton64(1), // mLastObjectIDUsed (which is the root directory) + 0, // mBlocksUsed + 0, // mBlocksInOldFiles + 0, // mBlocksInDeletedFiles + 0, // mBlocksInDirectories + box_hton64(BlockSoftLimit), // mBlocksSoftLimit + box_hton64(BlockHardLimit), // mBlocksHardLimit + 0, // mCurrentMarkNumber + 0, // mOptionsPresent + 0 // mNumberDeletedDirectories + }; + + // Generate the filename + ASSERT(rRootDir[rRootDir.size() - 1] == '/' || + rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); + std::string fn(rRootDir + INFO_FILENAME); + + // Open the file for writing + RaidFileWrite rf(DiscSet, fn); + rf.Open(false); // no overwriting, as this is a new file + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Done. +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::Load(int32_t, const std::string &, int, bool) +// Purpose: Loads the info from disc, given the root information. Can be marked as read only. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID) +{ + // Generate the filename + std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME); + + // Open the file for reading (passing on optional request for revision ID) + std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID)); + + // Read in a header + info_StreamFormat hdr; + if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Check it + if(ntohl(hdr.mMagicValue) != INFO_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != AccountID) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + + // Make new object + std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); + + // Put in basic location info + info->mAccountID = AccountID; + info->mDiscSet = DiscSet; + info->mFilename = fn; + info->mReadOnly = ReadOnly; + + // Insert info from file + info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker); + info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed); + info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); + info->mBlocksInOldFiles = box_ntoh64(hdr.mBlocksInOldFiles); + info->mBlocksInDeletedFiles = box_ntoh64(hdr.mBlocksInDeletedFiles); + info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories); + info->mBlocksSoftLimit = box_ntoh64(hdr.mBlocksSoftLimit); + info->mBlocksHardLimit = box_ntoh64(hdr.mBlocksHardLimit); + + // Load up array of deleted objects + int64_t numDelObj = box_ntoh64(hdr.mNumberDeletedDirectories); + + // Then load them in + if(numDelObj > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int64_t toload = numDelObj; + while(toload > 0) + { + // How many in this one? + int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload)); + + if(!rf->ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Add them + for(int t = 0; t < b; ++t) + { + info->mDeletedDirectories.push_back(box_ntoh64(objs[t])); + } + + // Number loaded + toload -= b; + } + } + + // Final check + if(static_cast<int64_t>(info->mDeletedDirectories.size()) != numDelObj) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + + // return it to caller + return info; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::CreateForRegeneration(...) +// Purpose: Return an object which can be used to save for regeneration. +// Created: 23/4/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration(int32_t AccountID, const std::string &rRootDir, + int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles, + int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit) +{ + // Generate the filename + std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME); + + // Make new object + std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); + + // Put in basic info + info->mAccountID = AccountID; + info->mDiscSet = DiscSet; + info->mFilename = fn; + info->mReadOnly = false; + + // Insert info starting info + info->mClientStoreMarker = 0; + info->mLastObjectIDUsed = LastObjectID; + info->mBlocksUsed = BlocksUsed; + info->mBlocksInOldFiles = BlocksInOldFiles; + info->mBlocksInDeletedFiles = BlocksInDeletedFiles; + info->mBlocksInDirectories = BlocksInDirectories; + info->mBlocksSoftLimit = BlockSoftLimit; + info->mBlocksHardLimit = BlockHardLimit; + + // return it to caller + return info; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::Save() +// Purpose: Save modified info back to disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::Save() +{ + // Make sure we're initialised (although should never come to this) + if(mFilename.empty() || mAccountID == -1 || mDiscSet == -1) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Can we do this? + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Then... open a write file + RaidFileWrite rf(mDiscSet, mFilename); + rf.Open(true); // allow overwriting + + // Make header + info_StreamFormat hdr; + hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); + hdr.mAccountID = htonl(mAccountID); + hdr.mClientStoreMarker = box_hton64(mClientStoreMarker); + hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed); + hdr.mBlocksUsed = box_hton64(mBlocksUsed); + hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles); + hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles); + hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories); + hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit); + hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit); + hdr.mCurrentMarkNumber = 0; + hdr.mOptionsPresent = 0; + hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size()); + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Write the deleted object list + if(mDeletedDirectories.size() > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int tosave = mDeletedDirectories.size(); + std::vector<int64_t>::iterator i(mDeletedDirectories.begin()); + while(tosave > 0) + { + // How many in this one? + int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); + + // Add them + for(int t = 0; t < b; ++t) + { + ASSERT(i != mDeletedDirectories.end()); + objs[t] = box_hton64((*i)); + i++; + } + + // Write + rf.Write(objs, b * sizeof(int64_t)); + + // Number saved + tosave -= b; + } + } + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Mark is as not modified + mIsModified = false; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksUsed(int32_t) +// Purpose: Change number of blocks used, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksUsed + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksUsed += Delta; + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksInOldFiles(int32_t) +// Purpose: Change number of blocks in old files, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInOldFiles + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInOldFiles += Delta; + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksInDeletedFiles(int32_t) +// Purpose: Change number of blocks in deleted files, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInDeletedFiles + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInDeletedFiles += Delta; + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksInDirectories(int32_t) +// Purpose: Change number of blocks in directories, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInDirectories + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInDirectories += Delta; + + mIsModified = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::CorrectAllUsedValues(int64_t, int64_t, int64_t, int64_t) +// Purpose: Set all the usage counts to specific values -- use for correcting in housekeeping +// if something when wrong during the backup connection, and the store info wasn't +// saved back to disc. +// Created: 15/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Set the values + mBlocksUsed = Used; + mBlocksInOldFiles = InOldFiles; + mBlocksInDeletedFiles = InDeletedFiles; + mBlocksInDirectories = InDirectories; + + mIsModified = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::AddDeletedDirectory(int64_t) +// Purpose: Add a directory ID to the deleted list +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::AddDeletedDirectory(int64_t DirID) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + mDeletedDirectories.push_back(DirID); + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::RemovedDeletedDirectory(int64_t) +// Purpose: Remove a directory from the deleted list +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::RemovedDeletedDirectory(int64_t DirID) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + std::vector<int64_t>::iterator i(std::find(mDeletedDirectories.begin(), mDeletedDirectories.end(), DirID)); + if(i == mDeletedDirectories.end()) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoDirNotInList) + } + mDeletedDirectories.erase(i); + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeLimits(int64_t, int64_t) +// Purpose: Change the soft and hard limits +// Created: 15/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + mBlocksSoftLimit = BlockSoftLimit; + mBlocksHardLimit = BlockHardLimit; + + mIsModified = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::AllocateObjectID() +// Purpose: Allocate an ID for a new object in the store. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreInfo::AllocateObjectID() +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if(mLastObjectIDUsed < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotInitialised) + } + + // Return the next object ID + return ++mLastObjectIDUsed; + + mIsModified = true; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::SetClientStoreMarker(int64_t) +// Purpose: Sets the client store marker +// Created: 2003/10/29 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + mClientStoreMarker = ClientStoreMarker; + + mIsModified = true; +} + + + diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h new file mode 100644 index 00000000..a94ca9d6 --- /dev/null +++ b/lib/backupstore/BackupStoreInfo.h @@ -0,0 +1,111 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreInfo.h +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREINFO__H +#define BACKUPSTOREINFO__H + +#include <memory> +#include <string> +#include <vector> + +class BackupStoreCheck; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreInfo +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +class BackupStoreInfo +{ + friend class BackupStoreCheck; +public: + ~BackupStoreInfo(); +private: + // Creation through static functions only + BackupStoreInfo(); + // No copying allowed + BackupStoreInfo(const BackupStoreInfo &); + +public: + // Create a New account, saving a blank info object to the disc + static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit); + + // Load it from the store + static std::auto_ptr<BackupStoreInfo> Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0); + + // Has info been modified? + bool IsModified() const {return mIsModified;} + + // Save modified infomation back to store + void Save(); + + // Data access functions + int32_t GetAccountID() const {return mAccountID;} + int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;} + int64_t GetBlocksUsed() const {return mBlocksUsed;} + int64_t GetBlocksInOldFiles() const {return mBlocksInOldFiles;} + int64_t GetBlocksInDeletedFiles() const {return mBlocksInDeletedFiles;} + int64_t GetBlocksInDirectories() const {return mBlocksInDirectories;} + const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;} + int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;} + int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;} + bool IsReadOnly() const {return mReadOnly;} + int GetDiscSetNumber() const {return mDiscSet;} + + // Data modification functions + void ChangeBlocksUsed(int64_t Delta); + void ChangeBlocksInOldFiles(int64_t Delta); + void ChangeBlocksInDeletedFiles(int64_t Delta); + void ChangeBlocksInDirectories(int64_t Delta); + void CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories); + void AddDeletedDirectory(int64_t DirID); + void RemovedDeletedDirectory(int64_t DirID); + void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit); + + // Object IDs + int64_t AllocateObjectID(); + + // Client marker set and get + int64_t GetClientStoreMarker() {return mClientStoreMarker;} + void SetClientStoreMarker(int64_t ClientStoreMarker); + +private: + static std::auto_ptr<BackupStoreInfo> CreateForRegeneration(int32_t AccountID, const std::string &rRootDir, + int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles, + int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit); + +private: + // Location information + int32_t mAccountID; + int mDiscSet; + std::string mFilename; + bool mReadOnly; + bool mIsModified; + + // Client infomation + int64_t mClientStoreMarker; + + // Account information + int64_t mLastObjectIDUsed; + int64_t mBlocksUsed; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; + int64_t mBlocksSoftLimit; + int64_t mBlocksHardLimit; + std::vector<int64_t> mDeletedDirectories; +}; + + +#endif // BACKUPSTOREINFO__H + + diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp new file mode 100644 index 00000000..f6db2ca4 --- /dev/null +++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp @@ -0,0 +1,321 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreRefCountDatabase.cpp +// Purpose: Backup store object reference count database storage +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <algorithm> + +#include "BackupStoreRefCountDatabase.h" +#include "BackupStoreException.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreAccounts.h" +#include "RaidFileController.h" +#include "RaidFileUtil.h" +#include "RaidFileException.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +#define REFCOUNT_MAGIC_VALUE 0x52656643 // RefC +#define REFCOUNT_FILENAME "refcount" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::BackupStoreRefCountDatabase() +// Purpose: Default constructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const + BackupStoreAccountDatabase::Entry& rAccount) +: mAccount(rAccount), + mFilename(GetFilename(rAccount)), + mReadOnly(true), + mIsModified(false) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase +// Purpose: Destructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase() +{ +} + +std::string BackupStoreRefCountDatabase::GetFilename(const + BackupStoreAccountDatabase::Entry& rAccount) +{ + std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount); + ASSERT(RootDir[RootDir.size() - 1] == '/' || + RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); + + std::string fn(RootDir + "refcount.db"); + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet())); + return RaidFileUtil::MakeWriteFileName(rdiscSet, fn); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Create(int32_t, +// const std::string &, int, bool) +// Purpose: Create a new database, overwriting an existing +// one only if AllowOverwrite is true. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreRefCountDatabase::Create(const + BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite) +{ + // Initial header + refcount_StreamFormat hdr; + hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE); + hdr.mAccountID = htonl(rAccount.GetID()); + + // Generate the filename + std::string Filename = GetFilename(rAccount); + + // Open the file for writing + if (FileExists(Filename) && !AllowOverwrite) + { + BOX_ERROR("Attempted to overwrite refcount database file: " << + Filename); + THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile); + } + + int flags = O_CREAT | O_BINARY | O_RDWR; + if (!AllowOverwrite) + { + flags |= O_EXCL; + } + + std::auto_ptr<FileStream> DatabaseFile(new FileStream(Filename, flags)); + + // Write header + DatabaseFile->Write(&hdr, sizeof(hdr)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Load(int32_t AccountID, +// BackupStoreAccountDatabase& rAccountDatabase, +// bool ReadOnly); +// Purpose: Loads the info from disc, given the root +// information. Can be marked as read only. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( + const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly) +{ + // Generate the filename + std::string filename = GetFilename(rAccount); + int flags = ReadOnly ? O_RDONLY : O_RDWR; + + // Open the file for read/write + std::auto_ptr<FileStream> dbfile(new FileStream(filename, + flags | O_BINARY)); + + // Read in a header + refcount_StreamFormat hdr; + if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Check it + if(ntohl(hdr.mMagicValue) != REFCOUNT_MAGIC_VALUE || + (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID()) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + + // Make new object + std::auto_ptr<BackupStoreRefCountDatabase> refcount(new BackupStoreRefCountDatabase(rAccount)); + + // Put in basic location info + refcount->mReadOnly = ReadOnly; + refcount->mapDatabaseFile = dbfile; + + // return it to caller + return refcount; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Save() +// Purpose: Save modified info back to disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +/* +void BackupStoreRefCountDatabase::Save() +{ + // Make sure we're initialised (although should never come to this) + if(mFilename.empty() || mAccount.GetID() == 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Can we do this? + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Then... open a write file + RaidFileWrite rf(mAccount.GetDiscSet(), mFilename); + rf.Open(true); // allow overwriting + + // Make header + info_StreamFormat hdr; + hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); + hdr.mAccountID = htonl(mAccountID); + hdr.mClientStoreMarker = box_hton64(mClientStoreMarker); + hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed); + hdr.mBlocksUsed = box_hton64(mBlocksUsed); + hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles); + hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles); + hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories); + hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit); + hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit); + hdr.mCurrentMarkNumber = 0; + hdr.mOptionsPresent = 0; + hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size()); + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Write the deleted object list + if(mDeletedDirectories.size() > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int tosave = mDeletedDirectories.size(); + std::vector<int64_t>::iterator i(mDeletedDirectories.begin()); + while(tosave > 0) + { + // How many in this one? + int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); + + // Add them + for(int t = 0; t < b; ++t) + { + ASSERT(i != mDeletedDirectories.end()); + objs[t] = box_hton64((*i)); + i++; + } + + // Write + rf.Write(objs, b * sizeof(int64_t)); + + // Number saved + tosave -= b; + } + } + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Mark is as not modified + mIsModified = false; +} +*/ + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::GetRefCount(int64_t +// ObjectID) +// Purpose: Get the number of references to the specified object +// out of the database +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- +BackupStoreRefCountDatabase::refcount_t +BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const +{ + IOStream::pos_type offset = GetOffset(ObjectID); + + if (GetSize() < offset + GetEntrySize()) + { + BOX_ERROR("attempted read of unknown refcount for object " << + BOX_FORMAT_OBJECTID(ObjectID)); + THROW_EXCEPTION(BackupStoreException, + UnknownObjectRefCountRequested); + } + + mapDatabaseFile->Seek(offset, SEEK_SET); + + refcount_t refcount; + if (mapDatabaseFile->Read(&refcount, sizeof(refcount)) != + sizeof(refcount)) + { + BOX_LOG_SYS_ERROR("short read on refcount database: " << + mFilename); + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo); + } + + return ntohl(refcount); +} + +int64_t BackupStoreRefCountDatabase::GetLastObjectIDUsed() const +{ + return (GetSize() - sizeof(refcount_StreamFormat)) / + sizeof(refcount_t); +} + +void BackupStoreRefCountDatabase::AddReference(int64_t ObjectID) +{ + refcount_t refcount; + + if (ObjectID > GetLastObjectIDUsed()) + { + // new object, assume no previous references + refcount = 0; + } + else + { + // read previous value from database + refcount = GetRefCount(ObjectID); + } + + refcount++; + + SetRefCount(ObjectID, refcount); +} + +void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID, + refcount_t NewRefCount) +{ + IOStream::pos_type offset = GetOffset(ObjectID); + mapDatabaseFile->Seek(offset, SEEK_SET); + refcount_t RefCountNetOrder = htonl(NewRefCount); + mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder)); +} + +bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) +{ + refcount_t refcount = GetRefCount(ObjectID); // must exist in database + ASSERT(refcount > 0); + refcount--; + SetRefCount(ObjectID, refcount); + return (refcount > 0); +} + diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h new file mode 100644 index 00000000..93c79afb --- /dev/null +++ b/lib/backupstore/BackupStoreRefCountDatabase.h @@ -0,0 +1,128 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreRefCountDatabase.h +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREREFCOUNTDATABASE__H +#define BACKUPSTOREREFCOUNTDATABASE__H + +#include <memory> +#include <string> +#include <vector> + +#include "BackupStoreAccountDatabase.h" +#include "FileStream.h" + +class BackupStoreCheck; +class BackupStoreContext; + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + uint32_t mMagicValue; // also the version number + uint32_t mAccountID; +} refcount_StreamFormat; + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreRefCountDatabase +// Purpose: Backup store reference count database storage +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- +class BackupStoreRefCountDatabase +{ + friend class BackupStoreCheck; + friend class BackupStoreContext; + friend class HousekeepStoreAccount; + +public: + ~BackupStoreRefCountDatabase(); +private: + // Creation through static functions only + BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry& + rAccount); + // No copying allowed + BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &); + +public: + // Create a new database for a new account. This method will refuse + // to overwrite any existing file. + static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount) + { + Create(rAccount, false); + } + + // Load it from the store + static std::auto_ptr<BackupStoreRefCountDatabase> Load(const + BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); + + typedef uint32_t refcount_t; + + // Data access functions + refcount_t GetRefCount(int64_t ObjectID) const; + int64_t GetLastObjectIDUsed() const; + + // Data modification functions + void AddReference(int64_t ObjectID); + // RemoveReference returns false if refcount drops to zero + bool RemoveReference(int64_t ObjectID); + +private: + // Create a new database for an existing account. Used during + // account checking if opening the old database throws an exception. + // This method will overwrite any existing file. + static void CreateForRegeneration(const + BackupStoreAccountDatabase::Entry& rAccount) + { + Create(rAccount, true); + } + + static void Create(const BackupStoreAccountDatabase::Entry& rAccount, + bool AllowOverwrite); + + static std::string GetFilename(const BackupStoreAccountDatabase::Entry& + rAccount); + IOStream::pos_type GetSize() const + { + return mapDatabaseFile->GetPosition() + + mapDatabaseFile->BytesLeftToRead(); + } + IOStream::pos_type GetEntrySize() const + { + return sizeof(refcount_t); + } + IOStream::pos_type GetOffset(int64_t ObjectID) const + { + return ((ObjectID - 1) * GetEntrySize()) + + sizeof(refcount_StreamFormat); + } + void SetRefCount(int64_t ObjectID, refcount_t NewRefCount); + + // Location information + BackupStoreAccountDatabase::Entry mAccount; + std::string mFilename; + bool mReadOnly; + bool mIsModified; + std::auto_ptr<FileStream> mapDatabaseFile; +}; + +#endif // BACKUPSTOREREFCOUNTDATABASE__H diff --git a/lib/backupstore/StoreStructure.cpp b/lib/backupstore/StoreStructure.cpp new file mode 100644 index 00000000..45a1ce91 --- /dev/null +++ b/lib/backupstore/StoreStructure.cpp @@ -0,0 +1,95 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreStructure.cpp +// Purpose: +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "RaidFileController.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StoreStructure::MakeObjectFilename(int64_t, const std::string &, int, std::string &, bool) +// Purpose: Builds the object filename for a given object, given a root. Optionally ensure that the +// directory exists. +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void StoreStructure::MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists) +{ + const static char *hex = "0123456789abcdef"; + + // Set output to root string + rFilenameOut = rStoreRoot; + + // get the id value from the stored object ID so we can do + // bitwise operations on it. + uint64_t id = (uint64_t)ObjectID; + + // get leafname, shift the bits which make up the leafname off + unsigned int leafname(id & STORE_ID_SEGMENT_MASK); + id >>= STORE_ID_SEGMENT_LENGTH; + + // build pathname + while(id != 0) + { + // assumes that the segments are no bigger than 8 bits + int v = id & STORE_ID_SEGMENT_MASK; + rFilenameOut += hex[(v & 0xf0) >> 4]; + rFilenameOut += hex[v & 0xf]; + rFilenameOut += DIRECTORY_SEPARATOR_ASCHAR; + + // shift the bits we used off the pathname + id >>= STORE_ID_SEGMENT_LENGTH; + } + + // Want to make sure this exists? + if(EnsureDirectoryExists) + { + if(!RaidFileRead::DirectoryExists(DiscSet, rFilenameOut)) + { + // Create it + RaidFileWrite::CreateDirectory(DiscSet, rFilenameOut, true /* recusive */); + } + } + + // append the filename + rFilenameOut += 'o'; + rFilenameOut += hex[(leafname & 0xf0) >> 4]; + rFilenameOut += hex[leafname & 0xf]; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StoreStructure::MakeWriteLockFilename(const std::string &, int, std::string &) +// Purpose: Generate the on disc filename of the write lock file +// Created: 15/12/03 +// +// -------------------------------------------------------------------------- +void StoreStructure::MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut) +{ + // Find the disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(DiscSet)); + + // Make the filename + std::string writeLockFile(rdiscSet[0] + DIRECTORY_SEPARATOR + rStoreRoot + "write.lock"); + + // Return it to the caller + rFilenameOut = writeLockFile; +} + + diff --git a/lib/backupstore/StoreStructure.h b/lib/backupstore/StoreStructure.h new file mode 100644 index 00000000..ffbe83dd --- /dev/null +++ b/lib/backupstore/StoreStructure.h @@ -0,0 +1,32 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreStructure.h +// Purpose: Functions for placing files in the store +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef STORESTRUCTURE__H +#define STORESTRUCTURE__H + +#include <string> + +#ifdef BOX_RELEASE_BUILD + #define STORE_ID_SEGMENT_LENGTH 8 + #define STORE_ID_SEGMENT_MASK 0xff +#else + // Debug we'll use lots and lots of directories to stress things + #define STORE_ID_SEGMENT_LENGTH 2 + #define STORE_ID_SEGMENT_MASK 0x03 +#endif + + +namespace StoreStructure +{ + void MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists); + void MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut); +}; + +#endif // STORESTRUCTURE__H + diff --git a/lib/common/Archive.h b/lib/common/Archive.h new file mode 100644 index 00000000..b70f12c4 --- /dev/null +++ b/lib/common/Archive.h @@ -0,0 +1,161 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Archive.h +// Purpose: Backup daemon state archive +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- + +#ifndef ARCHIVE__H +#define ARCHIVE__H + +#include <vector> +#include <string> +#include <memory> + +#include "IOStream.h" +#include "Guards.h" + +#define ARCHIVE_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2) + +#define ARCHIVE_MAGIC_VALUE_RECURSE 0x4449525F +#define ARCHIVE_MAGIC_VALUE_NOOP 0x5449525F + +class Archive +{ +public: + Archive(IOStream &Stream, int Timeout) + : mrStream(Stream) + { + mTimeout = Timeout; + } +private: + // no copying + Archive(const Archive &); + Archive & operator=(const Archive &); +public: + ~Archive() + { + } + // + // + // + void Write(bool Item) + { + Write((int) Item); + } + void Write(int Item) + { + int32_t privItem = htonl(Item); + mrStream.Write(&privItem, sizeof(privItem)); + } + void Write(int64_t Item) + { + int64_t privItem = box_hton64(Item); + mrStream.Write(&privItem, sizeof(privItem)); + } + void Write(uint64_t Item) + { + uint64_t privItem = box_hton64(Item); + mrStream.Write(&privItem, sizeof(privItem)); + } + void Write(uint8_t Item) + { + int privItem = Item; + Write(privItem); + } + void Write(const std::string &Item) + { + int size = Item.size(); + Write(size); + mrStream.Write(Item.c_str(), size); + } + // + // + // + void Read(bool &rItemOut) + { + int privItem; + Read(privItem); + + if (privItem) + { + rItemOut = true; + } + else + { + rItemOut = false; + } + } + void Read(int &rItemOut) + { + int32_t privItem; + if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + } + rItemOut = ntohl(privItem); + } + void Read(int64_t &rItemOut) + { + int64_t privItem; + if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + } + rItemOut = box_ntoh64(privItem); + } + void Read(uint64_t &rItemOut) + { + uint64_t privItem; + if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + } + rItemOut = box_ntoh64(privItem); + } + void Read(uint8_t &rItemOut) + { + int privItem; + Read(privItem); + rItemOut = privItem; + } + void Read(std::string &rItemOut) + { + int size; + Read(size); + + // Assume most strings are relatively small + char buf[256]; + if(size < (int) sizeof(buf)) + { + // Fetch rest of pPayload, relying on the Protocol to error on stupidly large sizes for us + if(!mrStream.ReadFullBuffer(buf, size, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + } + // assign to this string, storing the header and the extra payload + rItemOut.assign(buf, size); + } + else + { + // Block of memory to hold it + MemoryBlockGuard<char*> dataB(size); + char *ppayload = dataB; + + // Fetch rest of pPayload, relying on the Protocol to error on stupidly large sizes for us + if(!mrStream.ReadFullBuffer(ppayload, size, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + } + // assign to this string, storing the header and the extra pPayload + rItemOut.assign(ppayload, size); + } + } +private: + IOStream &mrStream; + int mTimeout; +}; + +#endif // ARCHIVE__H diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h new file mode 100644 index 00000000..e40224da --- /dev/null +++ b/lib/common/BannerText.h @@ -0,0 +1,18 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BannerText.h +// Purpose: Banner text for daemons and utilities +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef BANNERTEXT__H +#define BANNERTEXT__H + +#define BANNER_TEXT(UtilityName) \ + "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers and " \ + "contributors 2003-2010" + +#endif // BANNERTEXT__H + diff --git a/lib/common/BeginStructPackForWire.h b/lib/common/BeginStructPackForWire.h new file mode 100644 index 00000000..e73bb886 --- /dev/null +++ b/lib/common/BeginStructPackForWire.h @@ -0,0 +1,23 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BeginStructPackForWire.h +// Purpose: Begin structure packing for wire +// Created: 25/11/03 +// +// -------------------------------------------------------------------------- + +// No header guard -- this is intentional + +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS + +#pragma pack(1) + +#else + + logical error -- check BoxPlatform.h and including file + +#endif + + + diff --git a/lib/common/Box.h b/lib/common/Box.h new file mode 100644 index 00000000..158fab7b --- /dev/null +++ b/lib/common/Box.h @@ -0,0 +1,185 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Box.h +// Purpose: Main header file for the Box project +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef BOX__H +#define BOX__H + +// Use the same changes as gcc3 for gcc4 +#ifdef PLATFORM_GCC4 + #define PLATFORM_GCC3 +#endif + +#include "BoxPlatform.h" + +// uncomment this line to enable full memory leak finding on all +// malloc-ed blocks (at least, ones used by the STL) +//#define MEMLEAKFINDER_FULL_MALLOC_MONITORING + +// Show backtraces on exceptions in release builds until further notice +// (they are only logged at TRACE level anyway) +#ifdef HAVE_EXECINFO_H + #define SHOW_BACKTRACE_ON_EXCEPTION +#endif + +#ifdef SHOW_BACKTRACE_ON_EXCEPTION + #include "Utils.h" + #define OPTIONAL_DO_BACKTRACE DumpStackBacktrace(); +#else + #define OPTIONAL_DO_BACKTRACE +#endif + +#include "CommonException.h" +#include "Logging.h" + +#ifndef BOX_RELEASE_BUILD + + extern bool AssertFailuresToSyslog; + #define ASSERT_FAILS_TO_SYSLOG_ON {AssertFailuresToSyslog = true;} + void BoxDebugAssertFailed(const char *cond, const char *file, int line); + #define ASSERT(cond) \ + { \ + if(!(cond)) \ + { \ + BoxDebugAssertFailed(#cond, __FILE__, __LINE__); \ + THROW_EXCEPTION_MESSAGE(CommonException, \ + AssertFailed, #cond); \ + } \ + } + + // Note that syslog tracing is independent of BoxDebugTraceOn, + // but stdout tracing is not + extern bool BoxDebugTraceToSyslog; + #define TRACE_TO_SYSLOG(x) {BoxDebugTraceToSyslog = x;} + extern bool BoxDebugTraceToStdout; + #define TRACE_TO_STDOUT(x) {BoxDebugTraceToStdout = x;} + + extern bool BoxDebugTraceOn; + int BoxDebug_printf(const char *format, ...); + int BoxDebugTrace(const char *format, ...); + + #ifndef PLATFORM_DISABLE_MEM_LEAK_TESTING + #define BOX_MEMORY_LEAK_TESTING + #endif + + // Exception names + #define EXCEPTION_CODENAMES_EXTENDED + +#else + #define ASSERT_FAILS_TO_SYSLOG_ON + #define ASSERT(cond) + + #define TRACE_TO_SYSLOG(x) {} + #define TRACE_TO_STDOUT(x) {} + + // Box Backup builds release get extra information for exception logging + #define EXCEPTION_CODENAMES_EXTENDED + #define EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION + +#endif + +#ifdef BOX_MEMORY_LEAK_TESTING + // Memory leak testing + #include "MemLeakFinder.h" + #define DEBUG_NEW new(__FILE__,__LINE__) + #define MEMLEAKFINDER_NOT_A_LEAK(x) memleakfinder_notaleak(x); + #define MEMLEAKFINDER_NO_LEAKS MemLeakSuppressionGuard _guard; + #define MEMLEAKFINDER_INIT memleakfinder_init(); + #define MEMLEAKFINDER_START {memleakfinder_global_enable = true;} + #define MEMLEAKFINDER_STOP {memleakfinder_global_enable = false;} +#else + #define DEBUG_NEW new + #define MEMLEAKFINDER_NOT_A_LEAK(x) + #define MEMLEAKFINDER_NO_LEAKS + #define MEMLEAKFINDER_INIT + #define MEMLEAKFINDER_START + #define MEMLEAKFINDER_STOP +#endif + +#define THROW_EXCEPTION(type, subtype) \ + { \ + if(!HideExceptionMessageGuard::ExceptionsHidden()) \ + { \ + OPTIONAL_DO_BACKTRACE \ + BOX_WARNING("Exception thrown: " \ + #type "(" #subtype ") " \ + "at " __FILE__ "(" << __LINE__ << ")") \ + } \ + throw type(type::subtype); \ + } + +#define THROW_EXCEPTION_MESSAGE(type, subtype, message) \ + { \ + std::ostringstream _box_throw_line; \ + _box_throw_line << message; \ + if(!HideExceptionMessageGuard::ExceptionsHidden()) \ + { \ + OPTIONAL_DO_BACKTRACE \ + BOX_WARNING("Exception thrown: " \ + #type "(" #subtype ") (" << message << \ + ") at " __FILE__ "(" << __LINE__ << ")") \ + } \ + throw type(type::subtype, _box_throw_line.str()); \ + } + +// extra macros for converting to network byte order +#ifdef HAVE_NETINET_IN_H + #include <netinet/in.h> +#endif + +// Always define a swap64 function, as it's useful. +inline uint64_t box_swap64(uint64_t x) +{ + return ((x & 0xff) << 56 | + (x & 0xff00LL) << 40 | + (x & 0xff0000LL) << 24 | + (x & 0xff000000LL) << 8 | + (x & 0xff00000000LL) >> 8 | + (x & 0xff0000000000LL) >> 24 | + (x & 0xff000000000000LL) >> 40 | + (x & 0xff00000000000000LL) >> 56); +} + +#ifdef WORDS_BIGENDIAN + #define box_hton64(x) (x) + #define box_ntoh64(x) (x) +#elif defined(HAVE_BSWAP64) + #ifdef HAVE_SYS_ENDIAN_H + #include <sys/endian.h> + #endif + #ifdef HAVE_ASM_BYTEORDER_H + #include <asm/byteorder.h> + #endif + + #define box_hton64(x) BSWAP64(x) + #define box_ntoh64(x) BSWAP64(x) +#else + #define box_hton64(x) box_swap64(x) + #define box_ntoh64(x) box_swap64(x) +#endif + +// overloaded auto-conversion functions +inline uint64_t hton(uint64_t in) { return box_hton64(in); } +inline uint32_t hton(uint32_t in) { return htonl(in); } +inline uint16_t hton(uint16_t in) { return htons(in); } +inline uint8_t hton(uint8_t in) { return in; } +inline int64_t hton(int64_t in) { return box_hton64(in); } +inline int32_t hton(int32_t in) { return htonl(in); } +inline int16_t hton(int16_t in) { return htons(in); } +inline int8_t hton(int8_t in) { return in; } +inline uint64_t ntoh(uint64_t in) { return box_ntoh64(in); } +inline uint32_t ntoh(uint32_t in) { return ntohl(in); } +inline uint16_t ntoh(uint16_t in) { return ntohs(in); } +inline uint8_t ntoh(uint8_t in) { return in; } +inline int64_t ntoh(int64_t in) { return box_ntoh64(in); } +inline int32_t ntoh(int32_t in) { return ntohl(in); } +inline int16_t ntoh(int16_t in) { return ntohs(in); } +inline int8_t ntoh(int8_t in) { return in; } + +#endif // BOX__H + diff --git a/lib/common/BoxConfig-MSVC.h b/lib/common/BoxConfig-MSVC.h new file mode 100644 index 00000000..bb3ffb30 --- /dev/null +++ b/lib/common/BoxConfig-MSVC.h @@ -0,0 +1,402 @@ +/* lib/common/BoxConfig.h. Generated by configure. */ +/* lib/common/BoxConfig.h.in. Generated from configure.ac by autoheader. */ +/* Hacked by hand to work for MSVC by Chris Wilson */ + +/* Define to major version for BDB_VERSION */ +/* #undef BDB_VERSION_MAJOR */ + +/* Define to minor version for BDB_VERSION */ +/* #undef BDB_VERSION_MINOR */ + +/* Define to point version for BDB_VERSION */ +/* #undef BDB_VERSION_POINT */ + +/* Name of the 64 bit endian swapping function */ +/* #undef BSWAP64 */ + +/* Define to 1 if the `closedir' function returns void instead of `int'. */ +#define CLOSEDIR_VOID 1 + +/* Define to 1 if non-aligned int16 access will fail */ +/* #undef HAVE_ALIGNED_ONLY_INT16 */ + +/* Define to 1 if non-aligned int32 access will fail */ +/* #undef HAVE_ALIGNED_ONLY_INT32 */ + +/* Define to 1 if non-aligned int64 access will fail */ +/* #undef HAVE_ALIGNED_ONLY_INT64 */ + +/* Define to 1 if you have the <asm/byteorder.h> header file. */ +/* #undef HAVE_ASM_BYTEORDER_H */ + +/* Define to 1 if BSWAP64 is defined to the name of a valid 64 bit endian + swapping function */ +/* #undef HAVE_BSWAP64 */ + +/* Define to 1 if you have the <db.h> header file. */ +/* #undef HAVE_DB_H */ + +/* Define to 1 if you have the declaration of `F_SETLK', and to 0 if you + don't. */ +#define HAVE_DECL_F_SETLK 0 + +/* Define to 1 if you have the declaration of `INFTIM', and to 0 if you don't. + */ +#define HAVE_DECL_INFTIM 0 + +/* Define to 1 if you have the declaration of `O_EXLOCK', and to 0 if you + don't. */ +#define HAVE_DECL_O_EXLOCK 0 + +/* Define to 1 if you have the declaration of `SO_PEERCRED', and to 0 if you + don't. */ +#define HAVE_DECL_SO_PEERCRED 0 + +/* Define to 1 if you have the declaration of `XATTR_NOFOLLOW', and to 0 if + you don't. */ +#define HAVE_DECL_XATTR_NOFOLLOW 0 + +/* Define to 1 if you have the declaration of `O_BINARY', and to 0 if you + don't. */ +#define HAVE_DECL_O_BINARY 1 + +/* Define to 1 if #define of pragmas works */ +/* #undef HAVE_DEFINE_PRAGMA */ + +/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'. + */ +/* #undef HAVE_DIRENT_H */ + +/* Define to 1 if you have the <editline/readline.h> header file. */ +/* #undef HAVE_EDITLINE_READLINE_H */ + +/* define if the compiler supports exceptions */ +#define HAVE_EXCEPTIONS + +/* Define to 1 if you have the <execinfo.h> header file. */ +/* #undef HAVE_EXECINFO_H */ + +/* Define to 1 if you have the `flock' function. */ +/* #undef HAVE_FLOCK */ + +/* Define to 1 if you have the `getmntent' function. */ +/* #undef HAVE_GETMNTENT */ + +/* Define to 1 if you have the `getpeereid' function. */ +/* #undef HAVE_GETPEEREID */ + +/* Define to 1 if you have the `getpid' function. */ +// #define HAVE_GETPID 1 + +/* Define to 1 if you have the `getxattr' function. */ +/* #undef HAVE_GETXATTR */ + +/* Define to 1 if you have the <history.h> header file. */ +/* #undef HAVE_HISTORY_H */ + +/* Define to 1 if you have the <inttypes.h> header file. */ +// #define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `kqueue' function. */ +/* #undef HAVE_KQUEUE */ + +/* Define to 1 if you have the `lchown' function. */ +/* #undef HAVE_LCHOWN */ + +/* Define to 1 if you have the `lgetxattr' function. */ +/* #undef HAVE_LGETXATTR */ + +/* Define to 1 if you have the `crypto' library (-lcrypto). */ +#define HAVE_LIBCRYPTO 1 + +/* Define if you have a readline compatible library */ +/* #undef HAVE_LIBREADLINE */ + +/* Define to 1 if you have the `ssl' library (-lssl). */ +#define HAVE_LIBSSL 1 + +/* Define to 1 if you have the `z' library (-lz). */ +#define HAVE_LIBZ 1 + +/* Define to 1 if you have the `listxattr' function. */ +/* #undef HAVE_LISTXATTR */ + +/* Define to 1 if you have the `llistxattr' function. */ +/* #undef HAVE_LLISTXATTR */ + +/* Define to 1 if syscall lseek requires a dummy middle parameter */ +/* #undef HAVE_LSEEK_DUMMY_PARAM */ + +/* Define to 1 if you have the `lsetxattr' function. */ +/* #undef HAVE_LSETXATTR */ + +/* Define to 1 if you have the <memory.h> header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the <mntent.h> header file. */ +/* #undef HAVE_MNTENT_H */ + +/* Define to 1 if this platform supports mounts */ +/* #undef HAVE_MOUNTS */ + +/* define if the compiler implements namespaces */ +#define HAVE_NAMESPACES + +/* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the <netinet/in.h> header file. */ +/* #undef HAVE_NETINET_IN_H */ + +/* Define to 1 if SSL is pre-0.9.7 */ +/* #undef HAVE_OLD_SSL */ + +/* Define to 1 if you have the <openssl/ssl.h> header file. */ +#define HAVE_OPENSSL_SSL_H 1 + +/* Define to 1 if you have the <process.h> header file. */ +#define HAVE_PROCESS_H 1 + +/* Define to 1 if you have the <pwd.h> header file. */ +/* #undef HAVE_PWD_H */ + +/* Define to 1 (and set RANDOM_DEVICE) if a random device is available */ +/* #undef HAVE_RANDOM_DEVICE */ + +/* Define to 1 if you have the <readline.h> header file. */ +/* #undef HAVE_READLINE_H */ + +/* Define if your readline library has add_history */ +/* #undef HAVE_READLINE_HISTORY */ + +/* Define to 1 if you have the <readline/history.h> header file. */ +/* #undef HAVE_READLINE_HISTORY_H */ + +/* Define to 1 if you have the <readline/readline.h> header file. */ +/* #undef HAVE_READLINE_READLINE_H */ + +/* Define to 1 if you have the <regex.h> header file. */ +/* #undef HAVE_REGEX_H */ +#define HAVE_PCREPOSIX_H 1 +#define HAVE_REGEX_SUPPORT 1 + +/* Define to 1 if you have the `setproctitle' function. */ +/* #undef HAVE_SETPROCTITLE */ + +/* Define to 1 if you have the `setxattr' function. */ +/* #undef HAVE_SETXATTR */ + +/* Define to 1 if you have the <signal.h> header file. */ +#define HAVE_SIGNAL_H 1 + +/* Define to 1 if SSL is available */ +#define HAVE_SSL 1 + +/* Define to 1 if you have the `statfs' function. */ +/* #undef HAVE_STATFS */ + +/* Define to 1 if `stat' has the bug that it succeeds when given the + zero-length file name argument. */ +/* #undef HAVE_STAT_EMPTY_STRING_BUG */ + +/* Define to 1 if stdbool.h conforms to C99. */ +#define HAVE_STDBOOL_H 1 + +/* Define to 1 if you have the <stdint.h> header file. */ +// #define HAVE_STDINT_H 1 + +/* Define to 1 if you have the <stdlib.h> header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the <strings.h> header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the <string.h> header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if `d_type' is member of `struct dirent'. */ +/* #undef HAVE_STRUCT_DIRENT_D_TYPE */ + +/* Define to 1 if `mnt_dir' is member of `struct mntent'. */ +/* #undef HAVE_STRUCT_MNTENT_MNT_DIR */ + +/* Define to 1 if `mnt_mountp' is member of `struct mnttab'. */ +/* #undef HAVE_STRUCT_MNTTAB_MNT_MOUNTP */ + +/* Define to 1 if `sin_len' is member of `struct sockaddr_in'. */ +/* #undef HAVE_STRUCT_SOCKADDR_IN_SIN_LEN */ + +/* Define to 1 if `f_mntonname' is member of `struct statfs'. */ +/* #undef HAVE_STRUCT_STATFS_F_MNTONNAME */ + +/* Define to 1 if `st_flags' is member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_FLAGS */ + +/* Define to 1 if `st_mtimespec' is member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_MTIMESPEC */ + +/* Define to 1 if you have the `syscall' function. */ +/* #undef HAVE_SYSCALL */ + +/* Define to 1 if you have the <syslog.h> header file. */ +/* #undef HAVE_SYSLOG_H */ + +/* Define to 1 if you have the <sys/dir.h> header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the <sys/endian.h> header file. */ +/* #undef HAVE_SYS_ENDIAN_H */ + +/* Define to 1 if you have the <sys/mnttab.h> header file. */ +/* #undef HAVE_SYS_MNTTAB_H */ + +/* Define to 1 if you have the <sys/mount.h> header file. */ +/* #undef HAVE_SYS_MOUNT_H */ + +/* Define to 1 if you have the <sys/ndir.h> header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the <sys/param.h> header file. */ +// #define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the <sys/socket.h> header file. */ +/* #undef HAVE_SYS_SOCKET_H */ + +/* Define to 1 if you have the <sys/stat.h> header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the <sys/syscall.h> header file. */ +/* #undef HAVE_SYS_SYSCALL_H */ + +/* Define to 1 if you have the <sys/time.h> header file. */ +// #define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the <sys/types.h> header file. */ +// #define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the <sys/wait.h> header file. */ +/* #undef HAVE_SYS_WAIT_H */ + +/* Define to 1 if you have the <sys/xattr.h> header file. */ +/* #undef HAVE_SYS_XATTR_H */ + +/* Define to 1 if you have the <time.h> header file. */ +#define HAVE_TIME_H 1 + +/* Define to 1 if the system has the type `uint16_t'. */ +#define HAVE_UINT16_T 1 + +/* Define to 1 if the system has the type `uint32_t'. */ +#define HAVE_UINT32_T 1 + +/* Define to 1 if the system has the type `uint64_t'. */ +#define HAVE_UINT64_T 1 + +/* Define to 1 if the system has the type `uint8_t'. */ +#define HAVE_UINT8_T 1 + +/* Define to 1 if you have the <unistd.h> header file. */ +// #define HAVE_UNISTD_H 1 + +/* Define to 1 if the system has the type `u_int16_t'. */ +/* #undef HAVE_U_INT16_T */ + +/* Define to 1 if the system has the type `u_int32_t'. */ +/* #undef HAVE_U_INT32_T */ + +/* Define to 1 if the system has the type `u_int64_t'. */ +/* #undef HAVE_U_INT64_T */ + +/* Define to 1 if the system has the type `u_int8_t'. */ +/* #undef HAVE_U_INT8_T */ + +/* Define to 1 if struct dirent.d_type is valid */ +/* #undef HAVE_VALID_DIRENT_D_TYPE */ + +/* Define to 1 if the system has the type `_Bool'. */ +/* #undef HAVE__BOOL */ + +/* Define to 1 if you have the `__syscall' function. */ +/* #undef HAVE___SYSCALL */ + +/* Define to 1 if __syscall is available but needs a definition */ +/* #undef HAVE___SYSCALL_NEED_DEFN */ + +/* max value of long long calculated by configure */ +/* #undef LLONG_MAX */ + +/* min value of long long calculated by configure */ +/* #undef LLONG_MIN */ + +/* Define to 1 if `lstat' dereferences a symlink specified with a trailing + slash. */ +/* #undef LSTAT_FOLLOWS_SLASHED_SYMLINK */ + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "box@fluffy.co.uk" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "Box Backup" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "Box Backup 0.11" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "box-backup" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "0.09" + +/* Define to the filename of the random device (and set HAVE_RANDOM_DEVICE) */ +/* #undef RANDOM_DEVICE */ + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* TMP directory name */ +#define TEMP_DIRECTORY_NAME "/tmp" + +/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your <sys/time.h> declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* Define to 1 if your processor stores words with the most significant byte + first (like Motorola and SPARC, unlike Intel and VAX). */ +/* #undef WORDS_BIGENDIAN */ + +/* Number of bits in a file offset, on hosts where this is settable. */ +/* #undef _FILE_OFFSET_BITS */ + +/* Define for large files, on AIX-style hosts. */ +/* #undef _LARGE_FILES */ + +/* Define to 1 if __USE_MALLOC is required work around STL memory leaks */ +/* #undef __USE_MALLOC */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `int' if <sys/types.h> doesn't define. */ +#define gid_t int + +/* Define to `int' if <sys/types.h> does not define. */ +/* #undef mode_t */ + +/* Define to `long' if <sys/types.h> does not define. */ +/* #undef off_t */ + +/* Define to `int' if <sys/types.h> does not define. */ +/* #undef pid_t */ + +/* Define to `unsigned' if <sys/types.h> does not define. */ +/* #undef size_t */ + +/* Define to `int' if <sys/types.h> doesn't define. */ +#define uid_t int diff --git a/lib/common/BoxException.cpp b/lib/common/BoxException.cpp new file mode 100644 index 00000000..2503ca63 --- /dev/null +++ b/lib/common/BoxException.cpp @@ -0,0 +1,21 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxException.cpp +// Purpose: Exception +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BoxException.h" + +#include "MemLeakFindOn.h" + +BoxException::BoxException() +{ +} + +BoxException::~BoxException() throw () +{ +} diff --git a/lib/common/BoxException.h b/lib/common/BoxException.h new file mode 100644 index 00000000..a8f5d7a6 --- /dev/null +++ b/lib/common/BoxException.h @@ -0,0 +1,38 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxException.h +// Purpose: Exception +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- + +#ifndef BOXEXCEPTION__H +#define BOXEXCEPTION__H + +#include <exception> +#include <string> + +// -------------------------------------------------------------------------- +// +// Class +// Name: BoxException +// Purpose: Exception +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +class BoxException : public std::exception +{ +public: + BoxException(); + ~BoxException() throw (); + + virtual unsigned int GetType() const throw() = 0; + virtual unsigned int GetSubType() const throw() = 0; + +private: +}; + + +#endif // BOXEXCEPTION__H + diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h new file mode 100644 index 00000000..617aa031 --- /dev/null +++ b/lib/common/BoxPlatform.h @@ -0,0 +1,201 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxPlatform.h +// Purpose: Specifies what each platform supports in more detail, and includes +// extra files to get basic support for types. +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- + +#ifndef BOXPLATFORM__H +#define BOXPLATFORM__H + +#ifdef WIN32 +#define DIRECTORY_SEPARATOR "\\" +#define DIRECTORY_SEPARATOR_ASCHAR '\\' +#else +#define DIRECTORY_SEPARATOR "/" +#define DIRECTORY_SEPARATOR_ASCHAR '/' +#endif + +#define PLATFORM_DEV_NULL "/dev/null" + +#ifdef _MSC_VER +#include "BoxConfig-MSVC.h" +#include "BoxVersion.h" +#else +#include "BoxConfig.h" +#endif + +#ifdef WIN32 + #ifdef __MSVCRT_VERSION__ + #if __MSVCRT_VERSION__ < 0x0601 + #error Must include Box.h before sys/types.h + #endif + #else + // need msvcrt version 6.1 or higher for _gmtime64() + // must define this before importing <sys/types.h> + #define __MSVCRT_VERSION__ 0x0601 + #endif +#endif + +#ifdef HAVE_SYS_TYPES_H + #include <sys/types.h> +#endif +#ifdef HAVE_INTTYPES_H + #include <inttypes.h> +#else + #ifdef HAVE_STDINT_H + #include <stdint.h> + #endif +#endif + +// Slight hack; disable interception in raidfile test on Darwin and Windows +#if defined __APPLE__ || defined WIN32 + // TODO: Replace with autoconf test + #define PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE +#endif + +// Disable memory testing under Darwin, it just doesn't like it very much. +#ifdef __APPLE__ + // TODO: We really should get some decent leak detection code. + #define PLATFORM_DISABLE_MEM_LEAK_TESTING +#endif + +// Darwin also has a weird idea of permissions and dates on symlinks: +// perms are fixed at creation time by your umask, and dates can't be +// changed. This breaks unit tests if we try to compare these things. +// See: http://lists.apple.com/archives/darwin-kernel/2006/Dec/msg00057.html +#ifdef __APPLE__ + #define PLATFORM_DISABLE_SYMLINK_ATTRIB_COMPARE +#endif + +// Find out if credentials on UNIX sockets can be obtained +#ifdef HAVE_GETPEEREID + // +#elif HAVE_DECL_SO_PEERCRED + // +#elif defined HAVE_UCRED_H && HAVE_GETPEERUCRED + // +#else + #define PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET +#endif + +#ifdef HAVE_DEFINE_PRAGMA + // set packing to one bytes (can't use push/pop on gcc) + #define BEGIN_STRUCTURE_PACKING_FOR_WIRE #pragma pack(1) + + // Use default packing + #define END_STRUCTURE_PACKING_FOR_WIRE #pragma pack() +#else + #define STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#endif + +// Handle differing xattr APIs +#ifdef HAVE_SYS_XATTR_H + #if !defined(HAVE_LLISTXATTR) && defined(HAVE_LISTXATTR) && HAVE_DECL_XATTR_NOFOLLOW + #define llistxattr(a,b,c) listxattr(a,b,c,XATTR_NOFOLLOW) + #endif + #if !defined(HAVE_LGETXATTR) && defined(HAVE_GETXATTR) && HAVE_DECL_XATTR_NOFOLLOW + #define lgetxattr(a,b,c,d) getxattr(a,b,c,d,0,XATTR_NOFOLLOW) + #endif + #if !defined(HAVE_LSETXATTR) && defined(HAVE_SETXATTR) && HAVE_DECL_XATTR_NOFOLLOW + #define lsetxattr(a,b,c,d,e) setxattr(a,b,c,d,0,(e)|XATTR_NOFOLLOW) + #endif +#endif + +#if defined WIN32 && !defined __MINGW32__ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + + typedef unsigned __int8 u_int8_t; + typedef unsigned __int16 u_int16_t; + typedef unsigned __int32 u_int32_t; + typedef unsigned __int64 u_int64_t; + + #define HAVE_U_INT8_T + #define HAVE_U_INT16_T + #define HAVE_U_INT32_T + #define HAVE_U_INT64_T +#endif // WIN32 && !__MINGW32__ + +// Define missing types +#ifndef HAVE_UINT8_T + typedef u_int8_t uint8_t; +#endif + +#ifndef HAVE_UINT16_T + typedef u_int16_t uint16_t; +#endif + +#ifndef HAVE_UINT32_T + typedef u_int32_t uint32_t; +#endif + +#ifndef HAVE_UINT64_T + typedef u_int64_t uint64_t; +#endif + +#ifndef HAVE_U_INT8_T + typedef uint8_t u_int8_t; +#endif + +#ifndef HAVE_U_INT16_T + typedef uint16_t u_int16_t; +#endif + +#ifndef HAVE_U_INT32_T + typedef uint32_t u_int32_t; +#endif + +#ifndef HAVE_U_INT64_T + typedef uint64_t u_int64_t; +#endif + +#if !HAVE_DECL_INFTIM + #define INFTIM -1 +#endif + +// for Unix compatibility with Windows :-) +#ifndef O_BINARY + #define O_BINARY 0 +#endif + +#ifdef WIN32 + typedef u_int64_t InodeRefType; +#else + typedef ino_t InodeRefType; +#endif + +#ifdef WIN32 + #define WIN32_LEAN_AND_MEAN +#endif + +#include "emu.h" + +#ifdef WIN32 + #define INVALID_FILE INVALID_HANDLE_VALUE + typedef HANDLE tOSFileHandle; +#else + #define INVALID_FILE -1 + typedef int tOSFileHandle; +#endif + +// Solaris has no dirfd(x) macro or function, and we need one for +// intercept tests. We cannot define macros with arguments directly +// using AC_DEFINE, so do it here instead of in configure.ac. + +#if ! defined PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE && ! HAVE_DECL_DIRFD + #ifdef HAVE_DIR_D_FD + #define dirfd(x) (x)->d_fd + #elif defined HAVE_DIR_DD_FD + #define dirfd(x) (x)->dd_fd + #else + #error No way to get file descriptor from DIR structure + #endif +#endif + +#endif // BOXPLATFORM__H diff --git a/lib/common/BoxPortsAndFiles.h.in b/lib/common/BoxPortsAndFiles.h.in new file mode 100644 index 00000000..41bad0ba --- /dev/null +++ b/lib/common/BoxPortsAndFiles.h.in @@ -0,0 +1,44 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxPortsAndFiles.h +// Purpose: Central list of which tcp/ip ports and hardcoded file locations +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BOXPORTSANDFILES__H +#define BOXPORTSANDFILES__H + +#define BOX_PORT_BASE 2200 + + +// Backup store daemon +#define BOX_PORT_BBSTORED (BOX_PORT_BASE+1) +#define BOX_PORT_BBSTORED_TEST 22011 + +// directory within the RAIDFILE root for the backup store daemon +#define BOX_RAIDFILE_ROOT_BBSTORED "backup" + +// configuration file paths +#ifdef WIN32 + // no default config file path, use these macros to call + // GetDefaultConfigFilePath() instead. + + #define BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE \ + GetDefaultConfigFilePath("bbackupd.conf").c_str() + #define BOX_GET_DEFAULT_RAIDFILE_CONFIG_FILE \ + GetDefaultConfigFilePath("raidfile.conf").c_str() + #define BOX_GET_DEFAULT_BBSTORED_CONFIG_FILE \ + GetDefaultConfigFilePath("bbstored.conf").c_str() +#else +#define BOX_FILE_BBACKUPD_DEFAULT_CONFIG "@sysconfdir_expanded@/boxbackup/bbackupd.conf" +#define BOX_FILE_RAIDFILE_DEFAULT_CONFIG "@sysconfdir_expanded@/boxbackup/raidfile.conf" +#define BOX_FILE_BBSTORED_DEFAULT_CONFIG "@sysconfdir_expanded@/boxbackup/bbstored.conf" +#define BOX_FILE_BBACKUPD_OLD_CONFIG "@sysconfdir_expanded@/box/bbackupd.conf" +#define BOX_FILE_RAIDFILE_OLD_CONFIG "@sysconfdir_expanded@/box/raidfile.conf" +#define BOX_FILE_BBSTORED_OLD_CONFIG "@sysconfdir_expanded@/box/bbstored.conf" +#endif + +#endif // BOXPORTSANDFILES__H + diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp new file mode 100644 index 00000000..d05c0a6c --- /dev/null +++ b/lib/common/BoxTime.cpp @@ -0,0 +1,96 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxTime.cpp +// Purpose: Time for the box +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_SYS_TIME_H + #include <sys/time.h> +#endif + +#ifdef HAVE_TIME_H + #include <time.h> +#endif + +#include <errno.h> +#include <string.h> + +#include "BoxTime.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: GetCurrentBoxTime() +// Purpose: Returns the current time as a box time. +// (1 sec precision, or better if supported by system) +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +box_time_t GetCurrentBoxTime() +{ + #ifdef HAVE_GETTIMEOFDAY + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) + { + BOX_LOG_SYS_ERROR("Failed to gettimeofday(), " + "dropping precision"); + } + else + { + box_time_t timeNow = (tv.tv_sec * MICRO_SEC_IN_SEC_LL) + + tv.tv_usec; + return timeNow; + } + #endif + + return SecondsToBoxTime(time(0)); +} + +std::string FormatTime(box_time_t time, bool includeDate, bool showMicros) +{ + std::ostringstream buf; + + time_t seconds = BoxTimeToSeconds(time); + int micros = BoxTimeToMicroSeconds(time) % MICRO_SEC_IN_SEC; + + struct tm tm_now, *tm_ptr = &tm_now; + + #ifdef WIN32 + if ((tm_ptr = localtime(&seconds)) != NULL) + #else + if (localtime_r(&seconds, &tm_now) != NULL) + #endif + { + buf << std::setfill('0'); + + if (includeDate) + { + buf << std::setw(4) << (tm_ptr->tm_year + 1900) << "-" << + std::setw(2) << (tm_ptr->tm_mon + 1) << "-" << + std::setw(2) << (tm_ptr->tm_mday) << " "; + } + + buf << std::setw(2) << tm_ptr->tm_hour << ":" << + std::setw(2) << tm_ptr->tm_min << ":" << + std::setw(2) << tm_ptr->tm_sec; + + if (showMicros) + { + buf << "." << std::setw(6) << micros; + } + } + else + { + buf << strerror(errno); + } + + return buf.str(); +} + diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h new file mode 100644 index 00000000..6681bbbd --- /dev/null +++ b/lib/common/BoxTime.h @@ -0,0 +1,46 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxTime.h +// Purpose: How time is represented +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BOXTIME__H +#define BOXTIME__H + +// Time is presented as an unsigned 64 bit integer, in microseconds +typedef uint64_t box_time_t; + +#define NANO_SEC_IN_SEC (1000000000LL) +#define NANO_SEC_IN_USEC (1000) +#define NANO_SEC_IN_USEC_LL (1000LL) +#define MICRO_SEC_IN_SEC (1000000) +#define MICRO_SEC_IN_SEC_LL (1000000LL) +#define MILLI_SEC_IN_NANO_SEC (1000) +#define MILLI_SEC_IN_NANO_SEC_LL (1000LL) + +box_time_t GetCurrentBoxTime(); + +inline box_time_t SecondsToBoxTime(time_t Seconds) +{ + return ((box_time_t)Seconds * MICRO_SEC_IN_SEC_LL); +} +inline time_t BoxTimeToSeconds(box_time_t Time) +{ + return Time / MICRO_SEC_IN_SEC_LL; +} +inline uint64_t BoxTimeToMilliSeconds(box_time_t Time) +{ + return Time / MILLI_SEC_IN_NANO_SEC_LL; +} +inline uint64_t BoxTimeToMicroSeconds(box_time_t Time) +{ + return Time; +} + +std::string FormatTime(box_time_t time, bool includeDate, + bool showMicros = false); + +#endif // BOXTIME__H diff --git a/lib/common/BoxTimeToText.cpp b/lib/common/BoxTimeToText.cpp new file mode 100644 index 00000000..dbeefe1b --- /dev/null +++ b/lib/common/BoxTimeToText.cpp @@ -0,0 +1,76 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxTimeToText.cpp +// Purpose: Convert box time to text +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <time.h> +#include <stdio.h> + +#include "BoxTimeToText.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BoxTimeToISO8601String(box_time_t, bool) +// Purpose: Convert a 64 bit box time to a ISO 8601 compliant +// string, either in local or UTC time +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +std::string BoxTimeToISO8601String(box_time_t Time, bool localTime) +{ + time_t timeInSecs = BoxTimeToSeconds(Time); + char str[128]; // more than enough space + +#ifdef WIN32 + struct tm *time; + __time64_t winTime = timeInSecs; + + if(localTime) + { + time = _localtime64(&winTime); + } + else + { + time = _gmtime64(&winTime); + } + + if(time == NULL) + { + // ::sprintf(str, "%016I64x ", bob); + return std::string("unable to convert time"); + } + + sprintf(str, "%04d-%02d-%02dT%02d:%02d:%02d", time->tm_year + 1900, + time->tm_mon + 1, time->tm_mday, time->tm_hour, + time->tm_min, time->tm_sec); +#else // ! WIN32 + struct tm time; + + if(localTime) + { + localtime_r(&timeInSecs, &time); + } + else + { + gmtime_r(&timeInSecs, &time); + } + + sprintf(str, "%04d-%02d-%02dT%02d:%02d:%02d", time.tm_year + 1900, + time.tm_mon + 1, time.tm_mday, time.tm_hour, + time.tm_min, time.tm_sec); +#endif // WIN32 + + return std::string(str); +} + + diff --git a/lib/common/BoxTimeToText.h b/lib/common/BoxTimeToText.h new file mode 100644 index 00000000..21fa5d57 --- /dev/null +++ b/lib/common/BoxTimeToText.h @@ -0,0 +1,19 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxTimeToText.h +// Purpose: Convert box time to text +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#ifndef BOXTIMETOTEXT__H +#define BOXTIMETOTEXT__H + +#include <string> +#include "BoxTime.h" + +std::string BoxTimeToISO8601String(box_time_t Time, bool localTime); + +#endif // BOXTIMETOTEXT__H + diff --git a/lib/common/BoxTimeToUnix.h b/lib/common/BoxTimeToUnix.h new file mode 100644 index 00000000..f8a8797e --- /dev/null +++ b/lib/common/BoxTimeToUnix.h @@ -0,0 +1,34 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxTimeToUnix.h +// Purpose: Convert times in 64 bit values to UNIX structures +// Created: 2003/10/07 +// +// -------------------------------------------------------------------------- + +#ifndef FILEMODIFICATIONTIMETOTIMEVAL__H +#define FILEMODIFICATIONTIMETOTIMEVAL__H + +#ifdef WIN32 +#include <time.h> +#else +#include <sys/time.h> +#endif + +#include "BoxTime.h" + +inline void BoxTimeToTimeval(box_time_t Time, struct timeval &tv) +{ + tv.tv_sec = (long)(Time / MICRO_SEC_IN_SEC_LL); + tv.tv_usec = (long)(Time % MICRO_SEC_IN_SEC_LL); +} + +inline void BoxTimeToTimespec(box_time_t Time, struct timespec &tv) +{ + tv.tv_sec = (time_t)(Time / MICRO_SEC_IN_SEC_LL); + tv.tv_nsec = ((long)(Time % MICRO_SEC_IN_SEC_LL)) * NANO_SEC_IN_USEC; +} + +#endif // FILEMODIFICATIONTIMETOTIMEVAL__H + diff --git a/lib/common/BufferedStream.cpp b/lib/common/BufferedStream.cpp new file mode 100644 index 00000000..b58253f3 --- /dev/null +++ b/lib/common/BufferedStream.cpp @@ -0,0 +1,207 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BufferedStream.cpp +// Purpose: Buffering read-only wrapper around IOStreams +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BufferedStream.h" +#include "CommonException.h" + +#include <string.h> + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::BufferedStream(const char *, int, int) +// Purpose: Constructor, set up buffer +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +BufferedStream::BufferedStream(IOStream& rSource) +: mrSource(rSource), mBufferSize(0), mBufferPosition(0) +{ } + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::Read(void *, int) +// Purpose: Reads bytes from the file +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +int BufferedStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + if (mBufferSize == mBufferPosition) + { + // buffer is empty, fill it. + + int numBytesRead = mrSource.Read(mBuffer, sizeof(mBuffer), + Timeout); + + if (numBytesRead < 0) + { + return numBytesRead; + } + + mBufferSize = numBytesRead; + } + + int sizeToReturn = mBufferSize - mBufferPosition; + + if (sizeToReturn > NBytes) + { + sizeToReturn = NBytes; + } + + memcpy(pBuffer, mBuffer + mBufferPosition, sizeToReturn); + mBufferPosition += sizeToReturn; + + if (mBufferPosition == mBufferSize) + { + // clear out the buffer + mBufferSize = 0; + mBufferPosition = 0; + } + + return sizeToReturn; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::BytesLeftToRead() +// Purpose: Returns number of bytes to read (may not be most efficient function ever) +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +IOStream::pos_type BufferedStream::BytesLeftToRead() +{ + return mrSource.BytesLeftToRead() + mBufferSize - mBufferPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::Write(void *, int) +// Purpose: Writes bytes to the underlying stream (not supported) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::GetPosition() +// Purpose: Get position in stream +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type BufferedStream::GetPosition() const +{ + return mrSource.GetPosition() - mBufferSize + mBufferPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::Seek(pos_type, int) +// Purpose: Seeks within file, as lseek, invalidate buffer +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + switch (SeekType) + { + case SeekType_Absolute: + { + // just go there + mrSource.Seek(Offset, SeekType); + } + break; + + case SeekType_Relative: + { + // Actual underlying file position is + // (mBufferSize - mBufferPosition) ahead of us. + // Need to subtract that amount from the seek + // to seek forward that much less, putting the + // real pointer in the right place. + mrSource.Seek(Offset - mBufferSize + mBufferPosition, + SeekType); + } + break; + + case SeekType_End: + { + // Actual underlying file position is + // (mBufferSize - mBufferPosition) ahead of us. + // Need to add that amount to the seek + // to seek backwards that much more, putting the + // real pointer in the right place. + mrSource.Seek(Offset + mBufferSize - mBufferPosition, + SeekType); + } + } + + // always clear the buffer for now (may be slightly wasteful) + mBufferSize = 0; + mBufferPosition = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::Close() +// Purpose: Closes the underlying stream (not needed) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedStream::Close() +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::StreamDataLeft() +// Purpose: Any data left to write? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool BufferedStream::StreamDataLeft() +{ + return mrSource.StreamDataLeft(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::StreamClosed() +// Purpose: Is the stream closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool BufferedStream::StreamClosed() +{ + return mrSource.StreamClosed(); +} + diff --git a/lib/common/BufferedStream.h b/lib/common/BufferedStream.h new file mode 100644 index 00000000..079c482a --- /dev/null +++ b/lib/common/BufferedStream.h @@ -0,0 +1,43 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BufferedStream.h +// Purpose: Buffering read-only wrapper around IOStreams +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- + +#ifndef BUFFEREDSTREAM__H +#define BUFFEREDSTREAM__H + +#include "IOStream.h" + +class BufferedStream : public IOStream +{ +private: + IOStream& mrSource; + char mBuffer[4096]; + int mBufferSize; + int mBufferPosition; + +public: + BufferedStream(IOStream& rSource); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + BufferedStream(const BufferedStream &rToCopy) + : mrSource(rToCopy.mrSource) { /* do not call */ } +}; + +#endif // BUFFEREDSTREAM__H + + diff --git a/lib/common/BufferedWriteStream.cpp b/lib/common/BufferedWriteStream.cpp new file mode 100644 index 00000000..797be00d --- /dev/null +++ b/lib/common/BufferedWriteStream.cpp @@ -0,0 +1,181 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BufferedWriteStream.cpp +// Purpose: Buffering write-only wrapper around IOStreams +// Created: 2010/09/13 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BufferedWriteStream.h" +#include "CommonException.h" + +#include <string.h> + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::BufferedWriteStream(const char *, int, int) +// Purpose: Constructor, set up buffer +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +BufferedWriteStream::BufferedWriteStream(IOStream& rSink) +: mrSink(rSink), mBufferPosition(0) +{ } + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::Read(void *, int) +// Purpose: Reads bytes from the file - throws exception +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +int BufferedWriteStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::BytesLeftToRead() +// Purpose: Returns number of bytes to read (may not be most efficient function ever) +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +IOStream::pos_type BufferedWriteStream::BytesLeftToRead() +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::Write(void *, int) +// Purpose: Writes bytes to the underlying stream (not supported) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedWriteStream::Write(const void *pBuffer, int NBytes) +{ + int numBytesRemain = NBytes; + + do + { + int maxWritable = sizeof(mBuffer) - mBufferPosition; + int numBytesToWrite = (numBytesRemain < maxWritable) ? + numBytesRemain : maxWritable; + + if(numBytesToWrite > 0) + { + memcpy(mBuffer + mBufferPosition, pBuffer, + numBytesToWrite); + mBufferPosition += numBytesToWrite; + pBuffer = ((const char *)pBuffer) + numBytesToWrite; + numBytesRemain -= numBytesToWrite; + } + + if(numBytesRemain > 0) + { + Flush(); + } + } + while(numBytesRemain > 0); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::GetPosition() +// Purpose: Get position in stream +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type BufferedWriteStream::GetPosition() const +{ + return mrSink.GetPosition() + mBufferPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::Seek(pos_type, int) +// Purpose: Seeks within file, as lseek, invalidate buffer +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedWriteStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + // Always flush the buffer before seeking + Flush(); + + mrSink.Seek(Offset, SeekType); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::Flush(); +// Purpose: Write out current buffer contents and invalidate +// Created: 2010/09/13 +// +// -------------------------------------------------------------------------- +void BufferedWriteStream::Flush(int Timeout) +{ + if(mBufferPosition > 0) + { + mrSink.Write(mBuffer, mBufferPosition); + } + + mBufferPosition = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::Close() +// Purpose: Closes the underlying stream (not needed) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedWriteStream::Close() +{ + Flush(); + mrSink.Close(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::StreamDataLeft() +// Purpose: Any data left to write? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool BufferedWriteStream::StreamDataLeft() +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::StreamClosed() +// Purpose: Is the stream closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool BufferedWriteStream::StreamClosed() +{ + return mrSink.StreamClosed(); +} + diff --git a/lib/common/BufferedWriteStream.h b/lib/common/BufferedWriteStream.h new file mode 100644 index 00000000..7a1c8c17 --- /dev/null +++ b/lib/common/BufferedWriteStream.h @@ -0,0 +1,44 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BufferedWriteStream.h +// Purpose: Buffering write-only wrapper around IOStreams +// Created: 2010/09/13 +// +// -------------------------------------------------------------------------- + +#ifndef BUFFEREDWRITESTREAM__H +#define BUFFEREDWRITESTREAM__H + +#include "IOStream.h" + +class BufferedWriteStream : public IOStream +{ +private: + IOStream& mrSink; + char mBuffer[4096]; + int mBufferPosition; + +public: + BufferedWriteStream(IOStream& rSource); + virtual ~BufferedWriteStream() { Close(); } + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Flush(int Timeout = IOStream::TimeOutInfinite); + virtual void Close(); + + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + BufferedWriteStream(const BufferedWriteStream &rToCopy) + : mrSink(rToCopy.mrSink) { /* do not call */ } +}; + +#endif // BUFFEREDWRITESTREAM__H + + diff --git a/lib/common/CollectInBufferStream.cpp b/lib/common/CollectInBufferStream.cpp new file mode 100644 index 00000000..90e2e7bc --- /dev/null +++ b/lib/common/CollectInBufferStream.cpp @@ -0,0 +1,274 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CollectInBufferStream.cpp +// Purpose: Collect data in a buffer, and then read it out. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "CollectInBufferStream.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +#define INITIAL_BUFFER_SIZE 1024 +#define MAX_BUFFER_ADDITION (1024*64) + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::CollectInBufferStream() +// Purpose: Constructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +CollectInBufferStream::CollectInBufferStream() + : mBuffer(INITIAL_BUFFER_SIZE), + mBufferSize(INITIAL_BUFFER_SIZE), + mBytesInBuffer(0), + mReadPosition(0), + mInWritePhase(true) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::~CollectInBufferStream() +// Purpose: Destructor +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +CollectInBufferStream::~CollectInBufferStream() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::Read(void *, int, int) +// Purpose: As interface. But only works in read phase +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +int CollectInBufferStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + + // Adjust to number of bytes left + if(NBytes > (mBytesInBuffer - mReadPosition)) + { + NBytes = (mBytesInBuffer - mReadPosition); + } + ASSERT(NBytes >= 0); + if(NBytes <= 0) return 0; // careful now + + // Copy in the requested number of bytes and adjust the read pointer + ::memcpy(pBuffer, ((char*)mBuffer) + mReadPosition, NBytes); + mReadPosition += NBytes; + + return NBytes; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::BytesLeftToRead() +// Purpose: As interface. But only works in read phase +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +IOStream::pos_type CollectInBufferStream::BytesLeftToRead() +{ + if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + + return (mBytesInBuffer - mReadPosition); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::Write(void *, int) +// Purpose: As interface. But only works in write phase +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void CollectInBufferStream::Write(const void *pBuffer, int NBytes) +{ + if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + + // Enough space in the buffer + if((mBytesInBuffer + NBytes) > mBufferSize) + { + // Need to reallocate... what's the block size we'll use? + int allocateBlockSize = mBufferSize; + if(allocateBlockSize > MAX_BUFFER_ADDITION) + { + allocateBlockSize = MAX_BUFFER_ADDITION; + } + + // Write it the easy way. Although it's not the most efficient... + int newSize = mBufferSize; + while(newSize < (mBytesInBuffer + NBytes)) + { + newSize += allocateBlockSize; + } + + // Reallocate buffer + mBuffer.Resize(newSize); + + // Store new size + mBufferSize = newSize; + } + + // Copy in data and adjust counter + ::memcpy(((char*)mBuffer) + mBytesInBuffer, pBuffer, NBytes); + mBytesInBuffer += NBytes; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::GetPosition() +// Purpose: In write phase, returns the number of bytes written, in read +// phase, the number of bytes to go +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +IOStream::pos_type CollectInBufferStream::GetPosition() const +{ + return mInWritePhase?mBytesInBuffer:mReadPosition; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::Seek(pos_type, int) +// Purpose: As interface. But read phase only. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void CollectInBufferStream::Seek(pos_type Offset, int SeekType) +{ + if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + + int newPos = 0; + switch(SeekType) + { + case IOStream::SeekType_Absolute: + newPos = Offset; + break; + case IOStream::SeekType_Relative: + newPos = mReadPosition + Offset; + break; + case IOStream::SeekType_End: + newPos = mBytesInBuffer + Offset; + break; + default: + THROW_EXCEPTION(CommonException, IOStreamBadSeekType) + break; + } + + // Make sure it doesn't go over + if(newPos > mBytesInBuffer) + { + newPos = mBytesInBuffer; + } + // or under + if(newPos < 0) + { + newPos = 0; + } + + // Set the new read position + mReadPosition = newPos; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::StreamDataLeft() +// Purpose: As interface +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool CollectInBufferStream::StreamDataLeft() +{ + return mInWritePhase?(false):(mReadPosition < mBytesInBuffer); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::StreamClosed() +// Purpose: As interface +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool CollectInBufferStream::StreamClosed() +{ + return !mInWritePhase; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::SetForReading() +// Purpose: Switch to read phase, after all data written +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void CollectInBufferStream::SetForReading() +{ + if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + + // Move to read phase + mInWritePhase = false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::GetBuffer() +// Purpose: Returns the buffer +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void *CollectInBufferStream::GetBuffer() const +{ + return mBuffer.GetPtr(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::GetSize() +// Purpose: Returns the buffer size +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +int CollectInBufferStream::GetSize() const +{ + return mBytesInBuffer; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CollectInBufferStream::Reset() +// Purpose: Reset the stream, so it is empty and ready to be written to. +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +void CollectInBufferStream::Reset() +{ + mInWritePhase = true; + mBytesInBuffer = 0; + mReadPosition = 0; +} + diff --git a/lib/common/CollectInBufferStream.h b/lib/common/CollectInBufferStream.h new file mode 100644 index 00000000..d73af8db --- /dev/null +++ b/lib/common/CollectInBufferStream.h @@ -0,0 +1,60 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CollectInBufferStream.h +// Purpose: Collect data in a buffer, and then read it out. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef COLLECTINBUFFERSTREAM__H +#define COLLECTINBUFFERSTREAM__H + +#include "IOStream.h" +#include "Guards.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: CollectInBufferStream +// Purpose: Collect data in a buffer, and then read it out. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class CollectInBufferStream : public IOStream +{ +public: + CollectInBufferStream(); + ~CollectInBufferStream(); +private: + // No copying + CollectInBufferStream(const CollectInBufferStream &); + CollectInBufferStream(const IOStream &); +public: + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(pos_type Offset, int SeekType); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + void SetForReading(); + + void Reset(); + + void *GetBuffer() const; + int GetSize() const; + bool IsSetForReading() const {return !mInWritePhase;} + +private: + MemoryBlockGuard<char*> mBuffer; + int mBufferSize; + int mBytesInBuffer; + int mReadPosition; + bool mInWritePhase; +}; + +#endif // COLLECTINBUFFERSTREAM__H + diff --git a/lib/common/CommonException.h b/lib/common/CommonException.h new file mode 100644 index 00000000..a0eb3bf5 --- /dev/null +++ b/lib/common/CommonException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CommonException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef COMMONEXCEPTION__H +#define COMMONEXCEPTION__H + +// Compatibility header with old non-autogen exception scheme +#include "autogen_CommonException.h" + +#endif // COMMONEXCEPTION__H + diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt new file mode 100644 index 00000000..b2819886 --- /dev/null +++ b/lib/common/CommonException.txt @@ -0,0 +1,47 @@ + +# NOTE: Exception descriptions are for public distributions of Box Backup only -- do not rely for other applications. + + +EXCEPTION Common 1 + +Internal 0 +AssertFailed 1 +OSFileOpenError 2 Can't open a file -- attempted to load a non-existant config file or bad file referenced within? +OSFileCloseError 3 +FileAlreadyClosed 4 +BadArguments 5 +ConfigNoKey 6 +ConfigNoSubConfig 7 +GetLineNoHandle 8 +OSFileError 9 Error accessing a file. Check permissions. +GetLineEOF 10 +ConfigBadIntValue 11 +GetLineTooLarge 12 Protects against very large lines using up lots of memory. +NotSupported 13 +OSFileReadError 14 +OSFileWriteError 15 +FileClosed 16 +IOStreamBadSeekType 17 +CantWriteToPartialReadStream 18 +CollectInBufferStreamNotInCorrectPhase 19 +NamedLockAlreadyLockingSomething 20 +NamedLockNotHeld 21 +StreamableMemBlockIncompleteRead 22 +MemBlockStreamNotSupported 23 +StreamDoesntHaveRequiredProperty 24 +CannotWriteToReadGatherStream 25 +ReadGatherStreamAddingBadBlock 26 +CouldNotLookUpUsername 27 +CouldNotRestoreProcessUser 28 +CouldNotChangeProcessUser 29 +RegexNotSupportedOnThisPlatform 30 Your platform does not have built in regular expression libraries. +BadRegularExpression 31 +CouldNotCreateKQueue 32 +KEventErrorAdd 33 +KEventErrorWait 34 +KEventErrorRemove 35 +KQueueNotSupportedOnThisPlatform 36 +IOStreamGetLineNotEnoughDataToIgnore 37 Bad value passed to IOStreamGetLine::IgnoreBufferedData() +TempDirPathTooLong 38 Your temporary directory path is too long. Check the TMP and TEMP environment variables. +ArchiveBlockIncompleteRead 39 The Store Object Info File is too short or corrupted, and will be rewritten automatically when the next backup completes. +AccessDenied 40 Access to the file or directory was denied. Please check the permissions. diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp new file mode 100644 index 00000000..f49f3c6e --- /dev/null +++ b/lib/common/Configuration.cpp @@ -0,0 +1,920 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Configuration.cpp +// Purpose: Reading configuration files +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <limits.h> +#include <stdlib.h> +#include <string.h> + +#include <sstream> + +#include "Configuration.h" +#include "CommonException.h" +#include "Guards.h" +#include "FdGetLine.h" + +#include "MemLeakFindOn.h" + +#include <cstring> + +// utility whitespace function +inline bool iw(int c) +{ + return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded +} + +// boolean values +static const char *sValueBooleanStrings[] = {"yes", "true", "no", "false", 0}; +static const bool sValueBooleanValue[] = {true, true, false, false}; + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + void *testFunction +) +: mName(name), + mHasDefaultValue(false), + mFlags(flags), + mTestFunction(testFunction) +{ } + +// to allow passing NULL for default ListenAddresses + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + NoDefaultValue_t t, + void *testFunction +) +: mName(name), + mHasDefaultValue(false), + mFlags(flags), + mTestFunction(testFunction) +{ } + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + std::string defaultValue, + void *testFunction +) +: mName(name), + mDefaultValue(defaultValue), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ } + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + const char *defaultValue, + void *testFunction +) +: mName(name), + mDefaultValue(defaultValue), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ } + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + int defaultValue, + void *testFunction +) +: mName(name), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ + ASSERT(flags & ConfigTest_IsInt); + std::ostringstream val; + val << defaultValue; + mDefaultValue = val.str(); +} + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + bool defaultValue, + void *testFunction +) +: mName(name), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ + ASSERT(flags & ConfigTest_IsBool); + mDefaultValue = defaultValue ? "yes" : "no"; +} + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + const ConfigurationVerifyKey& rToCopy +) +: mName(rToCopy.mName), + mDefaultValue(rToCopy.mDefaultValue), + mHasDefaultValue(rToCopy.mHasDefaultValue), + mFlags(rToCopy.mFlags), + mTestFunction(rToCopy.mTestFunction) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::Configuration(const std::string &) +// Purpose: Constructor +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +Configuration::Configuration(const std::string &rName) + : mName(rName) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::Configuration(const Configuration &) +// Purpose: Copy constructor +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +Configuration::Configuration(const Configuration &rToCopy) + : mName(rToCopy.mName), + mKeys(rToCopy.mKeys), + mSubConfigurations(rToCopy.mSubConfigurations) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::~Configuration() +// Purpose: Destructor +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +Configuration::~Configuration() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::LoadAndVerify(const std::string &, const ConfigurationVerify *, std::string &) +// Purpose: Loads a configuration file from disc, checks it. Returns NULL if it was faulting, in which +// case they'll be an error message. +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +std::auto_ptr<Configuration> Configuration::LoadAndVerify( + const std::string& rFilename, + const ConfigurationVerify *pVerify, + std::string &rErrorMsg) +{ + // Just to make sure + rErrorMsg.erase(); + + // Open the file + FileHandleGuard<O_RDONLY> file(rFilename); + + // GetLine object + FdGetLine getline(file); + + // Object to create + std::auto_ptr<Configuration> apConfig( + new Configuration(std::string("<root>"))); + + try + { + // Load + LoadInto(*apConfig, getline, rErrorMsg, true); + + if(!rErrorMsg.empty()) + { + // An error occured, return now + BOX_ERROR("Error in Configuration::LoadInto: " << + rErrorMsg); + return std::auto_ptr<Configuration>(0); + } + + // Verify? + if(pVerify) + { + if(!apConfig->Verify(*pVerify, std::string(), rErrorMsg)) + { + BOX_ERROR("Error verifying configuration: " << + rErrorMsg); + return std::auto_ptr<Configuration>(0); + } + } + } + catch(...) + { + // Clean up + throw; + } + + // Success. Return result. + return apConfig; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: LoadInto(Configuration &, FdGetLine &, std::string &, bool) +// Purpose: Private. Load configuration information from the file into the config object. +// Returns 'abort' flag, if error, will be appended to rErrorMsg. +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel) +{ + bool startBlockExpected = false; + std::string blockName; + + //TRACE1("BLOCK: |%s|\n", rConfig.mName.c_str()); + + while(!rGetLine.IsEOF()) + { + std::string line(rGetLine.GetLine(true)); /* preprocess out whitespace and comments */ + + if(line.empty()) + { + // Ignore blank lines + continue; + } + + // Line an open block string? + if(line == "{") + { + if(startBlockExpected) + { + // New config object + Configuration subConfig(blockName); + + // Continue processing into this block + if(!LoadInto(subConfig, rGetLine, rErrorMsg, false)) + { + // Abort error + return false; + } + + startBlockExpected = false; + + // Store... + rConfig.AddSubConfig(blockName, subConfig); + } + else + { + rErrorMsg += "Unexpected start block in " + + rConfig.mName + "\n"; + } + } + else + { + // Close block? + if(line == "}") + { + if(RootLevel) + { + // error -- root level doesn't have a close + rErrorMsg += "Root level has close block -- forgot to terminate subblock?\n"; + // but otherwise ignore + } + else + { + //TRACE0("ENDBLOCK\n"); + return true; // All very good and nice + } + } + // Either a key, or a sub block beginning + else + { + // Can't be a start block + if(startBlockExpected) + { + rErrorMsg += "Block " + blockName + " wasn't started correctly (no '{' on line of it's own)\n"; + startBlockExpected = false; + } + + // Has the line got an = in it? + unsigned int equals = 0; + for(; equals < line.size(); ++equals) + { + if(line[equals] == '=') + { + // found! + break; + } + } + if(equals < line.size()) + { + // Make key value pair + unsigned int keyend = equals; + while(keyend > 0 && iw(line[keyend-1])) + { + keyend--; + } + unsigned int valuestart = equals+1; + while(valuestart < line.size() && iw(line[valuestart])) + { + valuestart++; + } + if(keyend > 0 && valuestart <= line.size()) + { + std::string key(line.substr(0, keyend)); + std::string value(line.substr(valuestart)); + rConfig.AddKeyValue(key, value); + } + else + { + rErrorMsg += "Invalid configuration key: " + line + "\n"; + } + } + else + { + // Start of sub block + blockName = line; + startBlockExpected = true; + } + } + } + } + + // End of file? + if(!RootLevel && rGetLine.IsEOF()) + { + // Error if EOF and this isn't the root level + rErrorMsg += "File ended without terminating all subblocks\n"; + } + + return true; +} + +void Configuration::AddKeyValue(const std::string& rKey, + const std::string& rValue) +{ + // Check for duplicate values + if(mKeys.find(rKey) != mKeys.end()) + { + // Multi-values allowed here, but checked later on + mKeys[rKey] += MultiValueSeparator; + mKeys[rKey] += rValue; + } + else + { + // Store + mKeys[rKey] = rValue; + } +} + +void Configuration::AddSubConfig(const std::string& rName, + const Configuration& rSubConfig) +{ + mSubConfigurations.push_back( + std::pair<std::string, Configuration>(rName, rSubConfig)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::KeyExists(const std::string&) +// Purpose: Checks to see if a key exists +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +bool Configuration::KeyExists(const std::string& rKeyName) const +{ + return mKeys.find(rKeyName) != mKeys.end(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetKeyValue(const std::string&) +// Purpose: Returns the value of a configuration variable +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +const std::string &Configuration::GetKeyValue(const std::string& rKeyName) const +{ + std::map<std::string, std::string>::const_iterator i(mKeys.find(rKeyName)); + + if(i == mKeys.end()) + { + BOX_ERROR("Missing configuration key: " << rKeyName); + THROW_EXCEPTION(CommonException, ConfigNoKey) + } + else + { + return i->second; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetKeyValueInt(const std::string& rKeyName) +// Purpose: Gets a key value as an integer +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +int Configuration::GetKeyValueInt(const std::string& rKeyName) const +{ + std::map<std::string, std::string>::const_iterator i(mKeys.find(rKeyName)); + + if(i == mKeys.end()) + { + THROW_EXCEPTION(CommonException, ConfigNoKey) + } + else + { + long value = ::strtol((i->second).c_str(), NULL, + 0 /* C style handling */); + if(value == LONG_MAX || value == LONG_MIN) + { + THROW_EXCEPTION(CommonException, ConfigBadIntValue) + } + return (int)value; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetKeyValueUint32(const std::string& rKeyName) +// Purpose: Gets a key value as a 32-bit unsigned integer +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +uint32_t Configuration::GetKeyValueUint32(const std::string& rKeyName) const +{ + std::map<std::string, std::string>::const_iterator i(mKeys.find(rKeyName)); + + if(i == mKeys.end()) + { + THROW_EXCEPTION(CommonException, ConfigNoKey) + } + else + { + errno = 0; + long value = ::strtoul((i->second).c_str(), NULL, + 0 /* C style handling */); + if(errno != 0) + { + THROW_EXCEPTION(CommonException, ConfigBadIntValue) + } + return (int)value; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetKeyValueBool(const std::string&) +// Purpose: Gets a key value as a boolean +// Created: 17/2/04 +// +// -------------------------------------------------------------------------- +bool Configuration::GetKeyValueBool(const std::string& rKeyName) const +{ + std::map<std::string, std::string>::const_iterator i(mKeys.find(rKeyName)); + + if(i == mKeys.end()) + { + THROW_EXCEPTION(CommonException, ConfigNoKey) + } + else + { + bool value = false; + + // Anything this is called for should have been verified as having a correct + // string in the verification section. However, this does default to false + // if it isn't in the string table. + + for(int l = 0; sValueBooleanStrings[l] != 0; ++l) + { + if(::strcasecmp((i->second).c_str(), sValueBooleanStrings[l]) == 0) + { + // Found. + value = sValueBooleanValue[l]; + break; + } + } + + return value; + } + +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetKeyNames() +// Purpose: Returns list of key names +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +std::vector<std::string> Configuration::GetKeyNames() const +{ + std::map<std::string, std::string>::const_iterator i(mKeys.begin()); + + std::vector<std::string> r; + + for(; i != mKeys.end(); ++i) + { + r.push_back(i->first); + } + + return r; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::SubConfigurationExists(const +// std::string&) +// Purpose: Checks to see if a sub configuration exists +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +bool Configuration::SubConfigurationExists(const std::string& rSubName) const +{ + // Attempt to find it... + std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin()); + + for(; i != mSubConfigurations.end(); ++i) + { + // This the one? + if(i->first == rSubName) + { + // Yes. + return true; + } + } + + // didn't find it. + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetSubConfiguration(const +// std::string&) +// Purpose: Gets a sub configuration +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +const Configuration &Configuration::GetSubConfiguration(const std::string& + rSubName) const +{ + // Attempt to find it... + std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin()); + + for(; i != mSubConfigurations.end(); ++i) + { + // This the one? + if(i->first == rSubName) + { + // Yes. + return i->second; + } + } + + THROW_EXCEPTION(CommonException, ConfigNoSubConfig) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetSubConfiguration(const +// std::string&) +// Purpose: Gets a sub configuration for editing +// Created: 2008/08/12 +// +// -------------------------------------------------------------------------- +Configuration &Configuration::GetSubConfigurationEditable(const std::string& + rSubName) +{ + // Attempt to find it... + + for(SubConfigListType::iterator + i = mSubConfigurations.begin(); + i != mSubConfigurations.end(); ++i) + { + // This the one? + if(i->first == rSubName) + { + // Yes. + return i->second; + } + } + + THROW_EXCEPTION(CommonException, ConfigNoSubConfig) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetSubConfigurationNames() +// Purpose: Return list of sub configuration names +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +std::vector<std::string> Configuration::GetSubConfigurationNames() const +{ + std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin()); + + std::vector<std::string> r; + + for(; i != mSubConfigurations.end(); ++i) + { + r.push_back(i->first); + } + + return r; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::Verify(const ConfigurationVerify &, const std::string &, std::string &) +// Purpose: Checks that the configuration is valid according to the +// supplied verifier +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +bool Configuration::Verify(const ConfigurationVerify &rVerify, + const std::string &rLevel, std::string &rErrorMsg) +{ + bool ok = true; + + // First... check the keys + if(rVerify.mpKeys != 0) + { + const ConfigurationVerifyKey *pvkey = rVerify.mpKeys; + + bool todo = true; + do + { + // Can the key be found? + if(KeyExists(pvkey->Name())) + { + // Get value + const std::string &rval = GetKeyValue(pvkey->Name()); + const char *val = rval.c_str(); + + // Check it's a number? + if((pvkey->Flags() & ConfigTest_IsInt) == ConfigTest_IsInt) + { + // Test it... + char *end; + long r = ::strtol(val, &end, 0); + if(r == LONG_MIN || r == LONG_MAX || end != (val + rval.size())) + { + // not a good value + ok = false; + rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is not a valid integer.\n"; + } + } + + // Check it's a number? + if(pvkey->Flags() & ConfigTest_IsUint32) + { + // Test it... + char *end; + errno = 0; + uint32_t r = ::strtoul(val, &end, 0); + if(errno != 0 || end != (val + rval.size())) + { + // not a good value + ok = false; + rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is not a valid unsigned 32-bit integer.\n"; + } + } + + // Check it's a bool? + if((pvkey->Flags() & ConfigTest_IsBool) == ConfigTest_IsBool) + { + // See if it's one of the allowed strings. + bool found = false; + for(int l = 0; sValueBooleanStrings[l] != 0; ++l) + { + if(::strcasecmp(val, sValueBooleanStrings[l]) == 0) + { + // Found. + found = true; + break; + } + } + + // Error if it's not one of them. + if(!found) + { + ok = false; + rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is not a valid boolean value.\n"; + } + } + + // Check for multi valued statments where they're not allowed + if((pvkey->Flags() & ConfigTest_MultiValueAllowed) == 0) + { + // Check to see if this key is a multi-value -- it shouldn't be + if(rval.find(MultiValueSeparator) != rval.npos) + { + ok = false; + rErrorMsg += rLevel + mName +"." + pvkey->Name() + " (key) multi value not allowed (duplicated key?).\n"; + } + } + } + else + { + // Is it required to exist? + if((pvkey->Flags() & ConfigTest_Exists) == ConfigTest_Exists) + { + // Should exist, but doesn't. + ok = false; + rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is missing.\n"; + } + else if(pvkey->HasDefaultValue()) + { + mKeys[pvkey->Name()] = + pvkey->DefaultValue(); + } + } + + if((pvkey->Flags() & ConfigTest_LastEntry) == ConfigTest_LastEntry) + { + // No more! + todo = false; + } + + // next + pvkey++; + + } while(todo); + + // Check for additional keys + for(std::map<std::string, std::string>::const_iterator i = mKeys.begin(); + i != mKeys.end(); ++i) + { + // Is the name in the list? + const ConfigurationVerifyKey *scan = rVerify.mpKeys; + bool found = false; + while(scan) + { + if(scan->Name() == i->first) + { + found = true; + break; + } + + // Next? + if((scan->Flags() & ConfigTest_LastEntry) == ConfigTest_LastEntry) + { + break; + } + scan++; + } + + if(!found) + { + // Shouldn't exist, but does. + ok = false; + rErrorMsg += rLevel + mName + "." + i->first + " (key) is not a known key. Check spelling and placement.\n"; + } + } + } + + // Then the sub configurations + if(rVerify.mpSubConfigurations) + { + // Find the wildcard entry, if it exists, and check that required subconfigs are there + const ConfigurationVerify *wildcardverify = 0; + + const ConfigurationVerify *scan = rVerify.mpSubConfigurations; + while(scan) + { + if(scan->mName.length() > 0 && scan->mName[0] == '*') + { + wildcardverify = scan; + } + + // Required? + if((scan->Tests & ConfigTest_Exists) == ConfigTest_Exists) + { + if(scan->mName.length() > 0 && + scan->mName[0] == '*') + { + // Check something exists + if(mSubConfigurations.size() < 1) + { + // A sub config should exist, but doesn't. + ok = false; + rErrorMsg += rLevel + mName + ".* (block) is missing (a block must be present).\n"; + } + } + else + { + // Check real thing exists + if(!SubConfigurationExists(scan->mName)) + { + // Should exist, but doesn't. + ok = false; + rErrorMsg += rLevel + mName + "." + scan->mName + " (block) is missing.\n"; + } + } + } + + // Next? + if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) + { + break; + } + scan++; + } + + // Go through the sub configurations, one by one + for(SubConfigListType::iterator + i = mSubConfigurations.begin(); + i != mSubConfigurations.end(); ++i) + { + // Can this be found? + const ConfigurationVerify *subverify = 0; + + const ConfigurationVerify *scan = rVerify.mpSubConfigurations; + const char *name = i->first.c_str(); + ASSERT(name); + while(scan) + { + if(scan->mName == name) + { + // found it! + subverify = scan; + } + + // Next? + if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) + { + break; + } + scan++; + } + + // Use wildcard? + if(subverify == 0) + { + subverify = wildcardverify; + } + + // Verify + if(subverify) + { + // override const-ness here... + if(!i->second.Verify(*subverify, mName + '.', + rErrorMsg)) + { + ok = false; + } + } + } + } + + return ok; +} + + diff --git a/lib/common/Configuration.h b/lib/common/Configuration.h new file mode 100644 index 00000000..4828b315 --- /dev/null +++ b/lib/common/Configuration.h @@ -0,0 +1,147 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Configuration +// Purpose: Reading configuration files +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- + +#ifndef CONFIGURATION__H +#define CONFIGURATION__H + +#include <map> +#include <list> +#include <vector> +#include <string> +#include <memory> + +// For defining tests +enum +{ + ConfigTest_LastEntry = 1, + ConfigTest_Exists = 2, + ConfigTest_IsInt = 4, + ConfigTest_IsUint32 = 8, + ConfigTest_MultiValueAllowed = 16, + ConfigTest_IsBool = 32 +}; + +class ConfigurationVerifyKey +{ +public: + typedef enum + { + NoDefaultValue = 1 + } NoDefaultValue_t; + + ConfigurationVerifyKey(std::string name, int flags, + void *testFunction = NULL); + // to allow passing ConfigurationVerifyKey::NoDefaultValue + // for default ListenAddresses + ConfigurationVerifyKey(std::string name, int flags, + NoDefaultValue_t t, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + std::string defaultValue, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + const char* defaultValue, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + int defaultValue, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + bool defaultValue, void *testFunction = NULL); + const std::string& Name() const { return mName; } + const std::string& DefaultValue() const { return mDefaultValue; } + const bool HasDefaultValue() const { return mHasDefaultValue; } + const int Flags() const { return mFlags; } + const void* TestFunction() const { return mTestFunction; } + ConfigurationVerifyKey(const ConfigurationVerifyKey& rToCopy); + +private: + ConfigurationVerifyKey& operator=(const ConfigurationVerifyKey& + noAssign); + + std::string mName; // "*" for all other keys (not implemented yet) + std::string mDefaultValue; // default for when it's not present + bool mHasDefaultValue; + int mFlags; + void *mTestFunction; // set to zero for now, will implement later +}; + +class ConfigurationVerify +{ +public: + std::string mName; // "*" for all other sub config names + const ConfigurationVerify *mpSubConfigurations; + const ConfigurationVerifyKey *mpKeys; + int Tests; + void *TestFunction; // set to zero for now, will implement later +}; + +class FdGetLine; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Configuration +// Purpose: Loading, checking, and representing configuration files +// Created: 2003/07/23 +// +// -------------------------------------------------------------------------- +class Configuration +{ +public: + Configuration(const std::string &rName); + Configuration(const Configuration &rToCopy); + ~Configuration(); + + enum + { + // The character to separate multi-values + MultiValueSeparator = '\x01' + }; + + static std::auto_ptr<Configuration> LoadAndVerify( + const std::string& rFilename, + const ConfigurationVerify *pVerify, + std::string &rErrorMsg); + + static std::auto_ptr<Configuration> Load( + const std::string& rFilename, + std::string &rErrorMsg) + { return LoadAndVerify(rFilename, 0, rErrorMsg); } + + bool KeyExists(const std::string& rKeyName) const; + const std::string &GetKeyValue(const std::string& rKeyName) const; + int GetKeyValueInt(const std::string& rKeyName) const; + uint32_t GetKeyValueUint32(const std::string& rKeyName) const; + bool GetKeyValueBool(const std::string& rKeyName) const; + std::vector<std::string> GetKeyNames() const; + + bool SubConfigurationExists(const std::string& rSubName) const; + const Configuration &GetSubConfiguration(const std::string& rSubName) const; + Configuration &GetSubConfigurationEditable(const std::string& rSubName); + std::vector<std::string> GetSubConfigurationNames() const; + + void AddKeyValue(const std::string& rKey, const std::string& rValue); + void AddSubConfig(const std::string& rName, const Configuration& rSubConfig); + + bool Verify(const ConfigurationVerify &rVerify, std::string &rErrorMsg) + { + return Verify(rVerify, std::string(), rErrorMsg); + } + +private: + std::string mName; + // Order of keys not preserved + std::map<std::string, std::string> mKeys; + // Order of sub blocks preserved + typedef std::list<std::pair<std::string, Configuration> > SubConfigListType; + SubConfigListType mSubConfigurations; + + static bool LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel); + bool Verify(const ConfigurationVerify &rVerify, const std::string &rLevel, + std::string &rErrorMsg); +}; + +#endif // CONFIGURATION__H + diff --git a/lib/common/Conversion.h b/lib/common/Conversion.h new file mode 100644 index 00000000..cba5bb08 --- /dev/null +++ b/lib/common/Conversion.h @@ -0,0 +1,98 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Conversion.h +// Purpose: Convert between various types +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- + +#ifndef CONVERSION__H +#define CONVERSION__H + +#include <string> + +namespace BoxConvert +{ + // -------------------------------------------------------------------------- + // + // Function + // Name: BoxConvert::Convert<to_type, from_type>(to_type &, from_type) + // Purpose: Convert from types to types + // Created: 9/4/04 + // + // -------------------------------------------------------------------------- + template<typename to_type, typename from_type> + inline to_type Convert(from_type From) + { + // Default conversion, simply use C++ conversion + return From; + } + + // Specialise for string -> integer + int32_t _ConvertStringToInt(const char *pString, int Size); + template<> + inline int32_t Convert<int32_t, const std::string &>(const std::string &rFrom) + { + return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 32); + } + template<> + inline int16_t Convert<int16_t, const std::string &>(const std::string &rFrom) + { + return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 16); + } + template<> + inline int8_t Convert<int8_t, const std::string &>(const std::string &rFrom) + { + return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 8); + } + template<> + inline int32_t Convert<int32_t, const char *>(const char *pFrom) + { + return BoxConvert::_ConvertStringToInt(pFrom, 32); + } + template<> + inline int16_t Convert<int16_t, const char *>(const char *pFrom) + { + return BoxConvert::_ConvertStringToInt(pFrom, 16); + } + template<> + inline int8_t Convert<int8_t, const char *>(const char *pFrom) + { + return BoxConvert::_ConvertStringToInt(pFrom, 8); + } + + // Specialise for integer -> string + void _ConvertIntToString(std::string &rTo, int32_t From); + template<> + inline std::string Convert<std::string, int32_t>(int32_t From) + { + std::string r; + BoxConvert::_ConvertIntToString(r, From); + return r; + } + template<> + inline std::string Convert<std::string, int16_t>(int16_t From) + { + std::string r; + BoxConvert::_ConvertIntToString(r, From); + return r; + } + template<> + inline std::string Convert<std::string, int8_t>(int8_t From) + { + std::string r; + BoxConvert::_ConvertIntToString(r, From); + return r; + } + + // Specialise for bool -> string + template<> + inline std::string Convert<std::string, bool>(bool From) + { + return std::string(From?"true":"false"); + } +}; + +#endif // CONVERSION__H + diff --git a/lib/common/ConversionException.txt b/lib/common/ConversionException.txt new file mode 100644 index 00000000..91b5fa9a --- /dev/null +++ b/lib/common/ConversionException.txt @@ -0,0 +1,8 @@ + +EXCEPTION Conversion 12 + +Internal 0 +CannotConvertEmptyStringToInt 1 +BadStringRepresentationOfInt 2 +IntOverflowInConvertFromString 3 +BadIntSize 4 diff --git a/lib/common/ConversionString.cpp b/lib/common/ConversionString.cpp new file mode 100644 index 00000000..2d0a8d58 --- /dev/null +++ b/lib/common/ConversionString.cpp @@ -0,0 +1,129 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ConversionString.cpp +// Purpose: Conversions to and from strings +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <stdlib.h> +#include <limits.h> +#include <errno.h> + +#include "Conversion.h" +#include "autogen_ConversionException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BoxConvert::_ConvertStringToInt(const char *, int) +// Purpose: Convert from string to integer, with range checking. +// Always does signed -- no point in unsigned as C++ type checking +// isn't up to handling it properly. +// If a null pointer is passed in, then returns 0. +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- +int32_t BoxConvert::_ConvertStringToInt(const char *pString, int Size) +{ + // Handle null strings gracefully. + if(pString == 0) + { + return 0; + } + + // Check for initial validity + if(*pString == '\0') + { + THROW_EXCEPTION(ConversionException, CannotConvertEmptyStringToInt) + } + + // Convert. + char *numEnd = 0; + errno = 0; // Some platforms don't reset it. + long r = ::strtol(pString, &numEnd, 0); + + // Check that all the characters were used + if(*numEnd != '\0') + { + THROW_EXCEPTION(ConversionException, BadStringRepresentationOfInt) + } + + // Error check + if(r == 0 && errno == EINVAL) + { + THROW_EXCEPTION(ConversionException, BadStringRepresentationOfInt) + } + + // Range check from strtol + if((r == LONG_MIN || r == LONG_MAX) && errno == ERANGE) + { + THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString) + } + + // Check range for size of integer + switch(Size) + { + case 32: + { + // No extra checking needed if long is an int32 + if(sizeof(long) > sizeof(int32_t)) + { + if(r <= (0 - 0x7fffffffL) || r > 0x7fffffffL) + { + THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString) + } + } + break; + } + + case 16: + { + if(r <= (0 - 0x7fff) || r > 0x7fff) + { + THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString) + } + break; + } + + case 8: + { + if(r <= (0 - 0x7f) || r > 0x7f) + { + THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString) + } + break; + } + + default: + { + THROW_EXCEPTION(ConversionException, BadIntSize) + break; + } + } + + // Return number + return r; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BoxConvert::_ConvertIntToString(std::string &, int32_t) +// Purpose: Convert signed interger to a string +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- +void BoxConvert::_ConvertIntToString(std::string &rTo, int32_t From) +{ + char text[64]; // size more than enough + ::sprintf(text, "%d", (int)From); + rTo = text; +} + diff --git a/lib/common/DebugAssertFailed.cpp b/lib/common/DebugAssertFailed.cpp new file mode 100644 index 00000000..e498d641 --- /dev/null +++ b/lib/common/DebugAssertFailed.cpp @@ -0,0 +1,37 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: AssertFailed.cpp +// Purpose: Assert failure code +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- + +#ifndef BOX_RELEASE_BUILD + +#include "Box.h" + +#include <stdio.h> + +#ifdef WIN32 + #include "emu.h" +#else + #include <syslog.h> +#endif + +#include "MemLeakFindOn.h" + +bool AssertFailuresToSyslog = false; + +void BoxDebugAssertFailed(const char *cond, const char *file, int line) +{ + printf("ASSERT FAILED: [%s] at %s(%d)\n", cond, file, line); + if(AssertFailuresToSyslog) + { + ::syslog(LOG_ERR, "ASSERT FAILED: [%s] at %s(%d)\n", cond, file, line); + } +} + + +#endif // BOX_RELEASE_BUILD + diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp new file mode 100644 index 00000000..72891cd1 --- /dev/null +++ b/lib/common/DebugMemLeakFinder.cpp @@ -0,0 +1,552 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemLeakFinder.cpp +// Purpose: Memory leak finder +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + + +#ifndef BOX_RELEASE_BUILD + +#include "Box.h" + +#undef malloc +#undef realloc +#undef free + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <map> +#include <stdio.h> +#include <string.h> +#include <set> +#include <cstdlib> // for std::atexit + +#include "MemLeakFinder.h" + +static bool memleakfinder_initialised = false; +bool memleakfinder_global_enable = false; + +typedef struct +{ + size_t size; + const char *file; + int line; +} MallocBlockInfo; + +typedef struct +{ + size_t size; + const char *file; + int line; + bool array; +} ObjectInfo; + +namespace +{ + static std::map<void *, MallocBlockInfo> sMallocBlocks; + static std::map<void *, ObjectInfo> sObjectBlocks; + static bool sTrackingDataDestroyed = false; + + static class DestructionWatchdog + { + public: + ~DestructionWatchdog() + { + sTrackingDataDestroyed = true; + } + } + sWatchdog; + + static bool sTrackMallocInSection = false; + static std::set<void *> sSectionMallocBlocks; + static bool sTrackObjectsInSection = false; + static std::map<void *, ObjectInfo> sSectionObjectBlocks; + + static std::set<void *> sNotLeaks; + + void *sNotLeaksPre[1024]; + size_t sNotLeaksPreNum = 0; +} + +void memleakfinder_init() +{ + ASSERT(!memleakfinder_initialised); + + { + // allocates a permanent buffer on Solaris. + // not a leak? + std::ostringstream oss; + } + + memleakfinder_initialised = true; +} + +MemLeakSuppressionGuard::MemLeakSuppressionGuard() +{ + ASSERT(memleakfinder_global_enable); + memleakfinder_global_enable = false; +} + +MemLeakSuppressionGuard::~MemLeakSuppressionGuard() +{ + ASSERT(!memleakfinder_global_enable); + memleakfinder_global_enable = true; +} + +// these functions may well allocate memory, which we don't want to track. +static int sInternalAllocDepth = 0; + +class InternalAllocGuard +{ + public: + InternalAllocGuard () { sInternalAllocDepth++; } + ~InternalAllocGuard() { sInternalAllocDepth--; } +}; + +void memleakfinder_malloc_add_block(void *b, size_t size, const char *file, int line) +{ + InternalAllocGuard guard; + + if(b != 0) + { + MallocBlockInfo i; + i.size = size; + i.file = file; + i.line = line; + sMallocBlocks[b] = i; + + if(sTrackMallocInSection) + { + sSectionMallocBlocks.insert(b); + } + } +} + +void *memleakfinder_malloc(size_t size, const char *file, int line) +{ + InternalAllocGuard guard; + + void *b = std::malloc(size); + if(!memleakfinder_global_enable) return b; + if(!memleakfinder_initialised) return b; + + memleakfinder_malloc_add_block(b, size, file, line); + + //TRACE4("malloc(), %d, %s, %d, %08x\n", size, file, line, b); + return b; +} + +void *memleakfinder_realloc(void *ptr, size_t size) +{ + InternalAllocGuard guard; + + if(!memleakfinder_global_enable || !memleakfinder_initialised) + { + return std::realloc(ptr, size); + } + + // Check it's been allocated + std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); + if(ptr && i == sMallocBlocks.end()) + { + BOX_WARNING("Block " << ptr << " realloc()ated, but not " + "in list. Error? Or allocated in startup static " + "objects?"); + } + + void *b = std::realloc(ptr, size); + + if(ptr && i!=sMallocBlocks.end()) + { + // Worked? + if(b != 0) + { + // Update map + MallocBlockInfo inf = i->second; + inf.size = size; + sMallocBlocks.erase(i); + sMallocBlocks[b] = inf; + + if(sTrackMallocInSection) + { + std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr)); + if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si); + sSectionMallocBlocks.insert(b); + } + } + } + else + { + memleakfinder_malloc_add_block(b, size, "FOUND-IN-REALLOC", 0); + } + + //TRACE3("realloc(), %d, %08x->%08x\n", size, ptr, b); + return b; +} + +void memleakfinder_free(void *ptr) +{ + InternalAllocGuard guard; + + if(memleakfinder_global_enable && memleakfinder_initialised) + { + // Check it's been allocated + std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); + if(i != sMallocBlocks.end()) + { + sMallocBlocks.erase(i); + } + else + { + BOX_WARNING("Block " << ptr << " freed, but not " + "known. Error? Or allocated in startup " + "static allocation?"); + } + + if(sTrackMallocInSection) + { + std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr)); + if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si); + } + } + + //TRACE1("free(), %08x\n", ptr); + std::free(ptr); +} + + +void memleakfinder_notaleak_insert_pre() +{ + InternalAllocGuard guard; + + if(!memleakfinder_global_enable) return; + if(!memleakfinder_initialised) return; + + for(size_t l = 0; l < sNotLeaksPreNum; l++) + { + sNotLeaks.insert(sNotLeaksPre[l]); + } + + sNotLeaksPreNum = 0; +} + +bool is_leak(void *ptr) +{ + InternalAllocGuard guard; + + ASSERT(memleakfinder_initialised); + memleakfinder_notaleak_insert_pre(); + return sNotLeaks.find(ptr) == sNotLeaks.end(); +} + +void memleakfinder_notaleak(void *ptr) +{ + InternalAllocGuard guard; + + ASSERT(!sTrackingDataDestroyed); + + memleakfinder_notaleak_insert_pre(); + if(memleakfinder_global_enable && memleakfinder_initialised) + { + sNotLeaks.insert(ptr); + } + else + { + if ( sNotLeaksPreNum < + sizeof(sNotLeaksPre)/sizeof(*sNotLeaksPre) ) + sNotLeaksPre[sNotLeaksPreNum++] = ptr; + } +/* { + std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); + if(i != sMallocBlocks.end()) sMallocBlocks.erase(i); + } + { + std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr)); + if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si); + } + { + std::map<void *, ObjectInfo>::iterator i(sObjectBlocks.find(ptr)); + if(i != sObjectBlocks.end()) sObjectBlocks.erase(i); + }*/ +} + + + +// start monitoring a section of code +void memleakfinder_startsectionmonitor() +{ + InternalAllocGuard guard; + + ASSERT(memleakfinder_initialised); + ASSERT(!sTrackingDataDestroyed); + + sTrackMallocInSection = true; + sSectionMallocBlocks.clear(); + sTrackObjectsInSection = true; + sSectionObjectBlocks.clear(); +} + +// trace all blocks allocated and still allocated since memleakfinder_startsectionmonitor() called +void memleakfinder_traceblocksinsection() +{ + InternalAllocGuard guard; + + ASSERT(memleakfinder_initialised); + ASSERT(!sTrackingDataDestroyed); + + std::set<void *>::iterator s(sSectionMallocBlocks.begin()); + for(; s != sSectionMallocBlocks.end(); ++s) + { + std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.find(*s)); + if(i == sMallocBlocks.end()) + { + BOX_WARNING("Logical error in section block finding"); + } + else + { + BOX_TRACE("Block " << i->first << " size " << + i->second.size << " allocated at " << + i->second.file << ":" << i->second.line); + } + } + for(std::map<void *, ObjectInfo>::const_iterator i(sSectionObjectBlocks.begin()); i != sSectionObjectBlocks.end(); ++i) + { + BOX_TRACE("Object" << (i->second.array?" []":"") << " " << + i->first << " size " << i->second.size << + " allocated at " << i->second.file << + ":" << i->second.line); + } +} + +int memleakfinder_numleaks() +{ + InternalAllocGuard guard; + + ASSERT(memleakfinder_initialised); + ASSERT(!sTrackingDataDestroyed); + + int n = 0; + + for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i) + { + if(is_leak(i->first)) ++n; + } + + for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i) + { + const ObjectInfo& rInfo = i->second; + if(is_leak(i->first)) ++n; + } + + return n; +} + +void memleakfinder_reportleaks_file(FILE *file) +{ + InternalAllocGuard guard; + + ASSERT(!sTrackingDataDestroyed); + + for(std::map<void *, MallocBlockInfo>::const_iterator + i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i) + { + if(is_leak(i->first)) + { + ::fprintf(file, "Block %p size %d allocated at " + "%s:%d\n", i->first, i->second.size, + i->second.file, i->second.line); + } + } + + for(std::map<void *, ObjectInfo>::const_iterator + i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i) + { + if(is_leak(i->first)) + { + ::fprintf(file, "Object%s %p size %d allocated at " + "%s:%d\n", i->second.array?" []":"", + i->first, i->second.size, i->second.file, + i->second.line); + } + } +} + +void memleakfinder_reportleaks() +{ + InternalAllocGuard guard; + + // report to stdout + memleakfinder_reportleaks_file(stdout); +} + +void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext) +{ + InternalAllocGuard guard; + + FILE *file = ::fopen(filename, "a"); + if(file != 0) + { + if(memleakfinder_numleaks() > 0) + { +#ifdef HAVE_GETPID + fprintf(file, "MEMORY LEAKS FROM PROCESS %d (%s)\n", getpid(), markertext); +#else + fprintf(file, "MEMORY LEAKS (%s)\n", markertext); +#endif + memleakfinder_reportleaks_file(file); + } + + ::fclose(file); + } + else + { + BOX_WARNING("Couldn't open memory leak results file " << + filename << " for appending"); + } +} + +static char atexit_filename[512]; +static char atexit_markertext[512]; +static bool atexit_registered = false; + +extern "C" void memleakfinder_atexit() +{ + memleakfinder_reportleaks_appendfile(atexit_filename, atexit_markertext); +} + +void memleakfinder_setup_exit_report(const char *filename, const char *markertext) +{ + ::strncpy(atexit_filename, filename, sizeof(atexit_filename)-1); + ::strncpy(atexit_markertext, markertext, sizeof(atexit_markertext)-1); + atexit_filename[sizeof(atexit_filename)-1] = 0; + atexit_markertext[sizeof(atexit_markertext)-1] = 0; + if(!atexit_registered) + { + std::atexit(memleakfinder_atexit); + atexit_registered = true; + } +} + + + + +void add_object_block(void *block, size_t size, const char *file, int line, bool array) +{ + InternalAllocGuard guard; + + if(!memleakfinder_global_enable) return; + if(!memleakfinder_initialised) return; + ASSERT(!sTrackingDataDestroyed); + + if(block != 0) + { + ObjectInfo i; + i.size = size; + i.file = file; + i.line = line; + i.array = array; + sObjectBlocks[block] = i; + + if(sTrackObjectsInSection) + { + sSectionObjectBlocks[block] = i; + } + } +} + +void remove_object_block(void *block) +{ + InternalAllocGuard guard; + + if(!memleakfinder_global_enable) return; + if(!memleakfinder_initialised) return; + if(sTrackingDataDestroyed) return; + + std::map<void *, ObjectInfo>::iterator i(sObjectBlocks.find(block)); + if(i != sObjectBlocks.end()) + { + sObjectBlocks.erase(i); + } + + if(sTrackObjectsInSection) + { + std::map<void *, ObjectInfo>::iterator i(sSectionObjectBlocks.find(block)); + if(i != sSectionObjectBlocks.end()) + { + sSectionObjectBlocks.erase(i); + } + } + + // If it's not in the list, just ignore it, as lots of stuff goes this way... +} + +static void *internal_new(size_t size, const char *file, int line) +{ + void *r; + + { + InternalAllocGuard guard; + r = std::malloc(size); + } + + if (sInternalAllocDepth == 0) + { + InternalAllocGuard guard; + add_object_block(r, size, file, line, false); + //TRACE4("new(), %d, %s, %d, %08x\n", size, file, line, r); + } + + return r; +} + +void *operator new(size_t size, const char *file, int line) +{ + return internal_new(size, file, line); +} + +void *operator new[](size_t size, const char *file, int line) +{ + return internal_new(size, file, line); +} + +// where there is no doctor... need to override standard new() too +// http://www.relisoft.com/book/tech/9new.html +// disabled because it causes hangs on FC2 in futex() in test/common +// while reading files. reason unknown. +/* +void *operator new(size_t size) +{ + return internal_new(size, "standard libraries", 0); +} +*/ + +void *operator new[](size_t size) +{ + return internal_new(size, "standard libraries", 0); +} + +void internal_delete(void *ptr) +{ + InternalAllocGuard guard; + + std::free(ptr); + remove_object_block(ptr); + //TRACE1("delete[]() called, %08x\n", ptr); +} + +void operator delete[](void *ptr) throw () +{ + internal_delete(ptr); +} + +void operator delete(void *ptr) throw () +{ + internal_delete(ptr); +} + +#endif // BOX_RELEASE_BUILD diff --git a/lib/common/DebugPrintf.cpp b/lib/common/DebugPrintf.cpp new file mode 100644 index 00000000..1335d473 --- /dev/null +++ b/lib/common/DebugPrintf.cpp @@ -0,0 +1,83 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: DebugPrintf.cpp +// Purpose: Implementation of a printf function, to avoid a stdio.h include in Box.h +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- + +#ifndef BOX_RELEASE_BUILD + +#include "Box.h" + +#include <stdio.h> +#include <stdarg.h> + +#ifdef WIN32 + #include "emu.h" +#else + #include <syslog.h> +#endif + +#include "MemLeakFindOn.h" + +// Use this apparently superflous printf function to avoid having to +// include stdio.h in every file in the project. + +int BoxDebug_printf(const char *format, ...) +{ + va_list ap; + va_start(ap, format); + int r = vprintf(format, ap); + va_end(ap); + return r; +} + + +bool BoxDebugTraceOn = true; +bool BoxDebugTraceToStdout = true; +bool BoxDebugTraceToSyslog = false; + +int BoxDebugTrace(const char *format, ...) +{ + char text[512]; + int r = 0; + if(BoxDebugTraceOn || BoxDebugTraceToSyslog) + { + va_list ap; + va_start(ap, format); + r = vsnprintf(text, sizeof(text), format, ap); + va_end(ap); + } + + // Send to stdout if trace is on and std out is enabled + if(BoxDebugTraceOn && BoxDebugTraceToStdout) + { + printf("%s", text); + } + + // But tracing to syslog is independent of tracing being on or not + if(BoxDebugTraceToSyslog) + { +#ifdef WIN32 + // Remove trailing '\n', if it's there + if(r > 0 && text[r-1] == '\n') + { + text[r-1] = '\0'; +#else + if(r > 0 && text[r] == '\n') + { + text[r] = '\0'; +#endif + --r; + } + // Log it + ::syslog(LOG_INFO, "TRACE: %s", text); + } + + return r; +} + + +#endif // BOX_RELEASE_BUILD diff --git a/lib/common/EndStructPackForWire.h b/lib/common/EndStructPackForWire.h new file mode 100644 index 00000000..82637f33 --- /dev/null +++ b/lib/common/EndStructPackForWire.h @@ -0,0 +1,23 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: EndStructPackForWire.h +// Purpose: End structure packing for wire +// Created: 25/11/03 +// +// -------------------------------------------------------------------------- + +// No header guard -- this is intentional + +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS + +#pragma pack() + +#else + + logical error -- check BoxPlatform.h and including file + +#endif + + + diff --git a/lib/common/EventWatchFilesystemObject.cpp b/lib/common/EventWatchFilesystemObject.cpp new file mode 100644 index 00000000..43533fc8 --- /dev/null +++ b/lib/common/EventWatchFilesystemObject.cpp @@ -0,0 +1,112 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: EventWatchFilesystemObject.cpp +// Purpose: WaitForEvent compatible object for watching directories +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <errno.h> +#include <fcntl.h> + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include "EventWatchFilesystemObject.h" +#include "autogen_CommonException.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: EventWatchFilesystemObject::EventWatchFilesystemObject +// (const char *) +// Purpose: Constructor -- opens the file object +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) +#ifdef HAVE_KQUEUE + : mDescriptor(::open(Filename, O_RDONLY /*O_EVTONLY*/, 0)) +#endif +{ +#ifdef HAVE_KQUEUE + if(mDescriptor == -1) + { + BOX_LOG_SYS_ERROR("EventWatchFilesystemObject: " + "Failed to open file '" << Filename << "'"); + THROW_EXCEPTION(CommonException, OSFileOpenError) + } +#else + THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform) +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: EventWatchFilesystemObject::~EventWatchFilesystemObject() +// Purpose: Destructor +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +EventWatchFilesystemObject::~EventWatchFilesystemObject() +{ + if(mDescriptor != -1) + { + ::close(mDescriptor); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: EventWatchFilesystemObject::EventWatchFilesystemObject +// (const EventWatchFilesystemObject &) +// Purpose: Copy constructor +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +EventWatchFilesystemObject::EventWatchFilesystemObject( + const EventWatchFilesystemObject &rToCopy) + : mDescriptor(::dup(rToCopy.mDescriptor)) +{ + if(mDescriptor == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } +} + + +#ifdef HAVE_KQUEUE +// -------------------------------------------------------------------------- +// +// Function +// Name: EventWatchFilesystemObject::FillInKEvent(struct kevent &, int) +// Purpose: For WaitForEvent +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +void EventWatchFilesystemObject::FillInKEvent(struct kevent &rEvent, + int Flags) const +{ + EV_SET(&rEvent, mDescriptor, EVFILT_VNODE, EV_CLEAR, + NOTE_DELETE | NOTE_WRITE, 0, (void*)this); +} +#else +void EventWatchFilesystemObject::FillInPoll(int &fd, short &events, + int Flags) const +{ + THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform) +} +#endif + diff --git a/lib/common/EventWatchFilesystemObject.h b/lib/common/EventWatchFilesystemObject.h new file mode 100644 index 00000000..f9175a49 --- /dev/null +++ b/lib/common/EventWatchFilesystemObject.h @@ -0,0 +1,48 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: EventWatchFilesystemObject.h +// Purpose: WaitForEvent compatible object for watching directories +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef EVENTWATCHFILESYSTEMOBJECT__H +#define EVENTWATCHFILESYSTEMOBJECT__H + +#ifdef HAVE_KQUEUE + #include <sys/event.h> +#endif + + +// -------------------------------------------------------------------------- +// +// Class +// Name: EventWatchFilesystemObject +// Purpose: WaitForEvent compatible object for watching files and directories +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +class EventWatchFilesystemObject +{ +public: + EventWatchFilesystemObject(const char *Filename); + ~EventWatchFilesystemObject(); + EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy); +private: + // Assignment not allowed + EventWatchFilesystemObject &operator=(const EventWatchFilesystemObject &); +public: + +#ifdef HAVE_KQUEUE + void FillInKEvent(struct kevent &rEvent, int Flags = 0) const; +#else + void FillInPoll(int &fd, short &events, int Flags = 0) const; +#endif + +private: + int mDescriptor; +}; + +#endif // EventWatchFilesystemObject__H + diff --git a/lib/common/ExcludeList.cpp b/lib/common/ExcludeList.cpp new file mode 100644 index 00000000..edbf1a6a --- /dev/null +++ b/lib/common/ExcludeList.cpp @@ -0,0 +1,481 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ExcludeList.cpp +// Purpose: General purpose exclusion list +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_REGEX_SUPPORT + #ifdef HAVE_PCREPOSIX_H + #include <pcreposix.h> + #else + #include <regex.h> + #endif + #define EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED +#endif + +#include "ExcludeList.h" +#include "Utils.h" +#include "Configuration.h" +#include "Archive.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::ExcludeList() +// Purpose: Constructor. Generates an exclude list which will allow everything +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +ExcludeList::ExcludeList() + : mpAlwaysInclude(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::~ExcludeList() +// Purpose: Destructor +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +ExcludeList::~ExcludeList() +{ +#ifdef HAVE_REGEX_SUPPORT + // free regex memory + while(mRegex.size() > 0) + { + regex_t *pregex = mRegex.back(); + mRegex.pop_back(); + // Free regex storage, and the structure itself + ::regfree(pregex); + delete pregex; + } +#endif + + // Clean up exceptions list + if(mpAlwaysInclude != 0) + { + delete mpAlwaysInclude; + mpAlwaysInclude = 0; + } +} + +#ifdef WIN32 +std::string ExcludeList::ReplaceSlashesDefinite(const std::string& input) const +{ + std::string output = input; + + for (std::string::size_type pos = output.find("/"); + pos != std::string::npos; + pos = output.find("/")) + { + output.replace(pos, 1, DIRECTORY_SEPARATOR); + } + + for (std::string::iterator i = output.begin(); i != output.end(); i++) + { + *i = tolower(*i); + } + + return output; +} + +std::string ExcludeList::ReplaceSlashesRegex(const std::string& input) const +{ + std::string output = input; + + for (std::string::size_type pos = output.find("/"); + pos != std::string::npos; + pos = output.find("/")) + { + output.replace(pos, 1, "\\" DIRECTORY_SEPARATOR); + } + + for (std::string::iterator i = output.begin(); i != output.end(); i++) + { + *i = tolower(*i); + } + + return output; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::AddDefiniteEntries(const std::string &) +// Purpose: Adds a number of definite entries to the exclude list -- ones which +// will be excluded if and only if the test string matches exactly. +// Uses the Configuration classes' multi-value conventions, with +// multiple entires in one string separated by Configuration::MultiValueSeparator +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +void ExcludeList::AddDefiniteEntries(const std::string &rEntries) +{ + // Split strings up + std::vector<std::string> ens; + SplitString(rEntries, Configuration::MultiValueSeparator, ens); + + // Add to set of excluded strings + for(std::vector<std::string>::const_iterator i(ens.begin()); i != ens.end(); ++i) + { + if(i->size() > 0) + { + std::string entry = *i; + + // Convert any forward slashes in the string + // to backslashes + + #ifdef WIN32 + entry = ReplaceSlashesDefinite(entry); + #endif + + if (entry.size() > 0 && entry[entry.size() - 1] == + DIRECTORY_SEPARATOR_ASCHAR) + { + BOX_WARNING("Exclude entry ends in path " + "separator, will never match: " + << entry); + } + + mDefinite.insert(entry); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::AddRegexEntries(const std::string &) +// Purpose: Adds a number of regular expression entries to the exclude list -- +// if the test expression matches any of these regex, it will be excluded. +// Uses the Configuration classes' multi-value conventions, with +// multiple entires in one string separated by Configuration::MultiValueSeparator +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +void ExcludeList::AddRegexEntries(const std::string &rEntries) +{ +#ifdef HAVE_REGEX_SUPPORT + + // Split strings up + std::vector<std::string> ens; + SplitString(rEntries, Configuration::MultiValueSeparator, ens); + + // Create and add new regular expressions + for(std::vector<std::string>::const_iterator i(ens.begin()); i != ens.end(); ++i) + { + if(i->size() > 0) + { + // Allocate memory + regex_t *pregex = new regex_t; + + try + { + std::string entry = *i; + + // Convert any forward slashes in the string + // to appropriately escaped backslashes + + #ifdef WIN32 + entry = ReplaceSlashesRegex(entry); + #endif + + // Compile + int errcode = ::regcomp(pregex, entry.c_str(), + REG_EXTENDED | REG_NOSUB); + + if (errcode != 0) + { + char buf[1024]; + regerror(errcode, pregex, buf, sizeof(buf)); + BOX_ERROR("Invalid regular expression: " << + entry << ": " << buf); + THROW_EXCEPTION(CommonException, BadRegularExpression) + } + + // Store in list of regular expressions + mRegex.push_back(pregex); + // Store in list of regular expression string for Serialize + mRegexStr.push_back(entry.c_str()); + } + catch(...) + { + delete pregex; + throw; + } + } + } + +#else + THROW_EXCEPTION(CommonException, RegexNotSupportedOnThisPlatform) +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::IsExcluded(const std::string &) +// Purpose: Returns true if the entry should be excluded +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +bool ExcludeList::IsExcluded(const std::string &rTest) const +{ + std::string test = rTest; + + #ifdef WIN32 + test = ReplaceSlashesDefinite(test); + #endif + + // Check against the always include list + if(mpAlwaysInclude != 0) + { + if(mpAlwaysInclude->IsExcluded(test)) + { + // Because the "always include" list says it's 'excluded' + // this means it should actually be included. + return false; + } + } + + // Is it in the set of definite entries? + if(mDefinite.find(test) != mDefinite.end()) + { + return true; + } + + // Check against regular expressions +#ifdef HAVE_REGEX_SUPPORT + for(std::vector<regex_t *>::const_iterator i(mRegex.begin()); i != mRegex.end(); ++i) + { + // Test against this expression + if(regexec(*i, test.c_str(), 0, 0 /* no match information required */, 0 /* no flags */) == 0) + { + // match happened + return true; + } + // In all other cases, including an error, just continue to the next expression + } +#endif + + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::SetAlwaysIncludeList(ExcludeList *) +// Purpose: Takes ownership of the list, deletes any pre-existing list. +// NULL is acceptable to delete the list. +// The AlwaysInclude list is a list of exceptions to the exclusions. +// Created: 19/2/04 +// +// -------------------------------------------------------------------------- +void ExcludeList::SetAlwaysIncludeList(ExcludeList *pAlwaysInclude) +{ + // Delete old list + if(mpAlwaysInclude != 0) + { + delete mpAlwaysInclude; + mpAlwaysInclude = 0; + } + + // Store the pointer + mpAlwaysInclude = pAlwaysInclude; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void ExcludeList::Deserialize(Archive & rArchive) +{ + // + // + // + mDefinite.clear(); + +#ifdef HAVE_REGEX_SUPPORT + // free regex memory + while(mRegex.size() > 0) + { + regex_t *pregex = mRegex.back(); + mRegex.pop_back(); + // Free regex storage, and the structure itself + ::regfree(pregex); + delete pregex; + } + + mRegexStr.clear(); +#endif + + // Clean up exceptions list + if(mpAlwaysInclude != 0) + { + delete mpAlwaysInclude; + mpAlwaysInclude = 0; + } + + // + // + // + int64_t iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + for (int v = 0; v < iCount; v++) + { + // load each one + std::string strItem; + rArchive.Read(strItem); + mDefinite.insert(strItem); + } + } + + // + // + // +#ifdef HAVE_REGEX_SUPPORT + rArchive.Read(iCount); + + if (iCount > 0) + { + for (int v = 0; v < iCount; v++) + { + std::string strItem; + rArchive.Read(strItem); + + // Allocate memory + regex_t* pregex = new regex_t; + + try + { + // Compile + if(::regcomp(pregex, strItem.c_str(), + REG_EXTENDED | REG_NOSUB) != 0) + { + THROW_EXCEPTION(CommonException, + BadRegularExpression) + } + + // Store in list of regular expressions + mRegex.push_back(pregex); + + // Store in list of regular expression strings + // for Serialize + mRegexStr.push_back(strItem); + } + catch(...) + { + delete pregex; + throw; + } + } + } +#endif // HAVE_REGEX_SUPPORT + + // + // + // + int64_t aMagicMarker = 0; + rArchive.Read(aMagicMarker); + + if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mpAlwaysInclude = new ExcludeList; + if (!mpAlwaysInclude) + { + throw std::bad_alloc(); + } + + mpAlwaysInclude->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(CommonException, Internal) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void ExcludeList::Serialize(Archive & rArchive) const +{ + // + // + // + int64_t iCount = mDefinite.size(); + rArchive.Write(iCount); + + for (std::set<std::string>::const_iterator i = mDefinite.begin(); + i != mDefinite.end(); i++) + { + rArchive.Write(*i); + } + + // + // + // +#ifdef HAVE_REGEX_SUPPORT + // don't even try to save compiled regular expressions, + // use string copies instead. + ASSERT(mRegex.size() == mRegexStr.size()); + + iCount = mRegexStr.size(); + rArchive.Write(iCount); + + for (std::vector<std::string>::const_iterator i = mRegexStr.begin(); + i != mRegexStr.end(); i++) + { + rArchive.Write(*i); + } +#endif // HAVE_REGEX_SUPPORT + + // + // + // + if (!mpAlwaysInclude) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpAlwaysInclude->Serialize(rArchive); + } +} diff --git a/lib/common/ExcludeList.h b/lib/common/ExcludeList.h new file mode 100644 index 00000000..3c41bd11 --- /dev/null +++ b/lib/common/ExcludeList.h @@ -0,0 +1,76 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ExcludeList.h +// Purpose: General purpose exclusion list +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef EXCLUDELIST__H +#define EXCLUDELIST__H + +#include <string> +#include <set> +#include <vector> + +// avoid including regex.h in lots of places +#ifndef EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED + typedef int regex_t; +#endif + +class Archive; + +// -------------------------------------------------------------------------- +// +// Class +// Name: ExcludeList +// Purpose: General purpose exclusion list +// Created: 28/1/04 +// +// -------------------------------------------------------------------------- +class ExcludeList +{ +public: + ExcludeList(); + ~ExcludeList(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; + + void AddDefiniteEntries(const std::string &rEntries); + void AddRegexEntries(const std::string &rEntries); + + // Add exceptions to the exclusions (takes ownership) + void SetAlwaysIncludeList(ExcludeList *pAlwaysInclude); + + // Test function + bool IsExcluded(const std::string &rTest) const; + + // Mainly for tests + unsigned int SizeOfDefiniteList() const {return mDefinite.size();} + unsigned int SizeOfRegexList() const +#ifdef HAVE_REGEX_SUPPORT + {return mRegex.size();} +#else + {return 0;} +#endif + +private: + std::set<std::string> mDefinite; +#ifdef HAVE_REGEX_SUPPORT + std::vector<regex_t *> mRegex; + std::vector<std::string> mRegexStr; // save original regular expression string-based source for Serialize +#endif + +#ifdef WIN32 + std::string ReplaceSlashesDefinite(const std::string& input) const; + std::string ReplaceSlashesRegex (const std::string& input) const; +#endif + + // For exceptions to the excludes + ExcludeList *mpAlwaysInclude; +}; + +#endif // EXCLUDELIST__H + diff --git a/lib/common/FdGetLine.cpp b/lib/common/FdGetLine.cpp new file mode 100644 index 00000000..9b53288b --- /dev/null +++ b/lib/common/FdGetLine.cpp @@ -0,0 +1,228 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FdGetLine.cpp +// Purpose: Line based file descriptor reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include "FdGetLine.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// utility whitespace function +inline bool iw(int c) +{ + return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FdGetLine::FdGetLine(int) +// Purpose: Constructor, taking file descriptor +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +FdGetLine::FdGetLine(int fd) + : mFileHandle(fd), + mLineNumber(0), + mBufferBegin(0), + mBytesInBuffer(0), + mPendingEOF(false), + mEOF(false) +{ + if(mFileHandle < 0) {THROW_EXCEPTION(CommonException, BadArguments)} + //printf("FdGetLine buffer size = %d\n", sizeof(mBuffer)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FdGetLine::~FdGetLine() +// Purpose: Destructor +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +FdGetLine::~FdGetLine() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FdGetLine::GetLine(bool) +// Purpose: Returns a file from the file. If Preprocess is true, leading +// and trailing whitespace is removed, and comments (after #) +// are deleted. +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +std::string FdGetLine::GetLine(bool Preprocess) +{ + if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)} + + // EOF? + if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)} + + std::string r; + + bool foundLineEnd = false; + + while(!foundLineEnd && !mEOF) + { + // Use any bytes left in the buffer + while(mBufferBegin < mBytesInBuffer) + { + int c = mBuffer[mBufferBegin++]; + if(c == '\r') + { + // Ignore nasty Windows line ending extra chars + } + else if(c == '\n') + { + // Line end! + foundLineEnd = true; + break; + } + else + { + // Add to string + r += c; + } + + // Implicit line ending at EOF + if(mBufferBegin >= mBytesInBuffer && mPendingEOF) + { + foundLineEnd = true; + } + } + + // Check size + if(r.size() > FDGETLINE_MAX_LINE_SIZE) + { + THROW_EXCEPTION(CommonException, GetLineTooLarge) + } + + // Read more in? + if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF) + { +#ifdef WIN32 + int bytes; + + if (mFileHandle == _fileno(stdin)) + { + bytes = console_read(mBuffer, sizeof(mBuffer)); + } + else + { + bytes = ::read(mFileHandle, mBuffer, + sizeof(mBuffer)); + } +#else // !WIN32 + int bytes = ::read(mFileHandle, mBuffer, sizeof(mBuffer)); +#endif // WIN32 + + // Error? + if(bytes == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Adjust buffer info + mBytesInBuffer = bytes; + mBufferBegin = 0; + + // EOF / closed? + if(bytes == 0) + { + mPendingEOF = true; + } + } + + // EOF? + if(mPendingEOF && mBufferBegin >= mBytesInBuffer) + { + // File is EOF, and now we've depleted the buffer completely, so tell caller as well. + mEOF = true; + } + } + + if(!Preprocess) + { + return r; + } + else + { + // Check for comment char, but char before must be whitespace + int end = 0; + int size = r.size(); + while(end < size) + { + if(r[end] == '#' && (end == 0 || (iw(r[end-1])))) + { + break; + } + end++; + } + + // Remove whitespace + int begin = 0; + while(begin < size && iw(r[begin])) + { + begin++; + } + if(!iw(r[end])) end--; + while(end > begin && iw(r[end])) + { + end--; + } + + // Return a sub string + return r.substr(begin, end - begin + 1); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FdGetLine::DetachFile() +// Purpose: Detaches the file handle, setting the file pointer correctly. +// Probably not good for sockets... +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +void FdGetLine::DetachFile() +{ + if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)} + + // Adjust file pointer + int bytesOver = mBufferBegin - mBufferBegin; + ASSERT(bytesOver >= 0); + if(bytesOver > 0) + { + if(::lseek(mFileHandle, 0 - bytesOver, SEEK_CUR) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + + // Unset file pointer + mFileHandle = -1; +} + + diff --git a/lib/common/FdGetLine.h b/lib/common/FdGetLine.h new file mode 100644 index 00000000..df43c3c9 --- /dev/null +++ b/lib/common/FdGetLine.h @@ -0,0 +1,65 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FdGetLine.h +// Purpose: Line based file descriptor reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- + +#ifndef FDGETLINE__H +#define FDGETLINE__H + +#include <string> + +#ifdef BOX_RELEASE_BUILD + #define FDGETLINE_BUFFER_SIZE 1024 +#elif defined WIN32 + // need enough space for at least one unicode character + // in UTF-8 when calling console_read() from bbackupquery + #define FDGETLINE_BUFFER_SIZE 5 +#else + #define FDGETLINE_BUFFER_SIZE 4 +#endif + +// Just a very large upper bound for line size to avoid +// people sending lots of data over sockets and causing memory problems. +#define FDGETLINE_MAX_LINE_SIZE (1024*256) + +// -------------------------------------------------------------------------- +// +// Class +// Name: FdGetLine +// Purpose: Line based file descriptor reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +class FdGetLine +{ +public: + FdGetLine(int fd); + ~FdGetLine(); +private: + FdGetLine(const FdGetLine &rToCopy); + +public: + std::string GetLine(bool Preprocess = false); + bool IsEOF() {return mEOF;} + int GetLineNumber() {return mLineNumber;} + + // Call to detach, setting file pointer correctly to last bit read. + // Only works for lseek-able file descriptors. + void DetachFile(); + +private: + char mBuffer[FDGETLINE_BUFFER_SIZE]; + int mFileHandle; + int mLineNumber; + int mBufferBegin; + int mBytesInBuffer; + bool mPendingEOF; + bool mEOF; +}; + +#endif // FDGETLINE__H + diff --git a/lib/common/FileModificationTime.cpp b/lib/common/FileModificationTime.cpp new file mode 100644 index 00000000..1109b15f --- /dev/null +++ b/lib/common/FileModificationTime.cpp @@ -0,0 +1,64 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FileModificationTime.cpp +// Purpose: Function for getting file modification time. +// Created: 2010/02/15 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/stat.h> + +#include "BoxTime.h" +#include "FileModificationTime.h" + +#include "MemLeakFindOn.h" + +box_time_t FileModificationTime(EMU_STRUCT_STAT &st) +{ +#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC + box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); +#else + box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#endif + + return datamodified; +} + +box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st) +{ + box_time_t statusmodified = +#ifdef HAVE_STRUCT_STAT_ST_MTIMESPEC + (((int64_t)st.st_ctimespec.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC + (((int64_t)st.st_ctim.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctim.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIMENSEC + (((int64_t)st.st_ctimensec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL)); +#else // no nanoseconds anywhere + (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL)); +#endif + + return statusmodified; +} + +box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st) +{ +#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC + box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); + box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL); +#else + box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); + box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#endif + + return (datamodified > statusmodified)?datamodified:statusmodified; +} + diff --git a/lib/common/FileModificationTime.h b/lib/common/FileModificationTime.h new file mode 100644 index 00000000..e6e6c172 --- /dev/null +++ b/lib/common/FileModificationTime.h @@ -0,0 +1,22 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FileModificationTime.h +// Purpose: Function for getting file modification time. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef FILEMODIFICATIONTIME__H +#define FILEMODIFICATIONTIME__H + +#include <sys/stat.h> + +#include "BoxTime.h" + +box_time_t FileModificationTime(EMU_STRUCT_STAT &st); +box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st); +box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st); + +#endif // FILEMODIFICATIONTIME__H + diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp new file mode 100644 index 00000000..5be8237c --- /dev/null +++ b/lib/common/FileStream.cpp @@ -0,0 +1,447 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FileStream.cpp +// Purpose: IOStream interface to files +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "FileStream.h" +#include "CommonException.h" +#include "Logging.h" + +#include <errno.h> + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::FileStream(const char *, int, int) +// Purpose: Constructor, opens file +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +FileStream::FileStream(const std::string& rFilename, int flags, int mode) +#ifdef WIN32 + : mOSFileHandle(::openfile(rFilename.c_str(), flags, mode)), +#else + : mOSFileHandle(::open(rFilename.c_str(), flags, mode)), +#endif + mIsEOF(false), + mFileName(rFilename) +{ + AfterOpen(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::FileStream(const char *, int, int) +// Purpose: Alternative constructor, takes a const char *, +// avoids const strings being interpreted as handles! +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +FileStream::FileStream(const char *pFilename, int flags, int mode) +#ifdef WIN32 + : mOSFileHandle(::openfile(pFilename, flags, mode)), +#else + : mOSFileHandle(::open(pFilename, flags, mode)), +#endif + mIsEOF(false), + mFileName(pFilename) +{ + AfterOpen(); +} + +void FileStream::AfterOpen() +{ +#ifdef WIN32 + if(mOSFileHandle == INVALID_HANDLE_VALUE) +#else + if(mOSFileHandle < 0) +#endif + { + MEMLEAKFINDER_NOT_A_LEAK(this); + + #ifdef WIN32 + BOX_LOG_WIN_WARNING_NUMBER("Failed to open file: " << + mFileName, winerrno); + #else + BOX_LOG_SYS_WARNING("Failed to open file: " << + mFileName); + #endif + + if(errno == EACCES) + { + THROW_EXCEPTION(CommonException, AccessDenied) + } + else + { + THROW_EXCEPTION(CommonException, OSFileOpenError) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::FileStream(tOSFileHandle) +// Purpose: Constructor, using existing file descriptor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +FileStream::FileStream(tOSFileHandle FileDescriptor) + : mOSFileHandle(FileDescriptor), + mIsEOF(false), + mFileName("HANDLE") +{ +#ifdef WIN32 + if(mOSFileHandle == INVALID_HANDLE_VALUE) +#else + if(mOSFileHandle < 0) +#endif + { + MEMLEAKFINDER_NOT_A_LEAK(this); + BOX_ERROR("FileStream: called with invalid file handle"); + THROW_EXCEPTION(CommonException, OSFileOpenError) + } +} + +#if 0 +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::FileStream(const FileStream &) +// Purpose: Copy constructor, creates a duplicate of the file handle +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +FileStream::FileStream(const FileStream &rToCopy) + : mOSFileHandle(::dup(rToCopy.mOSFileHandle)), + mIsEOF(rToCopy.mIsEOF) +{ +#ifdef WIN32 + if(mOSFileHandle == INVALID_HANDLE_VALUE) +#else + if(mOSFileHandle < 0) +#endif + { + MEMLEAKFINDER_NOT_A_LEAK(this); + BOX_ERROR("FileStream: copying unopened file"); + THROW_EXCEPTION(CommonException, OSFileOpenError) + } +} +#endif // 0 + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::~FileStream() +// Purpose: Destructor, closes file +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +FileStream::~FileStream() +{ + if(mOSFileHandle != INVALID_FILE) + { + Close(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::Read(void *, int) +// Purpose: Reads bytes from the file +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +int FileStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + if(mOSFileHandle == INVALID_FILE) + { + THROW_EXCEPTION(CommonException, FileClosed) + } + +#ifdef WIN32 + int r; + DWORD numBytesRead = 0; + BOOL valid = ReadFile( + this->mOSFileHandle, + pBuffer, + NBytes, + &numBytesRead, + NULL + ); + + if(valid) + { + r = numBytesRead; + } + else if(GetLastError() == ERROR_BROKEN_PIPE) + { + r = 0; + } + else + { + BOX_LOG_WIN_ERROR("Failed to read from file: " << mFileName); + r = -1; + } +#else + int r = ::read(mOSFileHandle, pBuffer, NBytes); + if(r == -1) + { + BOX_LOG_SYS_ERROR("Failed to read from file: " << mFileName); + } +#endif + + if(r == -1) + { + THROW_EXCEPTION(CommonException, OSFileReadError) + } + + if(r == 0) + { + mIsEOF = true; + } + + return r; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::BytesLeftToRead() +// Purpose: Returns number of bytes to read (may not be most efficient function ever) +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +IOStream::pos_type FileStream::BytesLeftToRead() +{ + EMU_STRUCT_STAT st; + if(EMU_FSTAT(mOSFileHandle, &st) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + return st.st_size - GetPosition(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::Write(void *, int) +// Purpose: Writes bytes to the file +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void FileStream::Write(const void *pBuffer, int NBytes) +{ + if(mOSFileHandle == INVALID_FILE) + { + THROW_EXCEPTION(CommonException, FileClosed) + } + +#ifdef WIN32 + DWORD numBytesWritten = 0; + BOOL res = WriteFile( + this->mOSFileHandle, + pBuffer, + NBytes, + &numBytesWritten, + NULL + ); + + if ((res == 0) || (numBytesWritten != (DWORD)NBytes)) + { + // DWORD err = GetLastError(); + THROW_EXCEPTION(CommonException, OSFileWriteError) + } +#else + if(::write(mOSFileHandle, pBuffer, NBytes) != NBytes) + { + BOX_LOG_SYS_ERROR("Failed to write to file: " << mFileName); + THROW_EXCEPTION(CommonException, OSFileWriteError) + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::GetPosition() +// Purpose: Get position in stream +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type FileStream::GetPosition() const +{ + if(mOSFileHandle == INVALID_FILE) + { + THROW_EXCEPTION(CommonException, FileClosed) + } + +#ifdef WIN32 + LARGE_INTEGER conv; + + conv.HighPart = 0; + conv.LowPart = 0; + + conv.LowPart = SetFilePointer(this->mOSFileHandle, 0, &conv.HighPart, FILE_CURRENT); + + return (IOStream::pos_type)conv.QuadPart; +#else // ! WIN32 + off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR); + if(p == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + return (IOStream::pos_type)p; +#endif // WIN32 +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::Seek(pos_type, int) +// Purpose: Seeks within file, as lseek +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void FileStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + if(mOSFileHandle == INVALID_FILE) + { + THROW_EXCEPTION(CommonException, FileClosed) + } + +#ifdef WIN32 + LARGE_INTEGER conv; + + conv.QuadPart = Offset; + DWORD retVal = SetFilePointer(this->mOSFileHandle, conv.LowPart, &conv.HighPart, ConvertSeekTypeToOSWhence(SeekType)); + + if(retVal == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) + { + THROW_EXCEPTION(CommonException, OSFileError) + } +#else // ! WIN32 + if(::lseek(mOSFileHandle, Offset, ConvertSeekTypeToOSWhence(SeekType)) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } +#endif // WIN32 + + // Not end of file any more! + mIsEOF = false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::Close() +// Purpose: Closes the underlying file +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void FileStream::Close() +{ + if(mOSFileHandle == INVALID_FILE) + { + THROW_EXCEPTION(CommonException, FileAlreadyClosed) + } + +#ifdef WIN32 + if(::CloseHandle(mOSFileHandle) == 0) +#else + if(::close(mOSFileHandle) != 0) +#endif + { + THROW_EXCEPTION(CommonException, OSFileCloseError) + } + + mOSFileHandle = INVALID_FILE; + mIsEOF = true; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::StreamDataLeft() +// Purpose: Any data left to write? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool FileStream::StreamDataLeft() +{ + return !mIsEOF; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::StreamClosed() +// Purpose: Is the stream closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool FileStream::StreamClosed() +{ + return mIsEOF; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::CompareWith(IOStream&, int) +// Purpose: Compare bytes in this file with other stream's data +// Created: 2009/01/03 +// +// -------------------------------------------------------------------------- +bool FileStream::CompareWith(IOStream& rOther, int Timeout) +{ + // Size + IOStream::pos_type mySize = BytesLeftToRead(); + IOStream::pos_type otherSize = 0; + + // Test the contents + char buf1[2048]; + char buf2[2048]; + while(StreamDataLeft() && rOther.StreamDataLeft()) + { + int readSize = rOther.Read(buf1, sizeof(buf1), Timeout); + otherSize += readSize; + + if(Read(buf2, readSize) != readSize || + ::memcmp(buf1, buf2, readSize) != 0) + { + return false; + } + } + + // Check read all the data from the server and file -- can't be + // equal if local and remote aren't the same length. Can't use + // StreamDataLeft() test on local file, because if it's the same + // size, it won't know it's EOF yet. + + if(rOther.StreamDataLeft() || otherSize != mySize) + { + return false; + } + + return true; +} diff --git a/lib/common/FileStream.h b/lib/common/FileStream.h new file mode 100644 index 00000000..9101a968 --- /dev/null +++ b/lib/common/FileStream.h @@ -0,0 +1,66 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FileStream.h +// Purpose: FileStream interface to files +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef FILESTREAM__H +#define FILESTREAM__H + +#include "IOStream.h" + +#include <fcntl.h> +#include <stdlib.h> +#include <sys/stat.h> + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +class FileStream : public IOStream +{ +public: + FileStream(const std::string& rFilename, + int flags = (O_RDONLY | O_BINARY), + int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + + // Ensure that const char * name doesn't end up as a handle + // on Windows! + + FileStream(const char *pFilename, + int flags = (O_RDONLY | O_BINARY), + int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + + FileStream(tOSFileHandle FileDescriptor); + + virtual ~FileStream(); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + bool CompareWith(IOStream& rOther, int Timeout = IOStream::TimeOutInfinite); + +private: + tOSFileHandle mOSFileHandle; + bool mIsEOF; + FileStream(const FileStream &rToCopy) { /* do not call */ } + void AfterOpen(); + + // for debugging.. + std::string mFileName; +}; + + +#endif // FILESTREAM__H + + diff --git a/lib/common/Guards.h b/lib/common/Guards.h new file mode 100644 index 00000000..cd2e4628 --- /dev/null +++ b/lib/common/Guards.h @@ -0,0 +1,121 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Guards.h +// Purpose: Classes which ensure things are closed/deleted properly when +// going out of scope. Easy exception proof code, etc +// Created: 2003/07/12 +// +// -------------------------------------------------------------------------- + +#ifndef GUARDS__H +#define GUARDS__H + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <errno.h> +#include <fcntl.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> + +#include <new> + +#include "CommonException.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +template <int flags = O_RDONLY | O_BINARY, int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)> +class FileHandleGuard +{ +public: + FileHandleGuard(const std::string& rFilename) + : mOSFileHandle(::open(rFilename.c_str(), flags, mode)) + { + if(mOSFileHandle < 0) + { + BOX_LOG_SYS_ERROR("FileHandleGuard: failed to open " + "file '" << rFilename << "'"); + THROW_EXCEPTION(CommonException, OSFileOpenError) + } + } + + ~FileHandleGuard() + { + if(mOSFileHandle >= 0) + { + Close(); + } + } + + void Close() + { + if(mOSFileHandle < 0) + { + THROW_EXCEPTION(CommonException, FileAlreadyClosed) + } + if(::close(mOSFileHandle) != 0) + { + THROW_EXCEPTION(CommonException, OSFileCloseError) + } + mOSFileHandle = -1; + } + + operator int() const + { + return mOSFileHandle; + } + +private: + int mOSFileHandle; +}; + +template<typename type> +class MemoryBlockGuard +{ +public: + MemoryBlockGuard(int BlockSize) + : mpBlock(::malloc(BlockSize)) + { + if(mpBlock == 0) + { + throw std::bad_alloc(); + } + } + + ~MemoryBlockGuard() + { + free(mpBlock); + } + + operator type() const + { + return (type)mpBlock; + } + + type GetPtr() const + { + return (type)mpBlock; + } + + void Resize(int NewSize) + { + void *ptrn = ::realloc(mpBlock, NewSize); + if(ptrn == 0) + { + throw std::bad_alloc(); + } + mpBlock = ptrn; + } + +private: + void *mpBlock; +}; + +#include "MemLeakFindOff.h" + +#endif // GUARDS__H + diff --git a/lib/common/IOStream.cpp b/lib/common/IOStream.cpp new file mode 100644 index 00000000..fc9d0bc3 --- /dev/null +++ b/lib/common/IOStream.cpp @@ -0,0 +1,251 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: IOStream.cpp +// Purpose: I/O Stream abstraction +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "IOStream.h" +#include "CommonException.h" +#include "Guards.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::IOStream() +// Purpose: Constructor +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +IOStream::IOStream() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::~IOStream() +// Purpose: Destructor +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +IOStream::~IOStream() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::Close() +// Purpose: Close the stream +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void IOStream::Close() +{ + // Do nothing by default -- let the destructor clear everything up. +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::Seek(int, int) +// Purpose: Seek in stream (if supported) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void IOStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + THROW_EXCEPTION(CommonException, NotSupported) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::GetPosition() +// Purpose: Returns current position in stream (if supported) +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type IOStream::GetPosition() const +{ + THROW_EXCEPTION(CommonException, NotSupported) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::ConvertSeekTypeToOSWhence(int) +// Purpose: Return an whence arg for lseek given a IOStream seek type +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +int IOStream::ConvertSeekTypeToOSWhence(int SeekType) +{ + // Should be nicely optimised out as values are choosen in header file to match OS values. + int ostype = SEEK_SET; + switch(SeekType) + { +#ifdef WIN32 + case SeekType_Absolute: + ostype = FILE_BEGIN; + break; + case SeekType_Relative: + ostype = FILE_CURRENT; + break; + case SeekType_End: + ostype = FILE_END; + break; +#else // ! WIN32 + case SeekType_Absolute: + ostype = SEEK_SET; + break; + case SeekType_Relative: + ostype = SEEK_CUR; + break; + case SeekType_End: + ostype = SEEK_END; + break; +#endif // WIN32 + + default: + THROW_EXCEPTION(CommonException, IOStreamBadSeekType) + } + + return ostype; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::ReadFullBuffer(void *, int, int) +// Purpose: Reads bytes into buffer, returning whether or not it managed to +// get all the bytes required. Exception and abort use of stream +// if this returns false. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool IOStream::ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout) +{ + int bytesToGo = NBytes; + char *buffer = (char*)pBuffer; + if(pNBytesRead) (*pNBytesRead) = 0; + + while(bytesToGo > 0) + { + int bytesRead = Read(buffer, bytesToGo, Timeout); + if(bytesRead == 0) + { + // Timeout or something + return false; + } + // Increment things + bytesToGo -= bytesRead; + buffer += bytesRead; + if(pNBytesRead) (*pNBytesRead) += bytesRead; + } + + // Got everything + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::WriteAllBuffered() +// Purpose: Ensures that any data which has been buffered is written to the stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void IOStream::WriteAllBuffered() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::BytesLeftToRead() +// Purpose: Numbers of bytes left to read in the stream, or +// IOStream::SizeOfStreamUnknown if this isn't known. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +IOStream::pos_type IOStream::BytesLeftToRead() +{ + return IOStream::SizeOfStreamUnknown; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::CopyStreamTo(IOStream &, int Timeout) +// Purpose: Copies the entire stream to another stream (reading from this, +// writing to rCopyTo). Returns whether the copy completed (ie +// StreamDataLeft() returns false) +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool IOStream::CopyStreamTo(IOStream &rCopyTo, int Timeout, int BufferSize) +{ + // Make sure there's something to do before allocating that buffer + if(!StreamDataLeft()) + { + return true; // complete, even though nothing happened + } + + // Buffer + MemoryBlockGuard<char*> buffer(BufferSize); + + // Get copying! + while(StreamDataLeft()) + { + // Read some data + int bytes = Read(buffer, BufferSize, Timeout); + if(bytes == 0 && StreamDataLeft()) + { + return false; // incomplete, timed out + } + + // Write some data + if(bytes != 0) + { + rCopyTo.Write(buffer, bytes); + } + } + + return true; // completed +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::Flush(int Timeout) +// Purpose: Read and discard all remaining data in stream. +// Useful for protocol streams which must be flushed +// to avoid breaking the protocol. +// Created: 2008/08/20 +// +// -------------------------------------------------------------------------- +void IOStream::Flush(int Timeout) +{ + char buffer[4096]; + + while(StreamDataLeft()) + { + Read(buffer, sizeof(buffer), Timeout); + } +} + +void IOStream::Write(const char *pBuffer) +{ + Write(pBuffer, strlen(pBuffer)); +} diff --git a/lib/common/IOStream.h b/lib/common/IOStream.h new file mode 100644 index 00000000..0b1cedd3 --- /dev/null +++ b/lib/common/IOStream.h @@ -0,0 +1,73 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: IOStream.h +// Purpose: I/O Stream abstraction +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef IOSTREAM__H +#define IOSTREAM__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: IOStream +// Purpose: Abstract interface to streams of data +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +class IOStream +{ +public: + IOStream(); + virtual ~IOStream(); + +private: + IOStream(const IOStream &rToCopy); /* forbidden */ + IOStream& operator=(const IOStream &rToCopy); /* forbidden */ + +public: + enum + { + TimeOutInfinite = -1, + SizeOfStreamUnknown = -1 + }; + + enum + { + SeekType_Absolute = 0, + SeekType_Relative = 1, + SeekType_End = 2 + }; + + // Timeout in milliseconds + // Read may return 0 -- does not mean end of stream. + typedef int64_t pos_type; + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite) = 0; + virtual pos_type BytesLeftToRead(); // may return IOStream::SizeOfStreamUnknown (and will for most stream types) + virtual void Write(const void *pBuffer, int NBytes) = 0; + virtual void Write(const char *pBuffer); + virtual void WriteAllBuffered(); + virtual pos_type GetPosition() const; + virtual void Seek(pos_type Offset, int SeekType); + virtual void Close(); + + // Has all data that can be read been read? + virtual bool StreamDataLeft() = 0; + // Has the stream been closed (writing not possible) + virtual bool StreamClosed() = 0; + + // Utility functions + bool ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout = IOStream::TimeOutInfinite); + bool CopyStreamTo(IOStream &rCopyTo, int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024); + void Flush(int Timeout = IOStream::TimeOutInfinite); + + static int ConvertSeekTypeToOSWhence(int SeekType); +}; + + +#endif // IOSTREAM__H + + diff --git a/lib/common/IOStreamGetLine.cpp b/lib/common/IOStreamGetLine.cpp new file mode 100644 index 00000000..27a77c29 --- /dev/null +++ b/lib/common/IOStreamGetLine.cpp @@ -0,0 +1,227 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: IOStreamGetLine.cpp +// Purpose: Line based file descriptor reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "IOStreamGetLine.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// utility whitespace function +inline bool iw(int c) +{ + return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStreamGetLine::IOStreamGetLine(int) +// Purpose: Constructor, taking file descriptor +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +IOStreamGetLine::IOStreamGetLine(IOStream &Stream) + : mrStream(Stream), + mLineNumber(0), + mBufferBegin(0), + mBytesInBuffer(0), + mPendingEOF(false), + mEOF(false) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStreamGetLine::~IOStreamGetLine() +// Purpose: Destructor +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +IOStreamGetLine::~IOStreamGetLine() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStreamGetLine::GetLine(std::string &, bool, int) +// Purpose: Gets a line from the file, returning it in rOutput. If Preprocess is true, leading +// and trailing whitespace is removed, and comments (after #) +// are deleted. +// Returns true if a line is available now, false if retrying may get a line (eg timeout, signal), +// and exceptions if it's EOF. +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +bool IOStreamGetLine::GetLine(std::string &rOutput, bool Preprocess, int Timeout) +{ + // EOF? + if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)} + + // Initialise string to stored into + std::string r(mPendingString); + mPendingString.erase(); + + bool foundLineEnd = false; + + while(!foundLineEnd && !mEOF) + { + // Use any bytes left in the buffer + while(mBufferBegin < mBytesInBuffer) + { + int c = mBuffer[mBufferBegin++]; + if(c == '\r') + { + // Ignore nasty Windows line ending extra chars + } + else if(c == '\n') + { + // Line end! + foundLineEnd = true; + break; + } + else + { + // Add to string + r += c; + } + + // Implicit line ending at EOF + if(mBufferBegin >= mBytesInBuffer && mPendingEOF) + { + foundLineEnd = true; + } + } + + // Check size + if(r.size() > IOSTREAMGETLINE_MAX_LINE_SIZE) + { + THROW_EXCEPTION(CommonException, GetLineTooLarge) + } + + // Read more in? + if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF) + { + int bytes = mrStream.Read(mBuffer, sizeof(mBuffer), Timeout); + + // Adjust buffer info + mBytesInBuffer = bytes; + mBufferBegin = 0; + + // EOF / closed? + if(!mrStream.StreamDataLeft()) + { + mPendingEOF = true; + } + + // No data returned? + if(bytes == 0 && mrStream.StreamDataLeft()) + { + // store string away + mPendingString = r; + // Return false; + return false; + } + } + + // EOF? + if(mPendingEOF && mBufferBegin >= mBytesInBuffer) + { + // File is EOF, and now we've depleted the buffer completely, so tell caller as well. + mEOF = true; + } + } + + if(!Preprocess) + { + rOutput = r; + return true; + } + else + { + // Check for comment char, but char before must be whitespace + int end = 0; + int size = r.size(); + while(end < size) + { + if(r[end] == '#' && (end == 0 || (iw(r[end-1])))) + { + break; + } + end++; + } + + // Remove whitespace + int begin = 0; + while(begin < size && iw(r[begin])) + { + begin++; + } + if(!iw(r[end])) end--; + while(end > begin && iw(r[end])) + { + end--; + } + + // Return a sub string + rOutput = r.substr(begin, end - begin + 1); + return true; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStreamGetLine::DetachFile() +// Purpose: Detaches the file handle, setting the file pointer correctly. +// Probably not good for sockets... +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +void IOStreamGetLine::DetachFile() +{ + // Adjust file pointer + int bytesOver = mBytesInBuffer - mBufferBegin; + ASSERT(bytesOver >= 0); + if(bytesOver > 0) + { + mrStream.Seek(0 - bytesOver, IOStream::SeekType_Relative); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStreamGetLine::IgnoreBufferedData(int) +// Purpose: Ignore buffered bytes (effectively removing them from the +// beginning of the buffered data.) +// Cannot remove more bytes than are currently in the buffer. +// Be careful when this is used! +// Created: 22/12/04 +// +// -------------------------------------------------------------------------- +void IOStreamGetLine::IgnoreBufferedData(int BytesToIgnore) +{ + int bytesInBuffer = mBytesInBuffer - mBufferBegin; + if(BytesToIgnore < 0 || BytesToIgnore > bytesInBuffer) + { + THROW_EXCEPTION(CommonException, IOStreamGetLineNotEnoughDataToIgnore) + } + mBufferBegin += BytesToIgnore; +} + + + diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h new file mode 100644 index 00000000..9a5d1818 --- /dev/null +++ b/lib/common/IOStreamGetLine.h @@ -0,0 +1,71 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: IOStreamGetLine.h +// Purpose: Line based file descriptor reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- + +#ifndef IOSTREAMGETLINE__H +#define IOSTREAMGETLINE__H + +#include <string> + +#include "IOStream.h" + +#ifdef BOX_RELEASE_BUILD + #define IOSTREAMGETLINE_BUFFER_SIZE 1024 +#else + #define IOSTREAMGETLINE_BUFFER_SIZE 4 +#endif + +// Just a very large upper bound for line size to avoid +// people sending lots of data over sockets and causing memory problems. +#define IOSTREAMGETLINE_MAX_LINE_SIZE (1024*256) + +// -------------------------------------------------------------------------- +// +// Class +// Name: IOStreamGetLine +// Purpose: Line based stream reading +// Created: 2003/07/24 +// +// -------------------------------------------------------------------------- +class IOStreamGetLine +{ +public: + IOStreamGetLine(IOStream &Stream); + ~IOStreamGetLine(); +private: + IOStreamGetLine(const IOStreamGetLine &rToCopy); + +public: + bool GetLine(std::string &rOutput, bool Preprocess = false, int Timeout = IOStream::TimeOutInfinite); + bool IsEOF() {return mEOF;} + int GetLineNumber() {return mLineNumber;} + + // Call to detach, setting file pointer correctly to last bit read. + // Only works for lseek-able file descriptors. + void DetachFile(); + + // For doing interesting stuff with the remaining data... + // Be careful with this! + const void *GetBufferedData() const {return mBuffer + mBufferBegin;} + int GetSizeOfBufferedData() const {return mBytesInBuffer - mBufferBegin;} + void IgnoreBufferedData(int BytesToIgnore); + IOStream &GetUnderlyingStream() {return mrStream;} + +private: + char mBuffer[IOSTREAMGETLINE_BUFFER_SIZE]; + IOStream &mrStream; + int mLineNumber; + int mBufferBegin; + int mBytesInBuffer; + bool mPendingEOF; + bool mEOF; + std::string mPendingString; +}; + +#endif // IOSTREAMGETLINE__H + diff --git a/lib/common/InvisibleTempFileStream.cpp b/lib/common/InvisibleTempFileStream.cpp new file mode 100644 index 00000000..abfcb5f6 --- /dev/null +++ b/lib/common/InvisibleTempFileStream.cpp @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: InvisibleTempFileStream.cpp +// Purpose: IOStream interface to temporary files that +// delete themselves +// Created: 2006/10/13 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "InvisibleTempFileStream.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: InvisibleTempFileStream::InvisibleTempFileStream +// (const char *, int, int) +// Purpose: Constructor, opens invisible file +// Created: 2006/10/13 +// +// -------------------------------------------------------------------------- +InvisibleTempFileStream::InvisibleTempFileStream(const char *Filename, int flags, int mode) +#ifdef WIN32 + : FileStream(Filename, flags | O_TEMPORARY, mode) +#else + : FileStream(Filename, flags, mode) +#endif +{ + #ifndef WIN32 + if(unlink(Filename) != 0) + { + MEMLEAKFINDER_NOT_A_LEAK(this); + THROW_EXCEPTION(CommonException, OSFileOpenError) + } + #endif +} diff --git a/lib/common/InvisibleTempFileStream.h b/lib/common/InvisibleTempFileStream.h new file mode 100644 index 00000000..a77d05e2 --- /dev/null +++ b/lib/common/InvisibleTempFileStream.h @@ -0,0 +1,35 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: InvisibleTempFileStream.h +// Purpose: FileStream interface to temporary files that +// delete themselves +// Created: 2006/10/13 +// +// -------------------------------------------------------------------------- + +#ifndef INVISIBLETEMPFILESTREAM__H +#define INVISIBLETEMPFILESTREAM__H + +#include "FileStream.h" + +class InvisibleTempFileStream : public FileStream +{ +public: + InvisibleTempFileStream(const char *Filename, +#ifdef WIN32 + int flags = (O_RDONLY | O_BINARY), +#else + int flags = O_RDONLY, +#endif + int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + +private: + InvisibleTempFileStream(const InvisibleTempFileStream &rToCopy) + : FileStream(INVALID_FILE) + { /* do not call */ } +}; + +#endif // INVISIBLETEMPFILESTREAM__H + + diff --git a/lib/common/Logging.cpp b/lib/common/Logging.cpp new file mode 100644 index 00000000..296443ea --- /dev/null +++ b/lib/common/Logging.cpp @@ -0,0 +1,518 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Logging.cpp +// Purpose: Generic logging core routines implementation +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <errno.h> +#include <time.h> +#include <string.h> // for stderror + +// c.f. http://bugs.debian.org/512510 +#include <cstdio> + +#ifdef HAVE_SYSLOG_H + #include <syslog.h> +#endif +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <cstring> +#include <iomanip> + +#include "BoxTime.h" +#include "Logging.h" + +bool Logging::sLogToSyslog = false; +bool Logging::sLogToConsole = false; +bool Logging::sContextSet = false; + +bool HideExceptionMessageGuard::sHiddenState = false; + +std::vector<Logger*> Logging::sLoggers; +std::string Logging::sContext; +Console* Logging::spConsole = NULL; +Syslog* Logging::spSyslog = NULL; +Log::Level Logging::sGlobalLevel = Log::EVERYTHING; +Logging Logging::sGlobalLogging; //automatic initialisation +std::string Logging::sProgramName; + +Logging::Logging() +{ + ASSERT(!spConsole); + ASSERT(!spSyslog); + spConsole = new Console(); + spSyslog = new Syslog(); + sLogToConsole = true; + sLogToSyslog = true; +} + +Logging::~Logging() +{ + sLogToConsole = false; + sLogToSyslog = false; + delete spConsole; + delete spSyslog; + spConsole = NULL; + spSyslog = NULL; +} + +void Logging::ToSyslog(bool enabled) +{ + if (!sLogToSyslog && enabled) + { + Add(spSyslog); + } + + if (sLogToSyslog && !enabled) + { + Remove(spSyslog); + } + + sLogToSyslog = enabled; +} + +void Logging::ToConsole(bool enabled) +{ + if (!sLogToConsole && enabled) + { + Add(spConsole); + } + + if (sLogToConsole && !enabled) + { + Remove(spConsole); + } + + sLogToConsole = enabled; +} + +void Logging::FilterConsole(Log::Level level) +{ + spConsole->Filter(level); +} + +void Logging::FilterSyslog(Log::Level level) +{ + spSyslog->Filter(level); +} + +void Logging::Add(Logger* pNewLogger) +{ + for (std::vector<Logger*>::iterator i = sLoggers.begin(); + i != sLoggers.end(); i++) + { + if (*i == pNewLogger) + { + return; + } + } + + sLoggers.insert(sLoggers.begin(), pNewLogger); +} + +void Logging::Remove(Logger* pOldLogger) +{ + for (std::vector<Logger*>::iterator i = sLoggers.begin(); + i != sLoggers.end(); i++) + { + if (*i == pOldLogger) + { + sLoggers.erase(i); + return; + } + } +} + +void Logging::Log(Log::Level level, const std::string& rFile, + int line, const std::string& rMessage) +{ + if (level > sGlobalLevel) + { + return; + } + + std::string newMessage; + + if (sContextSet) + { + newMessage += "[" + sContext + "] "; + } + + newMessage += rMessage; + + for (std::vector<Logger*>::iterator i = sLoggers.begin(); + i != sLoggers.end(); i++) + { + bool result = (*i)->Log(level, rFile, line, newMessage); + if (!result) + { + return; + } + } +} + +void Logging::LogToSyslog(Log::Level level, const std::string& rFile, + int line, const std::string& rMessage) +{ + if (!sLogToSyslog) + { + return; + } + + if (level > sGlobalLevel) + { + return; + } + + std::string newMessage; + + if (sContextSet) + { + newMessage += "[" + sContext + "] "; + } + + newMessage += rMessage; + + spSyslog->Log(level, rFile, line, newMessage); +} + +void Logging::SetContext(std::string context) +{ + sContext = context; + sContextSet = true; +} + +Log::Level Logging::GetNamedLevel(const std::string& rName) +{ + if (rName == "nothing") { return Log::NOTHING; } + else if (rName == "fatal") { return Log::FATAL; } + else if (rName == "error") { return Log::ERROR; } + else if (rName == "warning") { return Log::WARNING; } + else if (rName == "notice") { return Log::NOTICE; } + else if (rName == "info") { return Log::INFO; } + else if (rName == "trace") { return Log::TRACE; } + else if (rName == "everything") { return Log::EVERYTHING; } + else + { + BOX_ERROR("Unknown verbosity level: " << rName); + return Log::INVALID; + } +} + +void Logging::ClearContext() +{ + sContextSet = false; +} + +void Logging::SetProgramName(const std::string& rProgramName) +{ + sProgramName = rProgramName; + + for (std::vector<Logger*>::iterator i = sLoggers.begin(); + i != sLoggers.end(); i++) + { + (*i)->SetProgramName(rProgramName); + } +} + +void Logging::SetFacility(int facility) +{ + spSyslog->SetFacility(facility); +} + +Logger::Logger() +: mCurrentLevel(Log::EVERYTHING) +{ + Logging::Add(this); +} + +Logger::Logger(Log::Level Level) +: mCurrentLevel(Level) +{ + Logging::Add(this); +} + +Logger::~Logger() +{ + Logging::Remove(this); +} + +bool Console::sShowTime = false; +bool Console::sShowTimeMicros = false; +bool Console::sShowTag = false; +bool Console::sShowPID = false; +std::string Console::sTag; + +void Console::SetProgramName(const std::string& rProgramName) +{ + sTag = rProgramName; +} + +void Console::SetShowTag(bool enabled) +{ + sShowTag = enabled; +} + +void Console::SetShowTime(bool enabled) +{ + sShowTime = enabled; +} + +void Console::SetShowTimeMicros(bool enabled) +{ + sShowTimeMicros = enabled; +} + +void Console::SetShowPID(bool enabled) +{ + sShowPID = enabled; +} + +bool Console::Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage) +{ + if (level > GetLevel()) + { + return true; + } + + FILE* target = stdout; + + if (level <= Log::WARNING) + { + target = stderr; + } + + std::ostringstream buf; + + if (sShowTime) + { + buf << FormatTime(GetCurrentBoxTime(), false, sShowTimeMicros); + buf << " "; + } + + if (sShowTag) + { + if (sShowPID) + { + buf << "[" << sTag << " " << getpid() << "] "; + } + else + { + buf << "[" << sTag << "] "; + } + } + else if (sShowPID) + { + buf << "[" << getpid() << "] "; + } + + if (level <= Log::FATAL) + { + buf << "FATAL: "; + } + else if (level <= Log::ERROR) + { + buf << "ERROR: "; + } + else if (level <= Log::WARNING) + { + buf << "WARNING: "; + } + else if (level <= Log::NOTICE) + { + buf << "NOTICE: "; + } + else if (level <= Log::INFO) + { + buf << "INFO: "; + } + else if (level <= Log::TRACE) + { + buf << "TRACE: "; + } + + buf << rMessage; + + #ifdef WIN32 + std::string output = buf.str(); + ConvertUtf8ToConsole(output.c_str(), output); + fprintf(target, "%s\n", output.c_str()); + #else + fprintf(target, "%s\n", buf.str().c_str()); + #endif + + return true; +} + +bool Syslog::Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage) +{ + if (level > GetLevel()) + { + return true; + } + + int syslogLevel = LOG_ERR; + + switch(level) + { + case Log::NOTHING: /* fall through */ + case Log::INVALID: /* fall through */ + case Log::FATAL: syslogLevel = LOG_CRIT; break; + case Log::ERROR: syslogLevel = LOG_ERR; break; + case Log::WARNING: syslogLevel = LOG_WARNING; break; + case Log::NOTICE: syslogLevel = LOG_NOTICE; break; + case Log::INFO: syslogLevel = LOG_INFO; break; + case Log::TRACE: /* fall through */ + case Log::EVERYTHING: syslogLevel = LOG_DEBUG; break; + } + + std::string msg; + + if (level <= Log::FATAL) + { + msg = "FATAL: "; + } + else if (level <= Log::ERROR) + { + msg = "ERROR: "; + } + else if (level <= Log::WARNING) + { + msg = "WARNING: "; + } + else if (level <= Log::NOTICE) + { + msg = "NOTICE: "; + } + + msg += rMessage; + + syslog(syslogLevel, "%s", msg.c_str()); + + return true; +} + +Syslog::Syslog() : mFacility(LOG_LOCAL6) +{ + ::openlog("Box Backup", LOG_PID, mFacility); +} + +Syslog::~Syslog() +{ + ::closelog(); +} + +void Syslog::SetProgramName(const std::string& rProgramName) +{ + mName = rProgramName; + ::closelog(); + ::openlog(mName.c_str(), LOG_PID, mFacility); +} + +void Syslog::SetFacility(int facility) +{ + mFacility = facility; + ::closelog(); + ::openlog(mName.c_str(), LOG_PID, mFacility); +} + +int Syslog::GetNamedFacility(const std::string& rFacility) +{ + #define CASE_RETURN(x) if (rFacility == #x) { return LOG_ ## x; } + CASE_RETURN(LOCAL0) + CASE_RETURN(LOCAL1) + CASE_RETURN(LOCAL2) + CASE_RETURN(LOCAL3) + CASE_RETURN(LOCAL4) + CASE_RETURN(LOCAL5) + CASE_RETURN(LOCAL6) + CASE_RETURN(DAEMON) + #undef CASE_RETURN + + BOX_ERROR("Unknown log facility '" << rFacility << "', " + "using default LOCAL6"); + return LOG_LOCAL6; +} + +bool FileLogger::Log(Log::Level Level, const std::string& rFile, + int line, std::string& rMessage) +{ + if (Level > GetLevel()) + { + return true; + } + + /* avoid infinite loop if this throws an exception */ + Logging::Remove(this); + + std::ostringstream buf; + buf << FormatTime(GetCurrentBoxTime(), true, false); + buf << " "; + + if (Level <= Log::FATAL) + { + buf << "[FATAL] "; + } + else if (Level <= Log::ERROR) + { + buf << "[ERROR] "; + } + else if (Level <= Log::WARNING) + { + buf << "[WARNING] "; + } + else if (Level <= Log::NOTICE) + { + buf << "[NOTICE] "; + } + else if (Level <= Log::INFO) + { + buf << "[INFO] "; + } + else if (Level <= Log::TRACE) + { + buf << "[TRACE] "; + } + + buf << rMessage << "\n"; + std::string output = buf.str(); + + #ifdef WIN32 + ConvertUtf8ToConsole(output.c_str(), output); + #endif + + mLogFile.Write(output.c_str(), output.length()); + + Logging::Add(this); + return true; +} + +std::string PrintEscapedBinaryData(const std::string& rInput) +{ + std::ostringstream output; + + for (size_t i = 0; i < rInput.length(); i++) + { + if (isprint(rInput[i])) + { + output << rInput[i]; + } + else + { + output << "\\x" << std::hex << std::setw(2) << + std::setfill('0') << (int) rInput[i] << + std::dec; + } + } + + return output.str(); +} diff --git a/lib/common/Logging.h b/lib/common/Logging.h new file mode 100644 index 00000000..15400711 --- /dev/null +++ b/lib/common/Logging.h @@ -0,0 +1,346 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Logging.h +// Purpose: Generic logging core routines declarations and macros +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +#ifndef LOGGING__H +#define LOGGING__H + +#include <cerrno> +#include <cstring> +#include <iomanip> +#include <sstream> +#include <vector> + +#include "FileStream.h" + +#define BOX_LOG(level, stuff) \ +{ \ + std::ostringstream _box_log_line; \ + _box_log_line << stuff; \ + Logging::Log(level, __FILE__, __LINE__, _box_log_line.str()); \ +} + +#define BOX_SYSLOG(level, stuff) \ +{ \ + std::ostringstream _box_log_line; \ + _box_log_line << stuff; \ + Logging::LogToSyslog(level, __FILE__, __LINE__, _box_log_line.str()); \ +} + +#define BOX_FATAL(stuff) BOX_LOG(Log::FATAL, stuff) +#define BOX_ERROR(stuff) BOX_LOG(Log::ERROR, stuff) +#define BOX_WARNING(stuff) BOX_LOG(Log::WARNING, stuff) +#define BOX_NOTICE(stuff) BOX_LOG(Log::NOTICE, stuff) +#define BOX_INFO(stuff) BOX_LOG(Log::INFO, stuff) +#define BOX_TRACE(stuff) \ + if (Logging::IsEnabled(Log::TRACE)) \ + { BOX_LOG(Log::TRACE, stuff) } + +#define BOX_SYS_ERROR(stuff) \ + stuff << ": " << std::strerror(errno) << " (" << errno << ")" + +#define BOX_LOG_SYS_WARNING(stuff) \ + BOX_WARNING(BOX_SYS_ERROR(stuff)) +#define BOX_LOG_SYS_ERROR(stuff) \ + BOX_ERROR(BOX_SYS_ERROR(stuff)) +#define BOX_LOG_SYS_FATAL(stuff) \ + BOX_FATAL(BOX_SYS_ERROR(stuff)) + +#define LOG_AND_THROW_ERROR(message, filename, exception, subtype) \ + BOX_LOG_SYS_ERROR(message << ": " << filename); \ + THROW_EXCEPTION_MESSAGE(exception, subtype, \ + BOX_SYS_ERROR(message << ": " << filename)); + +inline std::string GetNativeErrorMessage() +{ +#ifdef WIN32 + return GetErrorMessage(GetLastError()); +#else + std::ostringstream _box_log_line; + _box_log_line << std::strerror(errno) << " (" << errno << ")"; + return _box_log_line.str(); +#endif +} + +#ifdef WIN32 + #define BOX_LOG_WIN_ERROR(stuff) \ + BOX_ERROR(stuff << ": " << GetErrorMessage(GetLastError())) + #define BOX_LOG_WIN_WARNING(stuff) \ + BOX_WARNING(stuff << ": " << GetErrorMessage(GetLastError())) + #define BOX_LOG_WIN_ERROR_NUMBER(stuff, number) \ + BOX_ERROR(stuff << ": " << GetErrorMessage(number)) + #define BOX_LOG_WIN_WARNING_NUMBER(stuff, number) \ + BOX_WARNING(stuff << ": " << GetErrorMessage(number)) + #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_WIN_ERROR(stuff) + #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_WIN_WARNING(stuff) +#else + #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_SYS_ERROR(stuff) + #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_SYS_WARNING(stuff) +#endif + +#define BOX_LOG_SOCKET_ERROR(_type, _name, _port, stuff) \ + BOX_LOG_NATIVE_ERROR(stuff << " (type " << _type << ", name " << \ + _name << ", port " << _port << ")") + +#define BOX_FORMAT_HEX32(number) \ + std::hex << \ + std::showbase << \ + std::internal << \ + std::setw(10) << \ + std::setfill('0') << \ + (number) << \ + std::dec + +#define BOX_FORMAT_ACCOUNT(accno) \ + BOX_FORMAT_HEX32(accno) + +#define BOX_FORMAT_OBJECTID(objectid) \ + std::hex << \ + std::showbase << \ + (objectid) << \ + std::dec + +#define BOX_FORMAT_TIMESPEC(timespec) \ + timespec.tv_sec << \ + std::setw(6) << \ + timespec.tv_usec + +#undef ERROR + +namespace Log +{ + enum Level + { + NOTHING = 1, + FATAL, + ERROR, + WARNING, + NOTICE, + INFO, + TRACE, + EVERYTHING, + INVALID = -1 + }; +} + +// -------------------------------------------------------------------------- +// +// Class +// Name: Logger +// Purpose: Abstract base class for log targets +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +class Logger +{ + private: + Log::Level mCurrentLevel; + + public: + Logger(); + Logger(Log::Level level); + virtual ~Logger(); + + virtual bool Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage) = 0; + + void Filter(Log::Level level) + { + mCurrentLevel = level; + } + + virtual const char* GetType() = 0; + Log::Level GetLevel() { return mCurrentLevel; } + + virtual void SetProgramName(const std::string& rProgramName) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Console +// Purpose: Console logging target +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +class Console : public Logger +{ + private: + static bool sShowTag; + static bool sShowTime; + static bool sShowTimeMicros; + static bool sShowPID; + static std::string sTag; + + public: + virtual bool Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage); + virtual const char* GetType() { return "Console"; } + virtual void SetProgramName(const std::string& rProgramName); + + static void SetShowTag(bool enabled); + static void SetShowTime(bool enabled); + static void SetShowTimeMicros(bool enabled); + static void SetShowPID(bool enabled); +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Syslog +// Purpose: Syslog (or Windows Event Viewer) logging target +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +class Syslog : public Logger +{ + private: + std::string mName; + int mFacility; + + public: + Syslog(); + virtual ~Syslog(); + + virtual bool Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage); + virtual const char* GetType() { return "Syslog"; } + virtual void SetProgramName(const std::string& rProgramName); + virtual void SetFacility(int facility); + static int GetNamedFacility(const std::string& rFacility); +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Logging +// Purpose: Static logging helper, keeps track of enabled loggers +// and distributes log messages to them. +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +class Logging +{ + private: + static std::vector<Logger*> sLoggers; + static bool sLogToSyslog, sLogToConsole; + static std::string sContext; + static bool sContextSet; + static Console* spConsole; + static Syslog* spSyslog; + static Log::Level sGlobalLevel; + static Logging sGlobalLogging; + static std::string sProgramName; + + public: + Logging (); + ~Logging(); + static void ToSyslog (bool enabled); + static void ToConsole (bool enabled); + static void FilterSyslog (Log::Level level); + static void FilterConsole (Log::Level level); + static void Add (Logger* pNewLogger); + static void Remove (Logger* pOldLogger); + static void Log(Log::Level level, const std::string& rFile, + int line, const std::string& rMessage); + static void LogToSyslog(Log::Level level, const std::string& rFile, + int line, const std::string& rMessage); + static void SetContext(std::string context); + static void ClearContext(); + static void SetGlobalLevel(Log::Level level) { sGlobalLevel = level; } + static Log::Level GetGlobalLevel() { return sGlobalLevel; } + static Log::Level GetNamedLevel(const std::string& rName); + static bool IsEnabled(Log::Level level) + { + return (int)sGlobalLevel >= (int)level; + } + static void SetProgramName(const std::string& rProgramName); + static std::string GetProgramName() { return sProgramName; } + static void SetFacility(int facility); + + class Guard + { + private: + Log::Level mOldLevel; + + public: + Guard(Log::Level newLevel) + { + mOldLevel = Logging::GetGlobalLevel(); + Logging::SetGlobalLevel(newLevel); + } + ~Guard() + { + Logging::SetGlobalLevel(mOldLevel); + } + }; + + class Tagger + { + private: + std::string mOldTag; + + public: + Tagger(const std::string& rTempTag) + { + mOldTag = Logging::GetProgramName(); + Logging::SetProgramName(mOldTag + " " + rTempTag); + } + ~Tagger() + { + Logging::SetProgramName(mOldTag); + } + }; +}; + +class FileLogger : public Logger +{ + private: + FileStream mLogFile; + FileLogger(const FileLogger& forbidden) + : mLogFile("") { /* do not call */ } + + public: + FileLogger(const std::string& rFileName, Log::Level Level) + : Logger(Level), + mLogFile(rFileName, O_WRONLY | O_CREAT | O_APPEND) + { } + + virtual bool Log(Log::Level Level, const std::string& rFile, + int Line, std::string& rMessage); + + virtual const char* GetType() { return "FileLogger"; } + virtual void SetProgramName(const std::string& rProgramName) { } +}; + +class HideExceptionMessageGuard +{ + public: + HideExceptionMessageGuard() + { + mOldHiddenState = sHiddenState; + sHiddenState = true; + } + ~HideExceptionMessageGuard() + { + sHiddenState = mOldHiddenState; + } + static bool ExceptionsHidden() { return sHiddenState; } + + private: + static bool sHiddenState; + bool mOldHiddenState; +}; + +std::string PrintEscapedBinaryData(const std::string& rInput); + +#endif // LOGGING__H diff --git a/lib/common/MainHelper.h b/lib/common/MainHelper.h new file mode 100644 index 00000000..d91bc2f9 --- /dev/null +++ b/lib/common/MainHelper.h @@ -0,0 +1,43 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MainHelper.h +// Purpose: Helper stuff for main() programs +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- + +#ifndef MAINHELPER__H +#define MAINHELPER__H + +#include <stdio.h> + +#include "BoxException.h" + +#define MAINHELPER_START \ + if(argc == 2 && ::strcmp(argv[1], "--version") == 0) \ + { printf(BOX_VERSION "\n"); return 0; } \ + MEMLEAKFINDER_INIT \ + MEMLEAKFINDER_START \ + try { +#define MAINHELPER_END \ + } catch(BoxException &e) { \ + printf("Exception: %s (%d/%d)\n", e.what(), e.GetType(), e.GetSubType()); \ + return 1; \ + } catch(std::exception &e) { \ + printf("Exception: %s\n", e.what()); \ + return 1; \ + } catch(...) { \ + printf("Exception: <UNKNOWN>\n"); \ + return 1; } + +#ifdef BOX_MEMORY_LEAK_TESTING + #define MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, marker) \ + memleakfinder_setup_exit_report(file, marker); +#else + #define MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, marker) +#endif // BOX_MEMORY_LEAK_TESTING + + +#endif // MAINHELPER__H + diff --git a/lib/common/Makefile.extra b/lib/common/Makefile.extra new file mode 100644 index 00000000..cc3f3a7a --- /dev/null +++ b/lib/common/Makefile.extra @@ -0,0 +1,11 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_CommonException.h autogen_CommonException.cpp: $(MAKEEXCEPTION) CommonException.txt + $(_PERL) $(MAKEEXCEPTION) CommonException.txt + +# AUTOGEN SEEDING +autogen_ConversionException.h autogen_ConversionException.cpp: $(MAKEEXCEPTION) ConversionException.txt + $(_PERL) $(MAKEEXCEPTION) ConversionException.txt + diff --git a/lib/common/MemBlockStream.cpp b/lib/common/MemBlockStream.cpp new file mode 100644 index 00000000..538a7ef8 --- /dev/null +++ b/lib/common/MemBlockStream.cpp @@ -0,0 +1,235 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemBlockStream.cpp +// Purpose: Stream out data from any memory block +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "MemBlockStream.h" +#include "CommonException.h" +#include "StreamableMemBlock.h" +#include "CollectInBufferStream.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::MemBlockStream() +// Purpose: Constructor (doesn't copy block, careful with lifetimes) +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream(const void *pBuffer, int Size) + : mpBuffer((char*)pBuffer), + mBytesInBuffer(Size), + mReadPosition(0) +{ + ASSERT(pBuffer != 0); + ASSERT(Size >= 0); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::MemBlockStream(const StreamableMemBlock &) +// Purpose: Constructor (doesn't copy block, careful with lifetimes) +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream(const StreamableMemBlock &rBlock) + : mpBuffer((char*)rBlock.GetBuffer()), + mBytesInBuffer(rBlock.GetSize()), + mReadPosition(0) +{ + ASSERT(mpBuffer != 0); + ASSERT(mBytesInBuffer >= 0); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::MemBlockStream(const StreamableMemBlock &) +// Purpose: Constructor (doesn't copy block, careful with lifetimes) +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream(const CollectInBufferStream &rBuffer) + : mpBuffer((char*)rBuffer.GetBuffer()), + mBytesInBuffer(rBuffer.GetSize()), + mReadPosition(0) +{ + ASSERT(mpBuffer != 0); + ASSERT(mBytesInBuffer >= 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::MemBlockStream(const MemBlockStream &) +// Purpose: Copy constructor +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream(const MemBlockStream &rToCopy) + : mpBuffer(rToCopy.mpBuffer), + mBytesInBuffer(rToCopy.mBytesInBuffer), + mReadPosition(0) +{ + ASSERT(mpBuffer != 0); + ASSERT(mBytesInBuffer >= 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::~MemBlockStream() +// Purpose: Destructor +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +MemBlockStream::~MemBlockStream() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::Read(void *, int, int) +// Purpose: As interface. But only works in read phase +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +int MemBlockStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Adjust to number of bytes left + if(NBytes > (mBytesInBuffer - mReadPosition)) + { + NBytes = (mBytesInBuffer - mReadPosition); + } + ASSERT(NBytes >= 0); + if(NBytes <= 0) return 0; // careful now + + // Copy in the requested number of bytes and adjust the read pointer + ::memcpy(pBuffer, mpBuffer + mReadPosition, NBytes); + mReadPosition += NBytes; + + return NBytes; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::BytesLeftToRead() +// Purpose: As interface. But only works in read phase +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +IOStream::pos_type MemBlockStream::BytesLeftToRead() +{ + return (mBytesInBuffer - mReadPosition); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::Write(void *, int) +// Purpose: As interface. But only works in write phase +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void MemBlockStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, MemBlockStreamNotSupported) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::GetPosition() +// Purpose: In write phase, returns the number of bytes written, in read +// phase, the number of bytes to go +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +IOStream::pos_type MemBlockStream::GetPosition() const +{ + return mReadPosition; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::Seek(pos_type, int) +// Purpose: As interface. +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void MemBlockStream::Seek(pos_type Offset, int SeekType) +{ + int newPos = 0; + switch(SeekType) + { + case IOStream::SeekType_Absolute: + newPos = Offset; + break; + case IOStream::SeekType_Relative: + newPos = mReadPosition + Offset; + break; + case IOStream::SeekType_End: + newPos = mBytesInBuffer + Offset; + break; + default: + THROW_EXCEPTION(CommonException, IOStreamBadSeekType) + break; + } + + // Make sure it doesn't go over + if(newPos > mBytesInBuffer) + { + newPos = mBytesInBuffer; + } + // or under + if(newPos < 0) + { + newPos = 0; + } + + // Set the new read position + mReadPosition = newPos; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::StreamDataLeft() +// Purpose: As interface +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +bool MemBlockStream::StreamDataLeft() +{ + return mReadPosition < mBytesInBuffer; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: MemBlockStream::StreamClosed() +// Purpose: As interface +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +bool MemBlockStream::StreamClosed() +{ + return true; +} + diff --git a/lib/common/MemBlockStream.h b/lib/common/MemBlockStream.h new file mode 100644 index 00000000..f78ff8e6 --- /dev/null +++ b/lib/common/MemBlockStream.h @@ -0,0 +1,52 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemBlockStream.h +// Purpose: Stream out data from any memory block +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- + +#ifndef MEMBLOCKSTREAM__H +#define MEMBLOCKSTREAM__H + +#include "IOStream.h" + +class StreamableMemBlock; +class CollectInBufferStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: MemBlockStream +// Purpose: Stream out data from any memory block -- be careful the lifetime +// of the block is greater than the lifetime of this stream. +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +class MemBlockStream : public IOStream +{ +public: + MemBlockStream(const void *pBuffer, int Size); + MemBlockStream(const StreamableMemBlock &rBlock); + MemBlockStream(const CollectInBufferStream &rBuffer); + MemBlockStream(const MemBlockStream &rToCopy); + ~MemBlockStream(); +public: + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(pos_type Offset, int SeekType); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + const char *mpBuffer; + int mBytesInBuffer; + int mReadPosition; +}; + +#endif // MEMBLOCKSTREAM__H + diff --git a/lib/common/MemLeakFindOff.h b/lib/common/MemLeakFindOff.h new file mode 100644 index 00000000..1cc98bac --- /dev/null +++ b/lib/common/MemLeakFindOff.h @@ -0,0 +1,27 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemLeakFindOff.h +// Purpose: Switch memory leak finding off +// Created: 13/1/04 +// +// -------------------------------------------------------------------------- + +// no header guard + +#ifdef BOX_MEMORY_LEAK_TESTING + +#undef new + +#ifndef MEMLEAKFINDER_FULL_MALLOC_MONITORING + #ifdef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED + #undef malloc + #undef realloc + #undef free + #undef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED + #endif +#endif + +#undef MEMLEAKFINDER_ENABLED + +#endif diff --git a/lib/common/MemLeakFindOn.h b/lib/common/MemLeakFindOn.h new file mode 100644 index 00000000..c20fe25a --- /dev/null +++ b/lib/common/MemLeakFindOn.h @@ -0,0 +1,25 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemLeakFindOn.h +// Purpose: Switch memory leak finding on +// Created: 13/1/04 +// +// -------------------------------------------------------------------------- + +// no header guard + +#ifdef BOX_MEMORY_LEAK_TESTING + +#define new DEBUG_NEW + +#ifndef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED + #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__) + #define realloc memleakfinder_realloc + #define free memleakfinder_free + #define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED +#endif + +#define MEMLEAKFINDER_ENABLED + +#endif diff --git a/lib/common/MemLeakFinder.h b/lib/common/MemLeakFinder.h new file mode 100644 index 00000000..ca207bd5 --- /dev/null +++ b/lib/common/MemLeakFinder.h @@ -0,0 +1,63 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MemLeakFinder.h +// Purpose: Memory leak finder +// Created: 12/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef MEMLEAKFINDER__H +#define MEMLEAKFINDER__H + +#ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING + // include stdlib now, to avoid problems with having the macros defined already + #include <cstdlib> +#endif + +// global enable flag +extern bool memleakfinder_global_enable; + +class MemLeakSuppressionGuard +{ + public: + MemLeakSuppressionGuard(); + ~MemLeakSuppressionGuard(); +}; + +extern "C" +{ + void *memleakfinder_malloc(size_t size, const char *file, int line); + void *memleakfinder_realloc(void *ptr, size_t size); + void memleakfinder_free(void *ptr); +} + +void memleakfinder_init(); + +int memleakfinder_numleaks(); + +void memleakfinder_reportleaks(); + +void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext); + +void memleakfinder_setup_exit_report(const char *filename, const char *markertext); + +void memleakfinder_startsectionmonitor(); + +void memleakfinder_traceblocksinsection(); + +void memleakfinder_notaleak(void *ptr); + +void *operator new (size_t size, const char *file, int line); +void *operator new[](size_t size, const char *file, int line); + +// define the malloc functions now, if required +#ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING + #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__) + #define realloc memleakfinder_realloc + #define free memleakfinder_free + #define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED +#endif + +#endif // MEMLEAKFINDER__H + diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp new file mode 100644 index 00000000..f96f80b5 --- /dev/null +++ b/lib/common/NamedLock.cpp @@ -0,0 +1,170 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: NamedLock.cpp +// Purpose: A global named lock, implemented as a lock file in +// file system +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <fcntl.h> +#include <errno.h> + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#ifdef HAVE_FLOCK + #include <sys/file.h> +#endif + +#include "NamedLock.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: NamedLock::NamedLock() +// Purpose: Constructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +NamedLock::NamedLock() + : mFileDescriptor(-1) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: NamedLock::~NamedLock() +// Purpose: Destructor (automatically unlocks if locked) +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +NamedLock::~NamedLock() +{ + if(mFileDescriptor != -1) + { + ReleaseLock(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: NamedLock::TryAndGetLock(const char *, int) +// Purpose: Tries to get a lock on the name in the file system. +// IMPORTANT NOTE: If a file exists with this name, it +// will be deleted. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) +{ + // Check + if(mFileDescriptor != -1) + { + THROW_EXCEPTION(CommonException, NamedLockAlreadyLockingSomething) + } + + // See if the lock can be got +#if HAVE_DECL_O_EXLOCK + int fd = ::open(rFilename.c_str(), + O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode); + if(fd != -1) + { + // Got a lock, lovely + mFileDescriptor = fd; + return true; + } + + // Failed. Why? + if(errno != EWOULDBLOCK) + { + // Not the expected error + THROW_EXCEPTION(CommonException, OSFileError) + } + + return false; +#else + int fd = ::open(rFilename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, mode); + if(fd == -1) + { + BOX_WARNING("Failed to open lockfile: " << rFilename); + THROW_EXCEPTION(CommonException, OSFileError) + } + +#ifdef HAVE_FLOCK + if(::flock(fd, LOCK_EX | LOCK_NB) != 0) + { + ::close(fd); + if(errno == EWOULDBLOCK) + { + return false; + } + else + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } +#elif HAVE_DECL_F_SETLK + struct flock desc; + desc.l_type = F_WRLCK; + desc.l_whence = SEEK_SET; + desc.l_start = 0; + desc.l_len = 0; + if(::fcntl(fd, F_SETLK, &desc) != 0) + { + ::close(fd); + if(errno == EAGAIN) + { + return false; + } + else + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } +#endif + + // Success + mFileDescriptor = fd; + + return true; +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: NamedLock::ReleaseLock() +// Purpose: Release the lock. Exceptions if the lock is not held +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void NamedLock::ReleaseLock() +{ + // Got a lock? + if(mFileDescriptor == -1) + { + THROW_EXCEPTION(CommonException, NamedLockNotHeld) + } + + // Close the file + if(::close(mFileDescriptor) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + // Mark as unlocked + mFileDescriptor = -1; +} + + + + diff --git a/lib/common/NamedLock.h b/lib/common/NamedLock.h new file mode 100644 index 00000000..534115db --- /dev/null +++ b/lib/common/NamedLock.h @@ -0,0 +1,41 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: NamedLock.h +// Purpose: A global named lock, implemented as a lock file in file system +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef NAMEDLOCK__H +#define NAMEDLOCK__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: NamedLock +// Purpose: A global named lock, implemented as a lock file in file system +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +class NamedLock +{ +public: + NamedLock(); + ~NamedLock(); +private: + // No copying allowed + NamedLock(const NamedLock &); + +public: + bool TryAndGetLock(const std::string& rFilename, int mode = 0755); + bool GotLock() {return mFileDescriptor != -1;} + void ReleaseLock(); + + +private: + int mFileDescriptor; +}; + +#endif // NAMEDLOCK__H + diff --git a/lib/common/PartialReadStream.cpp b/lib/common/PartialReadStream.cpp new file mode 100644 index 00000000..f2f79715 --- /dev/null +++ b/lib/common/PartialReadStream.cpp @@ -0,0 +1,138 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: PartialReadStream.h +// Purpose: Read part of another stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "PartialReadStream.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::PartialReadStream(IOStream &, +// pos_type) +// Purpose: Constructor, taking another stream and the number of +// bytes to be read from it. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +PartialReadStream::PartialReadStream(IOStream &rSource, + pos_type BytesToRead) + : mrSource(rSource), + mBytesLeft(BytesToRead) +{ + ASSERT(BytesToRead > 0); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::~PartialReadStream() +// Purpose: Destructor. Won't absorb any unread bytes. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +PartialReadStream::~PartialReadStream() +{ + // Warn in debug mode + if(mBytesLeft != 0) + { + BOX_TRACE("PartialReadStream destroyed with " << mBytesLeft << + " bytes remaining"); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::Read(void *, int, int) +// Purpose: As interface. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +int PartialReadStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Finished? + if(mBytesLeft <= 0) + { + return 0; + } + + // Asking for more than is allowed? + if(NBytes > mBytesLeft) + { + // Adjust downwards + NBytes = mBytesLeft; + } + + // Route the request to the source + int read = mrSource.Read(pBuffer, NBytes, Timeout); + ASSERT(read <= mBytesLeft); + + // Adjust the count + mBytesLeft -= read; + + // Return the number read + return read; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::BytesLeftToRead() +// Purpose: As interface. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +IOStream::pos_type PartialReadStream::BytesLeftToRead() +{ + return mBytesLeft; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::Write(const void *, int) +// Purpose: As interface. But will exception. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void PartialReadStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, CantWriteToPartialReadStream) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::StreamDataLeft() +// Purpose: As interface. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool PartialReadStream::StreamDataLeft() +{ + return mBytesLeft != 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: PartialReadStream::StreamClosed() +// Purpose: As interface. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +bool PartialReadStream::StreamClosed() +{ + // always closed + return true; +} + diff --git a/lib/common/PartialReadStream.h b/lib/common/PartialReadStream.h new file mode 100644 index 00000000..1b46b0bd --- /dev/null +++ b/lib/common/PartialReadStream.h @@ -0,0 +1,46 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: PartialReadStream.h +// Purpose: Read part of another stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- + +#ifndef PARTIALREADSTREAM__H +#define PARTIALREADSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: PartialReadStream +// Purpose: Read part of another stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +class PartialReadStream : public IOStream +{ +public: + PartialReadStream(IOStream &rSource, pos_type BytesToRead); + ~PartialReadStream(); +private: + // no copying allowed + PartialReadStream(const IOStream &); + PartialReadStream(const PartialReadStream &); + +public: + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + IOStream &mrSource; + pos_type mBytesLeft; +}; + +#endif // PARTIALREADSTREAM__H + diff --git a/lib/common/PathUtils.cpp b/lib/common/PathUtils.cpp new file mode 100644 index 00000000..924d47d2 --- /dev/null +++ b/lib/common/PathUtils.cpp @@ -0,0 +1,34 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: PathUtils.cpp +// Purpose: Platform-independent path manipulation +// Created: 2007/01/17 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include <string> + +// -------------------------------------------------------------------------- +// +// Function +// Name: MakeFullPath(const std::string& rDir, const std::string& rFile) +// Purpose: Combine directory and file name +// Created: 2006/08/10 +// +// -------------------------------------------------------------------------- +std::string MakeFullPath(const std::string& rDir, const std::string& rEntry) +{ + std::string result(rDir); + + if (result.size() > 0 && + result[result.size()-1] != DIRECTORY_SEPARATOR_ASCHAR) + { + result += DIRECTORY_SEPARATOR; + } + + result += rEntry; + + return result; +} diff --git a/lib/common/PathUtils.h b/lib/common/PathUtils.h new file mode 100644 index 00000000..1cf2e507 --- /dev/null +++ b/lib/common/PathUtils.h @@ -0,0 +1,26 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: PathUtils.h +// Purpose: Platform-independent path manipulation +// Created: 2007/01/17 +// +// -------------------------------------------------------------------------- + +#ifndef PATHUTILS_H +#define PATHUTILS_H + +#include <string> + +// -------------------------------------------------------------------------- +// +// Function +// Name: MakeFullPath(const std::string& rDir, const std::string& rFile) +// Purpose: Combine directory and file name +// Created: 2006/08/10 +// +// -------------------------------------------------------------------------- + +std::string MakeFullPath(const std::string& rDir, const std::string& rEntry); + +#endif // !PATHUTILS_H diff --git a/lib/common/ReadGatherStream.cpp b/lib/common/ReadGatherStream.cpp new file mode 100644 index 00000000..f50e6664 --- /dev/null +++ b/lib/common/ReadGatherStream.cpp @@ -0,0 +1,263 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ReadGatherStream.cpp +// Purpose: Build a stream (for reading only) out of a number of other streams. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "ReadGatherStream.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::ReadGatherStream(bool) +// Purpose: Constructor. Args says whether or not all the component streams will be deleted when this +// object is deleted. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +ReadGatherStream::ReadGatherStream(bool DeleteComponentStreamsOnDestruction) + : mDeleteComponentStreamsOnDestruction(DeleteComponentStreamsOnDestruction), + mCurrentPosition(0), + mTotalSize(0), + mCurrentBlock(0), + mPositionInCurrentBlock(0), + mSeekDoneForCurrent(false) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::~ReadGatherStream() +// Purpose: Destructor. Will delete all the stream objects, if required. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +ReadGatherStream::~ReadGatherStream() +{ + // Delete compoenent streams? + if(mDeleteComponentStreamsOnDestruction) + { + for(unsigned int l = 0; l < mComponents.size(); ++l) + { + delete mComponents[l]; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::AddComponent(IOStream *) +// Purpose: Add a component to this stream, returning the index +// of this component in the internal list. Use this +// with AddBlock() +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +int ReadGatherStream::AddComponent(IOStream *pStream) +{ + ASSERT(pStream != 0); + + // Just add the component to the list, returning it's index. + int index = mComponents.size(); + mComponents.push_back(pStream); + return index; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::AddBlock(int, pos_type, bool, pos_type) +// Purpose: Add a block to the list of blocks being gathered into one stream. +// Length is length of block to read from this component, Seek == true +// if a seek is required, and if true, SeekTo is the position (absolute) +// in the stream to be seeked to when this block is required. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +void ReadGatherStream::AddBlock(int Component, pos_type Length, bool Seek, pos_type SeekTo) +{ + // Check block + if(Component < 0 || Component >= (int)mComponents.size() || Length < 0 || SeekTo < 0) + { + THROW_EXCEPTION(CommonException, ReadGatherStreamAddingBadBlock); + } + + // Add to list + Block b; + b.mLength = Length; + b.mSeekTo = SeekTo; + b.mComponent = Component; + b.mSeek = Seek; + + mBlocks.push_back(b); + + // And update the total size + mTotalSize += Length; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::Read(void *, int, int) +// Purpose: As interface. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +int ReadGatherStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + int bytesToRead = NBytes; + uint8_t *buffer = (uint8_t*)pBuffer; + + while(bytesToRead > 0) + { + // Done? + if(mCurrentBlock >= mBlocks.size()) + { + // Stop now, as have finished the last block + return NBytes - bytesToRead; + } + + // Seek? + if(mPositionInCurrentBlock == 0 && mBlocks[mCurrentBlock].mSeek && !mSeekDoneForCurrent) + { + // Do seeks in this manner so that seeks are done regardless of whether the block + // has length > 0, and it will only be done once, and at as late a stage as possible. + + mComponents[mBlocks[mCurrentBlock].mComponent]->Seek(mBlocks[mCurrentBlock].mSeekTo, IOStream::SeekType_Absolute); + + mSeekDoneForCurrent = true; + } + + // Anything in the current block? + if(mPositionInCurrentBlock < mBlocks[mCurrentBlock].mLength) + { + // Read! + pos_type s = mBlocks[mCurrentBlock].mLength - mPositionInCurrentBlock; + if(s > bytesToRead) s = bytesToRead; + + pos_type r = mComponents[mBlocks[mCurrentBlock].mComponent]->Read(buffer, s, Timeout); + + // update variables + mPositionInCurrentBlock += r; + buffer += r; + bytesToRead -= r; + mCurrentPosition += r; + + if(r != s) + { + // Stream returned less than requested. To avoid blocking when not necessary, + // return now. + return NBytes - bytesToRead; + } + } + else + { + // Move to next block + ++mCurrentBlock; + mPositionInCurrentBlock = 0; + mSeekDoneForCurrent = false; + } + } + + return NBytes - bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::GetPosition() +// Purpose: As interface +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ReadGatherStream::GetPosition() const +{ + return mCurrentPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::BytesLeftToRead() +// Purpose: As interface +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ReadGatherStream::BytesLeftToRead() +{ + return mTotalSize - mCurrentPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::Write(const void *, int) +// Purpose: As interface. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +void ReadGatherStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, CannotWriteToReadGatherStream); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::StreamDataLeft() +// Purpose: As interface. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool ReadGatherStream::StreamDataLeft() +{ + if(mCurrentBlock >= mBlocks.size()) + { + // Done all the blocks + return false; + } + + if(mCurrentBlock == (mBlocks.size() - 1) + && mPositionInCurrentBlock >= mBlocks[mCurrentBlock].mLength) + { + // Are on the last block, and have got all the data from it. + return false; + } + + // Otherwise, there's more data to be read + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadGatherStream::StreamClosed() +// Purpose: As interface. But the stream is always closed. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +bool ReadGatherStream::StreamClosed() +{ + return true; +} + + diff --git a/lib/common/ReadGatherStream.h b/lib/common/ReadGatherStream.h new file mode 100644 index 00000000..613ede3e --- /dev/null +++ b/lib/common/ReadGatherStream.h @@ -0,0 +1,67 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ReadGatherStream.h +// Purpose: Build a stream (for reading only) out of a number of other streams. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef READGATHERSTREAM_H +#define READGATHERSTREAM_H + +#include "IOStream.h" + +#include <vector> + +// -------------------------------------------------------------------------- +// +// Class +// Name: ReadGatherStream +// Purpose: Build a stream (for reading only) out of a number of other streams. +// Created: 10/12/03 +// +// -------------------------------------------------------------------------- +class ReadGatherStream : public IOStream +{ +public: + ReadGatherStream(bool DeleteComponentStreamsOnDestruction); + ~ReadGatherStream(); +private: + ReadGatherStream(const ReadGatherStream &); + ReadGatherStream &operator=(const ReadGatherStream &); +public: + + int AddComponent(IOStream *pStream); + void AddBlock(int Component, pos_type Length, bool Seek = false, pos_type SeekTo = 0); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + virtual pos_type GetPosition() const; + +private: + bool mDeleteComponentStreamsOnDestruction; + std::vector<IOStream *> mComponents; + + typedef struct + { + pos_type mLength; + pos_type mSeekTo; + int mComponent; + bool mSeek; + } Block; + + std::vector<Block> mBlocks; + + pos_type mCurrentPosition; + pos_type mTotalSize; + unsigned int mCurrentBlock; + pos_type mPositionInCurrentBlock; + bool mSeekDoneForCurrent; +}; + + +#endif // READGATHERSTREAM_H diff --git a/lib/common/ReadLoggingStream.cpp b/lib/common/ReadLoggingStream.cpp new file mode 100644 index 00000000..54c99c95 --- /dev/null +++ b/lib/common/ReadLoggingStream.cpp @@ -0,0 +1,203 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ReadLoggingStream.cpp +// Purpose: Buffering wrapper around IOStreams +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "ReadLoggingStream.h" +#include "CommonException.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::ReadLoggingStream(const char *, int, int) +// Purpose: Constructor, set up buffer +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +ReadLoggingStream::ReadLoggingStream(IOStream& rSource, Logger& rLogger) +: mrSource(rSource), + mOffset(0), + mLength(mrSource.BytesLeftToRead()), + mTotalRead(0), + mStartTime(GetCurrentBoxTime()), + mrLogger(rLogger) +{ } + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::Read(void *, int) +// Purpose: Reads bytes from the file +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +int ReadLoggingStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + int numBytesRead = mrSource.Read(pBuffer, NBytes, Timeout); + + if (numBytesRead > 0) + { + mTotalRead += numBytesRead; + mOffset += numBytesRead; + } + + if (mLength == 0) + { + mrLogger.Log(numBytesRead, mOffset); + } + else if (mTotalRead == 0) + { + mrLogger.Log(numBytesRead, mOffset, mLength); + } + else + { + box_time_t timeNow = GetCurrentBoxTime(); + box_time_t elapsed = timeNow - mStartTime; + box_time_t finish = (elapsed * mLength) / mTotalRead; + // box_time_t remain = finish - elapsed; + mrLogger.Log(numBytesRead, mOffset, mLength, elapsed, finish); + } + + return numBytesRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::BytesLeftToRead() +// Purpose: Returns number of bytes to read (may not be most efficient function ever) +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ReadLoggingStream::BytesLeftToRead() +{ + return mLength - mOffset; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::Write(void *, int) +// Purpose: Writes bytes to the underlying stream (not supported) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ReadLoggingStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::GetPosition() +// Purpose: Get position in stream +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ReadLoggingStream::GetPosition() const +{ + return mOffset; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::Seek(pos_type, int) +// Purpose: Seeks within file, as lseek, invalidate buffer +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ReadLoggingStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + mrSource.Seek(Offset, SeekType); + + switch (SeekType) + { + case SeekType_Absolute: + { + // just go there + mOffset = Offset; + } + break; + + case SeekType_Relative: + { + // Actual underlying file position is + // (mBufferSize - mBufferPosition) ahead of us. + // Need to subtract that amount from the seek + // to seek forward that much less, putting the + // real pointer in the right place. + mOffset += Offset; + } + break; + + case SeekType_End: + { + // Actual underlying file position is + // (mBufferSize - mBufferPosition) ahead of us. + // Need to add that amount to the seek + // to seek backwards that much more, putting the + // real pointer in the right place. + mOffset = mLength - Offset; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::Close() +// Purpose: Closes the underlying stream (not needed) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ReadLoggingStream::Close() +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::StreamDataLeft() +// Purpose: Any data left to write? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool ReadLoggingStream::StreamDataLeft() +{ + return mrSource.StreamDataLeft(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::StreamClosed() +// Purpose: Is the stream closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool ReadLoggingStream::StreamClosed() +{ + return mrSource.StreamClosed(); +} + diff --git a/lib/common/ReadLoggingStream.h b/lib/common/ReadLoggingStream.h new file mode 100644 index 00000000..b23b542c --- /dev/null +++ b/lib/common/ReadLoggingStream.h @@ -0,0 +1,58 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ReadLoggingStream.h +// Purpose: Wrapper around IOStreams that logs read progress +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- + +#ifndef READLOGGINGSTREAM__H +#define READLOGGINGSTREAM__H + +#include "IOStream.h" +#include "BoxTime.h" + +class ReadLoggingStream : public IOStream +{ +public: + class Logger + { + public: + virtual ~Logger() { } + virtual void Log(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, + box_time_t finish) = 0; + virtual void Log(int64_t readSize, int64_t offset, + int64_t length) = 0; + virtual void Log(int64_t readSize, int64_t offset) = 0; + }; + +private: + IOStream& mrSource; + IOStream::pos_type mOffset, mLength, mTotalRead; + box_time_t mStartTime; + Logger& mrLogger; + +public: + ReadLoggingStream(IOStream& rSource, Logger& rLogger); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + ReadLoggingStream(const ReadLoggingStream &rToCopy) + : mrSource(rToCopy.mrSource), mrLogger(rToCopy.mrLogger) + { /* do not call */ } +}; + +#endif // READLOGGINGSTREAM__H + + diff --git a/lib/common/SelfFlushingStream.h b/lib/common/SelfFlushingStream.h new file mode 100644 index 00000000..36e9a4d3 --- /dev/null +++ b/lib/common/SelfFlushingStream.h @@ -0,0 +1,71 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SelfFlushingStream.h +// Purpose: A stream wrapper that always flushes the underlying +// stream, to ensure protocol safety. +// Created: 2008/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef SELFFLUSHINGSTREAM__H +#define SELFFLUSHINGSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: SelfFlushingStream +// Purpose: A stream wrapper that always flushes the underlying +// stream, to ensure protocol safety. +// Created: 2008/08/20 +// +// -------------------------------------------------------------------------- +class SelfFlushingStream : public IOStream +{ +public: + SelfFlushingStream(IOStream &rSource) + : mrSource(rSource) { } + + SelfFlushingStream(const SelfFlushingStream &rToCopy) + : mrSource(rToCopy.mrSource) { } + + ~SelfFlushingStream() + { + Flush(); + } + +private: + // no copying from IOStream allowed + SelfFlushingStream(const IOStream& rToCopy); + +public: + virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + return mrSource.Read(pBuffer, NBytes, Timeout); + } + virtual pos_type BytesLeftToRead() + { + return mrSource.BytesLeftToRead(); + } + virtual void Write(const void *pBuffer, int NBytes) + { + mrSource.Write(pBuffer, NBytes); + } + virtual bool StreamDataLeft() + { + return mrSource.StreamDataLeft(); + } + virtual bool StreamClosed() + { + return mrSource.StreamClosed(); + } + +private: + IOStream &mrSource; +}; + +#endif // SELFFLUSHINGSTREAM__H + diff --git a/lib/common/StreamableMemBlock.cpp b/lib/common/StreamableMemBlock.cpp new file mode 100644 index 00000000..cf431022 --- /dev/null +++ b/lib/common/StreamableMemBlock.cpp @@ -0,0 +1,364 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StreamableMemBlock.cpp +// Purpose: Memory blocks which can be loaded and saved from streams +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <new> +#include <cstdlib> +#include <string.h> + +#include "StreamableMemBlock.h" +#include "IOStream.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::StreamableMemBlock() +// Purpose: Constructor, making empty block +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +StreamableMemBlock::StreamableMemBlock() + : mpBuffer(0), + mSize(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::StreamableMemBlock(void *, int) +// Purpose: Create block, copying data from another bit of memory +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +StreamableMemBlock::StreamableMemBlock(void *pBuffer, int Size) + : mpBuffer(0), + mSize(0) +{ + AllocateBlock(Size); + ::memcpy(mpBuffer, pBuffer, Size); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::StreamableMemBlock(int) +// Purpose: Create block, initialising it to all zeros +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +StreamableMemBlock::StreamableMemBlock(int Size) + : mpBuffer(0), + mSize(0) +{ + AllocateBlock(Size); + ::memset(mpBuffer, 0, Size); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::StreamableMemBlock(const StreamableMemBlock &) +// Purpose: Copy constructor +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +StreamableMemBlock::StreamableMemBlock(const StreamableMemBlock &rToCopy) + : mpBuffer(0), + mSize(0) +{ + AllocateBlock(rToCopy.mSize); + ::memcpy(mpBuffer, rToCopy.mpBuffer, mSize); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::Set(void *, int) +// Purpose: Set the contents of the block +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::Set(void *pBuffer, int Size) +{ + FreeBlock(); + AllocateBlock(Size); + ::memcpy(mpBuffer, pBuffer, Size); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::Set(IOStream &) +// Purpose: Set from stream. Stream must support BytesLeftToRead() +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::Set(IOStream &rStream, int Timeout) +{ + // Get size + IOStream::pos_type size = rStream.BytesLeftToRead(); + if(size == IOStream::SizeOfStreamUnknown) + { + THROW_EXCEPTION(CommonException, StreamDoesntHaveRequiredProperty) + } + + // Allocate a new block (this way to be exception safe) + char *pblock = (char*)malloc(size); + if(pblock == 0) + { + throw std::bad_alloc(); + } + + try + { + // Read in + if(!rStream.ReadFullBuffer(pblock, size, 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + + // Free the block ready for replacement + FreeBlock(); + } + catch(...) + { + ::free(pblock); + throw; + } + + // store... + ASSERT(mpBuffer == 0); + mpBuffer = pblock; + mSize = size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::Set(const StreamableMemBlock &) +// Purpose: Set from other block. +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::Set(const StreamableMemBlock &rBlock) +{ + Set(rBlock.mpBuffer, rBlock.mSize); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::~StreamableMemBlock() +// Purpose: Destructor +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +StreamableMemBlock::~StreamableMemBlock() +{ + FreeBlock(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::FreeBlock() +// Purpose: Protected. Frees block of memory +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::FreeBlock() +{ + if(mpBuffer != 0) + { + ::free(mpBuffer); + } + mpBuffer = 0; + mSize = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::AllocateBlock(int) +// Purpose: Protected. Allocate the block of memory +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::AllocateBlock(int Size) +{ + ASSERT(mpBuffer == 0); + if(Size > 0) + { + mpBuffer = ::malloc(Size); + if(mpBuffer == 0) + { + throw std::bad_alloc(); + } + } + mSize = Size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::ResizeBlock(int) +// Purpose: Protected. Resizes the allocated block. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::ResizeBlock(int Size) +{ + ASSERT(Size > 0); + if(Size > 0) + { + void *pnewBuffer = ::realloc(mpBuffer, Size); + if(pnewBuffer == 0) + { + throw std::bad_alloc(); + } + mpBuffer = pnewBuffer; + } + mSize = Size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::ReadFromStream(IOStream &, int) +// Purpose: Read the block in from a stream +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::ReadFromStream(IOStream &rStream, int Timeout) +{ + // Get the size of the block + int32_t size_s; + if(!rStream.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + + int size = ntohl(size_s); + + + // Allocate a new block (this way to be exception safe) + char *pblock = (char*)malloc(size); + if(pblock == 0) + { + throw std::bad_alloc(); + } + + try + { + // Read in + if(!rStream.ReadFullBuffer(pblock, size, 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) + } + + // Free the block ready for replacement + FreeBlock(); + } + catch(...) + { + ::free(pblock); + throw; + } + + // store... + ASSERT(mpBuffer == 0); + mpBuffer = pblock; + mSize = size; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::WriteToStream(IOStream &) +// Purpose: Write the block to a stream +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::WriteToStream(IOStream &rStream) const +{ + int32_t sizenbo = htonl(mSize); + // Size + rStream.Write(&sizenbo, sizeof(sizenbo)); + // Buffer + if(mSize > 0) + { + rStream.Write(mpBuffer, mSize); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::WriteEmptyBlockToStream(IOStream &) +// Purpose: Writes an empty block to a stream. +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void StreamableMemBlock::WriteEmptyBlockToStream(IOStream &rStream) +{ + int32_t sizenbo = htonl(0); + rStream.Write(&sizenbo, sizeof(sizenbo)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::GetBuffer() +// Purpose: Get pointer to buffer +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +void *StreamableMemBlock::GetBuffer() const +{ + if(mSize == 0) + { + // Return something which isn't a null pointer + static const int validptr = 0; + return (void*)&validptr; + } + + // return the buffer + ASSERT(mpBuffer != 0); + return mpBuffer; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: StreamableMemBlock::operator==(const StreamableMemBlock &) +// Purpose: Test for equality of memory blocks +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +bool StreamableMemBlock::operator==(const StreamableMemBlock &rCompare) const +{ + if(mSize != rCompare.mSize) return false; + if(mSize == 0 && rCompare.mSize == 0) return true; // without memory comparison! + return ::memcmp(mpBuffer, rCompare.mpBuffer, mSize) == 0; +} + + diff --git a/lib/common/StreamableMemBlock.h b/lib/common/StreamableMemBlock.h new file mode 100644 index 00000000..250c0aea --- /dev/null +++ b/lib/common/StreamableMemBlock.h @@ -0,0 +1,71 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StreamableMemBlock.h +// Purpose: Memory blocks which can be loaded and saved from streams +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- + +#ifndef STREAMABLEMEMBLOCK__H +#define STREAMABLEMEMBLOCK__H + +class IOStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: StreamableMemBlock +// Purpose: Memory blocks which can be loaded and saved from streams +// Created: 2003/09/05 +// +// -------------------------------------------------------------------------- +class StreamableMemBlock +{ +public: + StreamableMemBlock(); + StreamableMemBlock(int Size); + StreamableMemBlock(void *pBuffer, int Size); + StreamableMemBlock(const StreamableMemBlock &rToCopy); + ~StreamableMemBlock(); + + void Set(const StreamableMemBlock &rBlock); + void Set(void *pBuffer, int Size); + void Set(IOStream &rStream, int Timeout); + StreamableMemBlock &operator=(const StreamableMemBlock &rBlock) + { + Set(rBlock); + return *this; + } + + void ReadFromStream(IOStream &rStream, int Timeout); + void WriteToStream(IOStream &rStream) const; + + static void WriteEmptyBlockToStream(IOStream &rStream); + + void *GetBuffer() const; + + // Size of block + int GetSize() const {return mSize;} + + // Buffer empty? + bool IsEmpty() const {return mSize == 0;} + + // Clear the contents of the block + void Clear() {FreeBlock();} + + bool operator==(const StreamableMemBlock &rCompare) const; + + void ResizeBlock(int Size); + +protected: // be careful with these! + void AllocateBlock(int Size); + void FreeBlock(); + +private: + void *mpBuffer; + int mSize; +}; + +#endif // STREAMABLEMEMBLOCK__H + diff --git a/lib/common/TemporaryDirectory.h b/lib/common/TemporaryDirectory.h new file mode 100644 index 00000000..9d52ecd9 --- /dev/null +++ b/lib/common/TemporaryDirectory.h @@ -0,0 +1,46 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: TemporaryDirectory.h +// Purpose: Location of temporary directory +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- + +#ifndef TEMPORARYDIRECTORY__H +#define TEMPORARYDIRECTORY__H + +#include <string> + +#ifdef WIN32 + #include <windows.h> +#endif + +// Prefix name with Box to avoid clashing with OS API names +std::string BoxGetTemporaryDirectoryName() +{ +#ifdef WIN32 + // http://msdn.microsoft.com/library/default.asp? + // url=/library/en-us/fileio/fs/creating_and_using_a_temporary_file.asp + + DWORD dwRetVal; + char lpPathBuffer[1024]; + DWORD dwBufSize = sizeof(lpPathBuffer); + + // Get the temp path. + dwRetVal = GetTempPath(dwBufSize, // length of the buffer + lpPathBuffer); // buffer for path + if (dwRetVal > dwBufSize) + { + THROW_EXCEPTION(CommonException, TempDirPathTooLong) + } + + return std::string(lpPathBuffer); +#elif defined TEMP_DIRECTORY_NAME + return std::string(TEMP_DIRECTORY_NAME); +#else + #error non-static temporary directory names not supported yet +#endif +} + +#endif // TEMPORARYDIRECTORY__H diff --git a/lib/common/Test.cpp b/lib/common/Test.cpp new file mode 100644 index 00000000..56638058 --- /dev/null +++ b/lib/common/Test.cpp @@ -0,0 +1,486 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Test.cpp +// Purpose: Useful stuff for tests +// Created: 2008/04/05 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <errno.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> + +#include <sys/stat.h> +#include <sys/types.h> + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include "Test.h" + +bool TestFileExists(const char *Filename) +{ + EMU_STRUCT_STAT st; + return EMU_STAT(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0; +} + +bool TestFileNotEmpty(const char *Filename) +{ + EMU_STRUCT_STAT st; + return EMU_STAT(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0 && + st.st_size > 0; +} + +bool TestDirExists(const char *Filename) +{ + EMU_STRUCT_STAT st; + return EMU_STAT(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == S_IFDIR; +} + +// -1 if doesn't exist +int TestGetFileSize(const char *Filename) +{ + EMU_STRUCT_STAT st; + if(EMU_STAT(Filename, &st) == 0) + { + return st.st_size; + } + return -1; +} + +std::string ConvertPaths(const std::string& rOriginal) +{ +#ifdef WIN32 + // convert UNIX paths to native + + std::string converted; + for (size_t i = 0; i < rOriginal.size(); i++) + { + if (rOriginal[i] == '/') + { + converted += '\\'; + } + else + { + converted += rOriginal[i]; + } + } + return converted; + +#else // !WIN32 + return rOriginal; +#endif +} + +int RunCommand(const std::string& rCommandLine) +{ + return ::system(ConvertPaths(rCommandLine).c_str()); +} + +#ifdef WIN32 +#include <windows.h> +#endif + +bool ServerIsAlive(int pid) +{ + #ifdef WIN32 + + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, + false, pid); + if (hProcess == NULL) + { + if (GetLastError() != ERROR_INVALID_PARAMETER) + { + BOX_ERROR("Failed to open process " << pid << + ": " << + GetErrorMessage(GetLastError())); + } + return false; + } + + DWORD exitCode; + BOOL result = GetExitCodeProcess(hProcess, &exitCode); + CloseHandle(hProcess); + + if (result == 0) + { + BOX_ERROR("Failed to get exit code for process " << + pid << ": " << + GetErrorMessage(GetLastError())) + return false; + } + + if (exitCode == STILL_ACTIVE) + { + return true; + } + + return false; + + #else // !WIN32 + + if(pid == 0) return false; + return ::kill(pid, 0) != -1; + + #endif // WIN32 +} + +int ReadPidFile(const char *pidFile) +{ + if(!TestFileNotEmpty(pidFile)) + { + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file " + "(perhaps one was already running?)"); + return -1; + } + + int pid = -1; + + FILE *f = fopen(pidFile, "r"); + if(f == NULL || fscanf(f, "%d", &pid) != 1) + { + TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); + return -1; + } + fclose(f); + + return pid; +} + +int LaunchServer(const std::string& rCommandLine, const char *pidFile) +{ + ::fprintf(stdout, "Starting server: %s\n", rCommandLine.c_str()); + +#ifdef WIN32 + + PROCESS_INFORMATION procInfo; + + STARTUPINFO startInfo; + startInfo.cb = sizeof(startInfo); + startInfo.lpReserved = NULL; + startInfo.lpDesktop = NULL; + startInfo.lpTitle = NULL; + startInfo.dwFlags = 0; + startInfo.cbReserved2 = 0; + startInfo.lpReserved2 = NULL; + + std::string cmd = ConvertPaths(rCommandLine); + CHAR* tempCmd = strdup(cmd.c_str()); + + DWORD result = CreateProcess + ( + NULL, // lpApplicationName, naughty! + tempCmd, // lpCommandLine + NULL, // lpProcessAttributes + NULL, // lpThreadAttributes + false, // bInheritHandles + 0, // dwCreationFlags + NULL, // lpEnvironment + NULL, // lpCurrentDirectory + &startInfo, // lpStartupInfo + &procInfo // lpProcessInformation + ); + + free(tempCmd); + + if (result == 0) + { + DWORD err = GetLastError(); + printf("Launch failed: %s: error %d\n", rCommandLine.c_str(), + (int)err); + TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + return -1; + } + + CloseHandle(procInfo.hProcess); + CloseHandle(procInfo.hThread); + + return WaitForServerStartup(pidFile, (int)procInfo.dwProcessId); + +#else // !WIN32 + + if(RunCommand(rCommandLine) != 0) + { + TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + return -1; + } + + return WaitForServerStartup(pidFile, 0); + +#endif // WIN32 +} + +int WaitForServerStartup(const char *pidFile, int pidIfKnown) +{ + #ifdef WIN32 + if (pidFile == NULL) + { + return pidIfKnown; + } + #else + // on other platforms there is no other way to get + // the PID, so a NULL pidFile doesn't make sense. + ASSERT(pidFile != NULL); + #endif + + // time for it to start up + if (Logging::GetGlobalLevel() >= Log::TRACE) + { + BOX_TRACE("Waiting for server to start"); + } + else + { + ::fprintf(stdout, "Waiting for server to start: "); + } + + for (int i = 0; i < 15; i++) + { + if (TestFileNotEmpty(pidFile)) + { + break; + } + + if (pidIfKnown && !ServerIsAlive(pidIfKnown)) + { + break; + } + + if (Logging::GetGlobalLevel() < Log::TRACE) + { + ::fprintf(stdout, "."); + ::fflush(stdout); + } + + ::sleep(1); + } + + // on Win32 we can check whether the process is alive + // without even checking the PID file + + if (pidIfKnown && !ServerIsAlive(pidIfKnown)) + { + if (Logging::GetGlobalLevel() >= Log::TRACE) + { + BOX_ERROR("server died!"); + } + else + { + ::fprintf(stdout, " server died!\n"); + } + + TEST_FAIL_WITH_MESSAGE("Server died!"); + return -1; + } + + if (!TestFileNotEmpty(pidFile)) + { + if (Logging::GetGlobalLevel() >= Log::TRACE) + { + BOX_ERROR("timed out!"); + } + else + { + ::fprintf(stdout, " timed out!\n"); + } + + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); + return -1; + } + + if (Logging::GetGlobalLevel() >= Log::TRACE) + { + BOX_TRACE("Server started"); + } + else + { + ::fprintf(stdout, " done.\n"); + } + + // wait a second for the pid to be written to the file + ::sleep(1); + + // read pid file + int pid = ReadPidFile(pidFile); + + // On Win32 we can check whether the PID in the pidFile matches + // the one returned by the system, which it always should. + + if (pidIfKnown && pid != pidIfKnown) + { + BOX_ERROR("Server wrote wrong pid to file (" << pidFile << + "): expected " << pidIfKnown << " but found " << + pid); + TEST_FAIL_WITH_MESSAGE("Server wrote wrong pid to file"); + return -1; + } + + return pid; +} + +void TestRemoteProcessMemLeaksFunc(const char *filename, + const char* file, int line) +{ +#ifdef BOX_MEMORY_LEAK_TESTING + // Does the file exist? + if(!TestFileExists(filename)) + { + if (failures == 0) + { + first_fail_file = file; + first_fail_line = line; + } + ++failures; + printf("FAILURE: MemLeak report not available (file %s) " + "at %s:%d\n", filename, file, line); + } + else + { + // Is it empty? + if(TestGetFileSize(filename) > 0) + { + if (failures == 0) + { + first_fail_file = file; + first_fail_line = line; + } + ++failures; + printf("FAILURE: Memory leaks found in other process " + "(file %s) at %s:%d\n==========\n", + filename, file, line); + FILE *f = fopen(filename, "r"); + char linebuf[512]; + while(::fgets(linebuf, sizeof(linebuf), f) != 0) + { + printf("%s", linebuf); + } + fclose(f); + printf("==========\n"); + } + + // Delete it + ::unlink(filename); + } +#endif +} + +void force_sync() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "force-sync") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void wait_for_sync_start() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "wait-for-sync") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void wait_for_sync_end() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "wait-for-end") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void sync_and_wait() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "sync-and-wait") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void terminate_bbackupd(int pid) +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "terminate") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + + for (int i = 0; i < 20; i++) + { + if (!ServerIsAlive(pid)) break; + fprintf(stdout, "."); + fflush(stdout); + sleep(1); + } + + TEST_THAT(!ServerIsAlive(pid)); + TestRemoteProcessMemLeaks("bbackupd.memleaks"); +} + + +// Wait a given number of seconds for something to complete +void wait_for_operation(int seconds, const char* message) +{ + if (Logging::GetGlobalLevel() >= Log::TRACE) + { + BOX_TRACE("Waiting " << seconds << " seconds for " << message); + } + else + { + printf("Waiting for %s: ", message); + fflush(stdout); + } + + for(int l = 0; l < seconds; ++l) + { + sleep(1); + if (Logging::GetGlobalLevel() < Log::TRACE) + { + printf("."); + fflush(stdout); + } + } + + if (Logging::GetGlobalLevel() >= Log::TRACE) + { + BOX_TRACE("Finished waiting for " << message); + } + else + { + printf(" done.\n"); + fflush(stdout); + } +} + +void safe_sleep(int seconds) +{ + BOX_TRACE("sleeping for " << seconds << " seconds"); + +#ifdef WIN32 + Sleep(seconds * 1000); +#else + struct timespec ts; + memset(&ts, 0, sizeof(ts)); + ts.tv_sec = seconds; + ts.tv_nsec = 0; + while (nanosleep(&ts, &ts) == -1 && errno == EINTR) + { + // FIXME evil hack for OSX, where ts.tv_sec contains + // a negative number interpreted as unsigned 32-bit + // when nanosleep() returns later than expected. + + int32_t secs = (int32_t) ts.tv_sec; + int64_t remain_ns = (secs * 1000000000) + ts.tv_nsec; + + if (remain_ns < 0) + { + BOX_WARNING("nanosleep interrupted " << + ((float)(0 - remain_ns) / 1000000000) << + " secs late"); + return; + } + + BOX_TRACE("nanosleep interrupted with " << + (remain_ns / 1000000000) << " secs remaining, " + "sleeping again"); + } +#endif +} + diff --git a/lib/common/Test.h b/lib/common/Test.h new file mode 100644 index 00000000..08ba4542 --- /dev/null +++ b/lib/common/Test.h @@ -0,0 +1,167 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Test.h +// Purpose: Useful stuff for tests +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- + +#ifndef TEST__H +#define TEST__H + +#include <cstring> + +#ifdef WIN32 +#define BBACKUPCTL "..\\..\\bin\\bbackupctl\\bbackupctl.exe" +#define BBACKUPD "..\\..\\bin\\bbackupd\\bbackupd.exe" +#define BBSTORED "..\\..\\bin\\bbstored\\bbstored.exe" +#define BBACKUPQUERY "..\\..\\bin\\bbackupquery\\bbackupquery.exe" +#define BBSTOREACCOUNTS "..\\..\\bin\\bbstoreaccounts\\bbstoreaccounts.exe" +#define TEST_RETURN(actual, expected) TEST_EQUAL(expected, actual); +#else +#define BBACKUPCTL "../../bin/bbackupctl/bbackupctl" +#define BBACKUPD "../../bin/bbackupd/bbackupd" +#define BBSTORED "../../bin/bbstored/bbstored" +#define BBACKUPQUERY "../../bin/bbackupquery/bbackupquery" +#define BBSTOREACCOUNTS "../../bin/bbstoreaccounts/bbstoreaccounts" +#define TEST_RETURN(actual, expected) TEST_EQUAL((expected << 8), actual); +#endif + +extern int failures; +extern int first_fail_line; +extern std::string first_fail_file; +extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; + +#define TEST_FAIL_WITH_MESSAGE(msg) \ +{ \ + if (failures == 0) \ + { \ + first_fail_file = __FILE__; \ + first_fail_line = __LINE__; \ + } \ + failures++; \ + BOX_ERROR("**** TEST FAILURE: " << msg << " at " << __FILE__ << \ + ":" << __LINE__); \ +} + +#define TEST_ABORT_WITH_MESSAGE(msg) {TEST_FAIL_WITH_MESSAGE(msg); return 1;} + +#define TEST_THAT(condition) {if(!(condition)) TEST_FAIL_WITH_MESSAGE("Condition [" #condition "] failed")} +#define TEST_THAT_ABORTONFAIL(condition) {if(!(condition)) TEST_ABORT_WITH_MESSAGE("Condition [" #condition "] failed")} + +// NOTE: The 0- bit is to allow this to work with stuff which has negative constants for flags (eg ConnectionException) +#define TEST_CHECK_THROWS(statement, excepttype, subtype) \ + { \ + bool didthrow = false; \ + HideExceptionMessageGuard hide; \ + try \ + { \ + statement; \ + } \ + catch(excepttype &e) \ + { \ + if(e.GetSubType() != ((unsigned int)excepttype::subtype) \ + && e.GetSubType() != (unsigned int)(0-excepttype::subtype)) \ + { \ + throw; \ + } \ + didthrow = true; \ + } \ + catch(...) \ + { \ + throw; \ + } \ + if(!didthrow) \ + { \ + TEST_FAIL_WITH_MESSAGE("Didn't throw exception " #excepttype "(" #subtype ")") \ + } \ + } + +// utility macro for comparing two strings in a line +#define TEST_EQUAL(_expected, _found) \ +{ \ + std::ostringstream _oss1; \ + _oss1 << _expected; \ + std::string _exp_str = _oss1.str(); \ + \ + std::ostringstream _oss2; \ + _oss2 << _found; \ + std::string _found_str = _oss2.str(); \ + \ + if(_exp_str != _found_str) \ + { \ + BOX_WARNING("Expected <" << _exp_str << "> but found <" << \ + _found_str << ">"); \ + \ + std::ostringstream _oss3; \ + _oss3 << #_found << " != " << #_expected; \ + \ + TEST_FAIL_WITH_MESSAGE(_oss3.str().c_str()); \ + } \ +} + +// utility macro for comparing two strings in a line +#define TEST_EQUAL_LINE(_expected, _found, _line) \ +{ \ + std::ostringstream _oss1; \ + _oss1 << _expected; \ + std::string _exp_str = _oss1.str(); \ + \ + std::ostringstream _oss2; \ + _oss2 << _found; \ + std::string _found_str = _oss2.str(); \ + \ + if(_exp_str != _found_str) \ + { \ + std::ostringstream _ossl; \ + _ossl << _line; \ + std::string _line_str = _ossl.str(); \ + printf("Expected <%s> but found <%s> in <%s>\n", \ + _exp_str.c_str(), _found_str.c_str(), _line_str.c_str()); \ + \ + std::ostringstream _oss3; \ + _oss3 << #_found << " != " << #_expected << " in " << _line; \ + \ + TEST_FAIL_WITH_MESSAGE(_oss3.str().c_str()); \ + } \ +} + + +// utility macro for testing a line +#define TEST_LINE(_condition, _line) \ + TEST_THAT(_condition); \ + if (!(_condition)) \ + { \ + printf("Test failed on <%s>\n", _line.c_str()); \ + } + +bool TestFileExists(const char *Filename); +bool TestDirExists(const char *Filename); + +// -1 if doesn't exist +int TestGetFileSize(const char *Filename); +std::string ConvertPaths(const std::string& rOriginal); +int RunCommand(const std::string& rCommandLine); +bool ServerIsAlive(int pid); +int ReadPidFile(const char *pidFile); +int LaunchServer(const std::string& rCommandLine, const char *pidFile); +int WaitForServerStartup(const char *pidFile, int pidIfKnown); + +#define TestRemoteProcessMemLeaks(filename) \ + TestRemoteProcessMemLeaksFunc(filename, __FILE__, __LINE__) + +void TestRemoteProcessMemLeaksFunc(const char *filename, + const char* file, int line); + +void force_sync(); +void wait_for_sync_start(); +void wait_for_sync_end(); +void sync_and_wait(); +void terminate_bbackupd(int pid); + +// Wait a given number of seconds for something to complete +void wait_for_operation(int seconds, const char* message); +void safe_sleep(int seconds); + +#endif // TEST__H diff --git a/lib/common/Timer.cpp b/lib/common/Timer.cpp new file mode 100644 index 00000000..137ad45f --- /dev/null +++ b/lib/common/Timer.cpp @@ -0,0 +1,626 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Timer.cpp +// Purpose: Generic timers which execute arbitrary code when +// they expire. +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- + +#ifdef WIN32 + #define _WIN32_WINNT 0x0500 +#endif + +#include "Box.h" + +#include <signal.h> +#include <cstring> + +#include "Timer.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +std::vector<Timer*>* Timers::spTimers = NULL; +bool Timers::sRescheduleNeeded = false; + +#define TIMER_ID "timer " << mName << " (" << this << ") " +#define TIMER_ID_OF(t) "timer " << (t).GetName() << " (" << &(t) << ") " + +typedef void (*sighandler_t)(int); + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::Init() +// Purpose: Initialise timers, prepare signal handler +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::Init() +{ + ASSERT(!spTimers); + + #if defined WIN32 && ! defined PLATFORM_CYGWIN + // no init needed + #else + struct sigaction newact, oldact; + newact.sa_handler = Timers::SignalHandler; + newact.sa_flags = SA_RESTART; + sigemptyset(&newact.sa_mask); + if (::sigaction(SIGALRM, &newact, &oldact) != 0) + { + BOX_ERROR("Failed to install signal handler"); + THROW_EXCEPTION(CommonException, Internal); + } + ASSERT(oldact.sa_handler == 0); + #endif // WIN32 && !PLATFORM_CYGWIN + + spTimers = new std::vector<Timer*>; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::Cleanup() +// Purpose: Clean up timers, stop signal handler +// Created: 6/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::Cleanup() +{ + ASSERT(spTimers); + if (!spTimers) + { + BOX_ERROR("Tried to clean up timers when not initialised!"); + return; + } + + #if defined WIN32 && ! defined PLATFORM_CYGWIN + // no cleanup needed + #else + struct itimerval timeout; + memset(&timeout, 0, sizeof(timeout)); + + int result = ::setitimer(ITIMER_REAL, &timeout, NULL); + ASSERT(result == 0); + + struct sigaction newact, oldact; + newact.sa_handler = SIG_DFL; + newact.sa_flags = SA_RESTART; + sigemptyset(&(newact.sa_mask)); + if (::sigaction(SIGALRM, &newact, &oldact) != 0) + { + BOX_ERROR("Failed to remove signal handler"); + THROW_EXCEPTION(CommonException, Internal); + } + ASSERT(oldact.sa_handler == Timers::SignalHandler); + #endif // WIN32 && !PLATFORM_CYGWIN + + spTimers->clear(); + delete spTimers; + spTimers = NULL; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::Add(Timer&) +// Purpose: Add a new timer to the set, and reschedule next wakeup +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::Add(Timer& rTimer) +{ + ASSERT(spTimers); + ASSERT(&rTimer); + spTimers->push_back(&rTimer); + Reschedule(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::Remove(Timer&) +// Purpose: Removes the timer from the set (preventing it from +// being called) and reschedule next wakeup +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::Remove(Timer& rTimer) +{ + ASSERT(spTimers); + ASSERT(&rTimer); + + bool restart = true; + while (restart) + { + restart = false; + + for (std::vector<Timer*>::iterator i = spTimers->begin(); + i != spTimers->end(); i++) + { + if (&rTimer == *i) + { + spTimers->erase(i); + restart = true; + break; + } + } + } + + Reschedule(); +} + +void Timers::RequestReschedule() +{ + sRescheduleNeeded = true; +} + +void Timers::RescheduleIfNeeded() +{ + if (sRescheduleNeeded) + { + Reschedule(); + } +} + +#define FORMAT_MICROSECONDS(t) \ + (int)(t / 1000000) << "." << \ + (int)(t % 1000000) << " seconds" + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::Reschedule() +// Purpose: Recalculate when the next wakeup is due +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::Reschedule() +{ + ASSERT(spTimers); + if (spTimers == NULL) + { + THROW_EXCEPTION(CommonException, Internal) + } + + #ifndef WIN32 + struct sigaction oldact; + if (::sigaction(SIGALRM, NULL, &oldact) != 0) + { + BOX_ERROR("Failed to check signal handler"); + THROW_EXCEPTION(CommonException, Internal) + } + + ASSERT(oldact.sa_handler == Timers::SignalHandler); + + if (oldact.sa_handler != Timers::SignalHandler) + { + BOX_ERROR("Signal handler was " << + (void *)oldact.sa_handler << + ", expected " << + (void *)Timers::SignalHandler); + THROW_EXCEPTION(CommonException, Internal) + } + #endif + + // Clear the reschedule-needed flag to false before we start. + // If a timer event occurs while we are scheduling, then we + // may or may not need to reschedule again, but this way + // we will do it anyway. + sRescheduleNeeded = false; + +#ifdef WIN32 + // win32 timers need no management +#else + box_time_t timeNow = GetCurrentBoxTime(); + + // scan for, trigger and remove expired timers. Removal requires + // us to restart the scan each time, due to std::vector semantics. + bool restart = true; + while (restart) + { + restart = false; + + for (std::vector<Timer*>::iterator i = spTimers->begin(); + i != spTimers->end(); i++) + { + Timer& rTimer = **i; + int64_t timeToExpiry = rTimer.GetExpiryTime() - timeNow; + + if (timeToExpiry <= 0) + { + /* + BOX_TRACE("timer " << *i << " has expired, " + "triggering it"); + */ + BOX_TRACE(TIMER_ID_OF(**i) "has expired, " + "triggering " << + FORMAT_MICROSECONDS(-timeToExpiry) << + " late"); + rTimer.OnExpire(); + spTimers->erase(i); + restart = true; + break; + } + else + { + /* + BOX_TRACE("timer " << *i << " has not " + "expired, triggering in " << + FORMAT_MICROSECONDS(timeToExpiry) << + " seconds"); + */ + } + } + } + + // Now the only remaining timers should all be in the future. + // Scan to find the next one to fire (earliest deadline). + + int64_t timeToNextEvent = 0; + std::string nameOfNextEvent; + + for (std::vector<Timer*>::iterator i = spTimers->begin(); + i != spTimers->end(); i++) + { + Timer& rTimer = **i; + int64_t timeToExpiry = rTimer.GetExpiryTime() - timeNow; + + ASSERT(timeToExpiry > 0) + if (timeToExpiry <= 0) + { + timeToExpiry = 1; + } + + if (timeToNextEvent == 0 || timeToNextEvent > timeToExpiry) + { + timeToNextEvent = timeToExpiry; + nameOfNextEvent = rTimer.GetName(); + } + } + + ASSERT(timeToNextEvent >= 0); + + if (timeToNextEvent == 0) + { + BOX_TRACE("timer: no more events, going to sleep."); + } + else + { + BOX_TRACE("timer: next event: " << nameOfNextEvent << + " expires in " << FORMAT_MICROSECONDS(timeToNextEvent)); + } + + struct itimerval timeout; + memset(&timeout, 0, sizeof(timeout)); + + timeout.it_value.tv_sec = BoxTimeToSeconds(timeToNextEvent); + timeout.it_value.tv_usec = (int) + (BoxTimeToMicroSeconds(timeToNextEvent) + % MICRO_SEC_IN_SEC); + + if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) + { + BOX_ERROR("Failed to initialise system timer\n"); + THROW_EXCEPTION(CommonException, Internal) + } +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::SignalHandler(unused) +// Purpose: Called as signal handler. Nothing is safe in a signal +// handler, not even traversing the list of timers, so +// just request a reschedule in future, which will do +// that for us, and trigger any expired timers at that +// time. +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::SignalHandler(int unused) +{ + // ASSERT(spTimers); + Timers::RequestReschedule(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Timer(size_t timeoutSecs, +// const std::string& rName) +// Purpose: Standard timer constructor, takes a timeout in +// seconds from now, and an optional name for +// logging purposes. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +Timer::Timer(size_t timeoutSecs, const std::string& rName) +: mExpires(GetCurrentBoxTime() + SecondsToBoxTime(timeoutSecs)), + mExpired(false), + mName(rName) +#ifdef WIN32 +, mTimerHandle(INVALID_HANDLE_VALUE) +#endif +{ + #ifndef BOX_RELEASE_BUILD + if (timeoutSecs == 0) + { + BOX_TRACE(TIMER_ID "initialised for " << timeoutSecs << + " secs, will not fire"); + } + else + { + BOX_TRACE(TIMER_ID "initialised for " << timeoutSecs << + " secs, to fire at " << FormatTime(mExpires, false, true)); + } + #endif + + if (timeoutSecs == 0) + { + mExpires = 0; + } + else + { + Timers::Add(*this); + Start(timeoutSecs * MICRO_SEC_IN_SEC_LL); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Start() +// Purpose: This internal function initialises an OS TimerQueue +// timer on Windows, while on Unixes there is only a +// single global timer, managed by the Timers class, +// so this method does nothing. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +void Timer::Start() +{ +#ifdef WIN32 + box_time_t timeNow = GetCurrentBoxTime(); + int64_t timeToExpiry = mExpires - timeNow; + + if (timeToExpiry <= 0) + { + BOX_WARNING(TIMER_ID << "fudging expiry from -" << + FORMAT_MICROSECONDS(-timeToExpiry)) + timeToExpiry = 1; + } + + Start(timeToExpiry); +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Start(int64_t delayInMicros) +// Purpose: This internal function initialises an OS TimerQueue +// timer on Windows, with a specified delay already +// calculated to save us doing it again. Like +// Timer::Start(), on Unixes it does nothing. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +void Timer::Start(int64_t delayInMicros) +{ +#ifdef WIN32 + // only call me once! + ASSERT(mTimerHandle == INVALID_HANDLE_VALUE); + + int64_t delayInMillis = delayInMicros / 1000; + + // Windows XP always seems to fire timers up to 20 ms late, + // at least on my test laptop. Not critical in practice, but our + // tests are precise enough that they will fail if we don't + // correct for it. + delayInMillis -= 20; + + // Set a system timer to call our timer routine + if (CreateTimerQueueTimer(&mTimerHandle, NULL, TimerRoutine, + (PVOID)this, delayInMillis, 0, WT_EXECUTEINTIMERTHREAD) + == FALSE) + { + BOX_ERROR(TIMER_ID "failed to create timer: " << + GetErrorMessage(GetLastError())); + mTimerHandle = INVALID_HANDLE_VALUE; + } +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Stop() +// Purpose: This internal function deletes the associated OS +// TimerQueue timer on Windows, and on Unixes does +// nothing. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +void Timer::Stop() +{ +#ifdef WIN32 + if (mTimerHandle != INVALID_HANDLE_VALUE) + { + if (DeleteTimerQueueTimer(NULL, mTimerHandle, + INVALID_HANDLE_VALUE) == FALSE) + { + BOX_ERROR(TIMER_ID "failed to delete timer: " << + GetErrorMessage(GetLastError())); + } + mTimerHandle = INVALID_HANDLE_VALUE; + } +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::~Timer() +// Purpose: Destructor for Timer objects. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +Timer::~Timer() +{ + #ifndef BOX_RELEASE_BUILD + BOX_TRACE(TIMER_ID "destroyed"); + #endif + + Timers::Remove(*this); + Stop(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Timer(Timer& rToCopy) +// Purpose: Copy constructor for Timer objects. Creates a new +// timer that will trigger at the same time as the +// original. The original will usually be discarded. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +Timer::Timer(const Timer& rToCopy) +: mExpires(rToCopy.mExpires), + mExpired(rToCopy.mExpired), + mName(rToCopy.mName) +#ifdef WIN32 +, mTimerHandle(INVALID_HANDLE_VALUE) +#endif +{ + #ifndef BOX_RELEASE_BUILD + if (mExpired) + { + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "already expired, will not fire"); + } + else if (mExpires == 0) + { + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "no expiry, will not fire"); + } + else + { + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "to fire at " << + (int)(mExpires / 1000000) << "." << + (int)(mExpires % 1000000)); + } + #endif + + if (!mExpired && mExpires != 0) + { + Timers::Add(*this); + Start(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::operator=(const Timer& rToCopy) +// Purpose: Assignment operator for Timer objects. Works +// exactly the same as the copy constructor, except +// that if the receiving timer is already running, +// it is stopped first. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +Timer& Timer::operator=(const Timer& rToCopy) +{ + #ifndef BOX_RELEASE_BUILD + if (rToCopy.mExpired) + { + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "already expired, will not fire"); + } + else if (rToCopy.mExpires == 0) + { + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "no expiry, will not fire"); + } + else + { + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "to fire at " << + (int)(rToCopy.mExpires / 1000000) << "." << + (int)(rToCopy.mExpires % 1000000)); + } + #endif + + Timers::Remove(*this); + Stop(); + + mExpires = rToCopy.mExpires; + mExpired = rToCopy.mExpired; + mName = rToCopy.mName; + + if (!mExpired && mExpires != 0) + { + Timers::Add(*this); + Start(); + } + + return *this; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::OnExpire() +// Purpose: Method called by Timers::Reschedule (on Unixes) +// on next poll after timer expires, or from +// Timer::TimerRoutine (on Windows) from a separate +// thread managed by the OS. Marks the timer as +// expired for future reference. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +void Timer::OnExpire() +{ + #ifndef BOX_RELEASE_BUILD + BOX_TRACE(TIMER_ID "fired"); + #endif + + mExpired = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::TimerRoutine(PVOID lpParam, +// BOOLEAN TimerOrWaitFired) +// Purpose: Static method called by the Windows OS when a +// TimerQueue timer expires. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +#ifdef WIN32 +VOID CALLBACK Timer::TimerRoutine(PVOID lpParam, + BOOLEAN TimerOrWaitFired) +{ + Timer* pTimer = (Timer*)lpParam; + pTimer->OnExpire(); + // is it safe to write to write debug output from a timer? + // e.g. to write to the Event Log? +} +#endif diff --git a/lib/common/Timer.h b/lib/common/Timer.h new file mode 100644 index 00000000..42b2e00f --- /dev/null +++ b/lib/common/Timer.h @@ -0,0 +1,89 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Timer.h +// Purpose: Generic timers which execute arbitrary code when +// they expire. +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- + +#ifndef TIMER__H +#define TIMER__H + +#ifdef HAVE_SYS_TIME_H + #include <sys/time.h> +#endif + +#include <vector> + +#include "BoxTime.h" + +#include "MemLeakFindOn.h" + +class Timer; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Timers +// Purpose: Static class to manage all timers and arrange +// efficient delivery of wakeup signals +// Created: 19/3/04 +// +// -------------------------------------------------------------------------- +class Timers +{ + private: + static std::vector<Timer*>* spTimers; + static void Reschedule(); + + static bool sRescheduleNeeded; + static void SignalHandler(int iUnused); + + public: + static void Init(); + static void Cleanup(); + static void Add (Timer& rTimer); + static void Remove(Timer& rTimer); + static void RequestReschedule(); + static void RescheduleIfNeeded(); +}; + +class Timer +{ +public: + Timer(size_t timeoutSecs, const std::string& rName = ""); + virtual ~Timer(); + Timer(const Timer &); + Timer &operator=(const Timer &); + + box_time_t GetExpiryTime() { return mExpires; } + virtual void OnExpire(); + bool HasExpired() + { + Timers::RescheduleIfNeeded(); + return mExpired; + } + + const std::string& GetName() const { return mName; } + +private: + box_time_t mExpires; + bool mExpired; + std::string mName; + + void Start(); + void Start(int64_t delayInMicros); + void Stop(); + + #ifdef WIN32 + HANDLE mTimerHandle; + static VOID CALLBACK TimerRoutine(PVOID lpParam, + BOOLEAN TimerOrWaitFired); + #endif +}; + +#include "MemLeakFindOff.h" + +#endif // TIMER__H diff --git a/lib/common/UnixUser.cpp b/lib/common/UnixUser.cpp new file mode 100644 index 00000000..f81b474c --- /dev/null +++ b/lib/common/UnixUser.cpp @@ -0,0 +1,123 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: UnixUser.cpp +// Purpose: Interface for managing the UNIX user of the current process +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_PWD_H + #include <pwd.h> +#endif + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include "UnixUser.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: UnixUser::UnixUser(const char *) +// Purpose: Constructor, initialises to info of given username +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +UnixUser::UnixUser(const char *Username) + : mUID(0), + mGID(0), + mRevertOnDestruction(false) +{ + // Get password info + struct passwd *pwd = ::getpwnam(Username); + if(pwd == 0) + { + THROW_EXCEPTION(CommonException, CouldNotLookUpUsername) + } + + // Store UID and GID + mUID = pwd->pw_uid; + mGID = pwd->pw_gid; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: UnixUser::UnixUser(uid_t, gid_t) +// Purpose: Construct from given UNIX user ID and group ID +// Created: 15/3/04 +// +// -------------------------------------------------------------------------- +UnixUser::UnixUser(uid_t UID, gid_t GID) + : mUID(UID), + mGID(GID), + mRevertOnDestruction(false) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: UnixUser::~UnixUser() +// Purpose: Destructor -- reverts to previous user if the change wasn't perminant +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +UnixUser::~UnixUser() +{ + if(mRevertOnDestruction) + { + // Revert to "real" user and group id of the process + if(::setegid(::getgid()) != 0 || ::seteuid(::getuid()) != 0) + { + THROW_EXCEPTION(CommonException, CouldNotRestoreProcessUser) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: UnixUser::ChangeProcessUser(bool) +// Purpose: Change the process user and group ID to the user. If Temporary == true +// the process username will be changed back when the object is destructed. +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +void UnixUser::ChangeProcessUser(bool Temporary) +{ + if(Temporary) + { + // Change temporarily (change effective only) + if(::setegid(mGID) != 0 || ::seteuid(mUID) != 0) + { + THROW_EXCEPTION(CommonException, CouldNotChangeProcessUser) + } + + // Mark for change on destruction + mRevertOnDestruction = true; + } + else + { + // Change permanently (change all UIDs and GIDs) + if(::setgid(mGID) != 0 || ::setuid(mUID) != 0) + { + THROW_EXCEPTION(CommonException, CouldNotChangeProcessUser) + } + } +} + + + + diff --git a/lib/common/UnixUser.h b/lib/common/UnixUser.h new file mode 100644 index 00000000..c895eb2a --- /dev/null +++ b/lib/common/UnixUser.h @@ -0,0 +1,37 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: UnixUser.h +// Purpose: Interface for managing the UNIX user of the current process +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- + +#ifndef UNIXUSER__H +#define UNIXUSER__H + +class UnixUser +{ +public: + UnixUser(const char *Username); + UnixUser(uid_t UID, gid_t GID); + ~UnixUser(); +private: + // no copying allowed + UnixUser(const UnixUser &); + UnixUser &operator=(const UnixUser &); +public: + + void ChangeProcessUser(bool Temporary = false); + + uid_t GetUID() {return mUID;} + gid_t GetGID() {return mGID;} + +private: + uid_t mUID; + gid_t mGID; + bool mRevertOnDestruction; +}; + +#endif // UNIXUSER__H + diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp new file mode 100644 index 00000000..6f21330d --- /dev/null +++ b/lib/common/Utils.cpp @@ -0,0 +1,315 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Utils.cpp +// Purpose: Utility function +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> + +#include <cstdlib> + +#ifdef SHOW_BACKTRACE_ON_EXCEPTION + #include <execinfo.h> + #include <stdlib.h> +#endif + +#ifdef HAVE_CXXABI_H + #include <cxxabi.h> +#endif + +#include "Utils.h" +#include "CommonException.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +std::string GetBoxBackupVersion() +{ + return BOX_VERSION; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SplitString(const std::string &, char, std::vector<std::string> &) +// Purpose: Splits a string at a given character +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput) +{ + // Split it up. + std::string::size_type b = 0; + std::string::size_type e = 0; + while(e = String.find_first_of(SplitOn, b), e != String.npos) + { + // Get this string + unsigned int len = e - b; + if(len >= 1) + { + rOutput.push_back(String.substr(b, len)); + } + b = e + 1; + } + // Last string + if(b < String.size()) + { + rOutput.push_back(String.substr(b)); + } +/*#ifndef BOX_RELEASE_BUILD + BOX_TRACE("Splitting string '" << String << " on " << (char)SplitOn); + for(unsigned int l = 0; l < rOutput.size(); ++l) + { + BOX_TRACE(l << " = '" << rOutput[l] << "'"); + } +#endif*/ +} + +#ifdef SHOW_BACKTRACE_ON_EXCEPTION +void DumpStackBacktrace() +{ + void *array[10]; + size_t size = backtrace (array, 10); + char **strings = backtrace_symbols (array, size); + + BOX_TRACE("Obtained " << size << " stack frames."); + + for(size_t i = 0; i < size; i++) + { + // Demangling code copied from + // cctbx_sources/boost_adaptbx/meta_ext.cpp, BSD license + + std::string mangled_frame = strings[i]; + std::string output_frame = strings[i]; // default + + #ifdef HAVE_CXXABI_H + int start = mangled_frame.find('('); + int end = mangled_frame.find('+', start); + std::string mangled_func = mangled_frame.substr(start + 1, + end - start - 1); + + int status; + +#include "MemLeakFindOff.h" + char* result = abi::__cxa_demangle(mangled_func.c_str(), + NULL, NULL, &status); +#include "MemLeakFindOn.h" + + if (result == NULL) + { + if (status == 0) + { + BOX_WARNING("Demangle failed but no error: " << + mangled_func); + } + else if (status == -1) + { + BOX_WARNING("Demangle failed with " + "memory allocation error: " << + mangled_func); + } + else if (status == -2) + { + // Probably non-C++ name, don't demangle + /* + BOX_WARNING("Demangle failed with " + "with invalid name: " << + mangled_func); + */ + } + else if (status == -3) + { + BOX_WARNING("Demangle failed with " + "with invalid argument: " << + mangled_func); + } + else + { + BOX_WARNING("Demangle failed with " + "with unknown error " << status << + ": " << mangled_func); + } + } + else + { + output_frame = mangled_frame.substr(0, start + 1) + + result + mangled_frame.substr(end); +#include "MemLeakFindOff.h" + std::free(result); +#include "MemLeakFindOn.h" + } + #endif // HAVE_CXXABI_H + + BOX_TRACE("Stack frame " << i << ": " << output_frame); + } + +#include "MemLeakFindOff.h" + std::free (strings); +#include "MemLeakFindOn.h" +} +#endif + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileExists(const std::string& rFilename) +// Purpose: Does a file exist? +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +bool FileExists(const std::string& rFilename, int64_t *pFileSize, + bool TreatLinksAsNotExisting) +{ + EMU_STRUCT_STAT st; + if(EMU_LSTAT(rFilename.c_str(), &st) != 0) + { + if(errno == ENOENT) + { + return false; + } + else + { + THROW_EXCEPTION(CommonException, OSFileError); + } + } + + // is it a file? + if((st.st_mode & S_IFDIR) == 0) + { + if(TreatLinksAsNotExisting && ((st.st_mode & S_IFLNK) != 0)) + { + return false; + } + + // Yes. Tell caller the size? + if(pFileSize != 0) + { + *pFileSize = st.st_size; + } + + return true; + } + else + { + return false; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ObjectExists(const std::string& rFilename) +// Purpose: Does a object exist, and if so, is it a file or a directory? +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +int ObjectExists(const std::string& rFilename) +{ + EMU_STRUCT_STAT st; + if(EMU_STAT(rFilename.c_str(), &st) != 0) + { + if(errno == ENOENT) + { + return ObjectExists_NoObject; + } + else + { + THROW_EXCEPTION(CommonException, OSFileError); + } + } + + // is it a file or a dir? + return ((st.st_mode & S_IFDIR) == 0)?ObjectExists_File:ObjectExists_Dir; +} + +std::string HumanReadableSize(int64_t Bytes) +{ + double readableValue = Bytes; + std::string units = " B"; + + if (readableValue > 1024) + { + readableValue /= 1024; + units = "kB"; + } + + if (readableValue > 1024) + { + readableValue /= 1024; + units = "MB"; + } + + if (readableValue > 1024) + { + readableValue /= 1024; + units = "GB"; + } + + std::ostringstream result; + result << std::fixed << std::setprecision(2) << readableValue << + " " << units; + return result.str(); +} + +std::string FormatUsageBar(int64_t Blocks, int64_t Bytes, int64_t Max, + bool MachineReadable) +{ + std::ostringstream result; + + + if (MachineReadable) + { + result << (Bytes >> 10) << " kB, " << + std::setprecision(0) << ((Bytes*100)/Max) << "%"; + } + else + { + // Bar graph + char bar[17]; + unsigned int b = (int)((Bytes * (sizeof(bar)-1)) / Max); + if(b > sizeof(bar)-1) {b = sizeof(bar)-1;} + for(unsigned int l = 0; l < b; l++) + { + bar[l] = '*'; + } + for(unsigned int l = b; l < sizeof(bar) - 1; l++) + { + bar[l] = ' '; + } + bar[sizeof(bar)-1] = '\0'; + + result << std::fixed << + std::setw(10) << Blocks << " blocks, " << + std::setw(10) << HumanReadableSize(Bytes) << ", " << + std::setw(3) << std::setprecision(0) << + ((Bytes*100)/Max) << "% |" << bar << "|"; + } + + return result.str(); +} + +std::string FormatUsageLineStart(const std::string& rName, + bool MachineReadable) +{ + std::ostringstream result; + + if (MachineReadable) + { + result << rName << ": "; + } + else + { + result << std::setw(20) << std::right << rName << ": "; + } + + return result.str(); +} diff --git a/lib/common/Utils.h b/lib/common/Utils.h new file mode 100644 index 00000000..8d98a520 --- /dev/null +++ b/lib/common/Utils.h @@ -0,0 +1,44 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Utils.h +// Purpose: Utility function +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef UTILS__H +#define UTILS__H + +#include <string> +#include <vector> + +#include "MemLeakFindOn.h" + +std::string GetBoxBackupVersion(); + +void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput); + +#ifdef SHOW_BACKTRACE_ON_EXCEPTION + void DumpStackBacktrace(); +#endif + +bool FileExists(const std::string& rFilename, int64_t *pFileSize = 0, + bool TreatLinksAsNotExisting = false); + +enum +{ + ObjectExists_NoObject = 0, + ObjectExists_File = 1, + ObjectExists_Dir = 2 +}; +int ObjectExists(const std::string& rFilename); +std::string HumanReadableSize(int64_t Bytes); +std::string FormatUsageBar(int64_t Blocks, int64_t Bytes, int64_t Max, + bool MachineReadable); +std::string FormatUsageLineStart(const std::string& rName, + bool MachineReadable); + +#include "MemLeakFindOff.h" + +#endif // UTILS__H diff --git a/lib/common/WaitForEvent.cpp b/lib/common/WaitForEvent.cpp new file mode 100644 index 00000000..5646bfbf --- /dev/null +++ b/lib/common/WaitForEvent.cpp @@ -0,0 +1,197 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: WaitForEvent.cpp +// Purpose: Generic waiting for events, using an efficient method (platform dependent) +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <errno.h> +#include <string.h> + +#include "WaitForEvent.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: WaitForEvent::WaitForEvent() +// Purpose: Constructor +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- +#ifdef HAVE_KQUEUE +WaitForEvent::WaitForEvent(int Timeout) + : mKQueue(::kqueue()), + mpTimeout(0) +{ + if(mKQueue == -1) + { + THROW_EXCEPTION(CommonException, CouldNotCreateKQueue) + } + + // Set the choosen timeout + SetTimeout(Timeout); +} +#else +WaitForEvent::WaitForEvent(int Timeout) + : mTimeout(Timeout), + mpPollInfo(0) +{ +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: WaitForEvent::~WaitForEvent() +// Purpose: Destructor +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- +WaitForEvent::~WaitForEvent() +{ +#ifdef HAVE_KQUEUE + ::close(mKQueue); + mKQueue = -1; +#else + if(mpPollInfo != 0) + { + ::free(mpPollInfo); + mpPollInfo = 0; + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: WaitForEvent::SetTimeout +// Purpose: Sets the timeout for future wait calls +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- +void WaitForEvent::SetTimeout(int Timeout) +{ +#ifdef HAVE_KQUEUE + // Generate timeout + if(Timeout != TimeoutInfinite) + { + mTimeout.tv_sec = Timeout / 1000; + mTimeout.tv_nsec = (Timeout % 1000) * 1000000; + } + + // Infinite or not? + mpTimeout = (Timeout != TimeoutInfinite)?(&mTimeout):(NULL); +#else + mTimeout = Timeout; +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WaitForEvent::Wait(int) +// Purpose: Wait for an event to take place. Returns a pointer to the object +// which has been signalled, or returns 0 for the timeout condition. +// Timeout specified in milliseconds. +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- +void *WaitForEvent::Wait() +{ +#ifdef HAVE_KQUEUE + // Event return structure + struct kevent e; + ::memset(&e, 0, sizeof(e)); + + switch(::kevent(mKQueue, NULL, 0, &e, 1, mpTimeout)) + { + case 0: + // Timeout + return 0; + break; + + case 1: + // Event happened! + return e.udata; + break; + + default: + // Interrupted system calls aren't an error, just equivalent to a timeout + if(errno != EINTR) + { + THROW_EXCEPTION(CommonException, KEventErrorWait) + } + return 0; + break; + } +#else + // Use poll() instead. + // Need to build the structures? + if(mpPollInfo == 0) + { + // Yes... + mpPollInfo = (struct pollfd *)::malloc((sizeof(struct pollfd) * mItems.size()) + 4); + if(mpPollInfo == 0) + { + throw std::bad_alloc(); + } + + // Build... + for(unsigned int l = 0; l < mItems.size(); ++l) + { + mpPollInfo[l].fd = mItems[l].fd; + mpPollInfo[l].events = mItems[l].events; + mpPollInfo[l].revents = 0; + } + } + + // Make sure everything is reset (don't really have to do this, but don't trust the OS) + for(unsigned int l = 0; l < mItems.size(); ++l) + { + mpPollInfo[l].revents = 0; + } + + // Poll! + switch(::poll(mpPollInfo, mItems.size(), mTimeout)) + { + case -1: + // Interrupted system calls aren't an error, just equivalent to a timeout + if(errno != EINTR) + { + THROW_EXCEPTION(CommonException, KEventErrorWait) + } + return 0; + break; + case 0: // timed out + return 0; + break; + default: // got some thing... + // control flows on... + break; + } + + // Find the item which was ready + for(unsigned int s = 0; s < mItems.size(); ++s) + { + if(mpPollInfo[s].revents & POLLIN) + { + return mItems[s].item; + break; + } + } +#endif + + return 0; +} + diff --git a/lib/common/WaitForEvent.h b/lib/common/WaitForEvent.h new file mode 100644 index 00000000..a80761ef --- /dev/null +++ b/lib/common/WaitForEvent.h @@ -0,0 +1,152 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: WaitForEvent.h +// Purpose: Generic waiting for events, using an efficient method (platform dependent) +// Created: 9/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef WAITFOREVENT__H +#define WAITFOREVENT__H + +#include <cstdlib> + +#ifdef HAVE_KQUEUE + #include <sys/event.h> + #include <sys/time.h> +#else + #include <vector> + #ifndef WIN32 + #include <poll.h> + #endif +#endif + +#include <cstdlib> + +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +class WaitForEvent +{ +public: + WaitForEvent(int Timeout = TimeoutInfinite); + ~WaitForEvent(); +private: + // No copying. + WaitForEvent(const WaitForEvent &); + WaitForEvent &operator=(const WaitForEvent &); +public: + + enum + { + TimeoutInfinite = -1 + }; + + void SetTimeout(int Timeout = TimeoutInfinite); + + void *Wait(); + +#ifndef HAVE_KQUEUE + typedef struct + { + int fd; + short events; + void *item; + } ItemInfo; +#endif + + // -------------------------------------------------------------------------- + // + // Function + // Name: WaitForEvent::Add(const Type &, int) + // Purpose: Adds an event to the list of items to wait on. The flags are passed to the object. + // Created: 9/3/04 + // + // -------------------------------------------------------------------------- + template<typename T> + void Add(const T *pItem, int Flags = 0) + { + ASSERT(pItem != 0); +#ifdef HAVE_KQUEUE + struct kevent e; + pItem->FillInKEvent(e, Flags); + // Fill in extra flags to say what to do + e.flags |= EV_ADD; + e.udata = (void*)pItem; + if(::kevent(mKQueue, &e, 1, NULL, 0, NULL) == -1) + { + THROW_EXCEPTION(CommonException, KEventErrorAdd) + } +#else + // Add item + ItemInfo i; + pItem->FillInPoll(i.fd, i.events, Flags); + i.item = (void*)pItem; + mItems.push_back(i); + // Delete any pre-prepared poll info, as it's now out of date + if(mpPollInfo != 0) + { + ::free(mpPollInfo); + mpPollInfo = 0; + } +#endif + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: WaitForEvent::Remove(const Type &, int) + // Purpose: Removes an event from the list of items to wait on. The flags are passed to the object. + // Created: 9/3/04 + // + // -------------------------------------------------------------------------- + template<typename T> + void Remove(const T *pItem, int Flags = 0) + { + ASSERT(pItem != 0); +#ifdef HAVE_KQUEUE + struct kevent e; + pItem->FillInKEvent(e, Flags); + // Fill in extra flags to say what to do + e.flags |= EV_DELETE; + e.udata = (void*)pItem; + if(::kevent(mKQueue, &e, 1, NULL, 0, NULL) == -1) + { + THROW_EXCEPTION(CommonException, KEventErrorRemove) + } +#else + if(mpPollInfo != 0) + { + ::free(mpPollInfo); + mpPollInfo = 0; + } + for(std::vector<ItemInfo>::iterator i(mItems.begin()); i != mItems.end(); ++i) + { + if(i->item == pItem) + { + mItems.erase(i); + return; + } + } +#endif + } + +private: +#ifdef HAVE_KQUEUE + int mKQueue; + struct timespec mTimeout; + struct timespec *mpTimeout; +#else + int mTimeout; + std::vector<ItemInfo> mItems; + struct pollfd *mpPollInfo; +#endif +}; + +#include "MemLeakFindOff.h" + +#endif // WAITFOREVENT__H + + diff --git a/lib/common/ZeroStream.cpp b/lib/common/ZeroStream.cpp new file mode 100644 index 00000000..9d87d76a --- /dev/null +++ b/lib/common/ZeroStream.cpp @@ -0,0 +1,170 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ZeroStream.cpp +// Purpose: An IOStream which returns all zeroes up to a certain size +// Created: 2007/04/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "ZeroStream.h" +#include "CommonException.h" + +#include <string.h> + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::ZeroStream(IOStream::pos_type) +// Purpose: Constructor +// Created: 2007/04/28 +// +// -------------------------------------------------------------------------- +ZeroStream::ZeroStream(IOStream::pos_type size) +: mSize(size), mPosition(0) +{ } + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::Read(void *, int) +// Purpose: Reads bytes from the file +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +int ZeroStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + ASSERT(NBytes > 0); + + int bytesToRead = NBytes; + + if (bytesToRead > mSize - mPosition) + { + bytesToRead = mSize - mPosition; + } + + memset(pBuffer, 0, bytesToRead); + mPosition += bytesToRead; + + return bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::BytesLeftToRead() +// Purpose: Returns number of bytes to read (may not be most efficient function ever) +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ZeroStream::BytesLeftToRead() +{ + return mSize - mPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::Write(void *, int) +// Purpose: Writes bytes to the underlying stream (not supported) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ZeroStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::GetPosition() +// Purpose: Get position in stream +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ZeroStream::GetPosition() const +{ + return mPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::Seek(pos_type, int) +// Purpose: Seeks within file, as lseek, invalidate buffer +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ZeroStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + switch (SeekType) + { + case SeekType_Absolute: + { + mPosition = Offset; + } + break; + + case SeekType_Relative: + { + mPosition += Offset; + } + break; + + case SeekType_End: + { + mPosition = mSize - Offset; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::Close() +// Purpose: Closes the underlying stream (not needed) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ZeroStream::Close() +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::StreamDataLeft() +// Purpose: Any data left to write? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool ZeroStream::StreamDataLeft() +{ + return false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::StreamClosed() +// Purpose: Is the stream closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool ZeroStream::StreamClosed() +{ + return false; +} + diff --git a/lib/common/ZeroStream.h b/lib/common/ZeroStream.h new file mode 100644 index 00000000..0119045b --- /dev/null +++ b/lib/common/ZeroStream.h @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ZeroStream.h +// Purpose: An IOStream which returns all zeroes up to a certain size +// Created: 2007/04/28 +// +// -------------------------------------------------------------------------- + +#ifndef ZEROSTREAM__H +#define ZEROSTREAM__H + +#include "IOStream.h" + +class ZeroStream : public IOStream +{ +private: + IOStream::pos_type mSize, mPosition; + +public: + ZeroStream(IOStream::pos_type mSize); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + ZeroStream(const ZeroStream &rToCopy); +}; + +#endif // ZEROSTREAM__H + + diff --git a/lib/common/makeexception.pl.in b/lib/common/makeexception.pl.in new file mode 100755 index 00000000..76b9b02b --- /dev/null +++ b/lib/common/makeexception.pl.in @@ -0,0 +1,283 @@ +#!@PERL@ + +# global exception list file +my $global_list = '../../ExceptionCodes.txt'; + + +my @exception; +my @exception_desc; +my $class; +my $class_number; + +# read the description! + +open EXCEPTION_DESC,$ARGV[0] or die "Can't open $ARGV[0]"; + +while(<EXCEPTION_DESC>) +{ + chomp; s/\A\s+//; s/#.+\Z//; s/\s+\Z//; s/\s+/ /g; + next unless m/\S/; + + if(m/\AEXCEPTION\s+(.+)\s+(\d+)\Z/) + { + $class = $1; + $class_number = $2; + } + else + { + my ($name,$number,$description) = split /\s+/,$_,3; + if($name eq '' || $number =~ m/\D/) + { + die "Bad line '$_'"; + } + if($exception[$number] ne '') + { + die "Duplicate exception number $number"; + } + $exception[$number] = $name; + $exception_desc[$number] = $description; + } +} + +die "Exception class and number not specified" unless $class ne '' && $class_number ne ''; + +close EXCEPTION_DESC; + +# write the code +print "Generating $class exception...\n"; + +open CPP,">autogen_${class}Exception.cpp" or die "Can't open cpp file for writing"; +open H,">autogen_${class}Exception.h" or die "Can't open h file for writing"; + +# write header file +my $guardname = uc 'AUTOGEN_'.$class.'EXCEPTION_H'; +print H <<__E; + +// Auto-generated file -- do not edit + +#ifndef $guardname +#define $guardname + +#include "BoxException.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: ${class}Exception +// Purpose: Exception +// Created: autogen +// +// -------------------------------------------------------------------------- +class ${class}Exception : public BoxException +{ +public: + ${class}Exception(unsigned int SubType, + const std::string& rMessage = "") + : mSubType(SubType), mMessage(rMessage) + { + } + + ${class}Exception(const ${class}Exception &rToCopy) + : mSubType(rToCopy.mSubType), mMessage(rToCopy.mMessage) + { + } + + ~${class}Exception() throw () + { + } + + enum + { + ExceptionType = $class_number + }; + + enum + { +__E + +for(my $e = 0; $e <= $#exception; $e++) +{ + if($exception[$e] ne '') + { + print H "\t\t".$exception[$e].' = '.$e.(($e==$#exception)?'':',')."\n" + } +} + +print H <<__E; + }; + + virtual unsigned int GetType() const throw(); + virtual unsigned int GetSubType() const throw(); + virtual const char *what() const throw(); + virtual const std::string& GetMessage() const + { + return mMessage; + } + +private: + unsigned int mSubType; + std::string mMessage; +}; + +#endif // $guardname +__E + +# ----------------------------------------------------------------------------------------------------------- + +print CPP <<__E; + +// Auto-generated file -- do not edit + +#include "Box.h" +#include "autogen_${class}Exception.h" + +#include "MemLeakFindOn.h" + +#ifdef EXCEPTION_CODENAMES_EXTENDED + #ifdef EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION +static const char *whats[] = { +__E + +my $last_seen = -1; +for(my $e = 0; $e <= $#exception; $e++) +{ + if($exception[$e] ne '') + { + for(my $s = $last_seen + 1; $s < $e; $s++) + { + print CPP "\t\"UNUSED\",\n" + } + my $ext = ($exception_desc[$e] ne '')?" ($exception_desc[$e])":''; + print CPP "\t\"${class} ".$exception[$e].$ext.'"'.(($e==$#exception)?'':',')."\n"; + $last_seen = $e; + } +} + +print CPP <<__E; +}; + #else +static const char *whats[] = { +__E + +$last_seen = -1; +for(my $e = 0; $e <= $#exception; $e++) +{ + if($exception[$e] ne '') + { + for(my $s = $last_seen + 1; $s < $e; $s++) + { + print CPP "\t\"UNUSED\",\n" + } + print CPP "\t\"${class} ".$exception[$e].'"'.(($e==$#exception)?'':',')."\n"; + $last_seen = $e; + } +} + +print CPP <<__E; +}; + #endif +#endif + +unsigned int ${class}Exception::GetType() const throw() +{ + return ${class}Exception::ExceptionType; +} + +unsigned int ${class}Exception::GetSubType() const throw() +{ + return mSubType; +} + +const char *${class}Exception::what() const throw() +{ +#ifdef EXCEPTION_CODENAMES_EXTENDED + if(mSubType < 0 || mSubType > (sizeof(whats) / sizeof(whats[0]))) + { + return "${class}"; + } + return whats[mSubType]; +#else + return "${class}"; +#endif +} + +__E + +close H; +close CPP; + +# update the global exception list +my $list_before; +my $list_after; +my $is_after = 0; +if(open CURRENT,$global_list) +{ + while(<CURRENT>) + { + next if m/\A#/; + + if(m/\AEXCEPTION TYPE (\w+) (\d+)/) + { + # check that the number isn't being reused + if($2 == $class_number && $1 ne $class) + { + die "Class number $class_number is being used by $class and $1 -- correct this.\n"; + } + if($2 > $class_number) + { + # This class comes after the current one (ensures numerical ordering) + $is_after = 1; + } + if($1 eq $class) + { + # skip this entry + while(<CURRENT>) + { + last if m/\AEND TYPE/; + } + $_ = ''; + } + } + + if($is_after) + { + $list_after .= $_; + } + else + { + $list_before .= $_; + } + } + + close CURRENT; +} + +open GLOBAL,">$global_list" or die "Can't open global exception code listing for writing"; + +print GLOBAL <<__E; +# +# automatically generated file, do not edit. +# +# This file lists all the exception codes used by the system. +# Use to look up more detailed descriptions of meanings of errors. +# +__E + +print GLOBAL $list_before; + +print GLOBAL "EXCEPTION TYPE $class $class_number\n"; +for(my $e = 0; $e <= $#exception; $e++) +{ + if($exception[$e] ne '') + { + my $ext = ($exception_desc[$e] ne '')?" - $exception_desc[$e]":''; + print GLOBAL "($class_number/$e) - ${class} ".$exception[$e].$ext."\n"; + } +} +print GLOBAL "END TYPE\n"; + +print GLOBAL $list_after; + +close GLOBAL; + + diff --git a/lib/compress/Compress.h b/lib/compress/Compress.h new file mode 100644 index 00000000..de38af2c --- /dev/null +++ b/lib/compress/Compress.h @@ -0,0 +1,197 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Compress.h +// Purpose: Interface to zlib compression +// Created: 5/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef COMPRESSCONTEXT__H +#define COMPRESSCONTEXT__H + +#include <zlib.h> + +#include "CompressException.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: Compress +// Purpose: Interface to zlib compression, only very slight wrapper. +// (Use CompressStream for a more friendly interface.) +// Created: 5/12/03 +// +// -------------------------------------------------------------------------- +template<bool Compressing> +class Compress +{ +public: + Compress() + : mFinished(false), + mFlush(Z_NO_FLUSH) + { + // initialise stream + mStream.zalloc = Z_NULL; + mStream.zfree = Z_NULL; + mStream.opaque = Z_NULL; + mStream.data_type = Z_BINARY; + + if((Compressing)?(deflateInit(&mStream, Z_DEFAULT_COMPRESSION)) + :(inflateInit(&mStream)) != Z_OK) + { + THROW_EXCEPTION(CompressException, InitFailed) + } + + mStream.avail_in = 0; + } + + ~Compress() + { + int r = 0; + if((r = ((Compressing)?(deflateEnd(&mStream)) + :(inflateEnd(&mStream)))) != Z_OK) + { + BOX_WARNING("zlib error code = " << r); + if(r == Z_DATA_ERROR) + { + BOX_WARNING("End of compress/decompress " + "without all input being consumed, " + "possible corruption?"); + } + else + { + THROW_EXCEPTION(CompressException, EndFailed) + } + } + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: Compress<Function>::InputRequired() + // Purpose: Input required yet? + // Created: 5/12/03 + // + // -------------------------------------------------------------------------- + bool InputRequired() + { + return mStream.avail_in <= 0; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: Compress<Function>::Input(const void *, int) + // Purpose: Set the input buffer ready for next output call. + // Created: 5/12/03 + // + // -------------------------------------------------------------------------- + void Input(const void *pInBuffer, int InLength) + { + // Check usage + if(mStream.avail_in != 0) + { + THROW_EXCEPTION(CompressException, BadUsageInputNotRequired) + } + + // Store info + mStream.next_in = (unsigned char *)pInBuffer; + mStream.avail_in = InLength; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: Compress<Function>::FinishInput() + // Purpose: When compressing, no more input will be given. + // Created: 5/12/03 + // + // -------------------------------------------------------------------------- + void FinishInput() + { + mFlush = Z_FINISH; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: Compress<Function>::Output(void *, int) + // Purpose: Get some output data + // Created: 5/12/03 + // + // -------------------------------------------------------------------------- + int Output(void *pOutBuffer, int OutLength, bool SyncFlush = false) + { + // need more input? + if(mStream.avail_in == 0 && mFlush != Z_FINISH && !SyncFlush) + { + return 0; + } + + // Buffers + mStream.next_out = (unsigned char *)pOutBuffer; + mStream.avail_out = OutLength; + + // Call one of the functions + int flush = mFlush; + if(SyncFlush && mFlush != Z_FINISH) + { + flush = Z_SYNC_FLUSH; + } + int ret = (Compressing)?(deflate(&mStream, flush)):(inflate(&mStream, flush)); + + if(SyncFlush && ret == Z_BUF_ERROR) + { + // No progress possible. Just return 0. + return 0; + } + + // Check errors + if(ret < 0) + { + BOX_WARNING("zlib error code = " << ret); + THROW_EXCEPTION(CompressException, TransformFailed) + } + + // Parse result + if(ret == Z_STREAM_END) + { + mFinished = true; + } + + // Return how much data was output + return OutLength - mStream.avail_out; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: Compress<Function>::OutputHasFinished() + // Purpose: No more output to be recieved + // Created: 5/12/03 + // + // -------------------------------------------------------------------------- + bool OutputHasFinished() + { + return mFinished; + } + + +private: + z_stream mStream; + bool mFinished; + int mFlush; +}; + +template<typename Integer> +Integer Compress_MaxSizeForCompressedData(Integer InLen) +{ + // Conservative rendition of the info found here: http://www.gzip.org/zlib/zlib_tech.html + int blocks = (InLen + 32*1024 - 1) / (32*1024); + return InLen + (blocks * 6) + 8; +} + + +#endif // COMPRESSCONTEXT__H + diff --git a/lib/compress/CompressException.h b/lib/compress/CompressException.h new file mode 100644 index 00000000..7e8c9ca2 --- /dev/null +++ b/lib/compress/CompressException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef COMPRESSEXCEPTION__H +#define COMPRESSEXCEPTION__H + +// Compatibility +#include "autogen_CompressException.h" + +#endif // COMPRESSEXCEPTION__H + diff --git a/lib/compress/CompressException.txt b/lib/compress/CompressException.txt new file mode 100644 index 00000000..278b76d3 --- /dev/null +++ b/lib/compress/CompressException.txt @@ -0,0 +1,12 @@ +EXCEPTION Compress 6 + +Internal 0 +InitFailed 1 +EndFailed 2 +BadUsageInputNotRequired 3 +TransformFailed 4 +CopyCompressStreamNotAllowed 5 +NullPointerPassedToCompressStream 6 +CompressStreamReadSupportNotRequested 7 Specify read in the constructor +CompressStreamWriteSupportNotRequested 8 Specify write in the constructor +CannotWriteToClosedCompressStream 9 diff --git a/lib/compress/CompressStream.cpp b/lib/compress/CompressStream.cpp new file mode 100644 index 00000000..9bb73e3d --- /dev/null +++ b/lib/compress/CompressStream.cpp @@ -0,0 +1,425 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CompressStream.h +// Purpose: Compressing stream +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> +#include <memory> + +#include "CompressStream.h" +#include "Compress.h" +#include "autogen_CompressException.h" + +#include "MemLeakFindOn.h" + +// How big a buffer to use +#ifndef BOX_RELEASE_BUILD + // debug! + #define BUFFER_SIZE 256 +#else + #define BUFFER_SIZE (32*1024) +#endif + +#define USE_READ_COMPRESSOR \ + CheckRead(); \ + Compress<false> *pDecompress = (Compress<false> *)mpReadCompressor; + +#define USE_WRITE_COMPRESSOR \ + CheckWrite(); \ + Compress<true> *pCompress = (Compress<true> *)mpWriteCompressor; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::CompressStream(IOStream *, bool, bool, bool, bool) +// Purpose: Constructor +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +CompressStream::CompressStream(IOStream *pStream, bool TakeOwnership, + bool DecompressRead, bool CompressWrite, bool PassThroughWhenNotCompressed) + : mpStream(pStream), + mHaveOwnership(TakeOwnership), + mDecompressRead(DecompressRead), + mCompressWrite(CompressWrite), + mPassThroughWhenNotCompressed(PassThroughWhenNotCompressed), + mpReadCompressor(0), + mpWriteCompressor(0), + mpBuffer(0), + mIsClosed(false) +{ + if(mpStream == 0) + { + THROW_EXCEPTION(CompressException, NullPointerPassedToCompressStream) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::~CompressStream() +// Purpose: Destructor +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +CompressStream::~CompressStream() +{ + // Clean up compressors + if(mpReadCompressor) + { + delete ((Compress<false>*)mpReadCompressor); + mpReadCompressor = 0; + } + if(mpWriteCompressor) + { + delete ((Compress<true>*)mpWriteCompressor); + mpWriteCompressor = 0; + } + + // Delete the stream, if we have ownership + if(mHaveOwnership) + { + delete mpStream; + mpStream = 0; + } + + // Free any buffer + if(mpBuffer != 0) + { + ::free(mpBuffer); + mpBuffer = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::CompressStream(const CompressStream &) +// Purpose: Copy constructor, will exception +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +CompressStream::CompressStream(const CompressStream &) +{ + THROW_EXCEPTION(CompressException, CopyCompressStreamNotAllowed) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::operator=(const CompressStream &) +// Purpose: Assignment operator, will exception +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +CompressStream &CompressStream::operator=(const CompressStream &) +{ + THROW_EXCEPTION(CompressException, CopyCompressStreamNotAllowed) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::Read(void *, int, int) +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +int CompressStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + USE_READ_COMPRESSOR + if(pDecompress == 0) + { + return mpStream->Read(pBuffer, NBytes, Timeout); + } + + // Where is the buffer? (note if writing as well, read buffer is second in a block of two buffer sizes) + void *pbuf = (mDecompressRead && mCompressWrite)?(((uint8_t*)mpBuffer) + BUFFER_SIZE):mpBuffer; + + // Any data left to go? + if(!pDecompress->InputRequired()) + { + // Output some data from the existing data read + return pDecompress->Output(pBuffer, NBytes, true /* write as much as possible */); + } + + // Read data into the buffer -- read as much as possible in one go + int s = mpStream->Read(pbuf, BUFFER_SIZE, Timeout); + if(s == 0) + { + return 0; + } + + // Give input to the compressor + pDecompress->Input(pbuf, s); + + // Output as much as possible + return pDecompress->Output(pBuffer, NBytes, true /* write as much as possible */); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::Write(const void *, int) +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::Write(const void *pBuffer, int NBytes) +{ + USE_WRITE_COMPRESSOR + if(pCompress == 0) + { + mpStream->Write(pBuffer, NBytes); + return; + } + + if(mIsClosed) + { + THROW_EXCEPTION(CompressException, CannotWriteToClosedCompressStream) + } + + // Give the data to the compressor + pCompress->Input(pBuffer, NBytes); + + // Write the data to the stream + WriteCompressedData(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::WriteAllBuffered() +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::WriteAllBuffered() +{ + if(mIsClosed) + { + THROW_EXCEPTION(CompressException, CannotWriteToClosedCompressStream) + } + + // Just ask compressed data to be written out, but with the sync flag set + WriteCompressedData(true); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::Close() +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::Close() +{ + if(mCompressWrite) + { + USE_WRITE_COMPRESSOR + if(pCompress != 0) + { + // Flush anything from the write buffer + pCompress->FinishInput(); + WriteCompressedData(); + + // Mark as definately closed + mIsClosed = true; + } + } + + // Close + mpStream->Close(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::WriteCompressedData(bool) +// Purpose: Private. Writes the output of the compressor to the stream, +// optionally doing a sync flush. +// Created: 28/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::WriteCompressedData(bool SyncFlush) +{ + USE_WRITE_COMPRESSOR + if(pCompress == 0) {THROW_EXCEPTION(CompressException, Internal)} + + int s = 0; + do + { + s = pCompress->Output(mpBuffer, BUFFER_SIZE, SyncFlush); + if(s > 0) + { + mpStream->Write(mpBuffer, s); + } + } while(s > 0); + // Check assumption -- all input has been consumed + if(!pCompress->InputRequired()) {THROW_EXCEPTION(CompressException, Internal)} +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::StreamDataLeft() +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +bool CompressStream::StreamDataLeft() +{ + USE_READ_COMPRESSOR + if(pDecompress == 0) + { + return mpStream->StreamDataLeft(); + } + + // Any bytes left in our buffer? + if(!pDecompress->InputRequired()) + { + // Still buffered data to decompress + return true; + } + + // Otherwise ask the stream + return mpStream->StreamDataLeft(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::StreamClosed() +// Purpose: As interface +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +bool CompressStream::StreamClosed() +{ + if(!mIsClosed && mpStream->StreamClosed()) + { + mIsClosed = true; + } + return mIsClosed; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::CheckRead() +// Purpose: Checks that everything is set up to read +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::CheckRead() +{ + // Has the read compressor already been created? + if(mpReadCompressor != 0) + { + return; + } + + // Need to create one? + if(mDecompressRead) + { + mpReadCompressor = new Compress<false>(); + // And make sure there's a buffer + CheckBuffer(); + } + else + { + // Not decompressing. Should be passing through data? + if(!mPassThroughWhenNotCompressed) + { + THROW_EXCEPTION(CompressException, CompressStreamReadSupportNotRequested) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::CheckWrite() +// Purpose: Checks that everything is set up to write +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::CheckWrite() +{ + // Has the read compressor already been created? + if(mpWriteCompressor != 0) + { + return; + } + + // Need to create one? + if(mCompressWrite) + { + mpWriteCompressor = new Compress<true>(); + // And make sure there's a buffer + CheckBuffer(); + } + else + { + // Not decompressing. Should be passing through data? + if(!mPassThroughWhenNotCompressed) + { + THROW_EXCEPTION(CompressException, CompressStreamWriteSupportNotRequested) + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CompressStream::CheckBuffer() +// Purpose: Allocates the buffer for (de)compression operations +// Created: 28/5/04 +// +// -------------------------------------------------------------------------- +void CompressStream::CheckBuffer() +{ + // Already done + if(mpBuffer != 0) + { + return; + } + + // Allocate the buffer -- which may actually be two buffers in one + // The temporary use buffer is first (used by write only, so only present if writing) + // and the read buffer follows. + int size = BUFFER_SIZE; + if(mDecompressRead && mCompressWrite) + { + size *= 2; + } + BOX_TRACE("Allocating CompressStream buffer, size " << size); + mpBuffer = ::malloc(size); + if(mpBuffer == 0) + { + throw std::bad_alloc(); + } +} + + diff --git a/lib/compress/CompressStream.h b/lib/compress/CompressStream.h new file mode 100644 index 00000000..7959e3dc --- /dev/null +++ b/lib/compress/CompressStream.h @@ -0,0 +1,62 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CompressStream.h +// Purpose: Compressing stream +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- + +#ifndef COMPRESSSTREAM__H +#define COMPRESSSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: CompressStream +// Purpose: Compressing stream +// Created: 27/5/04 +// +// -------------------------------------------------------------------------- +class CompressStream : public IOStream +{ +public: + CompressStream(IOStream *pStream, bool TakeOwnership, + bool DecompressRead, bool CompressWrite, bool PassThroughWhenNotCompressed = false); + ~CompressStream(); +private: + // No copying (have implementations which exception) + CompressStream(const CompressStream &); + CompressStream &operator=(const CompressStream &); +public: + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes); + virtual void WriteAllBuffered(); + virtual void Close(); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +protected: + void CheckRead(); + void CheckWrite(); + void CheckBuffer(); + void WriteCompressedData(bool SyncFlush = false); + +private: + IOStream *mpStream; + bool mHaveOwnership; + bool mDecompressRead; + bool mCompressWrite; + bool mPassThroughWhenNotCompressed; + // Avoid having to include Compress.h + void *mpReadCompressor; + void *mpWriteCompressor; + void *mpBuffer; + bool mIsClosed; +}; + +#endif // COMPRESSSTREAM__H + diff --git a/lib/compress/Makefile.extra b/lib/compress/Makefile.extra new file mode 100644 index 00000000..a29fcdb4 --- /dev/null +++ b/lib/compress/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_CompressException.h autogen_CompressException.cpp: $(MAKEEXCEPTION) CompressException.txt + $(_PERL) $(MAKEEXCEPTION) CompressException.txt + diff --git a/lib/crypto/CipherAES.cpp b/lib/crypto/CipherAES.cpp new file mode 100644 index 00000000..fad8b968 --- /dev/null +++ b/lib/crypto/CipherAES.cpp @@ -0,0 +1,163 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherAES.cpp +// Purpose: AES cipher description +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +// Only available in new versions of openssl +#ifndef HAVE_OLD_SSL + +#include <openssl/evp.h> + +#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE + +#include "CipherAES.h" +#include "CipherException.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherAES::CipherAES(CipherDescription::CipherMode, const void *, unsigned int, const void *) +// Purpose: Constructor -- note key material and IV are not copied. KeyLength in bytes. +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +CipherAES::CipherAES(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector) + : CipherDescription(), + mMode(Mode), + mpKey(pKey), + mKeyLength(KeyLength), + mpInitialisationVector(pInitialisationVector) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherAES::CipherAES(const CipherAES &) +// Purpose: Copy constructor +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +CipherAES::CipherAES(const CipherAES &rToCopy) + : CipherDescription(rToCopy), + mMode(rToCopy.mMode), + mpKey(rToCopy.mpKey), + mKeyLength(rToCopy.mKeyLength), + mpInitialisationVector(rToCopy.mpInitialisationVector) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ~CipherAES::CipherAES() +// Purpose: Destructor +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +CipherAES::~CipherAES() +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherAES::operator=(const CipherAES &) +// Purpose: Assignment operator +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +CipherAES &CipherAES::operator=(const CipherAES &rToCopy) +{ + CipherDescription::operator=(rToCopy); + + mMode = rToCopy.mMode; + mpKey = rToCopy.mpKey; + mKeyLength = rToCopy.mKeyLength; + mpInitialisationVector = rToCopy.mpInitialisationVector; + + return *this; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherAES::GetCipher() +// Purpose: Returns cipher object +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +const EVP_CIPHER *CipherAES::GetCipher() const +{ + switch(mMode) + { + case CipherDescription::Mode_ECB: + switch(mKeyLength) + { + case (128/8): return EVP_aes_128_ecb(); break; + case (192/8): return EVP_aes_192_ecb(); break; + case (256/8): return EVP_aes_256_ecb(); break; + default: + THROW_EXCEPTION(CipherException, EVPBadKeyLength) + break; + } + break; + + case CipherDescription::Mode_CBC: + switch(mKeyLength) + { + case (128/8): return EVP_aes_128_cbc(); break; + case (192/8): return EVP_aes_192_cbc(); break; + case (256/8): return EVP_aes_256_cbc(); break; + default: + THROW_EXCEPTION(CipherException, EVPBadKeyLength) + break; + } + break; + + default: + break; + } + + // Unknown! + THROW_EXCEPTION(CipherException, UnknownCipherMode) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherAES::SetupParameters(EVP_CIPHER_CTX *) +// Purpose: Set up various parameters for cipher +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +void CipherAES::SetupParameters(EVP_CIPHER_CTX *pCipherContext) const +{ + ASSERT(pCipherContext != 0); + + // Set key (key length is implied) + if(EVP_CipherInit_ex(pCipherContext, NULL, NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + +} + + + +#endif // n HAVE_OLD_SSL + diff --git a/lib/crypto/CipherAES.h b/lib/crypto/CipherAES.h new file mode 100644 index 00000000..50b96dc3 --- /dev/null +++ b/lib/crypto/CipherAES.h @@ -0,0 +1,50 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherAES.h +// Purpose: AES cipher description +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- + +#ifndef CIPHERAES__H +#define CIPHERAES__H + +// Only available in new versions of openssl +#ifndef HAVE_OLD_SSL + +#include "CipherDescription.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: CipherAES +// Purpose: AES cipher description +// Created: 27/4/04 +// +// -------------------------------------------------------------------------- +class CipherAES : public CipherDescription +{ +public: + CipherAES(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector = 0); + CipherAES(const CipherAES &rToCopy); + virtual ~CipherAES(); + CipherAES &operator=(const CipherAES &rToCopy); + + // Return OpenSSL cipher object + virtual const EVP_CIPHER *GetCipher() const; + + // Setup any other parameters + virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const; + +private: + CipherDescription::CipherMode mMode; + const void *mpKey; + unsigned int mKeyLength; + const void *mpInitialisationVector; +}; + +#endif // n HAVE_OLD_SSL + +#endif // CIPHERAES__H + diff --git a/lib/crypto/CipherBlowfish.cpp b/lib/crypto/CipherBlowfish.cpp new file mode 100644 index 00000000..e16cc6ed --- /dev/null +++ b/lib/crypto/CipherBlowfish.cpp @@ -0,0 +1,220 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherBlowfish.cpp +// Purpose: Blowfish cipher description +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <openssl/evp.h> + +#ifdef HAVE_OLD_SSL + #include <string.h> + #include <strings.h> +#endif + +#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE + +#include "CipherBlowfish.h" +#include "CipherException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherBlowfish::CipherBlowfish(CipherDescription::CipherMode, const void *, unsigned int, const void *) +// Purpose: Constructor -- note key material and IV are not copied. KeyLength in bytes. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherBlowfish::CipherBlowfish(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector) + : CipherDescription(), + mMode(Mode) +#ifndef HAVE_OLD_SSL + , mpKey(pKey), + mKeyLength(KeyLength), + mpInitialisationVector(pInitialisationVector) +{ +} +#else +{ + mKey.assign((const char *)pKey, KeyLength); + if(pInitialisationVector == 0) + { + bzero(mInitialisationVector, sizeof(mInitialisationVector)); + } + else + { + ::memcpy(mInitialisationVector, pInitialisationVector, sizeof(mInitialisationVector)); + } +} +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherBlowfish::CipherBlowfish(const CipherBlowfish &) +// Purpose: Copy constructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherBlowfish::CipherBlowfish(const CipherBlowfish &rToCopy) + : CipherDescription(rToCopy), + mMode(rToCopy.mMode), +#ifndef HAVE_OLD_SSL + mpKey(rToCopy.mpKey), + mKeyLength(rToCopy.mKeyLength), + mpInitialisationVector(rToCopy.mpInitialisationVector) +{ +} +#else + mKey(rToCopy.mKey) +{ + ::memcpy(mInitialisationVector, rToCopy.mInitialisationVector, sizeof(mInitialisationVector)); +} +#endif + + +#ifdef HAVE_OLD_SSL +// Hack functions to support old OpenSSL API +CipherDescription *CipherBlowfish::Clone() const +{ + return new CipherBlowfish(*this); +} +void CipherBlowfish::SetIV(const void *pIV) +{ + if(pIV == 0) + { + bzero(mInitialisationVector, sizeof(mInitialisationVector)); + } + else + { + ::memcpy(mInitialisationVector, pIV, sizeof(mInitialisationVector)); + } +} +#endif + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ~CipherBlowfish::CipherBlowfish() +// Purpose: Destructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherBlowfish::~CipherBlowfish() +{ +#ifdef HAVE_OLD_SSL + // Zero copy of key + for(unsigned int l = 0; l < mKey.size(); ++l) + { + mKey[l] = '\0'; + } +#endif +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherBlowfish::operator=(const CipherBlowfish &) +// Purpose: Assignment operator +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherBlowfish &CipherBlowfish::operator=(const CipherBlowfish &rToCopy) +{ + CipherDescription::operator=(rToCopy); + + mMode = rToCopy.mMode; +#ifndef HAVE_OLD_SSL + mpKey = rToCopy.mpKey; + mKeyLength = rToCopy.mKeyLength; + mpInitialisationVector = rToCopy.mpInitialisationVector; +#else + mKey = rToCopy.mKey; + ::memcpy(mInitialisationVector, rToCopy.mInitialisationVector, sizeof(mInitialisationVector)); +#endif + + return *this; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherBlowfish::GetCipher() +// Purpose: Returns cipher object +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +const EVP_CIPHER *CipherBlowfish::GetCipher() const +{ + switch(mMode) + { + case CipherDescription::Mode_ECB: + return EVP_bf_ecb(); + break; + + case CipherDescription::Mode_CBC: + return EVP_bf_cbc(); + break; + + case CipherDescription::Mode_CFB: + return EVP_bf_cfb(); + break; + + case CipherDescription::Mode_OFB: + return EVP_bf_ofb(); + break; + + default: + break; + } + + // Unknown! + THROW_EXCEPTION(CipherException, UnknownCipherMode) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherBlowfish::SetupParameters(EVP_CIPHER_CTX *) +// Purpose: Set up various parameters for cipher +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void CipherBlowfish::SetupParameters(EVP_CIPHER_CTX *pCipherContext) const +{ + ASSERT(pCipherContext != 0); + + // Set key length +#ifndef HAVE_OLD_SSL + if(EVP_CIPHER_CTX_set_key_length(pCipherContext, mKeyLength) != 1) +#else + if(EVP_CIPHER_CTX_set_key_length(pCipherContext, mKey.size()) != 1) +#endif + { + THROW_EXCEPTION(CipherException, EVPBadKeyLength) + } + // Set key +#ifndef HAVE_OLD_SSL + if(EVP_CipherInit_ex(pCipherContext, NULL, NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1) +#else + if(EVP_CipherInit(pCipherContext, NULL, (unsigned char*)mKey.c_str(), (unsigned char*)mInitialisationVector, -1) != 1) +#endif + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + +} + + + diff --git a/lib/crypto/CipherBlowfish.h b/lib/crypto/CipherBlowfish.h new file mode 100644 index 00000000..b3bcf028 --- /dev/null +++ b/lib/crypto/CipherBlowfish.h @@ -0,0 +1,60 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherBlowfish.h +// Purpose: Blowfish cipher description +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef CIPHERBLOWFISH__H +#define CIPHERBLOWFISH__H + +#ifdef HAVE_OLD_SSL + #include <string> +#endif + +#include "CipherDescription.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: CipherBlowfish +// Purpose: Description of Blowfish cipher parameters -- note that copies are not made of key material and IV, careful with object lifetimes. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +class CipherBlowfish : public CipherDescription +{ +public: + CipherBlowfish(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector = 0); + CipherBlowfish(const CipherBlowfish &rToCopy); + virtual ~CipherBlowfish(); + CipherBlowfish &operator=(const CipherBlowfish &rToCopy); + + // Return OpenSSL cipher object + virtual const EVP_CIPHER *GetCipher() const; + + // Setup any other parameters + virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const; + +#ifdef HAVE_OLD_SSL + CipherDescription *Clone() const; + void SetIV(const void *pIV); +#endif + +private: + CipherDescription::CipherMode mMode; +#ifndef HAVE_OLD_SSL + const void *mpKey; + unsigned int mKeyLength; + const void *mpInitialisationVector; +#else + std::string mKey; + uint8_t mInitialisationVector[8]; +#endif +}; + + +#endif // CIPHERBLOWFISH__H + diff --git a/lib/crypto/CipherContext.cpp b/lib/crypto/CipherContext.cpp new file mode 100644 index 00000000..e5cd9b0e --- /dev/null +++ b/lib/crypto/CipherContext.cpp @@ -0,0 +1,620 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherContext.cpp +// Purpose: Context for symmetric encryption / descryption +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE +#include "CipherContext.h" +#include "CipherDescription.h" +#include "CipherException.h" +#include "Random.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::CipherContext() +// Purpose: Constructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherContext::CipherContext() + : mInitialised(false), + mWithinTransform(false), + mPaddingOn(true) +#ifdef HAVE_OLD_SSL + , mFunction(Decrypt), + mpDescription(0) +#endif +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::~CipherContext() +// Purpose: Destructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherContext::~CipherContext() +{ + if(mInitialised) + { + // Clean up + EVP_CIPHER_CTX_cleanup(&ctx); + mInitialised = false; + } +#ifdef HAVE_OLD_SSL + if(mpDescription != 0) + { + delete mpDescription; + mpDescription = 0; + } +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::Init(CipherContext::CipherFunction, const CipherDescription &) +// Purpose: Initialises the context, specifying the direction for the encryption, and a +// description of the cipher to use, it's keys, etc +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDescription &rDescription) +{ + // Check for bad usage + if(mInitialised) + { + THROW_EXCEPTION(CipherException, AlreadyInitialised) + } + if(Function != Decrypt && Function != Encrypt) + { + THROW_EXCEPTION(CipherException, BadArguments) + } + + // Initialise the cipher +#ifndef HAVE_OLD_SSL + EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does + + if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL, Function) != 1) +#else + // Store function for later + mFunction = Function; + + // Use old version of init call + if(EVP_CipherInit(&ctx, rDescription.GetCipher(), NULL, NULL, Function) != 1) +#endif + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + + try + { +#ifndef HAVE_OLD_SSL + // Let the description set up everything else + rDescription.SetupParameters(&ctx); +#else + // With the old version, a copy needs to be taken first. + mpDescription = rDescription.Clone(); + // Mark it as not a leak, otherwise static cipher contexts + // cause spurious memory leaks to be reported + MEMLEAKFINDER_NOT_A_LEAK(mpDescription); + mpDescription->SetupParameters(&ctx); +#endif + } + catch(...) + { + EVP_CIPHER_CTX_cleanup(&ctx); + throw; + } + + // mark as initialised + mInitialised = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::Reset() +// Purpose: Reset the context, so it can be initialised again with a different key +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void CipherContext::Reset() +{ + if(mInitialised) + { + // Clean up + EVP_CIPHER_CTX_cleanup(&ctx); + mInitialised = false; + } +#ifdef HAVE_OLD_SSL + if(mpDescription != 0) + { + delete mpDescription; + mpDescription = 0; + } +#endif + mWithinTransform = false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::Begin() +// Purpose: Begin a transformation +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +void CipherContext::Begin() +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Warn if in a transformation (not an error, because a context might not have been finalised if an exception occured) + if(mWithinTransform) + { + BOX_WARNING("CipherContext::Begin called when context " + "flagged as within a transform"); + } + + // Initialise the cipher context again + if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + + // Mark as being within a transform + mWithinTransform = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::Transform(void *, int, const void *, int) +// Purpose: Transforms the data in the in buffer to the out buffer. If pInBuffer == 0 && InLength == 0 +// then Final() is called instead. +// Returns the number of bytes placed in the out buffer. +// There must be room in the out buffer for all the data in the in buffer. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::Transform(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + if(!mWithinTransform) + { + THROW_EXCEPTION(CipherException, BeginNotCalled) + } + + // Check parameters + if(pOutBuffer == 0 || OutLength < 0 || (pInBuffer != 0 && InLength <= 0) || (pInBuffer == 0 && InLength != 0)) + { + THROW_EXCEPTION(CipherException, BadArguments) + } + + // Is this the final call? + if(pInBuffer == 0) + { + return Final(pOutBuffer, OutLength); + } + + // Check output buffer size + if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx))) + { + THROW_EXCEPTION(CipherException, OutputBufferTooSmall); + } + + // Do the transform + int outLength = OutLength; + if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) + { + THROW_EXCEPTION(CipherException, EVPUpdateFailure) + } + + return outLength; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::Final(void *, int) +// Purpose: Transforms the data as per Transform, and returns the final data in the out buffer. +// Returns the number of bytes written in the out buffer. +// Two main causes of exceptions being thrown: 1) Data is corrupt, and so the end isn't +// padded properly. 2) Padding is off, and the data to be encrypted isn't a multiple +// of a block long. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::Final(void *pOutBuffer, int OutLength) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + if(!mWithinTransform) + { + THROW_EXCEPTION(CipherException, BeginNotCalled) + } + + // Check parameters + if(pOutBuffer == 0 || OutLength < 0) + { + THROW_EXCEPTION(CipherException, BadArguments) + } + + // Check output buffer size + if(OutLength < (2 * EVP_CIPHER_CTX_block_size(&ctx))) + { + THROW_EXCEPTION(CipherException, OutputBufferTooSmall); + } + + // Do the transform + int outLength = OutLength; +#ifndef HAVE_OLD_SSL + if(EVP_CipherFinal_ex(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } +#else + OldOpenSSLFinal((unsigned char*)pOutBuffer, outLength); +#endif + + mWithinTransform = false; + + return outLength; +} + + +#ifdef HAVE_OLD_SSL +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::OldOpenSSLFinal(unsigned char *, int &) +// Purpose: The old version of OpenSSL needs more work doing to finalise the cipher, +// and reset it so that it's ready for another go. +// Created: 27/3/04 +// +// -------------------------------------------------------------------------- +void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) +{ + // Old version needs to use a different form, and then set up the cipher again for next time around + int outLength = rOutLengthOut; + // Have to emulate padding off... + int blockSize = EVP_CIPHER_CTX_block_size(&ctx); + if(mPaddingOn) + { + // Just use normal final call + if(EVP_CipherFinal(&ctx, Buffer, &outLength) != 1) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } + } + else + { + // Padding is off. OpenSSL < 0.9.7 doesn't support this, so it has to be + // bodged in there. Which isn't nice. + if(mFunction == Decrypt) + { + // NASTY -- fiddling around with internals like this is bad. + // But only way to get this working on old versions of OpenSSL. + if(!EVP_EncryptUpdate(&ctx,Buffer,&outLength,ctx.buf,0) + || outLength != blockSize) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } + // Clean up + EVP_CIPHER_CTX_cleanup(&ctx); + } + else + { + // Check that the length is correct + if((ctx.buf_len % blockSize) != 0) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } + // For encryption, assume that the last block entirely is + // padding, and remove it. + char temp[1024]; + outLength = sizeof(temp); + if(EVP_CipherFinal(&ctx, Buffer, &outLength) != 1) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } + // Remove last block, assuming it's full of padded bytes only. + outLength -= blockSize; + // Copy anything to the main buffer + // (can't just use main buffer, because it might overwrite something important) + if(outLength > 0) + { + ::memcpy(Buffer, temp, outLength); + } + } + } + // Reinitialise the cipher for the next time around + if(EVP_CipherInit(&ctx, mpDescription->GetCipher(), NULL, NULL, mFunction) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + mpDescription->SetupParameters(&ctx); + + // Update length for caller + rOutLengthOut = outLength; +} +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::InSizeForOutBufferSize(int) +// Purpose: Returns the maximum amount of data that can be sent in +// given a output buffer size. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::InSizeForOutBufferSize(int OutLength) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Strictly speaking, the *2 is unnecessary. However... + // Final() is paranoid, and requires two input blocks of space to work. + return OutLength - (EVP_CIPHER_CTX_block_size(&ctx) * 2); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::MaxOutSizeForInBufferSize(int) +// Purpose: Returns the maximum output size for an input of a given length. +// Will tend to over estimate, as it needs to allow space for Final() to be called. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::MaxOutSizeForInBufferSize(int InLength) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Final() is paranoid, and requires two input blocks of space to work, and so we need to add + // three blocks on to be absolutely sure. + return InLength + (EVP_CIPHER_CTX_block_size(&ctx) * 3); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::TransformBlock(void *, int, const void *, int) +// Purpose: Transform one block to another all in one go, no Final required. +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Warn if in a transformation + if(mWithinTransform) + { + BOX_WARNING("CipherContext::TransformBlock called when " + "context flagged as within a transform"); + } + + // Check output buffer size + if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx))) + { + // Check if padding is off, in which case the buffer can be smaller + if(!mPaddingOn && OutLength <= InLength) + { + // This is OK. + } + else + { + THROW_EXCEPTION(CipherException, OutputBufferTooSmall); + } + } + + // Initialise the cipher context again + if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + + // Do the entire block + int outLength = 0; + try + { + // Update + outLength = OutLength; + if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) + { + THROW_EXCEPTION(CipherException, EVPUpdateFailure) + } + // Finalise + int outLength2 = OutLength - outLength; +#ifndef HAVE_OLD_SSL + if(EVP_CipherFinal_ex(&ctx, ((unsigned char*)pOutBuffer) + outLength, &outLength2) != 1) + { + THROW_EXCEPTION(CipherException, EVPFinalFailure) + } +#else + OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2); +#endif + outLength += outLength2; + } + catch(...) + { + // Finalise the context, so definately ready for the next caller + int outs = OutLength; +#ifndef HAVE_OLD_SSL + EVP_CipherFinal_ex(&ctx, (unsigned char*)pOutBuffer, &outs); +#else + OldOpenSSLFinal((unsigned char*)pOutBuffer, outs); +#endif + throw; + } + + return outLength; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::GetIVLength() +// Purpose: Returns the size of the IV for this context +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +int CipherContext::GetIVLength() +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + return EVP_CIPHER_CTX_iv_length(&ctx); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::SetIV(const void *) +// Purpose: Sets the IV for this context (must be correctly sized, use GetIVLength) +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +void CipherContext::SetIV(const void *pIV) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Warn if in a transformation + if(mWithinTransform) + { + BOX_WARNING("CipherContext::SetIV called when context " + "flagged as within a transform"); + } + + // Set IV + if(EVP_CipherInit(&ctx, NULL, NULL, (unsigned char *)pIV, -1) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + +#ifdef HAVE_OLD_SSL + // Update description + if(mpDescription != 0) + { + mpDescription->SetIV(pIV); + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::SetRandomIV(int &) +// Purpose: Set a random IV for the context, and return a pointer to the IV used, +// and the length of this IV in the rLengthOut arg. +// Created: 3/12/03 +// +// -------------------------------------------------------------------------- +const void *CipherContext::SetRandomIV(int &rLengthOut) +{ + if(!mInitialised) + { + THROW_EXCEPTION(CipherException, NotInitialised) + } + + // Warn if in a transformation + if(mWithinTransform) + { + BOX_WARNING("CipherContext::SetRandomIV called when " + "context flagged as within a transform"); + } + + // Get length of IV + unsigned int ivLen = EVP_CIPHER_CTX_iv_length(&ctx); + if(ivLen > sizeof(mGeneratedIV)) + { + THROW_EXCEPTION(CipherException, IVSizeImplementationLimitExceeded) + } + + // Generate some random data + Random::Generate(mGeneratedIV, ivLen); + + // Set IV + if(EVP_CipherInit(&ctx, NULL, NULL, mGeneratedIV, -1) != 1) + { + THROW_EXCEPTION(CipherException, EVPInitFailure) + } + +#ifdef HAVE_OLD_SSL + // Update description + if(mpDescription != 0) + { + mpDescription->SetIV(mGeneratedIV); + } +#endif + + // Return the IV and it's length + rLengthOut = ivLen; + return mGeneratedIV; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherContext::UsePadding(bool) +// Purpose: Set whether or not the context uses padding. +// Created: 12/12/03 +// +// -------------------------------------------------------------------------- +void CipherContext::UsePadding(bool Padding) +{ +#ifndef HAVE_OLD_SSL + if(EVP_CIPHER_CTX_set_padding(&ctx, Padding) != 1) + { + THROW_EXCEPTION(CipherException, EVPSetPaddingFailure) + } +#endif + mPaddingOn = Padding; +} + + + diff --git a/lib/crypto/CipherContext.h b/lib/crypto/CipherContext.h new file mode 100644 index 00000000..64ce52d8 --- /dev/null +++ b/lib/crypto/CipherContext.h @@ -0,0 +1,83 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherContext.h +// Purpose: Context for symmetric encryption / descryption +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef CIPHERCONTEXT__H +#define CIPHERCONTEXT__H + +#ifdef BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_FALSE + always include CipherContext.h first in any .cpp file +#endif +#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE +#include <openssl/evp.h> +class CipherDescription; + +#define CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH 32 + +// -------------------------------------------------------------------------- +// +// Class +// Name: CipherContext +// Purpose: Context for symmetric encryption / descryption +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +class CipherContext +{ +public: + CipherContext(); + ~CipherContext(); +private: + CipherContext(const CipherContext &); // no copying + CipherContext &operator=(const CipherContext &); // no assignment +public: + + typedef enum + { + Decrypt = 0, + Encrypt = 1 + } CipherFunction; + + void Init(CipherContext::CipherFunction Function, const CipherDescription &rDescription); + void Reset(); + + void Begin(); + int Transform(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength); + int Final(void *pOutBuffer, int OutLength); + int InSizeForOutBufferSize(int OutLength); + int MaxOutSizeForInBufferSize(int InLength); + + int TransformBlock(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength); + + bool IsInitialised() {return mInitialised;} + + int GetIVLength(); + void SetIV(const void *pIV); + const void *SetRandomIV(int &rLengthOut); + + void UsePadding(bool Padding = true); + +#ifdef HAVE_OLD_SSL + void OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut); +#endif + +private: + EVP_CIPHER_CTX ctx; + bool mInitialised; + bool mWithinTransform; + bool mPaddingOn; + uint8_t mGeneratedIV[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH]; +#ifdef HAVE_OLD_SSL + CipherFunction mFunction; + CipherDescription *mpDescription; +#endif +}; + + +#endif // CIPHERCONTEXT__H + diff --git a/lib/crypto/CipherDescription.cpp b/lib/crypto/CipherDescription.cpp new file mode 100644 index 00000000..f0ba6ec8 --- /dev/null +++ b/lib/crypto/CipherDescription.cpp @@ -0,0 +1,73 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherDescription.cpp +// Purpose: Pure virtual base class for describing ciphers +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <openssl/evp.h> + +#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE + +#include "CipherDescription.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherDescription::CipherDescription() +// Purpose: Constructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherDescription::CipherDescription() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherDescription::CipherDescription(const CipherDescription &) +// Purpose: Copy constructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherDescription::CipherDescription(const CipherDescription &rToCopy) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ~CipherDescription::CipherDescription() +// Purpose: Destructor +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherDescription::~CipherDescription() +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: CipherDescription::operator=(const CipherDescription &) +// Purpose: Assignment operator +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +CipherDescription &CipherDescription::operator=(const CipherDescription &rToCopy) +{ + return *this; +} + + diff --git a/lib/crypto/CipherDescription.h b/lib/crypto/CipherDescription.h new file mode 100644 index 00000000..f825eefa --- /dev/null +++ b/lib/crypto/CipherDescription.h @@ -0,0 +1,59 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherDescription.h +// Purpose: Pure virtual base class for describing ciphers +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef CIPHERDESCRIPTION__H +#define CIPHERDESCRIPTION__H + +#ifndef BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE + #define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_FALSE + class EVP_CIPHER; + class EVP_CIPHER_CTX; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: CipherDescription +// Purpose: Describes a cipher +// Created: 1/12/03 +// +// -------------------------------------------------------------------------- +class CipherDescription +{ +public: + CipherDescription(); + CipherDescription(const CipherDescription &rToCopy); + virtual ~CipherDescription(); + CipherDescription &operator=(const CipherDescription &rToCopy); + + // Return OpenSSL cipher object + virtual const EVP_CIPHER *GetCipher() const = 0; + + // Setup any other parameters + virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const = 0; + + // Mode parameter for cipher -- used in derived classes + typedef enum + { + Mode_ECB = 0, + Mode_CBC = 1, + Mode_CFB = 2, + Mode_OFB = 3 + } CipherMode; + +#ifdef HAVE_OLD_SSL + // For the old version of OpenSSL, we need to be able to store cipher descriptions. + virtual CipherDescription *Clone() const = 0; + // And to be able to store new IVs + virtual void SetIV(const void *pIV) = 0; +#endif +}; + +#endif // CIPHERDESCRIPTION__H + diff --git a/lib/crypto/CipherException.h b/lib/crypto/CipherException.h new file mode 100644 index 00000000..b0c9f8cd --- /dev/null +++ b/lib/crypto/CipherException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CipherException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef CIPHEREXCEPTION__H +#define CIPHEREXCEPTION__H + +// Compatibility +#include "autogen_CipherException.h" + +#endif // CIPHEREXCEPTION__H + diff --git a/lib/crypto/CipherException.txt b/lib/crypto/CipherException.txt new file mode 100644 index 00000000..abdbac87 --- /dev/null +++ b/lib/crypto/CipherException.txt @@ -0,0 +1,18 @@ +EXCEPTION Cipher 5 + +Internal 0 +UnknownCipherMode 1 +AlreadyInitialised 2 +BadArguments 3 +EVPInitFailure 4 +EVPUpdateFailure 5 +EVPFinalFailure 6 +NotInitialised 7 +OutputBufferTooSmall 8 +EVPBadKeyLength 9 +BeginNotCalled 10 +IVSizeImplementationLimitExceeded 11 +PseudoRandNotAvailable 12 +EVPSetPaddingFailure 13 +RandomInitFailed 14 Failed to read from random device +LengthRequestedTooLongForRandomHex 15 diff --git a/lib/crypto/MD5Digest.cpp b/lib/crypto/MD5Digest.cpp new file mode 100644 index 00000000..58cc90ee --- /dev/null +++ b/lib/crypto/MD5Digest.cpp @@ -0,0 +1,82 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MD5Digest.cpp +// Purpose: Simple interface for creating MD5 digests +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- + + +#include "Box.h" + +#include "MD5Digest.h" + +#include "MemLeakFindOn.h" + + +MD5Digest::MD5Digest() +{ + MD5_Init(&md5); + for(unsigned int l = 0; l < sizeof(mDigest); ++l) + { + mDigest[l] = 0; + } +} + +MD5Digest::~MD5Digest() +{ +} + +void MD5Digest::Add(const std::string &rString) +{ + MD5_Update(&md5, rString.c_str(), rString.size()); +} + +void MD5Digest::Add(const void *pData, int Length) +{ + MD5_Update(&md5, pData, Length); +} + +void MD5Digest::Finish() +{ + MD5_Final(mDigest, &md5); +} + +std::string MD5Digest::DigestAsString() +{ + std::string r; + + static const char *hex = "0123456789abcdef"; + + for(unsigned int l = 0; l < sizeof(mDigest); ++l) + { + r += hex[(mDigest[l] & 0xf0) >> 4]; + r += hex[(mDigest[l] & 0x0f)]; + } + + return r; +} + +int MD5Digest::CopyDigestTo(uint8_t *to) +{ + for(int l = 0; l < MD5_DIGEST_LENGTH; ++l) + { + to[l] = mDigest[l]; + } + + return MD5_DIGEST_LENGTH; +} + + +bool MD5Digest::DigestMatches(uint8_t *pCompareWith) const +{ + for(int l = 0; l < MD5_DIGEST_LENGTH; ++l) + { + if(pCompareWith[l] != mDigest[l]) + return false; + } + + return true; +} + diff --git a/lib/crypto/MD5Digest.h b/lib/crypto/MD5Digest.h new file mode 100644 index 00000000..1be01ea9 --- /dev/null +++ b/lib/crypto/MD5Digest.h @@ -0,0 +1,57 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MD5Digest.h +// Purpose: Simple interface for creating MD5 digests +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef MD5DIGEST_H +#define MD5DIGEST_H + +#include <openssl/md5.h> +#include <string> + +// -------------------------------------------------------------------------- +// +// Function +// Name: MD5Digest +// Purpose: Simple interface for creating MD5 digests +// Created: 8/12/03 +// +// -------------------------------------------------------------------------- +class MD5Digest +{ +public: + MD5Digest(); + virtual ~MD5Digest(); + + void Add(const std::string &rString); + void Add(const void *pData, int Length); + + void Finish(); + + std::string DigestAsString(); + uint8_t *DigestAsData(int *pLength = 0) + { + if(pLength) *pLength = sizeof(mDigest); + return mDigest; + } + + enum + { + DigestLength = MD5_DIGEST_LENGTH + }; + + int CopyDigestTo(uint8_t *to); + + bool DigestMatches(uint8_t *pCompareWith) const; + +private: + MD5_CTX md5; + uint8_t mDigest[MD5_DIGEST_LENGTH]; +}; + +#endif // MD5DIGEST_H + diff --git a/lib/crypto/Makefile.extra b/lib/crypto/Makefile.extra new file mode 100644 index 00000000..3c94389f --- /dev/null +++ b/lib/crypto/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_CipherException.cpp autogen_CipherException.h: $(MAKEEXCEPTION) CipherException.txt + $(_PERL) $(MAKEEXCEPTION) CipherException.txt + diff --git a/lib/crypto/Random.cpp b/lib/crypto/Random.cpp new file mode 100644 index 00000000..1d6a07f0 --- /dev/null +++ b/lib/crypto/Random.cpp @@ -0,0 +1,128 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Random.cpp +// Purpose: Random numbers +// Created: 31/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <openssl/rand.h> +#include <stdio.h> + +#include "Random.h" +#include "CipherException.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Random::Initialise() +// Purpose: Add additional randomness to the standard library initialisation +// Created: 18/6/04 +// +// -------------------------------------------------------------------------- +void Random::Initialise() +{ +#ifdef HAVE_RANDOM_DEVICE + if(::RAND_load_file(RANDOM_DEVICE, 1024) != 1024) + { + THROW_EXCEPTION(CipherException, RandomInitFailed) + } +#else + BOX_ERROR("No random device -- additional seeding of random number " + "generator not performed."); +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Random::Generate(void *, int) +// Purpose: Generate Length bytes of random data +// Created: 31/12/03 +// +// -------------------------------------------------------------------------- +void Random::Generate(void *pOutput, int Length) +{ + if(RAND_pseudo_bytes((uint8_t*)pOutput, Length) == -1) + { + THROW_EXCEPTION(CipherException, PseudoRandNotAvailable) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Random::GenerateHex(int) +// Purpose: Generate Length bytes of hex encoded data. Note that the +// maximum length requested is limited. (Returns a string +// 2 x Length characters long.) +// Created: 1/11/04 +// +// -------------------------------------------------------------------------- +std::string Random::GenerateHex(int Length) +{ + uint8_t r[256]; + if(Length > (int)sizeof(r)) + { + THROW_EXCEPTION(CipherException, LengthRequestedTooLongForRandomHex) + } + Random::Generate(r, Length); + + std::string o; + static const char *h = "0123456789abcdef"; + for(int l = 0; l < Length; ++l) + { + o += h[r[l] >> 4]; + o += h[r[l] & 0xf]; + } + + return o; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Random::RandomInt(int) +// Purpose: Return a random integer between 0 and MaxValue inclusive. +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +uint32_t Random::RandomInt(uint32_t MaxValue) +{ + uint32_t v = 0; + + // Generate a mask + uint32_t mask = 0; + while(mask < MaxValue) + { + mask = (mask << 1) | 1; + } + + do + { + // Generate a random number + uint32_t r = 0; + Random::Generate(&r, sizeof(r)); + + // Mask off relevant bits + v = r & mask; + + // Check that it's in the right range. + } while(v > MaxValue); + + // NOTE: don't do a mod, because this doesn't give a correct random distribution + + return v; +} + + + diff --git a/lib/crypto/Random.h b/lib/crypto/Random.h new file mode 100644 index 00000000..da3e4335 --- /dev/null +++ b/lib/crypto/Random.h @@ -0,0 +1,25 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Random.h +// Purpose: Random numbers +// Created: 31/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef RANDOM__H +#define RANDOM__H + +#include <string> + +namespace Random +{ + void Initialise(); + void Generate(void *pOutput, int Length); + std::string GenerateHex(int Length); + uint32_t RandomInt(uint32_t MaxValue); +}; + + +#endif // RANDOM__H + diff --git a/lib/crypto/RollingChecksum.cpp b/lib/crypto/RollingChecksum.cpp new file mode 100644 index 00000000..a2954e3a --- /dev/null +++ b/lib/crypto/RollingChecksum.cpp @@ -0,0 +1,62 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RollingChecksum.cpp +// Purpose: A simple rolling checksum over a block of data +// Created: 6/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "RollingChecksum.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: RollingChecksum::RollingChecksum(const void *, unsigned int) +// Purpose: Constructor -- does initial computation of the checksum. +// Created: 6/12/03 +// +// -------------------------------------------------------------------------- +RollingChecksum::RollingChecksum(const void * const data, const unsigned int Length) + : a(0), + b(0) +{ + const uint8_t *block = (const uint8_t *)data; + for(unsigned int x = Length; x >= 1; --x) + { + a += (*block); + b += x * (*block); + + ++block; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RollingChecksum::RollForwardSeveral(uint8_t*, uint8_t*, unsigned int, unsigned int) +// Purpose: Move the checksum forward a block, given a pointer to the first byte of the current block, +// and a pointer just after the last byte of the current block and the length of the block and of the skip. +// Created: 7/14/05 +// +// -------------------------------------------------------------------------- +void RollingChecksum::RollForwardSeveral(const uint8_t * const StartOfThisBlock, const uint8_t * const LastOfNextBlock, const unsigned int Length, const unsigned int Skip) +{ + // IMPLEMENTATION NOTE: Everything is implicitly mod 2^16 -- uint16_t's will overflow nicely. + unsigned int i; + uint16_t sumBegin=0, j,k; + + for(i=0; i < Skip; i++) + { + j = StartOfThisBlock[i]; + k = LastOfNextBlock[i]; + sumBegin += j; + a += (k - j); + b += a; + } + + b -= Length * sumBegin; +} diff --git a/lib/crypto/RollingChecksum.h b/lib/crypto/RollingChecksum.h new file mode 100644 index 00000000..be79c36f --- /dev/null +++ b/lib/crypto/RollingChecksum.h @@ -0,0 +1,107 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RollingChecksum.h +// Purpose: A simple rolling checksum over a block of data +// Created: 6/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef ROLLINGCHECKSUM__H +#define ROLLINGCHECKSUM__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: RollingChecksum +// Purpose: A simple rolling checksum over a block of data -- can move the block +// "forwards" in memory and get the next checksum efficiently. +// +// Implementation of http://rsync.samba.org/tech_report/node3.html +// Created: 6/12/03 +// +// -------------------------------------------------------------------------- +class RollingChecksum +{ +public: + RollingChecksum(const void * const data, const unsigned int Length); + + // -------------------------------------------------------------------------- + // + // Function + // Name: RollingChecksum::RollForward(uint8_t, uint8_t, unsigned int) + // Purpose: Move the checksum forward a block, given the first byte of the current block, + // last byte of the next block (it's rolling forward to) and the length of the block. + // Created: 6/12/03 + // + // -------------------------------------------------------------------------- + inline void RollForward(const uint8_t StartOfThisBlock, const uint8_t LastOfNextBlock, const unsigned int Length) + { + // IMPLEMENTATION NOTE: Everything is implicitly mod 2^16 -- uint16_t's will overflow nicely. + a -= StartOfThisBlock; + a += LastOfNextBlock; + b -= Length * StartOfThisBlock; + b += a; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: RollingChecksum::RollForwardSeveral(uint8_t*, uint8_t*, unsigned int, unsigned int) + // Purpose: Move the checksum forward a block, given a pointer to the first byte of the current block, + // and a pointer just after the last byte of the current block and the length of the block and of the skip. + // Created: 7/14/05 + // + // -------------------------------------------------------------------------- + void RollForwardSeveral(const uint8_t * const StartOfThisBlock, const uint8_t * const LastOfNextBlock, const unsigned int Length, const unsigned int Skip); + + // -------------------------------------------------------------------------- + // + // Function + // Name: RollingChecksum::GetChecksum() + // Purpose: Returns the checksum + // Created: 6/12/03 + // + // -------------------------------------------------------------------------- + inline uint32_t GetChecksum() const + { + return ((uint32_t)a) | (((uint32_t)b) << 16); + } + + // Components, just in case they're handy + inline uint16_t GetComponent1() const {return a;} + inline uint16_t GetComponent2() const {return b;} + + // -------------------------------------------------------------------------- + // + // Function + // Name: RollingChecksum::GetComponentForHashing() + // Purpose: Return the 16 bit component used for hashing and/or quick checks + // Created: 6/12/03 + // + // -------------------------------------------------------------------------- + inline uint16_t GetComponentForHashing() const + { + return b; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: RollingChecksum::ExtractHashingComponent(uint32_t) + // Purpose: Static. Given a full checksum, extract the component used in the hashing table. + // Created: 14/1/04 + // + // -------------------------------------------------------------------------- + static inline uint16_t ExtractHashingComponent(const uint32_t Checksum) + { + return Checksum >> 16; + } + +private: + uint16_t a; + uint16_t b; +}; + +#endif // ROLLINGCHECKSUM__H + diff --git a/lib/httpserver/HTTPException.txt b/lib/httpserver/HTTPException.txt new file mode 100644 index 00000000..52630cda --- /dev/null +++ b/lib/httpserver/HTTPException.txt @@ -0,0 +1,16 @@ +EXCEPTION HTTP 10 + +Internal 0 +RequestReadFailed 1 +RequestAlreadyBeenRead 2 +BadRequest 3 +UnknownResponseCodeUsed 4 +NoContentTypeSet 5 +POSTContentTooLong 6 +CannotSetRedirectIfReponseHasData 7 +CannotSetNotFoundIfReponseHasData 8 +NotImplemented 9 +RequestNotInitialised 10 +BadResponse 11 +ResponseReadFailed 12 +NoStreamConfigured 13 diff --git a/lib/httpserver/HTTPQueryDecoder.cpp b/lib/httpserver/HTTPQueryDecoder.cpp new file mode 100644 index 00000000..c49ac2ce --- /dev/null +++ b/lib/httpserver/HTTPQueryDecoder.cpp @@ -0,0 +1,159 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPQueryDecoder.cpp +// Purpose: Utility class to decode HTTP query strings +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> + +#include "HTTPQueryDecoder.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::HTTPQueryDecoder( +// HTTPRequest::Query_t &) +// Purpose: Constructor. Pass in the query contents you want +// to decode the query string into. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPQueryDecoder::HTTPQueryDecoder(HTTPRequest::Query_t &rDecodeInto) + : mrDecodeInto(rDecodeInto), + mInKey(true), + mEscapedState(0) +{ + // Insert the terminator for escaped characters + mEscaped[2] = '\0'; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::~HTTPQueryDecoder() +// Purpose: Destructor. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPQueryDecoder::~HTTPQueryDecoder() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::Decode(const char *, int) +// Purpose: Decode a chunk of query string -- call several times with +// the bits as they are received, and then call Finish() +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPQueryDecoder::DecodeChunk(const char *pQueryString, int QueryStringSize) +{ + for(int l = 0; l < QueryStringSize; ++l) + { + char c = pQueryString[l]; + + // BEFORE unescaping, check to see if we need to flip key / value + if(mEscapedState == 0) + { + if(mInKey && c == '=') + { + // Set to store characters in the value + mInKey = false; + continue; + } + else if(!mInKey && c == '&') + { + // Need to store the current key/value pair + mrDecodeInto.insert(HTTPRequest::QueryEn_t(mCurrentKey, mCurrentValue)); + // Blank the strings + mCurrentKey.erase(); + mCurrentValue.erase(); + + // Set to store characters in the key + mInKey = true; + continue; + } + } + + // Decode an escaped value? + if(mEscapedState == 1) + { + // Waiting for char one of the escaped hex value + mEscaped[0] = c; + mEscapedState = 2; + continue; + } + else if(mEscapedState == 2) + { + // Escaped value, decode it + mEscaped[1] = c; // str terminated in constructor + mEscapedState = 0; // stop being in escaped mode + long ch = ::strtol(mEscaped, NULL, 16); + if(ch <= 0 || ch > 255) + { + // Bad character, just ignore + continue; + } + + // Use this instead + c = (char)ch; + } + else if(c == '+') + { + c = ' '; + } + else if(c == '%') + { + mEscapedState = 1; + continue; + } + + // Store decoded value into the appropriate string + if(mInKey) + { + mCurrentKey += c; + } + else + { + mCurrentValue += c; + } + } + + // Don't do anything here with left over values, DecodeChunk might be called + // again. Let Finish() clean up. +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::Finish() +// Purpose: Finish the decoding. Necessary to get the last item! +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPQueryDecoder::Finish() +{ + // Insert any remaining value. + if(!mCurrentKey.empty()) + { + mrDecodeInto.insert(HTTPRequest::QueryEn_t(mCurrentKey, mCurrentValue)); + // Blank values, just in case + mCurrentKey.erase(); + mCurrentValue.erase(); + } +} + + diff --git a/lib/httpserver/HTTPQueryDecoder.h b/lib/httpserver/HTTPQueryDecoder.h new file mode 100644 index 00000000..ca5afe7e --- /dev/null +++ b/lib/httpserver/HTTPQueryDecoder.h @@ -0,0 +1,47 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPQueryDecoder.h +// Purpose: Utility class to decode HTTP query strings +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPQUERYDECODER__H +#define HTTPQUERYDECODER__H + +#include "HTTPRequest.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPQueryDecoder +// Purpose: Utility class to decode HTTP query strings +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPQueryDecoder +{ +public: + HTTPQueryDecoder(HTTPRequest::Query_t &rDecodeInto); + ~HTTPQueryDecoder(); +private: + // no copying + HTTPQueryDecoder(const HTTPQueryDecoder &); + HTTPQueryDecoder &operator=(const HTTPQueryDecoder &); +public: + + void DecodeChunk(const char *pQueryString, int QueryStringSize); + void Finish(); + +private: + HTTPRequest::Query_t &mrDecodeInto; + std::string mCurrentKey; + std::string mCurrentValue; + bool mInKey; + char mEscaped[4]; + int mEscapedState; +}; + +#endif // HTTPQUERYDECODER__H + diff --git a/lib/httpserver/HTTPRequest.cpp b/lib/httpserver/HTTPRequest.cpp new file mode 100644 index 00000000..4c5dc149 --- /dev/null +++ b/lib/httpserver/HTTPRequest.cpp @@ -0,0 +1,780 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPRequest.cpp +// Purpose: Request object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> +#include <strings.h> +#include <stdlib.h> +#include <stdio.h> + +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "HTTPQueryDecoder.h" +#include "autogen_HTTPException.h" +#include "IOStream.h" +#include "IOStreamGetLine.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +#define MAX_CONTENT_SIZE (128*1024) + +#define ENSURE_COOKIE_JAR_ALLOCATED \ + if(mpCookies == 0) {mpCookies = new CookieJar_t;} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::HTTPRequest() +// Purpose: Constructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPRequest::HTTPRequest() + : mMethod(Method_UNINITIALISED), + mHostPort(80), // default if not specified + mHTTPVersion(0), + mContentLength(-1), + mpCookies(0), + mClientKeepAliveRequested(false), + mExpectContinue(false), + mpStreamToReadFrom(NULL) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::HTTPRequest(enum Method, +// const std::string&) +// Purpose: Alternate constructor for hand-crafted requests +// Created: 03/01/09 +// +// -------------------------------------------------------------------------- +HTTPRequest::HTTPRequest(enum Method method, const std::string& rURI) + : mMethod(method), + mRequestURI(rURI), + mHostPort(80), // default if not specified + mHTTPVersion(HTTPVersion_1_1), + mContentLength(-1), + mpCookies(0), + mClientKeepAliveRequested(false), + mExpectContinue(false), + mpStreamToReadFrom(NULL) +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::~HTTPRequest() +// Purpose: Destructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPRequest::~HTTPRequest() +{ + // Clean up any cookies + if(mpCookies != 0) + { + delete mpCookies; + mpCookies = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::Receive(IOStreamGetLine &, int) +// Purpose: Read the request from an IOStreamGetLine (and +// attached stream). +// Returns false if there was no valid request, +// probably due to a kept-alive connection closing. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) +{ + // Check caller's logic + if(mMethod != Method_UNINITIALISED) + { + THROW_EXCEPTION(HTTPException, RequestAlreadyBeenRead) + } + + // Read the first line, which is of a different format to the rest of the lines + std::string requestLine; + if(!rGetLine.GetLine(requestLine, false /* no preprocessing */, Timeout)) + { + // Didn't get the request line, probably end of connection which had been kept alive + return false; + } + BOX_TRACE("Request line: " << requestLine); + + // Check the method + size_t p = 0; // current position in string + p = requestLine.find(' '); // end of first word + + if (p == std::string::npos) + { + // No terminating space, looks bad + p = requestLine.size(); + } + else + { + mHttpVerb = requestLine.substr(0, p); + if (mHttpVerb == "GET") + { + mMethod = Method_GET; + } + else if (mHttpVerb == "HEAD") + { + mMethod = Method_HEAD; + } + else if (mHttpVerb == "POST") + { + mMethod = Method_POST; + } + else if (mHttpVerb == "PUT") + { + mMethod = Method_PUT; + } + else + { + mMethod = Method_UNKNOWN; + } + } + + // Skip spaces to find URI + const char *requestLinePtr = requestLine.c_str(); + while(requestLinePtr[p] != '\0' && requestLinePtr[p] == ' ') + { + ++p; + } + + // Check there's a URI following... + if(requestLinePtr[p] == '\0') + { + // Didn't get the request line, probably end of connection which had been kept alive + return false; + } + + // Read the URI, unescaping any %XX hex codes + while(requestLinePtr[p] != ' ' && requestLinePtr[p] != '\0') + { + // End of URI, on to query string? + if(requestLinePtr[p] == '?') + { + // Put the rest into the query string, without escaping anything + ++p; + while(requestLinePtr[p] != ' ' && requestLinePtr[p] != '\0') + { + mQueryString += requestLinePtr[p]; + ++p; + } + break; + } + // Needs unescaping? + else if(requestLinePtr[p] == '+') + { + mRequestURI += ' '; + } + else if(requestLinePtr[p] == '%') + { + // Be tolerant about this... bad things are silently accepted, + // rather than throwing an error. + char code[4] = {0,0,0,0}; + code[0] = requestLinePtr[++p]; + if(code[0] != '\0') + { + code[1] = requestLinePtr[++p]; + } + + // Convert into a char code + long c = ::strtol(code, NULL, 16); + + // Accept it? + if(c > 0 && c <= 255) + { + mRequestURI += (char)c; + } + } + else + { + // Simple copy of character + mRequestURI += requestLinePtr[p]; + } + + ++p; + } + + // End of URL? + if(requestLinePtr[p] == '\0') + { + // Assume HTTP 0.9 + mHTTPVersion = HTTPVersion_0_9; + } + else + { + // Skip any more spaces + while(requestLinePtr[p] != '\0' && requestLinePtr[p] == ' ') + { + ++p; + } + + // Check to see if there's the right string next... + if(::strncmp(requestLinePtr + p, "HTTP/", 5) == 0) + { + // Find the version numbers + int major, minor; + if(::sscanf(requestLinePtr + p + 5, "%d.%d", &major, &minor) != 2) + { + THROW_EXCEPTION(HTTPException, BadRequest) + } + + // Store version + mHTTPVersion = (major * HTTPVersion__MajorMultiplier) + minor; + } + else + { + // Not good -- wrong string found + THROW_EXCEPTION(HTTPException, BadRequest) + } + } + + BOX_TRACE("HTTPRequest: method=" << mMethod << ", uri=" << + mRequestURI << ", version=" << mHTTPVersion); + + // If HTTP 1.1 or greater, assume keep-alive + if(mHTTPVersion >= HTTPVersion_1_1) + { + mClientKeepAliveRequested = true; + } + + // Decode query string? + if((mMethod == Method_GET || mMethod == Method_HEAD) && !mQueryString.empty()) + { + HTTPQueryDecoder decoder(mQuery); + decoder.DecodeChunk(mQueryString.c_str(), mQueryString.size()); + decoder.Finish(); + } + + // Now parse the headers + ParseHeaders(rGetLine, Timeout); + + std::string expected; + if (GetHeader("Expect", &expected)) + { + if (expected == "100-continue") + { + mExpectContinue = true; + } + } + + // Parse form data? + if(mMethod == Method_POST && mContentLength >= 0) + { + // Too long? Don't allow people to be nasty by sending lots of data + if(mContentLength > MAX_CONTENT_SIZE) + { + THROW_EXCEPTION(HTTPException, POSTContentTooLong) + } + + // Some data in the request to follow, parsing it bit by bit + HTTPQueryDecoder decoder(mQuery); + // Don't forget any data left in the GetLine object + int fromBuffer = rGetLine.GetSizeOfBufferedData(); + if(fromBuffer > mContentLength) fromBuffer = mContentLength; + if(fromBuffer > 0) + { + BOX_TRACE("Decoding " << fromBuffer << " bytes of " + "data from getline buffer"); + decoder.DecodeChunk((const char *)rGetLine.GetBufferedData(), fromBuffer); + // And tell the getline object to ignore the data we just used + rGetLine.IgnoreBufferedData(fromBuffer); + } + // Then read any more data, as required + int bytesToGo = mContentLength - fromBuffer; + while(bytesToGo > 0) + { + char buf[4096]; + int toRead = sizeof(buf); + if(toRead > bytesToGo) toRead = bytesToGo; + IOStream &rstream(rGetLine.GetUnderlyingStream()); + int r = rstream.Read(buf, toRead, Timeout); + if(r == 0) + { + // Timeout, just error + THROW_EXCEPTION(HTTPException, RequestReadFailed) + } + decoder.DecodeChunk(buf, r); + bytesToGo -= r; + } + // Finish off + decoder.Finish(); + } + else if (mContentLength > 0) + { + IOStream::pos_type bytesToCopy = rGetLine.GetSizeOfBufferedData(); + if (bytesToCopy > mContentLength) + { + bytesToCopy = mContentLength; + } + Write(rGetLine.GetBufferedData(), bytesToCopy); + SetForReading(); + mpStreamToReadFrom = &(rGetLine.GetUnderlyingStream()); + } + + return true; +} + +void HTTPRequest::ReadContent(IOStream& rStreamToWriteTo) +{ + Seek(0, SeekType_Absolute); + + CopyStreamTo(rStreamToWriteTo); + IOStream::pos_type bytesCopied = GetSize(); + + while (bytesCopied < mContentLength) + { + char buffer[1024]; + IOStream::pos_type bytesToCopy = sizeof(buffer); + if (bytesToCopy > mContentLength - bytesCopied) + { + bytesToCopy = mContentLength - bytesCopied; + } + bytesToCopy = mpStreamToReadFrom->Read(buffer, bytesToCopy); + rStreamToWriteTo.Write(buffer, bytesToCopy); + bytesCopied += bytesToCopy; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::Send(IOStream &, int) +// Purpose: Write the request to an IOStream using HTTP. +// Created: 03/01/09 +// +// -------------------------------------------------------------------------- +bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) +{ + switch (mMethod) + { + case Method_UNINITIALISED: + THROW_EXCEPTION(HTTPException, RequestNotInitialised); break; + case Method_UNKNOWN: + THROW_EXCEPTION(HTTPException, BadRequest); break; + case Method_GET: + rStream.Write("GET"); break; + case Method_HEAD: + rStream.Write("HEAD"); break; + case Method_POST: + rStream.Write("POST"); break; + case Method_PUT: + rStream.Write("PUT"); break; + } + + rStream.Write(" "); + rStream.Write(mRequestURI.c_str()); + rStream.Write(" "); + + switch (mHTTPVersion) + { + case HTTPVersion_0_9: rStream.Write("HTTP/0.9"); break; + case HTTPVersion_1_0: rStream.Write("HTTP/1.0"); break; + case HTTPVersion_1_1: rStream.Write("HTTP/1.1"); break; + default: + THROW_EXCEPTION(HTTPException, NotImplemented); + } + + rStream.Write("\n"); + std::ostringstream oss; + + if (mContentLength != -1) + { + oss << "Content-Length: " << mContentLength << "\n"; + } + + if (mContentType != "") + { + oss << "Content-Type: " << mContentType << "\n"; + } + + if (mHostName != "") + { + if (mHostPort != 80) + { + oss << "Host: " << mHostName << ":" << mHostPort << + "\n"; + } + else + { + oss << "Host: " << mHostName << "\n"; + } + } + + if (mpCookies) + { + THROW_EXCEPTION(HTTPException, NotImplemented); + } + + if (mClientKeepAliveRequested) + { + oss << "Connection: keep-alive\n"; + } + else + { + oss << "Connection: close\n"; + } + + for (std::vector<Header>::iterator i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + oss << i->first << ": " << i->second << "\n"; + } + + if (ExpectContinue) + { + oss << "Expect: 100-continue\n"; + } + + rStream.Write(oss.str().c_str()); + rStream.Write("\n"); + + return true; +} + +void HTTPRequest::SendWithStream(IOStream &rStreamToSendTo, int Timeout, + IOStream* pStreamToSend, HTTPResponse& rResponse) +{ + IOStream::pos_type size = pStreamToSend->BytesLeftToRead(); + + if (size != IOStream::SizeOfStreamUnknown) + { + mContentLength = size; + } + + Send(rStreamToSendTo, Timeout, true); + + rResponse.Receive(rStreamToSendTo, Timeout); + if (rResponse.GetResponseCode() != 100) + { + // bad response, abort now + return; + } + + pStreamToSend->CopyStreamTo(rStreamToSendTo, Timeout); + + // receive the final response + rResponse.Receive(rStreamToSendTo, Timeout); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::ParseHeaders(IOStreamGetLine &, int) +// Purpose: Private. Parse the headers of the request +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) +{ + std::string header; + bool haveHeader = false; + while(true) + { + if(rGetLine.IsEOF()) + { + // Header terminates unexpectedly + THROW_EXCEPTION(HTTPException, BadRequest) + } + + std::string currentLine; + if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) + { + // Timeout + THROW_EXCEPTION(HTTPException, RequestReadFailed) + } + + // Is this a continuation of the previous line? + bool processHeader = haveHeader; + if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) + { + // A continuation, don't process anything yet + processHeader = false; + } + //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); + + // Parse the header -- this will actually process the header + // from the previous run around the loop. + if(processHeader) + { + // Find where the : is in the line + const char *h = header.c_str(); + int p = 0; + while(h[p] != '\0' && h[p] != ':') + { + ++p; + } + // Skip white space + int dataStart = p + 1; + while(h[dataStart] == ' ' || h[dataStart] == '\t') + { + ++dataStart; + } + + std::string header_name(ToLowerCase(std::string(h, + p))); + + if (header_name == "content-length") + { + // Decode number + long len = ::strtol(h + dataStart, NULL, 10); // returns zero in error case, this is OK + if(len < 0) len = 0; + // Store + mContentLength = len; + } + else if (header_name == "content-type") + { + // Store rest of string as content type + mContentType = h + dataStart; + } + else if (header_name == "host") + { + // Store host header + mHostName = h + dataStart; + + // Is there a port number to split off? + std::string::size_type colon = mHostName.find_first_of(':'); + if(colon != std::string::npos) + { + // There's a port in the string... attempt to turn it into an int + mHostPort = ::strtol(mHostName.c_str() + colon + 1, 0, 10); + + // Truncate the string to just the hostname + mHostName = mHostName.substr(0, colon); + + BOX_TRACE("Host: header, hostname = " << + "'" << mHostName << "', host " + "port = " << mHostPort); + } + } + else if (header_name == "cookie") + { + // Parse cookies + ParseCookies(header, dataStart); + } + else if (header_name == "connection") + { + // Connection header, what is required? + const char *v = h + dataStart; + if(::strcasecmp(v, "close") == 0) + { + mClientKeepAliveRequested = false; + } + else if(::strcasecmp(v, "keep-alive") == 0) + { + mClientKeepAliveRequested = true; + } + // else don't understand, just assume default for protocol version + } + else + { + mExtraHeaders.push_back(Header(header_name, + h + dataStart)); + } + + // Unset have header flag, as it's now been processed + haveHeader = false; + } + + // Store the chunk of header the for next time round + if(haveHeader) + { + header += currentLine; + } + else + { + header = currentLine; + haveHeader = true; + } + + // End of headers? + if(currentLine.empty()) + { + // All done! + break; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::ParseCookies(const std::string &, int) +// Purpose: Parse the cookie header +// Created: 20/8/04 +// +// -------------------------------------------------------------------------- +void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) +{ + const char *data = rHeader.c_str() + DataStarts; + const char *pos = data; + const char *itemStart = pos; + std::string name; + + enum + { + s_NAME, s_VALUE, s_VALUE_QUOTED, s_FIND_NEXT_NAME + } state = s_NAME; + + do + { + switch(state) + { + case s_NAME: + { + if(*pos == '=') + { + // Found the name. Store + name.assign(itemStart, pos - itemStart); + // Looking at values now + state = s_VALUE; + if((*(pos + 1)) == '"') + { + // Actually it's a quoted value, skip over that + ++pos; + state = s_VALUE_QUOTED; + } + // Record starting point for this item + itemStart = pos + 1; + } + } + break; + + case s_VALUE: + { + if(*pos == ';' || *pos == ',' || *pos == '\0') + { + // Name ends + ENSURE_COOKIE_JAR_ALLOCATED + std::string value(itemStart, pos - itemStart); + (*mpCookies)[name] = value; + // And move to the waiting stage + state = s_FIND_NEXT_NAME; + } + } + break; + + case s_VALUE_QUOTED: + { + if(*pos == '"') + { + // That'll do nicely, save it + ENSURE_COOKIE_JAR_ALLOCATED + std::string value(itemStart, pos - itemStart); + (*mpCookies)[name] = value; + // And move to the waiting stage + state = s_FIND_NEXT_NAME; + } + } + break; + + case s_FIND_NEXT_NAME: + { + // Skip over terminators and white space to get to the next name + if(*pos != ';' && *pos != ',' && *pos != ' ' && *pos != '\t') + { + // Name starts here + itemStart = pos; + state = s_NAME; + } + } + break; + + default: + // Ooops + THROW_EXCEPTION(HTTPException, Internal) + break; + } + } + while(*(pos++) != 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::GetCookie(const char *, std::string &) const +// Purpose: Fetch a cookie's value. If cookie not present, returns false +// and string is unaltered. +// Created: 20/8/04 +// +// -------------------------------------------------------------------------- +bool HTTPRequest::GetCookie(const char *CookieName, std::string &rValueOut) const +{ + // Got any cookies? + if(mpCookies == 0) + { + return false; + } + + // See if it's there + CookieJar_t::const_iterator v(mpCookies->find(std::string(CookieName))); + if(v != mpCookies->end()) + { + // Return the value + rValueOut = v->second; + return true; + } + + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::GetCookie(const char *) +// Purpose: Return a string for the given cookie, or the null string if the +// cookie has not been recieved. +// Created: 22/8/04 +// +// -------------------------------------------------------------------------- +const std::string &HTTPRequest::GetCookie(const char *CookieName) const +{ + static const std::string noCookie; + + // Got any cookies? + if(mpCookies == 0) + { + return noCookie; + } + + // See if it's there + CookieJar_t::const_iterator v(mpCookies->find(std::string(CookieName))); + if(v != mpCookies->end()) + { + // Return the value + return v->second; + } + + return noCookie; +} + + + diff --git a/lib/httpserver/HTTPRequest.h b/lib/httpserver/HTTPRequest.h new file mode 100644 index 00000000..25effb70 --- /dev/null +++ b/lib/httpserver/HTTPRequest.h @@ -0,0 +1,189 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPRequest.h +// Purpose: Request object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPREQUEST__H +#define HTTPREQUEST__H + +#include <string> +#include <map> + +#include "CollectInBufferStream.h" + +class HTTPResponse; +class IOStream; +class IOStreamGetLine; + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPRequest +// Purpose: Request object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPRequest : public CollectInBufferStream +{ +public: + enum Method + { + Method_UNINITIALISED = -1, + Method_UNKNOWN = 0, + Method_GET = 1, + Method_HEAD = 2, + Method_POST = 3, + Method_PUT = 4 + }; + + HTTPRequest(); + HTTPRequest(enum Method method, const std::string& rURI); + ~HTTPRequest(); +private: + // no copying + HTTPRequest(const HTTPRequest &); + HTTPRequest &operator=(const HTTPRequest &); +public: + typedef std::multimap<std::string, std::string> Query_t; + typedef Query_t::value_type QueryEn_t; + typedef std::pair<std::string, std::string> Header; + + enum + { + HTTPVersion__MajorMultiplier = 1000, + HTTPVersion_0_9 = 9, + HTTPVersion_1_0 = 1000, + HTTPVersion_1_1 = 1001 + }; + + bool Receive(IOStreamGetLine &rGetLine, int Timeout); + bool Send(IOStream &rStream, int Timeout, bool ExpectContinue = false); + void SendWithStream(IOStream &rStreamToSendTo, int Timeout, + IOStream* pStreamToSend, HTTPResponse& rResponse); + void ReadContent(IOStream& rStreamToWriteTo); + + typedef std::map<std::string, std::string> CookieJar_t; + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPResponse::Get*() + // Purpose: Various Get accessors + // Created: 26/3/04 + // + // -------------------------------------------------------------------------- + enum Method GetMethod() const {return mMethod;} + const std::string &GetRequestURI() const {return mRequestURI;} + + // Note: the HTTPRequest generates and parses the Host: header + // Do not attempt to set one yourself with AddHeader(). + const std::string &GetHostName() const {return mHostName;} + void SetHostName(const std::string& rHostName) + { + mHostName = rHostName; + } + + const int GetHostPort() const {return mHostPort;} + const std::string &GetQueryString() const {return mQueryString;} + int GetHTTPVersion() const {return mHTTPVersion;} + const Query_t &GetQuery() const {return mQuery;} + int GetContentLength() const {return mContentLength;} + const std::string &GetContentType() const {return mContentType;} + const CookieJar_t *GetCookies() const {return mpCookies;} // WARNING: May return NULL + bool GetCookie(const char *CookieName, std::string &rValueOut) const; + const std::string &GetCookie(const char *CookieName) const; + bool GetHeader(const std::string& rName, std::string* pValueOut) const + { + std::string header = ToLowerCase(rName); + + for (std::vector<Header>::const_iterator + i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + if (i->first == header) + { + *pValueOut = i->second; + return true; + } + } + + return false; + } + std::vector<Header> GetHeaders() { return mExtraHeaders; } + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPRequest::GetClientKeepAliveRequested() + // Purpose: Returns true if the client requested that the connection + // should be kept open for further requests. + // Created: 22/12/04 + // + // -------------------------------------------------------------------------- + bool GetClientKeepAliveRequested() const {return mClientKeepAliveRequested;} + void SetClientKeepAliveRequested(bool keepAlive) + { + mClientKeepAliveRequested = keepAlive; + } + + void AddHeader(const std::string& rName, const std::string& rValue) + { + mExtraHeaders.push_back(Header(ToLowerCase(rName), rValue)); + } + bool IsExpectingContinue() const { return mExpectContinue; } + const char* GetVerb() const + { + if (!mHttpVerb.empty()) + { + return mHttpVerb.c_str(); + } + switch (mMethod) + { + case Method_UNINITIALISED: return "Uninitialized"; + case Method_UNKNOWN: return "Unknown"; + case Method_GET: return "GET"; + case Method_HEAD: return "HEAD"; + case Method_POST: return "POST"; + case Method_PUT: return "PUT"; + } + return "Bad"; + } + +private: + void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout); + void ParseCookies(const std::string &rHeader, int DataStarts); + + enum Method mMethod; + std::string mRequestURI; + std::string mHostName; + int mHostPort; + std::string mQueryString; + int mHTTPVersion; + Query_t mQuery; + int mContentLength; + std::string mContentType; + CookieJar_t *mpCookies; + bool mClientKeepAliveRequested; + std::vector<Header> mExtraHeaders; + bool mExpectContinue; + IOStream* mpStreamToReadFrom; + std::string mHttpVerb; + + std::string ToLowerCase(const std::string& rInput) const + { + std::string output = rInput; + for (std::string::iterator c = output.begin(); + c != output.end(); c++) + { + *c = tolower(*c); + } + return output; + } +}; + +#endif // HTTPREQUEST__H + diff --git a/lib/httpserver/HTTPResponse.cpp b/lib/httpserver/HTTPResponse.cpp new file mode 100644 index 00000000..1a8c8447 --- /dev/null +++ b/lib/httpserver/HTTPResponse.cpp @@ -0,0 +1,648 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPResponse.cpp +// Purpose: Response object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <string.h> + +#include "HTTPResponse.h" +#include "IOStreamGetLine.h" +#include "autogen_HTTPException.h" + +#include "MemLeakFindOn.h" + +// Static variables +std::string HTTPResponse::msDefaultURIPrefix; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::HTTPResponse(IOStream*) +// Purpose: Constructor for response to be sent to a stream +// Created: 04/01/09 +// +// -------------------------------------------------------------------------- +HTTPResponse::HTTPResponse(IOStream* pStreamToSendTo) + : mResponseCode(HTTPResponse::Code_NoContent), + mResponseIsDynamicContent(true), + mKeepAlive(false), + mContentLength(-1), + mpStreamToSendTo(pStreamToSendTo) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::HTTPResponse() +// Purpose: Constructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPResponse::HTTPResponse() + : mResponseCode(HTTPResponse::Code_NoContent), + mResponseIsDynamicContent(true), + mKeepAlive(false), + mContentLength(-1), + mpStreamToSendTo(NULL) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::~HTTPResponse() +// Purpose: Destructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPResponse::~HTTPResponse() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::ResponseCodeToString(int) +// Purpose: Return string equivalent of the response code, +// suitable for Status: headers +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +const char *HTTPResponse::ResponseCodeToString(int ResponseCode) +{ + switch(ResponseCode) + { + case Code_OK: return "200 OK"; break; + case Code_NoContent: return "204 No Content"; break; + case Code_MovedPermanently: return "301 Moved Permanently"; break; + case Code_Found: return "302 Found"; break; + case Code_NotModified: return "304 Not Modified"; break; + case Code_TemporaryRedirect: return "307 Temporary Redirect"; break; + case Code_MethodNotAllowed: return "400 Method Not Allowed"; break; + case Code_Unauthorized: return "401 Unauthorized"; break; + case Code_Forbidden: return "403 Forbidden"; break; + case Code_NotFound: return "404 Not Found"; break; + case Code_InternalServerError: return "500 Internal Server Error"; break; + case Code_NotImplemented: return "501 Not Implemented"; break; + default: + { + THROW_EXCEPTION(HTTPException, UnknownResponseCodeUsed) + } + } + return "500 Internal Server Error"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetResponseCode(int) +// Purpose: Set the response code to be returned +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetResponseCode(int Code) +{ + mResponseCode = Code; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetContentType(const char *) +// Purpose: Set content type +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetContentType(const char *ContentType) +{ + mContentType = ContentType; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::Send(IOStream &, bool) +// Purpose: Build the response, and send via the stream. +// Optionally omitting the content. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::Send(bool OmitContent) +{ + if (!mpStreamToSendTo) + { + THROW_EXCEPTION(HTTPException, NoStreamConfigured); + } + + if (GetSize() != 0 && mContentType.empty()) + { + THROW_EXCEPTION(HTTPException, NoContentTypeSet); + } + + // Build and send header + { + std::string header("HTTP/1.1 "); + header += ResponseCodeToString(mResponseCode); + header += "\r\nContent-Type: "; + header += mContentType; + header += "\r\nContent-Length: "; + { + char len[32]; + ::sprintf(len, "%d", OmitContent?(0):(GetSize())); + header += len; + } + // Extra headers... + for(std::vector<std::pair<std::string, std::string> >::const_iterator i(mExtraHeaders.begin()); i != mExtraHeaders.end(); ++i) + { + header += "\r\n"; + header += i->first + ": " + i->second; + } + // NOTE: a line ending must be included here in all cases + // Control whether the response is cached + if(mResponseIsDynamicContent) + { + // dynamic is private and can't be cached + header += "\r\nCache-Control: no-cache, private"; + } + else + { + // static is allowed to be cached for a day + header += "\r\nCache-Control: max-age=86400"; + } + if(mKeepAlive) + { + header += "\r\nConnection: keep-alive\r\n\r\n"; + } + else + { + header += "\r\nConnection: close\r\n\r\n"; + } + // NOTE: header ends with blank line in all cases + + // Write to stream + mpStreamToSendTo->Write(header.c_str(), header.size()); + } + + // Send content + if(!OmitContent) + { + mpStreamToSendTo->Write(GetBuffer(), GetSize()); + } +} + +void HTTPResponse::SendContinue() +{ + mpStreamToSendTo->Write("HTTP/1.1 100 Continue\r\n"); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::ParseHeaders(IOStreamGetLine &, int) +// Purpose: Private. Parse the headers of the response +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) +{ + std::string header; + bool haveHeader = false; + while(true) + { + if(rGetLine.IsEOF()) + { + // Header terminates unexpectedly + THROW_EXCEPTION(HTTPException, BadRequest) + } + + std::string currentLine; + if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) + { + // Timeout + THROW_EXCEPTION(HTTPException, RequestReadFailed) + } + + // Is this a continuation of the previous line? + bool processHeader = haveHeader; + if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) + { + // A continuation, don't process anything yet + processHeader = false; + } + //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); + + // Parse the header -- this will actually process the header + // from the previous run around the loop. + if(processHeader) + { + // Find where the : is in the line + const char *h = header.c_str(); + int p = 0; + while(h[p] != '\0' && h[p] != ':') + { + ++p; + } + // Skip white space + int dataStart = p + 1; + while(h[dataStart] == ' ' || h[dataStart] == '\t') + { + ++dataStart; + } + + if(p == sizeof("Content-Length")-1 + && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0) + { + // Decode number + long len = ::strtol(h + dataStart, NULL, 10); // returns zero in error case, this is OK + if(len < 0) len = 0; + // Store + mContentLength = len; + } + else if(p == sizeof("Content-Type")-1 + && ::strncasecmp(h, "Content-Type", sizeof("Content-Type")-1) == 0) + { + // Store rest of string as content type + mContentType = h + dataStart; + } + else if(p == sizeof("Cookie")-1 + && ::strncasecmp(h, "Cookie", sizeof("Cookie")-1) == 0) + { + THROW_EXCEPTION(HTTPException, NotImplemented); + /* + // Parse cookies + ParseCookies(header, dataStart); + */ + } + else if(p == sizeof("Connection")-1 + && ::strncasecmp(h, "Connection", sizeof("Connection")-1) == 0) + { + // Connection header, what is required? + const char *v = h + dataStart; + if(::strcasecmp(v, "close") == 0) + { + mKeepAlive = false; + } + else if(::strcasecmp(v, "keep-alive") == 0) + { + mKeepAlive = true; + } + // else don't understand, just assume default for protocol version + } + else + { + std::string headerName = header.substr(0, p); + AddHeader(headerName, h + dataStart); + } + + // Unset have header flag, as it's now been processed + haveHeader = false; + } + + // Store the chunk of header the for next time round + if(haveHeader) + { + header += currentLine; + } + else + { + header = currentLine; + haveHeader = true; + } + + // End of headers? + if(currentLine.empty()) + { + // All done! + break; + } + } +} + +void HTTPResponse::Receive(IOStream& rStream, int Timeout) +{ + IOStreamGetLine rGetLine(rStream); + + if(rGetLine.IsEOF()) + { + // Connection terminated unexpectedly + THROW_EXCEPTION(HTTPException, BadResponse) + } + + std::string statusLine; + if(!rGetLine.GetLine(statusLine, false /* no preprocess */, Timeout)) + { + // Timeout + THROW_EXCEPTION(HTTPException, ResponseReadFailed) + } + + if (statusLine.substr(0, 7) != "HTTP/1." || + statusLine[8] != ' ') + { + // Status line terminated unexpectedly + BOX_ERROR("Bad response status line: " << statusLine); + THROW_EXCEPTION(HTTPException, BadResponse) + } + + if (statusLine[5] == '1' && statusLine[7] == '1') + { + // HTTP/1.1 default is to keep alive + mKeepAlive = true; + } + + // Decode the status code + long status = ::strtol(statusLine.substr(9, 3).c_str(), NULL, 10); + // returns zero in error case, this is OK + if (status < 0) status = 0; + // Store + mResponseCode = status; + + // 100 Continue responses have no headers, terminating newline, or body + if (status == 100) + { + return; + } + + ParseHeaders(rGetLine, Timeout); + + // push back whatever bytes we have left + // rGetLine.DetachFile(); + if (mContentLength > 0) + { + if (mContentLength < rGetLine.GetSizeOfBufferedData()) + { + // very small response, not good! + THROW_EXCEPTION(HTTPException, NotImplemented); + } + + mContentLength -= rGetLine.GetSizeOfBufferedData(); + + Write(rGetLine.GetBufferedData(), + rGetLine.GetSizeOfBufferedData()); + } + + while (mContentLength != 0) // could be -1 as well + { + char buffer[4096]; + int readSize = sizeof(buffer); + if (mContentLength > 0 && mContentLength < readSize) + { + readSize = mContentLength; + } + readSize = rStream.Read(buffer, readSize, Timeout); + if (readSize == 0) + { + break; + } + mContentLength -= readSize; + Write(buffer, readSize); + } + + SetForReading(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const char *) +// Purpose: Add header, given entire line +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +/* +void HTTPResponse::AddHeader(const char *EntireHeaderLine) +{ + mExtraHeaders.push_back(std::string(EntireHeaderLine)); +} +*/ + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const std::string &) +// Purpose: Add header, given entire line +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +/* +void HTTPResponse::AddHeader(const std::string &rEntireHeaderLine) +{ + mExtraHeaders.push_back(rEntireHeaderLine); +} +*/ + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const char *, const char *) +// Purpose: Add header, given header name and it's value +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::AddHeader(const char *pHeader, const char *pValue) +{ + mExtraHeaders.push_back(Header(pHeader, pValue)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const char *, const std::string &) +// Purpose: Add header, given header name and it's value +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::AddHeader(const char *pHeader, const std::string &rValue) +{ + mExtraHeaders.push_back(Header(pHeader, rValue)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const std::string &, const std::string &) +// Purpose: Add header, given header name and it's value +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::AddHeader(const std::string &rHeader, const std::string &rValue) +{ + mExtraHeaders.push_back(Header(rHeader, rValue)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetCookie(const char *, const char *, const char *, int) +// Purpose: Sets a cookie, using name, value, path and expiry time. +// Created: 20/8/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetCookie(const char *Name, const char *Value, const char *Path, int ExpiresAt) +{ + if(ExpiresAt != 0) + { + THROW_EXCEPTION(HTTPException, NotImplemented) + } + + // Appears you shouldn't use quotes when you generate set-cookie headers. + // Oh well. It was fun finding that out. +/* std::string h("Set-Cookie: "); + h += Name; + h += "=\""; + h += Value; + h += "\"; Version=\"1\"; Path=\""; + h += Path; + h += "\""; +*/ + std::string h; + h += Name; + h += "="; + h += Value; + h += "; Version=1; Path="; + h += Path; + + mExtraHeaders.push_back(Header("Set-Cookie", h)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetAsRedirect(const char *, bool) +// Purpose: Sets the response objects to be a redirect to another page. +// If IsLocalURL == true, the default prefix will be added. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetAsRedirect(const char *RedirectTo, bool IsLocalURI) +{ + if(mResponseCode != HTTPResponse::Code_NoContent + || !mContentType.empty() + || GetSize() != 0) + { + THROW_EXCEPTION(HTTPException, CannotSetRedirectIfReponseHasData) + } + + // Set response code + mResponseCode = Code_Found; + + // Set location to redirect to + std::string header; + if(IsLocalURI) header += msDefaultURIPrefix; + header += RedirectTo; + mExtraHeaders.push_back(Header("Location", header)); + + // Set up some default content + mContentType = "text/html"; + #define REDIRECT_HTML_1 "<html><head><title>Redirection</title></head>\n<body><p><a href=\"" + #define REDIRECT_HTML_2 "\">Redirect to content</a></p></body></html>\n" + Write(REDIRECT_HTML_1, sizeof(REDIRECT_HTML_1) - 1); + if(IsLocalURI) Write(msDefaultURIPrefix.c_str(), msDefaultURIPrefix.size()); + Write(RedirectTo, ::strlen(RedirectTo)); + Write(REDIRECT_HTML_2, sizeof(REDIRECT_HTML_2) - 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetAsNotFound(const char *) +// Purpose: Set the response object to be a standard page not found 404 response. +// Created: 7/4/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetAsNotFound(const char *URI) +{ + if(mResponseCode != HTTPResponse::Code_NoContent + || mExtraHeaders.size() != 0 + || !mContentType.empty() + || GetSize() != 0) + { + THROW_EXCEPTION(HTTPException, CannotSetNotFoundIfReponseHasData) + } + + // Set response code + mResponseCode = Code_NotFound; + + // Set data + mContentType = "text/html"; + #define NOT_FOUND_HTML_1 "<html><head><title>404 Not Found</title></head>\n<body><h1>404 Not Found</h1>\n<p>The URI <i>" + #define NOT_FOUND_HTML_2 "</i> was not found on this server.</p></body></html>\n" + Write(NOT_FOUND_HTML_1, sizeof(NOT_FOUND_HTML_1) - 1); + WriteStringDefang(std::string(URI)); + Write(NOT_FOUND_HTML_2, sizeof(NOT_FOUND_HTML_2) - 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::WriteStringDefang(const char *, unsigned int) +// Purpose: Writes a string 'defanged', ie has HTML special characters escaped +// so that people can't output arbitary HTML by playing with +// URLs and form parameters, and it's safe to write strings into +// HTML element attribute values. +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::WriteStringDefang(const char *String, unsigned int StringLen) +{ + while(StringLen > 0) + { + unsigned int toWrite = 0; + while(toWrite < StringLen + && String[toWrite] != '<' + && String[toWrite] != '>' + && String[toWrite] != '&' + && String[toWrite] != '"') + { + ++toWrite; + } + if(toWrite > 0) + { + Write(String, toWrite); + StringLen -= toWrite; + String += toWrite; + } + + // Is it a bad character next? + while(StringLen > 0) + { + bool notSpecial = false; + switch(*String) + { + case '<': Write("<", 4); break; + case '>': Write(">", 4); break; + case '&': Write("&", 5); break; + case '"': Write(""", 6); break; + default: + // Stop this loop + notSpecial = true; + break; + } + if(notSpecial) break; + ++String; + --StringLen; + } + } +} + + diff --git a/lib/httpserver/HTTPResponse.h b/lib/httpserver/HTTPResponse.h new file mode 100644 index 00000000..04051958 --- /dev/null +++ b/lib/httpserver/HTTPResponse.h @@ -0,0 +1,175 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPResponse.h +// Purpose: Response object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPRESPONSE__H +#define HTTPRESPONSE__H + +#include <string> +#include <vector> + +#include "CollectInBufferStream.h" + +class IOStreamGetLine; + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPResponse +// Purpose: Response object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPResponse : public CollectInBufferStream +{ +public: + HTTPResponse(IOStream* pStreamToSendTo); + HTTPResponse(); + ~HTTPResponse(); + + // allow copying, but be very careful with the response stream, + // you can only read it once! (this class doesn't police it). + HTTPResponse(const HTTPResponse& rOther) + : mResponseCode(rOther.mResponseCode), + mResponseIsDynamicContent(rOther.mResponseIsDynamicContent), + mKeepAlive(rOther.mKeepAlive), + mContentType(rOther.mContentType), + mExtraHeaders(rOther.mExtraHeaders), + mContentLength(rOther.mContentLength), + mpStreamToSendTo(rOther.mpStreamToSendTo) + { + Write(rOther.GetBuffer(), rOther.GetSize()); + } + + HTTPResponse &operator=(const HTTPResponse &rOther) + { + Reset(); + Write(rOther.GetBuffer(), rOther.GetSize()); + mResponseCode = rOther.mResponseCode; + mResponseIsDynamicContent = rOther.mResponseIsDynamicContent; + mKeepAlive = rOther.mKeepAlive; + mContentType = rOther.mContentType; + mExtraHeaders = rOther.mExtraHeaders; + mContentLength = rOther.mContentLength; + mpStreamToSendTo = rOther.mpStreamToSendTo; + return *this; + } + + typedef std::pair<std::string, std::string> Header; + + void SetResponseCode(int Code); + int GetResponseCode() { return mResponseCode; } + void SetContentType(const char *ContentType); + const std::string& GetContentType() { return mContentType; } + + void SetAsRedirect(const char *RedirectTo, bool IsLocalURI = true); + void SetAsNotFound(const char *URI); + + void Send(bool OmitContent = false); + void SendContinue(); + void Receive(IOStream& rStream, int Timeout = IOStream::TimeOutInfinite); + + // void AddHeader(const char *EntireHeaderLine); + // void AddHeader(const std::string &rEntireHeaderLine); + void AddHeader(const char *Header, const char *Value); + void AddHeader(const char *Header, const std::string &rValue); + void AddHeader(const std::string &rHeader, const std::string &rValue); + bool GetHeader(const std::string& rName, std::string* pValueOut) const + { + for (std::vector<Header>::const_iterator + i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + if (i->first == rName) + { + *pValueOut = i->second; + return true; + } + } + return false; + } + std::string GetHeaderValue(const std::string& rName) + { + std::string value; + if (!GetHeader(rName, &value)) + { + THROW_EXCEPTION(CommonException, ConfigNoKey); + } + return value; + } + + // Set dynamic content flag, default is content is dynamic + void SetResponseIsDynamicContent(bool IsDynamic) {mResponseIsDynamicContent = IsDynamic;} + // Set keep alive control, default is to mark as to be closed + void SetKeepAlive(bool KeepAlive) {mKeepAlive = KeepAlive;} + + void SetCookie(const char *Name, const char *Value, const char *Path = "/", int ExpiresAt = 0); + + enum + { + Code_OK = 200, + Code_NoContent = 204, + Code_MovedPermanently = 301, + Code_Found = 302, // redirection + Code_NotModified = 304, + Code_TemporaryRedirect = 307, + Code_MethodNotAllowed = 400, + Code_Unauthorized = 401, + Code_Forbidden = 403, + Code_NotFound = 404, + Code_InternalServerError = 500, + Code_NotImplemented = 501 + }; + + static const char *ResponseCodeToString(int ResponseCode); + + void WriteStringDefang(const char *String, unsigned int StringLen); + void WriteStringDefang(const std::string &rString) {WriteStringDefang(rString.c_str(), rString.size());} + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPResponse::WriteString(const std::string &) + // Purpose: Write a string to the response (simple sugar function) + // Created: 9/4/04 + // + // -------------------------------------------------------------------------- + void WriteString(const std::string &rString) + { + Write(rString.c_str(), rString.size()); + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPResponse::SetDefaultURIPrefix(const std::string &) + // Purpose: Set default prefix used to local redirections + // Created: 26/3/04 + // + // -------------------------------------------------------------------------- + static void SetDefaultURIPrefix(const std::string &rPrefix) + { + msDefaultURIPrefix = rPrefix; + } + +private: + int mResponseCode; + bool mResponseIsDynamicContent; + bool mKeepAlive; + std::string mContentType; + std::vector<Header> mExtraHeaders; + int mContentLength; // only used when reading response from stream + IOStream* mpStreamToSendTo; // nonzero only when constructed with a stream + + static std::string msDefaultURIPrefix; + + void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout); +}; + +#endif // HTTPRESPONSE__H + diff --git a/lib/httpserver/HTTPServer.cpp b/lib/httpserver/HTTPServer.cpp new file mode 100644 index 00000000..be1db687 --- /dev/null +++ b/lib/httpserver/HTTPServer.cpp @@ -0,0 +1,247 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPServer.cpp +// Purpose: HTTP server class +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> + +#include "HTTPServer.h" +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "IOStreamGetLine.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::HTTPServer() +// Purpose: Constructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPServer::HTTPServer() + : mTimeout(20000) // default timeout leaves a little while for clients to get the second request in. +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::~HTTPServer() +// Purpose: Destructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPServer::~HTTPServer() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::DaemonName() +// Purpose: As interface, generic name for daemon +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +const char *HTTPServer::DaemonName() const +{ + return "generic-httpserver"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::GetConfigVerify() +// Purpose: As interface -- return most basic config so it's only necessary to +// provide this if you want to add extra directives. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify *HTTPServer::GetConfigVerify() const +{ + static ConfigurationVerifyKey verifyserverkeys[] = + { + HTTPSERVER_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) // no default addresses + }; + + static ConfigurationVerify verifyserver[] = + { + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } + }; + + static ConfigurationVerifyKey verifyrootkeys[] = + { + HTTPSERVER_VERIFY_ROOT_KEYS + }; + + static ConfigurationVerify verify = + { + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + }; + + return &verify; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::Run() +// Purpose: As interface. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::Run() +{ + // Do some configuration stuff + const Configuration &conf(GetConfiguration()); + HTTPResponse::SetDefaultURIPrefix(conf.GetKeyValue("AddressPrefix")); + + // Let the base class do the work + ServerStream<SocketStream, 80>::Run(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::Connection(SocketStream &) +// Purpose: As interface, handle connection +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::Connection(SocketStream &rStream) +{ + // Create a get line object to use + IOStreamGetLine getLine(rStream); + + // Notify dervived claases + HTTPConnectionOpening(); + + bool handleRequests = true; + while(handleRequests) + { + // Parse the request + HTTPRequest request; + if(!request.Receive(getLine, mTimeout)) + { + // Didn't get request, connection probably closed. + break; + } + + // Generate a response + HTTPResponse response(&rStream); + + try + { + Handle(request, response); + } + catch(BoxException &e) + { + char exceptionCode[256]; + ::sprintf(exceptionCode, "%s (%d/%d)", e.what(), + e.GetType(), e.GetSubType()); + SendInternalErrorResponse(exceptionCode, response); + } + catch(...) + { + SendInternalErrorResponse("unknown", response); + } + + // Keep alive? + if(request.GetClientKeepAliveRequested()) + { + // Mark the response to the client as supporting keepalive + response.SetKeepAlive(true); + } + else + { + // Stop now + handleRequests = false; + } + + // Send the response (omit any content if this is a HEAD method request) + response.Send(request.GetMethod() == HTTPRequest::Method_HEAD); + } + + // Notify derived classes + HTTPConnectionClosing(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::SendInternalErrorResponse(const char*, +// HTTPResponse&) +// Purpose: Generates an error message in the provided response +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::SendInternalErrorResponse(const std::string& rErrorMsg, + HTTPResponse& rResponse) +{ + #define ERROR_HTML_1 "<html><head><title>Internal Server Error</title></head>\n" \ + "<h1>Internal Server Error</h1>\n" \ + "<p>An error, type " + #define ERROR_HTML_2 " occured when processing the request.</p>" \ + "<p>Please try again later.</p>" \ + "</body>\n</html>\n" + + // Generate the error page + // rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError); + rResponse.SetContentType("text/html"); + rResponse.Write(ERROR_HTML_1, sizeof(ERROR_HTML_1) - 1); + rResponse.IOStream::Write(rErrorMsg.c_str()); + rResponse.Write(ERROR_HTML_2, sizeof(ERROR_HTML_2) - 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::HTTPConnectionOpening() +// Purpose: Override to get notifications of connections opening +// Created: 22/12/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::HTTPConnectionOpening() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::HTTPConnectionClosing() +// Purpose: Override to get notifications of connections closing +// Created: 22/12/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::HTTPConnectionClosing() +{ +} + + diff --git a/lib/httpserver/HTTPServer.h b/lib/httpserver/HTTPServer.h new file mode 100644 index 00000000..d9f74949 --- /dev/null +++ b/lib/httpserver/HTTPServer.h @@ -0,0 +1,81 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPServer.h +// Purpose: HTTP server class +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPSERVER__H +#define HTTPSERVER__H + +#include "ServerStream.h" +#include "SocketStream.h" + +class HTTPRequest; +class HTTPResponse; + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPServer +// Purpose: HTTP server +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPServer : public ServerStream<SocketStream, 80> +{ +public: + HTTPServer(); + ~HTTPServer(); +private: + // no copying + HTTPServer(const HTTPServer &); + HTTPServer &operator=(const HTTPServer &); +public: + + int GetTimeout() const {return mTimeout;} + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPServer::Handle(const HTTPRequest &, HTTPResponse &) + // Purpose: Response to a request, filling in the response object for sending + // at some point in the future. + // Created: 26/3/04 + // + // -------------------------------------------------------------------------- + virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) = 0; + + // For notifications to derived classes + virtual void HTTPConnectionOpening(); + virtual void HTTPConnectionClosing(); + +protected: + void SendInternalErrorResponse(const std::string& rErrorMsg, + HTTPResponse& rResponse); + int GetTimeout() { return mTimeout; } + +private: + int mTimeout; // Timeout for read operations + const char *DaemonName() const; + const ConfigurationVerify *GetConfigVerify() const; + void Run(); + void Connection(SocketStream &rStream); +}; + +// Root level +#define HTTPSERVER_VERIFY_ROOT_KEYS \ + ConfigurationVerifyKey("AddressPrefix", \ + ConfigTest_Exists | ConfigTest_LastEntry) + +// AddressPrefix is, for example, http://localhost:1080 -- ie the beginning of the URI +// This is used for handling redirections. + +// Server level +#define HTTPSERVER_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ + SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) + +#endif // HTTPSERVER__H + diff --git a/lib/httpserver/Makefile.extra b/lib/httpserver/Makefile.extra new file mode 100644 index 00000000..ef47f398 --- /dev/null +++ b/lib/httpserver/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_HTTPException.h autogen_HTTPException.cpp: $(MAKEEXCEPTION) HTTPException.txt + $(_PERL) $(MAKEEXCEPTION) HTTPException.txt + diff --git a/lib/httpserver/S3Client.cpp b/lib/httpserver/S3Client.cpp new file mode 100644 index 00000000..cd5988d5 --- /dev/null +++ b/lib/httpserver/S3Client.cpp @@ -0,0 +1,243 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: S3Client.cpp +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <cstring> + +// #include <cstdio> +// #include <ctime> + +#include <openssl/hmac.h> + +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "HTTPServer.h" +#include "autogen_HTTPException.h" +#include "IOStream.h" +#include "Logging.h" +#include "S3Client.h" +#include "decode.h" +#include "encode.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::GetObject(const std::string& rObjectURI) +// Purpose: Retrieve the object with the specified URI (key) +// from your S3 bucket. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::GetObject(const std::string& rObjectURI) +{ + return FinishAndSendRequest(HTTPRequest::Method_GET, rObjectURI); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::PutObject(const std::string& rObjectURI, +// IOStream& rStreamToSend, const char* pContentType) +// Purpose: Upload the stream to S3, creating or overwriting the +// object with the specified URI (key) in your S3 +// bucket. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::PutObject(const std::string& rObjectURI, + IOStream& rStreamToSend, const char* pContentType) +{ + return FinishAndSendRequest(HTTPRequest::Method_PUT, rObjectURI, + &rStreamToSend, pContentType); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::FinishAndSendRequest( +// HTTPRequest::Method Method, +// const std::string& rRequestURI, +// IOStream* pStreamToSend, +// const char* pStreamContentType) +// Purpose: Internal method which creates an HTTP request to S3, +// populates the date and authorization header fields, +// and sends it to S3 (or the simulator), attaching +// the specified stream if any to the request. Opens a +// connection to the server if necessary, which may +// throw a ConnectionException. Returns the HTTP +// response returned by S3, which may be a 500 error. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, + const std::string& rRequestURI, IOStream* pStreamToSend, + const char* pStreamContentType) +{ + HTTPRequest request(Method, rRequestURI); + request.SetHostName(mHostName); + + std::ostringstream date; + time_t tt = time(NULL); + struct tm *tp = gmtime(&tt); + if (!tp) + { + BOX_ERROR("Failed to get current time"); + THROW_EXCEPTION(HTTPException, Internal); + } + const char *dow[] = {"Sun","Mon","Tue","Wed","Thu","Fri","Sat"}; + date << dow[tp->tm_wday] << ", "; + const char *month[] = {"Jan","Feb","Mar","Apr","May","Jun", + "Jul","Aug","Sep","Oct","Nov","Dec"}; + date << std::internal << std::setfill('0') << + std::setw(2) << tp->tm_mday << " " << + month[tp->tm_mon] << " " << + (tp->tm_year + 1900) << " "; + date << std::setw(2) << tp->tm_hour << ":" << + std::setw(2) << tp->tm_min << ":" << + std::setw(2) << tp->tm_sec << " GMT"; + request.AddHeader("Date", date.str()); + + if (pStreamContentType) + { + request.AddHeader("Content-Type", pStreamContentType); + } + + std::string s3suffix = ".s3.amazonaws.com"; + std::string bucket; + if (mHostName.size() > s3suffix.size()) + { + std::string suffix = mHostName.substr(mHostName.size() - + s3suffix.size(), s3suffix.size()); + if (suffix == s3suffix) + { + bucket = mHostName.substr(0, mHostName.size() - + s3suffix.size()); + } + } + + std::ostringstream data; + data << request.GetVerb() << "\n"; + data << "\n"; /* Content-MD5 */ + data << request.GetContentType() << "\n"; + data << date.str() << "\n"; + + if (! bucket.empty()) + { + data << "/" << bucket; + } + + data << request.GetRequestURI(); + std::string data_string = data.str(); + + unsigned char digest_buffer[EVP_MAX_MD_SIZE]; + unsigned int digest_size = sizeof(digest_buffer); + /* unsigned char* mac = */ HMAC(EVP_sha1(), + mSecretKey.c_str(), mSecretKey.size(), + (const unsigned char*)data_string.c_str(), + data_string.size(), digest_buffer, &digest_size); + std::string digest((const char *)digest_buffer, digest_size); + + base64::encoder encoder; + std::string auth_code = "AWS " + mAccessKey + ":" + + encoder.encode(digest); + + if (auth_code[auth_code.size() - 1] == '\n') + { + auth_code = auth_code.substr(0, auth_code.size() - 1); + } + + request.AddHeader("Authorization", auth_code); + + if (mpSimulator) + { + if (pStreamToSend) + { + pStreamToSend->CopyStreamTo(request); + } + + request.SetForReading(); + CollectInBufferStream response_buffer; + HTTPResponse response(&response_buffer); + + mpSimulator->Handle(request, response); + return response; + } + else + { + try + { + if (!mapClientSocket.get()) + { + mapClientSocket.reset(new SocketStream()); + mapClientSocket->Open(Socket::TypeINET, + mHostName, mPort); + } + return SendRequest(request, pStreamToSend, + pStreamContentType); + } + catch (ConnectionException &ce) + { + if (ce.GetType() == ConnectionException::SocketWriteError) + { + // server may have disconnected us, + // try to reconnect, just once + mapClientSocket->Open(Socket::TypeINET, + mHostName, mPort); + return SendRequest(request, pStreamToSend, + pStreamContentType); + } + else + { + throw; + } + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::SendRequest(HTTPRequest& rRequest, +// IOStream* pStreamToSend, +// const char* pStreamContentType) +// Purpose: Internal method which sends a pre-existing HTTP +// request to S3. Attaches the specified stream if any +// to the request. Opens a connection to the server if +// necessary, which may throw a ConnectionException. +// Returns the HTTP response returned by S3, which may +// be a 500 error. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::SendRequest(HTTPRequest& rRequest, + IOStream* pStreamToSend, const char* pStreamContentType) +{ + HTTPResponse response; + + if (pStreamToSend) + { + rRequest.SendWithStream(*mapClientSocket, + 30000 /* milliseconds */, + pStreamToSend, response); + } + else + { + rRequest.Send(*mapClientSocket, 30000 /* milliseconds */); + response.Receive(*mapClientSocket, 30000 /* milliseconds */); + } + + return response; +} diff --git a/lib/httpserver/S3Client.h b/lib/httpserver/S3Client.h new file mode 100644 index 00000000..3c4126ac --- /dev/null +++ b/lib/httpserver/S3Client.h @@ -0,0 +1,72 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: S3Client.h +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- + +#ifndef S3CLIENT__H +#define S3CLIENT__H + +#include <string> +#include <map> + +#include "HTTPRequest.h" +#include "SocketStream.h" + +class HTTPResponse; +class HTTPServer; +class IOStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: S3Client +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- +class S3Client +{ + public: + S3Client(HTTPServer* pSimulator, const std::string& rHostName, + const std::string& rAccessKey, const std::string& rSecretKey) + : mpSimulator(pSimulator), + mHostName(rHostName), + mAccessKey(rAccessKey), + mSecretKey(rSecretKey) + { } + + S3Client(std::string HostName, int Port, const std::string& rAccessKey, + const std::string& rSecretKey) + : mpSimulator(NULL), + mHostName(HostName), + mPort(Port), + mAccessKey(rAccessKey), + mSecretKey(rSecretKey) + { } + + HTTPResponse GetObject(const std::string& rObjectURI); + HTTPResponse PutObject(const std::string& rObjectURI, + IOStream& rStreamToSend, const char* pContentType = NULL); + + private: + HTTPServer* mpSimulator; + std::string mHostName; + int mPort; + std::auto_ptr<SocketStream> mapClientSocket; + std::string mAccessKey, mSecretKey; + + HTTPResponse FinishAndSendRequest(HTTPRequest::Method Method, + const std::string& rRequestURI, + IOStream* pStreamToSend = NULL, + const char* pStreamContentType = NULL); + HTTPResponse SendRequest(HTTPRequest& rRequest, + IOStream* pStreamToSend = NULL, + const char* pStreamContentType = NULL); +}; + +#endif // S3CLIENT__H + diff --git a/lib/httpserver/S3Simulator.cpp b/lib/httpserver/S3Simulator.cpp new file mode 100644 index 00000000..4f6bb3e6 --- /dev/null +++ b/lib/httpserver/S3Simulator.cpp @@ -0,0 +1,309 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: S3Client.cpp +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <algorithm> +#include <cstring> + +// #include <cstdio> +// #include <ctime> + +#include <openssl/hmac.h> + +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "autogen_HTTPException.h" +#include "IOStream.h" +#include "Logging.h" +#include "S3Simulator.h" +#include "decode.h" +#include "encode.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::GetConfigVerify() +// Purpose: Returns additional configuration options for the +// S3 simulator. Currently the access key, secret key +// and store directory can be configured. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify* S3Simulator::GetConfigVerify() const +{ + static ConfigurationVerifyKey verifyserverkeys[] = + { + HTTPSERVER_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) // no default addresses + }; + + static ConfigurationVerify verifyserver[] = + { + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } + }; + + static ConfigurationVerifyKey verifyrootkeys[] = + { + ConfigurationVerifyKey("AccessKey", ConfigTest_Exists), + ConfigurationVerifyKey("SecretKey", ConfigTest_Exists), + ConfigurationVerifyKey("StoreDirectory", ConfigTest_Exists), + HTTPSERVER_VERIFY_ROOT_KEYS + }; + + static ConfigurationVerify verify = + { + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + }; + + return &verify; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::Handle(HTTPRequest &rRequest, +// HTTPResponse &rResponse) +// Purpose: Handles any incoming S3 request, by checking +// authorization and then dispatching to one of the +// private Handle* methods. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + // if anything goes wrong, return a 500 error + rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError); + rResponse.SetContentType("text/plain"); + + try + { + const Configuration& rConfig(GetConfiguration()); + std::string access_key = rConfig.GetKeyValue("AccessKey"); + std::string secret_key = rConfig.GetKeyValue("SecretKey"); + + std::string md5, date, bucket; + rRequest.GetHeader("content-md5", &md5); + rRequest.GetHeader("date", &date); + + std::string host = rRequest.GetHostName(); + std::string s3suffix = ".s3.amazonaws.com"; + if (host.size() > s3suffix.size()) + { + std::string suffix = host.substr(host.size() - + s3suffix.size(), s3suffix.size()); + if (suffix == s3suffix) + { + bucket = host.substr(0, host.size() - + s3suffix.size()); + } + } + + std::ostringstream data; + data << rRequest.GetVerb() << "\n"; + data << md5 << "\n"; + data << rRequest.GetContentType() << "\n"; + data << date << "\n"; + + // header names are already in lower case, i.e. canonical form + + std::vector<HTTPRequest::Header> headers = rRequest.GetHeaders(); + std::sort(headers.begin(), headers.end()); + + for (std::vector<HTTPRequest::Header>::iterator + i = headers.begin(); i != headers.end(); i++) + { + if (i->first.substr(0, 5) == "x-amz") + { + data << i->first << ":" << i->second << "\n"; + } + } + + if (! bucket.empty()) + { + data << "/" << bucket; + } + + data << rRequest.GetRequestURI(); + std::string data_string = data.str(); + + unsigned char digest_buffer[EVP_MAX_MD_SIZE]; + unsigned int digest_size = sizeof(digest_buffer); + /* unsigned char* mac = */ HMAC(EVP_sha1(), + secret_key.c_str(), secret_key.size(), + (const unsigned char*)data_string.c_str(), + data_string.size(), digest_buffer, &digest_size); + std::string digest((const char *)digest_buffer, digest_size); + + base64::encoder encoder; + std::string expectedAuth = "AWS " + access_key + ":" + + encoder.encode(digest); + + if (expectedAuth[expectedAuth.size() - 1] == '\n') + { + expectedAuth = expectedAuth.substr(0, + expectedAuth.size() - 1); + } + + std::string actualAuth; + if (!rRequest.GetHeader("authorization", &actualAuth) || + actualAuth != expectedAuth) + { + rResponse.SetResponseCode(HTTPResponse::Code_Unauthorized); + SendInternalErrorResponse("Authentication Failed", + rResponse); + } + else if (rRequest.GetMethod() == HTTPRequest::Method_GET) + { + HandleGet(rRequest, rResponse); + } + else if (rRequest.GetMethod() == HTTPRequest::Method_PUT) + { + HandlePut(rRequest, rResponse); + } + else + { + rResponse.SetResponseCode(HTTPResponse::Code_MethodNotAllowed); + SendInternalErrorResponse("Unsupported Method", + rResponse); + } + } + catch (CommonException &ce) + { + SendInternalErrorResponse(ce.what(), rResponse); + } + catch (std::exception &e) + { + SendInternalErrorResponse(e.what(), rResponse); + } + catch (...) + { + SendInternalErrorResponse("Unknown exception", rResponse); + } + + if (rResponse.GetResponseCode() != 200 && + rResponse.GetSize() == 0) + { + // no error message written, provide a default + std::ostringstream s; + s << rResponse.GetResponseCode(); + SendInternalErrorResponse(s.str().c_str(), rResponse); + } + + return; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::HandleGet(HTTPRequest &rRequest, +// HTTPResponse &rResponse) +// Purpose: Handles an S3 GET request, i.e. downloading an +// existing object. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +void S3Simulator::HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + std::string path = GetConfiguration().GetKeyValue("StoreDirectory"); + path += rRequest.GetRequestURI(); + std::auto_ptr<FileStream> apFile; + + try + { + apFile.reset(new FileStream(path)); + } + catch (CommonException &ce) + { + if (ce.GetSubType() == CommonException::OSFileOpenError) + { + rResponse.SetResponseCode(HTTPResponse::Code_NotFound); + } + else if (ce.GetSubType() == CommonException::AccessDenied) + { + rResponse.SetResponseCode(HTTPResponse::Code_Forbidden); + } + throw; + } + + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingRESTOperations.html + apFile->CopyStreamTo(rResponse); + rResponse.AddHeader("x-amz-id-2", "qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY"); + rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); + rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); + rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\""); + rResponse.AddHeader("Server", "AmazonS3"); + rResponse.SetResponseCode(HTTPResponse::Code_OK); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::HandlePut(HTTPRequest &rRequest, +// HTTPResponse &rResponse) +// Purpose: Handles an S3 PUT request, i.e. uploading data to +// create or replace an object. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +void S3Simulator::HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + std::string path = GetConfiguration().GetKeyValue("StoreDirectory"); + path += rRequest.GetRequestURI(); + std::auto_ptr<FileStream> apFile; + + try + { + apFile.reset(new FileStream(path, O_CREAT | O_WRONLY)); + } + catch (CommonException &ce) + { + if (ce.GetSubType() == CommonException::OSFileOpenError) + { + rResponse.SetResponseCode(HTTPResponse::Code_NotFound); + } + else if (ce.GetSubType() == CommonException::AccessDenied) + { + rResponse.SetResponseCode(HTTPResponse::Code_Forbidden); + } + throw; + } + + if (rRequest.IsExpectingContinue()) + { + rResponse.SendContinue(); + } + + rRequest.ReadContent(*apFile); + + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTObjectPUT.html + rResponse.AddHeader("x-amz-id-2", "LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7"); + rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); + rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); + rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\""); + rResponse.SetContentType(""); + rResponse.AddHeader("Server", "AmazonS3"); + rResponse.SetResponseCode(HTTPResponse::Code_OK); +} diff --git a/lib/httpserver/S3Simulator.h b/lib/httpserver/S3Simulator.h new file mode 100644 index 00000000..f80770ee --- /dev/null +++ b/lib/httpserver/S3Simulator.h @@ -0,0 +1,40 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: S3Simulator.h +// Purpose: Amazon S3 simulation HTTP server for S3 testing +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- + +#ifndef S3SIMULATOR__H +#define S3SIMULATOR__H + +#include "HTTPServer.h" + +class ConfigurationVerify; +class HTTPRequest; +class HTTPResponse; + +// -------------------------------------------------------------------------- +// +// Class +// Name: S3Simulator +// Purpose: Amazon S3 simulation HTTP server for S3 testing +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- +class S3Simulator : public HTTPServer +{ +public: + S3Simulator() { } + ~S3Simulator() { } + + const ConfigurationVerify* GetConfigVerify() const; + virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse); + virtual void HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse); + virtual void HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse); +}; + +#endif // S3SIMULATOR__H + diff --git a/lib/httpserver/cdecode.cpp b/lib/httpserver/cdecode.cpp new file mode 100644 index 00000000..e632f182 --- /dev/null +++ b/lib/httpserver/cdecode.cpp @@ -0,0 +1,92 @@ +/* +cdecoder.c - c source to a base64 decoding algorithm implementation + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +extern "C" +{ + +#include "cdecode.h" + +int base64_decode_value(char value_in) +{ + static const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; + static const char decoding_size = sizeof(decoding); + value_in -= 43; + if (value_in < 0 || value_in > decoding_size) return -1; + return decoding[(int)value_in]; +} + +void base64_init_decodestate(base64_decodestate* state_in) +{ + state_in->step = step_a; + state_in->plainchar = 0; +} + +int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in) +{ + const char* codechar = code_in; + char* plainchar = plaintext_out; + char fragment; + + *plainchar = state_in->plainchar; + + switch (state_in->step) + { + while (1) + { + case step_a: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_a; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar = (fragment & 0x03f) << 2; + case step_b: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_b; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x030) >> 4; + *plainchar = (fragment & 0x00f) << 4; + case step_c: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_c; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x03c) >> 2; + *plainchar = (fragment & 0x003) << 6; + case step_d: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_d; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x03f); + } + } + /* control should not reach here */ + return plainchar - plaintext_out; +} + +} diff --git a/lib/httpserver/cdecode.h b/lib/httpserver/cdecode.h new file mode 100644 index 00000000..d0d7f489 --- /dev/null +++ b/lib/httpserver/cdecode.h @@ -0,0 +1,28 @@ +/* +cdecode.h - c header for a base64 decoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_CDECODE_H +#define BASE64_CDECODE_H + +typedef enum +{ + step_a, step_b, step_c, step_d +} base64_decodestep; + +typedef struct +{ + base64_decodestep step; + char plainchar; +} base64_decodestate; + +void base64_init_decodestate(base64_decodestate* state_in); + +int base64_decode_value(char value_in); + +int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in); + +#endif /* BASE64_CDECODE_H */ diff --git a/lib/httpserver/cencode.cpp b/lib/httpserver/cencode.cpp new file mode 100644 index 00000000..b33c0683 --- /dev/null +++ b/lib/httpserver/cencode.cpp @@ -0,0 +1,113 @@ +/* +cencoder.c - c source to a base64 encoding algorithm implementation + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +extern "C" +{ + +#include "cencode.h" + +const int CHARS_PER_LINE = 72; + +void base64_init_encodestate(base64_encodestate* state_in) +{ + state_in->step = step_A; + state_in->result = 0; + state_in->stepcount = 0; +} + +char base64_encode_value(char value_in) +{ + static const char* encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + if (value_in > 63) return '='; + return encoding[(int)value_in]; +} + +int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in) +{ + const char* plainchar = plaintext_in; + const char* const plaintextend = plaintext_in + length_in; + char* codechar = code_out; + char result; + char fragment; + + result = state_in->result; + + switch (state_in->step) + { + while (1) + { + case step_A: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_A; + return codechar - code_out; + } + fragment = *plainchar++; + result = (fragment & 0x0fc) >> 2; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x003) << 4; + case step_B: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_B; + return codechar - code_out; + } + fragment = *plainchar++; + result |= (fragment & 0x0f0) >> 4; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x00f) << 2; + case step_C: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_C; + return codechar - code_out; + } + fragment = *plainchar++; + result |= (fragment & 0x0c0) >> 6; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x03f) >> 0; + *codechar++ = base64_encode_value(result); + + ++(state_in->stepcount); + if (state_in->stepcount == CHARS_PER_LINE/4) + { + *codechar++ = '\n'; + state_in->stepcount = 0; + } + } + } + /* control should not reach here */ + return codechar - code_out; +} + +int base64_encode_blockend(char* code_out, base64_encodestate* state_in) +{ + char* codechar = code_out; + + switch (state_in->step) + { + case step_B: + *codechar++ = base64_encode_value(state_in->result); + *codechar++ = '='; + *codechar++ = '='; + break; + case step_C: + *codechar++ = base64_encode_value(state_in->result); + *codechar++ = '='; + break; + case step_A: + break; + } + *codechar++ = '\n'; + + return codechar - code_out; +} + +} diff --git a/lib/httpserver/cencode.h b/lib/httpserver/cencode.h new file mode 100644 index 00000000..cf321312 --- /dev/null +++ b/lib/httpserver/cencode.h @@ -0,0 +1,32 @@ +/* +cencode.h - c header for a base64 encoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_CENCODE_H +#define BASE64_CENCODE_H + +typedef enum +{ + step_A, step_B, step_C +} base64_encodestep; + +typedef struct +{ + base64_encodestep step; + char result; + int stepcount; +} base64_encodestate; + +void base64_init_encodestate(base64_encodestate* state_in); + +char base64_encode_value(char value_in); + +int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in); + +int base64_encode_blockend(char* code_out, base64_encodestate* state_in); + +#endif /* BASE64_CENCODE_H */ + diff --git a/lib/httpserver/decode.h b/lib/httpserver/decode.h new file mode 100644 index 00000000..fe59ef7a --- /dev/null +++ b/lib/httpserver/decode.h @@ -0,0 +1,77 @@ +// :mode=c++: +/* +decode.h - c++ wrapper for a base64 decoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_DECODE_H +#define BASE64_DECODE_H + +#include <iostream> + +namespace base64 +{ + + extern "C" + { + #include "cdecode.h" + } + + struct decoder + { + base64_decodestate _state; + int _buffersize; + + decoder(int buffersize_in = 4096) + : _buffersize(buffersize_in) + {} + int decode(char value_in) + { + return base64_decode_value(value_in); + } + int decode(const char* code_in, const int length_in, char* plaintext_out) + { + return base64_decode_block(code_in, length_in, plaintext_out, &_state); + } + std::string decode(const std::string& input) + { + base64_init_decodestate(&_state); + char* output = new char[2*input.size()]; + int outlength = decode(input.c_str(), input.size(), + output); + std::string output_string(output, outlength); + base64_init_decodestate(&_state); + delete [] output; + return output_string; + } + void decode(std::istream& istream_in, std::ostream& ostream_in) + { + base64_init_decodestate(&_state); + // + const int N = _buffersize; + char* code = new char[N]; + char* plaintext = new char[N]; + int codelength; + int plainlength; + + do + { + istream_in.read((char*)code, N); + codelength = istream_in.gcount(); + plainlength = decode(code, codelength, plaintext); + ostream_in.write((const char*)plaintext, plainlength); + } + while (istream_in.good() && codelength > 0); + // + base64_init_decodestate(&_state); + + delete [] code; + delete [] plaintext; + } + }; + +} // namespace base64 + +#endif // BASE64_DECODE_H diff --git a/lib/httpserver/encode.h b/lib/httpserver/encode.h new file mode 100644 index 00000000..81957a0f --- /dev/null +++ b/lib/httpserver/encode.h @@ -0,0 +1,87 @@ +// :mode=c++: +/* +encode.h - c++ wrapper for a base64 encoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_ENCODE_H +#define BASE64_ENCODE_H + +#include <iostream> + +namespace base64 +{ + + extern "C" + { + #include "cencode.h" + } + + struct encoder + { + base64_encodestate _state; + int _buffersize; + + encoder(int buffersize_in = 4096) + : _buffersize(buffersize_in) + {} + int encode(char value_in) + { + return base64_encode_value(value_in); + } + int encode(const char* code_in, const int length_in, char* plaintext_out) + { + return base64_encode_block(code_in, length_in, plaintext_out, &_state); + } + int encode_end(char* plaintext_out) + { + return base64_encode_blockend(plaintext_out, &_state); + } + std::string encode(const std::string& input) + { + base64_init_encodestate(&_state); + char* output = new char[2*input.size()]; + int outlength = encode(input.c_str(), input.size(), + output); + outlength += encode_end(output + outlength); + std::string output_string(output, outlength); + base64_init_encodestate(&_state); + delete [] output; + return output_string; + } + void encode(std::istream& istream_in, std::ostream& ostream_in) + { + base64_init_encodestate(&_state); + // + const int N = _buffersize; + char* plaintext = new char[N]; + char* code = new char[2*N]; + int plainlength; + int codelength; + + do + { + istream_in.read(plaintext, N); + plainlength = istream_in.gcount(); + // + codelength = encode(plaintext, plainlength, code); + ostream_in.write(code, codelength); + } + while (istream_in.good() && plainlength > 0); + + codelength = encode_end(code); + ostream_in.write(code, codelength); + // + base64_init_encodestate(&_state); + + delete [] code; + delete [] plaintext; + } + }; + +} // namespace base64 + +#endif // BASE64_ENCODE_H + diff --git a/lib/intercept/intercept.cpp b/lib/intercept/intercept.cpp new file mode 100644 index 00000000..7a33b610 --- /dev/null +++ b/lib/intercept/intercept.cpp @@ -0,0 +1,673 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: intercept.cpp +// Purpose: Syscall interception code for the raidfile test +// Created: 2003/07/22 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "intercept.h" + +#ifdef HAVE_SYS_SYSCALL_H + #include <sys/syscall.h> +#endif +#include <sys/types.h> +#include <unistd.h> + +#ifdef HAVE_SYS_UIO_H + #include <sys/uio.h> +#endif + +#include <errno.h> +#include <stdarg.h> + +#ifdef HAVE_DLFCN_H +#include <dlfcn.h> +#endif + +#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + +#if !defined(HAVE_SYSCALL) && !defined(HAVE___SYSCALL) && !defined(HAVE___SYSCALL_NEED_DEFN) + #define PLATFORM_NO_SYSCALL +#endif + +#ifdef PLATFORM_NO_SYSCALL + // For some reason, syscall just doesn't work on Darwin + // so instead, we build functions using assembler in a varient + // of the technique used in the Darwin Libc + extern "C" int + TEST_open(const char *path, int flags, mode_t mode); + extern "C" int + TEST_close(int d); + extern "C" ssize_t + TEST_write(int d, const void *buf, size_t nbytes); + extern "C" ssize_t + TEST_read(int d, void *buf, size_t nbytes); + extern "C" ssize_t + TEST_readv(int d, const struct iovec *iov, int iovcnt); + extern "C" off_t + TEST_lseek(int fildes, off_t offset, int whence); +#else + // if we have __syscall, we should use it for everything + // (on FreeBSD 7 this is required for 64-bit alignment of off_t). + // if not, we should continue to use the old syscall(). + #ifdef HAVE___SYSCALL_NEED_DEFN + // Need this, not declared in syscall.h nor unistd.h + extern "C" off_t __syscall(quad_t number, ...); + #endif + #ifdef HAVE___SYSCALL + #undef syscall + #define syscall __syscall + #endif +#endif + +#include <string.h> +#include <stdio.h> + +#include "MemLeakFindOn.h" + +int intercept_count = 0; +const char *intercept_filename = 0; +int intercept_filedes = -1; +off_t intercept_errorafter = 0; +int intercept_errno = 0; +int intercept_syscall = 0; +off_t intercept_filepos = 0; +int intercept_delay_ms = 0; + +static opendir_t* opendir_real = NULL; +static readdir_t* readdir_real = NULL; +static readdir_t* readdir_hook = NULL; +static closedir_t* closedir_real = NULL; +static lstat_t* lstat_real = NULL; +static lstat_t* lstat_hook = NULL; +static const char* lstat_file = NULL; +static lstat_t* stat_real = NULL; +static lstat_t* stat_hook = NULL; +static const char* stat_file = NULL; + +static lstat_post_hook_t* lstat_post_hook = NULL; +static lstat_post_hook_t* stat_post_hook = NULL; + +#define SIZE_ALWAYS_ERROR -773 + +void intercept_clear_setup() +{ + intercept_count = 0; + intercept_filename = 0; + intercept_filedes = -1; + intercept_errorafter = 0; + intercept_syscall = 0; + intercept_filepos = 0; + intercept_delay_ms = 0; + readdir_hook = NULL; + stat_hook = NULL; + lstat_hook = NULL; + stat_post_hook = NULL; + lstat_post_hook = NULL; +} + +bool intercept_triggered() +{ + return intercept_count == 0; +} + +void intercept_setup_error(const char *filename, unsigned int errorafter, int errortoreturn, int syscalltoerror) +{ + BOX_TRACE("Setup for error: " << filename << + ", after " << errorafter << + ", err " << errortoreturn << + ", syscall " << syscalltoerror); + + intercept_count = 1; + intercept_filename = filename; + intercept_filedes = -1; + intercept_errorafter = errorafter; + intercept_syscall = syscalltoerror; + intercept_errno = errortoreturn; + intercept_filepos = 0; + intercept_delay_ms = 0; +} + +void intercept_setup_delay(const char *filename, unsigned int delay_after, + int delay_ms, int syscall_to_delay, int num_delays) +{ + BOX_TRACE("Setup for delay: " << filename << + ", after " << delay_after << + ", wait " << delay_ms << " ms" << + ", times " << num_delays << + ", syscall " << syscall_to_delay); + + intercept_count = num_delays; + intercept_filename = filename; + intercept_filedes = -1; + intercept_errorafter = delay_after; + intercept_syscall = syscall_to_delay; + intercept_errno = 0; + intercept_filepos = 0; + intercept_delay_ms = delay_ms; +} + +bool intercept_errornow(int d, int size, int syscallnum) +{ + ASSERT(intercept_count > 0) + + if (intercept_filedes == -1) + { + return false; // no error please! + } + + if (d != intercept_filedes) + { + return false; // no error please! + } + + if (syscallnum != intercept_syscall) + { + return false; // no error please! + } + + bool ret = false; // no error unless one of the conditions matches + + //printf("Checking for err, %d, %d, %d\n", d, size, syscallnum); + + if (intercept_delay_ms != 0) + { + BOX_TRACE("Delaying " << intercept_delay_ms << " ms " << + " for syscall " << syscallnum << + " at " << intercept_filepos); + + struct timespec tm; + tm.tv_sec = intercept_delay_ms / 1000; + tm.tv_nsec = (intercept_delay_ms % 1000) * 1000000; + while (nanosleep(&tm, &tm) != 0 && + errno == EINTR) { } + } + + if (size == SIZE_ALWAYS_ERROR) + { + // Looks good for an error! + BOX_TRACE("Returning error " << intercept_errno << + " for syscall " << syscallnum); + ret = true; + } + else if (intercept_filepos + size < intercept_errorafter) + { + return false; // no error please + } + else if (intercept_errno != 0) + { + BOX_TRACE("Returning error " << intercept_errno << + " for syscall " << syscallnum << + " at " << intercept_filepos); + ret = true; + } + + intercept_count--; + if (intercept_count == 0) + { + intercept_clear_setup(); + } + + return ret; +} + +int intercept_reterr() +{ + int err = intercept_errno; + intercept_clear_setup(); + return err; +} + +#define CHECK_FOR_FAKE_ERROR_COND(D, S, CALL, FAILRES) \ + if(intercept_count > 0) \ + { \ + if(intercept_errornow(D, S, CALL)) \ + { \ + errno = intercept_reterr(); \ + return FAILRES; \ + } \ + } + +#if defined _FILE_OFFSET_BITS && _FILE_OFFSET_BITS == 64 + #define DEFINE_ONLY_OPEN64 +#endif + +extern "C" int +#ifdef DEFINE_ONLY_OPEN64 + open64(const char *path, int flags, ...) +#else + open(const char *path, int flags, ...) +#endif // DEFINE_ONLY_OPEN64 +{ + if(intercept_count > 0) + { + if(intercept_filename != NULL && + intercept_syscall == SYS_open && + strcmp(path, intercept_filename) == 0) + { + errno = intercept_reterr(); + return -1; + } + } + + mode_t mode = 0; + if (flags & O_CREAT) + { + va_list ap; + va_start(ap, flags); + mode = va_arg(ap, int); + va_end(ap); + } + +#ifdef PLATFORM_NO_SYSCALL + int r = TEST_open(path, flags, mode); +#else + int r = syscall(SYS_open, path, flags, mode); +#endif + + if(intercept_filename != NULL && + intercept_count > 0 && + intercept_filedes == -1) + { + // Right file? + if(strcmp(intercept_filename, path) == 0) + { + intercept_filedes = r; + //printf("Found file to intercept, h = %d\n", r); + } + } + + return r; +} + +#ifndef DEFINE_ONLY_OPEN64 +extern "C" int +// open64(const char *path, int flags, mode_t mode) +// open64(const char *path, int flags, ...) +open64 (__const char *path, int flags, ...) +{ + mode_t mode = 0; + if (flags & O_CREAT) + { + va_list ap; + va_start(ap, flags); + mode = va_arg(ap, int); + va_end(ap); + } + + // With _FILE_OFFSET_BITS set to 64 this should really use (flags | + // O_LARGEFILE) here, but not actually necessary for the tests and not + // worth the trouble finding O_LARGEFILE + return open(path, flags, mode); +} +#endif // !DEFINE_ONLY_OPEN64 + +extern "C" int +close(int d) +{ + CHECK_FOR_FAKE_ERROR_COND(d, SIZE_ALWAYS_ERROR, SYS_close, -1); +#ifdef PLATFORM_NO_SYSCALL + int r = TEST_close(d); +#else + int r = syscall(SYS_close, d); +#endif + if(r == 0) + { + if(d == intercept_filedes) + { + intercept_filedes = -1; + } + } + return r; +} + +extern "C" ssize_t +write(int d, const void *buf, size_t nbytes) +{ + CHECK_FOR_FAKE_ERROR_COND(d, nbytes, SYS_write, -1); +#ifdef PLATFORM_NO_SYSCALL + int r = TEST_write(d, buf, nbytes); +#else + int r = syscall(SYS_write, d, buf, nbytes); +#endif + if(r != -1) + { + intercept_filepos += r; + } + return r; +} + +extern "C" ssize_t +read(int d, void *buf, size_t nbytes) +{ + CHECK_FOR_FAKE_ERROR_COND(d, nbytes, SYS_read, -1); +#ifdef PLATFORM_NO_SYSCALL + int r = TEST_read(d, buf, nbytes); +#else + int r = syscall(SYS_read, d, buf, nbytes); +#endif + if(r != -1) + { + intercept_filepos += r; + } + return r; +} + +extern "C" ssize_t +readv(int d, const struct iovec *iov, int iovcnt) +{ + // how many bytes? + int nbytes = 0; + for(int b = 0; b < iovcnt; ++b) + { + nbytes += iov[b].iov_len; + } + + CHECK_FOR_FAKE_ERROR_COND(d, nbytes, SYS_readv, -1); +#ifdef PLATFORM_NO_SYSCALL + int r = TEST_readv(d, iov, iovcnt); +#else + int r = syscall(SYS_readv, d, iov, iovcnt); +#endif + if(r != -1) + { + intercept_filepos += r; + } + return r; +} + +extern "C" off_t +lseek(int fildes, off_t offset, int whence) +{ + // random magic for lseek syscall, see /usr/src/lib/libc/sys/lseek.c + CHECK_FOR_FAKE_ERROR_COND(fildes, 0, SYS_lseek, -1); +#ifdef PLATFORM_NO_SYSCALL + int r = TEST_lseek(fildes, offset, whence); +#else + #ifdef HAVE_LSEEK_DUMMY_PARAM + off_t r = syscall(SYS_lseek, fildes, 0 /* extra 0 required here! */, offset, whence); + #elif defined(_FILE_OFFSET_BITS) + // Don't bother trying to call SYS__llseek on 32 bit since it is + // fiddly and not needed for the tests + off_t r = syscall(SYS_lseek, fildes, (uint32_t)offset, whence); + #else + off_t r = syscall(SYS_lseek, fildes, offset, whence); + #endif +#endif + if(r != -1) + { + intercept_filepos = r; + } + return r; +} + +void intercept_setup_readdir_hook(const char *dirname, readdir_t hookfn) +{ + if (hookfn != NULL && dirname == NULL) + { + dirname = intercept_filename; + ASSERT(dirname != NULL); + } + + if (hookfn != NULL) + { + BOX_TRACE("readdir hooked to " << hookfn << " for " << dirname); + } + else if (intercept_filename != NULL) + { + BOX_TRACE("readdir unhooked from " << readdir_hook << + " for " << intercept_filename); + } + + intercept_filename = dirname; + readdir_hook = hookfn; +} + +void intercept_setup_lstat_hook(const char *filename, lstat_t hookfn) +{ + /* + if (hookfn != NULL) + { + BOX_TRACE("lstat hooked to " << hookfn << " for " << filename); + } + else + { + BOX_TRACE("lstat unhooked from " << lstat_hook << " for " << + lstat_file); + } + */ + + lstat_file = filename; + lstat_hook = hookfn; +} + +void intercept_setup_lstat_post_hook(lstat_post_hook_t hookfn) +{ + /* + if (hookfn != NULL) + { + BOX_TRACE("lstat hooked to " << hookfn << " for " << filename); + } + else + { + BOX_TRACE("lstat unhooked from " << lstat_hook << " for " << + lstat_file); + } + */ + + lstat_post_hook = hookfn; +} + +void intercept_setup_stat_post_hook(lstat_post_hook_t hookfn) +{ + /* + if (hookfn != NULL) + { + BOX_TRACE("lstat hooked to " << hookfn << " for " << filename); + } + else + { + BOX_TRACE("lstat unhooked from " << lstat_hook << " for " << + lstat_file); + } + */ + + stat_post_hook = hookfn; +} + +static void * find_function(const char *pName) +{ + dlerror(); + void *result = NULL; + + #ifdef HAVE_LARGE_FILE_SUPPORT + { + // search for the 64-bit version first + std::string name64(pName); + name64 += "64"; + result = dlsym(RTLD_NEXT, name64.c_str()); + if (dlerror() == NULL && result != NULL) + { + return result; + } + } + #endif + + result = dlsym(RTLD_NEXT, pName); + const char *errmsg = (const char *)dlerror(); + + if (errmsg == NULL) + { + return result; + } + + BOX_ERROR("Failed to find real " << pName << " function: " << errmsg); + return NULL; +} + +extern "C" +DIR *opendir(const char *dirname) +{ + if (opendir_real == NULL) + { + opendir_real = (opendir_t*)find_function("opendir"); + } + + if (opendir_real == NULL) + { + perror("cannot find real opendir"); + return NULL; + } + + DIR* r = opendir_real(dirname); + + if (readdir_hook != NULL && + intercept_filename != NULL && + intercept_filedes == -1 && + strcmp(intercept_filename, dirname) == 0) + { + intercept_filedes = dirfd(r); + //printf("Found file to intercept, h = %d\n", r); + } + + return r; +} + +extern "C" +struct dirent *readdir(DIR *dir) +{ + if (readdir_hook != NULL && dirfd(dir) == intercept_filedes) + { + return readdir_hook(dir); + } + + if (readdir_real == NULL) + { + readdir_real = (readdir_t*)find_function("readdir"); + } + + if (readdir_real == NULL) + { + perror("cannot find real readdir"); + return NULL; + } + + return readdir_real(dir); +} + +extern "C" +int closedir(DIR *dir) +{ + if (dirfd(dir) == intercept_filedes) + { + intercept_filedes = -1; + } + + if (closedir_real == NULL) + { + closedir_real = (closedir_t*)find_function("closedir"); + } + + if (closedir_real == NULL) + { + perror("cannot find real closedir"); + errno = ENOSYS; + return -1; + } + + return closedir_real(dir); +} + +extern "C" int +#ifdef LINUX_WEIRD_LSTAT +__lxstat(int ver, const char *file_name, STAT_STRUCT *buf) +#else +lstat(const char *file_name, STAT_STRUCT *buf) +#endif +{ + if (lstat_real == NULL) + { + #ifdef LINUX_WEIRD_LSTAT + lstat_real = (lstat_t*)find_function("__lxstat"); + #else + lstat_real = (lstat_t*)find_function("lstat"); + #endif + } + + if (lstat_real == NULL) + { + perror("cannot find real lstat"); + errno = ENOSYS; + return -1; + } + + if (lstat_hook == NULL || strcmp(file_name, lstat_file) != 0) + { + #ifdef LINUX_WEIRD_LSTAT + int ret = lstat_real(ver, file_name, buf); + #else + int ret = lstat_real(file_name, buf); + #endif + if (lstat_post_hook != NULL) + { + ret = lstat_post_hook(ret, file_name, buf); + } + return ret; + } + + #ifdef LINUX_WEIRD_LSTAT + return lstat_hook(ver, file_name, buf); + #else + return lstat_hook(file_name, buf); + #endif +} + +extern "C" int +#ifdef LINUX_WEIRD_LSTAT +__xstat(int ver, const char *file_name, STAT_STRUCT *buf) +#else +stat(const char *file_name, STAT_STRUCT *buf) +#endif +{ + if (stat_real == NULL) + { + #ifdef LINUX_WEIRD_LSTAT + stat_real = (lstat_t*)find_function("__xstat"); + #else + stat_real = (lstat_t*)find_function("stat"); + #endif + } + + if (stat_real == NULL) + { + perror("cannot find real stat"); + errno = ENOSYS; + return -1; + } + + if (stat_hook == NULL || strcmp(file_name, stat_file) != 0) + { + #ifdef LINUX_WEIRD_LSTAT + int ret = stat_real(ver, file_name, buf); + #else + int ret = stat_real(file_name, buf); + #endif + if (stat_post_hook != NULL) + { + ret = stat_post_hook(ret, file_name, buf); + } + return ret; + } + + #ifdef LINUX_WEIRD_LSTAT + return stat_hook(ver, file_name, buf); + #else + return stat_hook(file_name, buf); + #endif +} + +#endif // n PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE diff --git a/lib/intercept/intercept.h b/lib/intercept/intercept.h new file mode 100644 index 00000000..80a17d3f --- /dev/null +++ b/lib/intercept/intercept.h @@ -0,0 +1,54 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: intercept.h +// Purpose: Syscall interception code for unit tests +// Created: 2006/11/29 +// +// -------------------------------------------------------------------------- + +#ifndef INTERCEPT_H +#define INTERCEPT_H +#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + +#include <dirent.h> + +#include <sys/types.h> +#include <sys/stat.h> + +extern "C" +{ + typedef DIR *(opendir_t) (const char *name); + typedef struct dirent *(readdir_t) (DIR *dir); + typedef struct dirent *(readdir_t) (DIR *dir); + typedef int (closedir_t)(DIR *dir); +#if defined __GNUC__ && __GNUC__ >= 2 + #define LINUX_WEIRD_LSTAT + #define STAT_STRUCT struct stat /* should be stat64 */ + typedef int (lstat_t) (int ver, const char *file_name, + STAT_STRUCT *buf); +#else + #define STAT_STRUCT struct stat + typedef int (lstat_t) (const char *file_name, + STAT_STRUCT *buf); +#endif +} + +typedef int (lstat_post_hook_t) (int old_ret, const char *file_name, + struct stat *buf); + +void intercept_setup_error(const char *filename, unsigned int errorafter, + int errortoreturn, int syscalltoerror); +void intercept_setup_delay(const char *filename, unsigned int delay_after, + int delay_ms, int syscall_to_delay, int num_delays); +bool intercept_triggered(); + +void intercept_setup_readdir_hook(const char *dirname, readdir_t hookfn); +void intercept_setup_lstat_hook (const char *filename, lstat_t hookfn); +void intercept_setup_lstat_post_hook(lstat_post_hook_t hookfn); +void intercept_setup_stat_post_hook (lstat_post_hook_t hookfn); + +void intercept_clear_setup(); + +#endif // !PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE +#endif // !INTERCEPT_H diff --git a/lib/raidfile/Makefile.extra b/lib/raidfile/Makefile.extra new file mode 100644 index 00000000..bf06ed2f --- /dev/null +++ b/lib/raidfile/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_RaidFileException.h autogen_RaidFileException.cpp: $(MAKEEXCEPTION) RaidFileException.txt + $(_PERL) $(MAKEEXCEPTION) RaidFileException.txt + diff --git a/lib/raidfile/RaidFileController.cpp b/lib/raidfile/RaidFileController.cpp new file mode 100644 index 00000000..2cc6976b --- /dev/null +++ b/lib/raidfile/RaidFileController.cpp @@ -0,0 +1,227 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileController.cpp +// Purpose: Controls config and daemon comms for RaidFile classes +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> + +#include "RaidFileController.h" +#include "RaidFileException.h" +#include "Configuration.h" + +#include "MemLeakFindOn.h" + +RaidFileController RaidFileController::mController; + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::RaidFileController() +// Purpose: Constructor +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +RaidFileController::RaidFileController() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::~RaidFileController() +// Purpose: Destructor +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +RaidFileController::~RaidFileController() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::RaidFileController() +// Purpose: Copy constructor +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +RaidFileController::RaidFileController(const RaidFileController &rController) +{ + THROW_EXCEPTION(RaidFileException, Internal) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::Initialise(const std::string&) +// Purpose: Initialises the system, loading the configuration file. +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +void RaidFileController::Initialise(const std::string& rConfigFilename) +{ + MEMLEAKFINDER_NO_LEAKS; + + static const ConfigurationVerifyKey verifykeys[] = + { + ConfigurationVerifyKey("SetNumber", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("BlockSize", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("Dir0", ConfigTest_Exists), + ConfigurationVerifyKey("Dir1", ConfigTest_Exists), + ConfigurationVerifyKey("Dir2", + ConfigTest_Exists | ConfigTest_LastEntry) + }; + + static const ConfigurationVerify subverify = + { + "*", + 0, + verifykeys, + ConfigTest_LastEntry, + 0 + }; + + static const ConfigurationVerify verify = + { + "RAID FILE CONFIG", + &subverify, + 0, + ConfigTest_LastEntry, + 0 + }; + + // Load the configuration + std::string err; + std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify( + rConfigFilename, &verify, err); + + if(pconfig.get() == 0 || !err.empty()) + { + BOX_ERROR("RaidFile configuration file errors: " << err); + THROW_EXCEPTION(RaidFileException, BadConfigFile) + } + + // Allow reinitializing the controller by remove any existing + // disc sets. Used by Boxi unit tests. + mSetList.clear(); + + // Use the values + int expectedSetNum = 0; + std::vector<std::string> confdiscs(pconfig->GetSubConfigurationNames()); + for(std::vector<std::string>::const_iterator i(confdiscs.begin()); i != confdiscs.end(); ++i) + { + const Configuration &disc(pconfig->GetSubConfiguration((*i).c_str())); + + int setNum = disc.GetKeyValueInt("SetNumber"); + if(setNum != expectedSetNum) + { + THROW_EXCEPTION(RaidFileException, BadConfigFile) + } + RaidFileDiscSet set(setNum, (unsigned int)disc.GetKeyValueInt("BlockSize")); + // Get the values of the directory keys + std::string d0(disc.GetKeyValue("Dir0")); + std::string d1(disc.GetKeyValue("Dir1")); + std::string d2(disc.GetKeyValue("Dir2")); + // Are they all different (using RAID) or all the same (not using RAID) + if(d0 != d1 && d1 != d2 && d0 != d2) + { + set.push_back(d0); + set.push_back(d1); + set.push_back(d2); + } + else if(d0 == d1 && d0 == d2) + { + // Just push the first one, which is the non-RAID place to store files + set.push_back(d0); + } + else + { + // One must be the same as another! Which is bad. + THROW_EXCEPTION(RaidFileException, BadConfigFile) + } + mSetList.push_back(set); + expectedSetNum++; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::GetDiscSet(int) +// Purpose: Returns the numbered disc set +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +RaidFileDiscSet &RaidFileController::GetDiscSet(unsigned int DiscSetNum) +{ + if(DiscSetNum < 0 || DiscSetNum >= mSetList.size()) + { + THROW_EXCEPTION(RaidFileException, NoSuchDiscSet) + } + + return mSetList[DiscSetNum]; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileDiscSet::GetSetNumForWriteFiles(const std::string &) +// Purpose: Returns the set number the 'temporary' written files should +// be stored on, given a filename. +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +int RaidFileDiscSet::GetSetNumForWriteFiles(const std::string &rFilename) const +{ + // Simple hash function, add up the ASCII values of all the characters, + // and get modulo number of partitions in the set. + std::string::const_iterator i(rFilename.begin()); + int h = 0; + for(; i != rFilename.end(); ++i) + { + h += (*i); + } + return h % size(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileController::DiscSetPathToFileSystemPath(unsigned int, const std::string &, int) +// Purpose: Given a Raid File style file name, return a filename for the physical filing system. +// DiscOffset is effectively the disc number (but remember files are rotated around the +// discs in a disc set) +// Created: 19/1/04 +// +// -------------------------------------------------------------------------- +std::string RaidFileController::DiscSetPathToFileSystemPath(unsigned int DiscSetNum, const std::string &rFilename, int DiscOffset) +{ + if(DiscSetNum < 0 || DiscSetNum >= mController.mSetList.size()) + { + THROW_EXCEPTION(RaidFileException, NoSuchDiscSet) + } + + // Work out which disc it's to be on + int disc = (mController.mSetList[DiscSetNum].GetSetNumForWriteFiles(rFilename) + DiscOffset) + % mController.mSetList[DiscSetNum].size(); + + // Make the string + std::string r((mController.mSetList[DiscSetNum])[disc]); + r += DIRECTORY_SEPARATOR_ASCHAR; + r += rFilename; + return r; +} + + + diff --git a/lib/raidfile/RaidFileController.h b/lib/raidfile/RaidFileController.h new file mode 100644 index 00000000..216bdf3a --- /dev/null +++ b/lib/raidfile/RaidFileController.h @@ -0,0 +1,108 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileController.h +// Purpose: Controls config and daemon comms for RaidFile classes +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +/* NOTE: will log to local5: include a line like + local5.info /var/log/raidfile + in /etc/syslog.conf +*/ + +#ifndef RAIDFILECONTROLLER__H +#define RAIDFILECONTROLLER__H + +#include <string> +#include <vector> + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileDiscSet +// Purpose: Describes a set of paritions for RAID like files. +// Use as list of directories containing the files. +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +class RaidFileDiscSet : public std::vector<std::string> +{ +public: + RaidFileDiscSet(int SetID, unsigned int BlockSize) + : mSetID(SetID), + mBlockSize(BlockSize) + { + } + RaidFileDiscSet(const RaidFileDiscSet &rToCopy) + : std::vector<std::string>(rToCopy), + mSetID(rToCopy.mSetID), + mBlockSize(rToCopy.mBlockSize) + { + } + + ~RaidFileDiscSet() + { + } + + int GetSetID() const {return mSetID;} + + int GetSetNumForWriteFiles(const std::string &rFilename) const; + + unsigned int GetBlockSize() const {return mBlockSize;} + + // Is this disc set a non-RAID disc set? (ie files never get transformed to raid storage) + bool IsNonRaidSet() const {return 1 == size();} + +private: + int mSetID; + unsigned int mBlockSize; +}; + +class _RaidFileController; // compiler warning avoidance + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileController +// Purpose: Manages the configuration of the RaidFile system, handles +// communication with the daemon. +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- +class RaidFileController +{ + friend class _RaidFileController; // to avoid compiler warning +private: + RaidFileController(); + RaidFileController(const RaidFileController &rController); +public: + ~RaidFileController(); + +public: + void Initialise(const std::string& rConfigFilename = + "/etc/boxbackup/raidfile.conf"); + int GetNumDiscSets() {return mSetList.size();} + + // -------------------------------------------------------------------------- + // + // Function + // Name: RaidFileController::GetController() + // Purpose: Gets the one and only controller object. + // Created: 2003/07/08 + // + // -------------------------------------------------------------------------- + static RaidFileController &GetController() {return mController;} + RaidFileDiscSet &GetDiscSet(unsigned int DiscSetNum); + + static std::string DiscSetPathToFileSystemPath(unsigned int DiscSetNum, const std::string &rFilename, int DiscOffset); + +private: + std::vector<RaidFileDiscSet> mSetList; + + static RaidFileController mController; +}; + +#endif // RAIDFILECONTROLLER__H + diff --git a/lib/raidfile/RaidFileException.h b/lib/raidfile/RaidFileException.h new file mode 100644 index 00000000..809d4d5a --- /dev/null +++ b/lib/raidfile/RaidFileException.h @@ -0,0 +1,17 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef RAIDFILEEXCEPTION__H +#define RAIDFILEEXCEPTION__H + +// Compatibility +#include "autogen_RaidFileException.h" + +#endif // RAIDFILEEXCEPTION__H + diff --git a/lib/raidfile/RaidFileException.txt b/lib/raidfile/RaidFileException.txt new file mode 100644 index 00000000..c7ddfcc5 --- /dev/null +++ b/lib/raidfile/RaidFileException.txt @@ -0,0 +1,28 @@ +EXCEPTION RaidFile 2 + +Internal 0 +CantOpenConfigFile 1 The raidfile.conf file is not accessible. Check that it is present in the default location or daemon configuration files point to the correct location. +BadConfigFile 2 +NoSuchDiscSet 3 +CannotOverwriteExistingFile 4 +AlreadyOpen 5 +ErrorOpeningWriteFile 6 +NotOpen 7 +OSError 8 Error when accessing an underlying file. Check file permissions allow files to be read and written in the configured raid directories. +WriteFileOpenOnTransform 9 +WrongNumberOfDiscsInSet 10 There should be three directories in each disc set. +RaidFileDoesntExist 11 Error when accessing a file on the store. Check the store with bbstoreaccounts check. +ErrorOpeningFileForRead 12 +FileIsDamagedNotRecoverable 13 +InvalidRaidFile 14 +DirectoryIncomplete 15 +UnexpectedFileInDirPlace 16 +FileExistsInDirectoryCreation 17 +UnsupportedReadWriteOrClose 18 +CanOnlyGetUsageBeforeCommit 19 +CanOnlyGetFileSizeBeforeCommit 20 +ErrorOpeningWriteFileOnTruncate 21 +FileIsCurrentlyOpenForWriting 22 +RequestedModifyUnreferencedFile 23 Internal error: the server attempted to modify a file which has no references. +RequestedModifyMultiplyReferencedFile 24 Internal error: the server attempted to modify a file which has multiple references. +RequestedDeleteReferencedFile 25 Internal error: the server attempted to delete a file which is still referenced. diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp new file mode 100644 index 00000000..0a79be57 --- /dev/null +++ b/lib/raidfile/RaidFileRead.cpp @@ -0,0 +1,1724 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileRead.cpp +// Purpose: Read Raid like Files +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <errno.h> +#include <fcntl.h> +#include <stdarg.h> +#include <unistd.h> + +#include <sys/stat.h> +#include <sys/types.h> + +#ifdef HAVE_SYS_UIO_H + #include <sys/uio.h> +#endif + +#ifdef HAVE_DIRENT_H + #include <dirent.h> +#endif + +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <map> +#include <memory> + +#include "RaidFileRead.h" +#include "RaidFileException.h" +#include "RaidFileController.h" +#include "RaidFileUtil.h" + +#include "MemLeakFindOn.h" + +#define READ_NUMBER_DISCS_REQUIRED 3 +#define READV_MAX_BLOCKS 64 + +// We want to use POSIX fstat() for now, not the emulated one +#undef fstat + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileRead_NonRaid +// Purpose: Internal class for reading RaidFiles which haven't been transformed +// into the RAID like form yet. +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +class RaidFileRead_NonRaid : public RaidFileRead +{ +public: + RaidFileRead_NonRaid(int SetNumber, const std::string &Filename, int OSFileHandle); + virtual ~RaidFileRead_NonRaid(); +private: + RaidFileRead_NonRaid(const RaidFileRead_NonRaid &rToCopy); + +public: + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + virtual pos_type GetFileSize() const; + virtual bool StreamDataLeft(); + +private: + int mOSFileHandle; + bool mEOF; +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid(int, const std::string &, const std::string &) +// Purpose: Constructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead_NonRaid::RaidFileRead_NonRaid(int SetNumber, const std::string &Filename, int OSFileHandle) + : RaidFileRead(SetNumber, Filename), + mOSFileHandle(OSFileHandle), + mEOF(false) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::~RaidFileRead_NonRaid() +// Purpose: Destructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead_NonRaid::~RaidFileRead_NonRaid() +{ + if(mOSFileHandle != -1) + { + Close(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::Read(const void *, int) +// Purpose: Reads bytes from the file +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +int RaidFileRead_NonRaid::Read(void *pBuffer, int NBytes, int Timeout) +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Read data + int bytesRead = ::read(mOSFileHandle, pBuffer, NBytes); + if(bytesRead == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Check for EOF + if(bytesRead == 0) + { + mEOF = true; + } + + return bytesRead; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::GetPosition() +// Purpose: Returns current position +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead::pos_type RaidFileRead_NonRaid::GetPosition() const +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Use lseek to find the current file position + off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR); + if(p == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + return p; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::Seek(pos_type, int) +// Purpose: Seek within the file +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +void RaidFileRead_NonRaid::Seek(IOStream::pos_type Offset, int SeekType) +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Seek... + if(::lseek(mOSFileHandle, Offset, ConvertSeekTypeToOSWhence(SeekType)) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Not EOF any more + mEOF = false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::Close() +// Purpose: Close the file (automatically done by destructor) +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +void RaidFileRead_NonRaid::Close() +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Close file... + if(::close(mOSFileHandle) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + mOSFileHandle = -1; + mEOF = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::GetFileSize() +// Purpose: Returns file size. +// Created: 2003/07/14 +// +// -------------------------------------------------------------------------- +RaidFileRead::pos_type RaidFileRead_NonRaid::GetFileSize() const +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // stat the file + struct stat st; + if(::fstat(mOSFileHandle, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + return st.st_size; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::StreamDataLeft() +// Purpose: Any data left? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool RaidFileRead_NonRaid::StreamDataLeft() +{ + return !mEOF; +} + +// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileRead_Raid +// Purpose: Internal class for reading RaidFiles have been transformed. +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +class RaidFileRead_Raid : public RaidFileRead +{ +public: + friend class RaidFileRead; + RaidFileRead_Raid(int SetNumber, const std::string &Filename, int Stripe1Handle, + int Stripe2Handle, int ParityHandle, pos_type FileSize, unsigned int BlockSize, + bool LastBlockHasSize); + virtual ~RaidFileRead_Raid(); +private: + RaidFileRead_Raid(const RaidFileRead_Raid &rToCopy); + +public: + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + virtual pos_type GetFileSize() const; + virtual bool StreamDataLeft(); + +private: + int ReadRecovered(void *pBuffer, int NBytes); + void AttemptToRecoverFromIOError(bool Stripe1); + void SetPosition(pos_type FilePosition); + static void MoveDamagedFileAlertDaemon(int SetNumber, const std::string &Filename, bool Stripe1); + +private: + int mStripe1Handle; + int mStripe2Handle; + int mParityHandle; + pos_type mFileSize; + unsigned int mBlockSize; + pos_type mCurrentPosition; + char *mRecoveryBuffer; + pos_type mRecoveryBufferStart; + bool mLastBlockHasSize; + bool mEOF; +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid(int, const std::string &, const std::string &) +// Purpose: Constructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead_Raid::RaidFileRead_Raid(int SetNumber, const std::string &Filename, int Stripe1Handle, int Stripe2Handle, int ParityHandle, pos_type FileSize, unsigned int BlockSize, bool LastBlockHasSize) + : RaidFileRead(SetNumber, Filename), + mStripe1Handle(Stripe1Handle), + mStripe2Handle(Stripe2Handle), + mParityHandle(ParityHandle), + mFileSize(FileSize), + mBlockSize(BlockSize), + mCurrentPosition(0), + mRecoveryBuffer(0), + mRecoveryBufferStart(-1), + mLastBlockHasSize(LastBlockHasSize), + mEOF(false) +{ + // Make sure size of the IOStream::pos_type matches the pos_type used + ASSERT(sizeof(pos_type) >= sizeof(off_t)); + + // Sanity check handles + if(mStripe1Handle != -1 && mStripe2Handle != -1) + { + // Everything is lovely, got two perfect files + } + else + { + // Check we have at least one stripe and a parity file + if((mStripe1Handle == -1 && mStripe2Handle == -1) || mParityHandle == -1) + { + // Should never have got this far + THROW_EXCEPTION(RaidFileException, Internal) + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::~RaidFileRead_Raid() +// Purpose: Destructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead_Raid::~RaidFileRead_Raid() +{ + Close(); + if(mRecoveryBuffer != 0) + { + ::free(mRecoveryBuffer); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::Read(const void *, int) +// Purpose: Reads bytes from the file +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +int RaidFileRead_Raid::Read(void *pBuffer, int NBytes, int Timeout) +{ + // How many more bytes could we read? + unsigned int maxRead = mFileSize - mCurrentPosition; + if((unsigned int)NBytes > maxRead) + { + NBytes = maxRead; + } + + // Return immediately if there's nothing to read, and set EOF + if(NBytes == 0) + { + mEOF = true; + return 0; + } + + // Can we use the normal file reading routine? + if(mStripe1Handle == -1 || mStripe2Handle == -1) + { + // File is damaged, try a the recovery read function + return ReadRecovered(pBuffer, NBytes); + } + + // Vectors for reading stuff from the files + struct iovec stripe1Reads[READV_MAX_BLOCKS]; + struct iovec stripe2Reads[READV_MAX_BLOCKS]; + struct iovec *stripeReads[2] = {stripe1Reads, stripe2Reads}; + unsigned int stripeReadsDataSize[2] = {0, 0}; + unsigned int stripeReadsSize[2] = {0, 0}; + int stripeHandles[2] = {mStripe1Handle, mStripe2Handle}; + + // Which block are we doing? + unsigned int currentBlock = mCurrentPosition / mBlockSize; + unsigned int bytesLeftInCurrentBlock = mBlockSize - (mCurrentPosition % mBlockSize); + ASSERT(bytesLeftInCurrentBlock > 0) + unsigned int leftToRead = NBytes; + char *bufferPtr = (char*)pBuffer; + + // Now... add some whole block entries in... + try + { + while(leftToRead > 0) + { + int whichStripe = (currentBlock & 1); + size_t rlen = mBlockSize; + // Adjust if it's the first block + if(bytesLeftInCurrentBlock != 0) + { + rlen = bytesLeftInCurrentBlock; + bytesLeftInCurrentBlock = 0; + } + // Adjust if we're out of bytes + if(rlen > leftToRead) + { + rlen = leftToRead; + } + stripeReads[whichStripe][stripeReadsSize[whichStripe]].iov_base = bufferPtr; + stripeReads[whichStripe][stripeReadsSize[whichStripe]].iov_len = rlen; + stripeReadsSize[whichStripe]++; + stripeReadsDataSize[whichStripe] += rlen; + leftToRead -= rlen; + bufferPtr += rlen; + currentBlock++; + + // Read data? + for(int s = 0; s < 2; ++s) + { + if((leftToRead == 0 || stripeReadsSize[s] >= READV_MAX_BLOCKS) && stripeReadsSize[s] > 0) + { + int r = ::readv(stripeHandles[s], stripeReads[s], stripeReadsSize[s]); + if(r == -1) + { + // Bad news... IO error? + if(errno == EIO) + { + // Attempt to recover from this failure + AttemptToRecoverFromIOError((s == 0) /* is stripe 1 */); + // Retry + return Read(pBuffer, NBytes, Timeout); + } + else + { + // Can't do anything, throw + THROW_EXCEPTION(RaidFileException, OSError) + } + } + else if(r != (int)stripeReadsDataSize[s]) + { + // Got the file sizes wrong/logic error! + THROW_EXCEPTION(RaidFileException, Internal) + } + stripeReadsSize[s] = 0; + stripeReadsDataSize[s] = 0; + } + } + } + } + catch(...) + { + // Get file pointers to right place (to meet exception safe stuff) + SetPosition(mCurrentPosition); + + throw; + } + + // adjust current position + mCurrentPosition += NBytes; + + return NBytes; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::MoveDamagedFileAlertDaemon(bool) +// Purpose: Moves a file into the damaged directory, and alerts the Daemon to recover it properly later. +// Created: 2003/07/22 +// +// -------------------------------------------------------------------------- +void RaidFileRead_Raid::MoveDamagedFileAlertDaemon(int SetNumber, const std::string &Filename, bool Stripe1) +{ + // Move the dodgy file away + // Get the controller and the disc set we're on + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size()) + { + THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet) + } + // Start disc + int startDisc = rdiscSet.GetSetNumForWriteFiles(Filename); + int errOnDisc = (startDisc + (Stripe1?0:1)) % READ_NUMBER_DISCS_REQUIRED; + + // Make a munged filename for renaming + std::string mungeFn(Filename + RAIDFILE_EXTENSION); + std::string awayName; + for(std::string::const_iterator i = mungeFn.begin(); i != mungeFn.end(); ++i) + { + char c = (*i); + if(c == DIRECTORY_SEPARATOR_ASCHAR) + { + awayName += '_'; + } + else if(c == '_') + { + awayName += "__"; + } + else + { + awayName += c; + } + } + // Make sure the error files directory exists + std::string dirname(rdiscSet[errOnDisc] + DIRECTORY_SEPARATOR ".raidfile-unreadable"); + int mdr = ::mkdir(dirname.c_str(), 0750); + if(mdr != 0 && errno != EEXIST) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Attempt to rename the file there -- ignore any return code here, as it's dubious anyway + std::string errorFile(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, errOnDisc)); + ::rename(errorFile.c_str(), (dirname + DIRECTORY_SEPARATOR_ASCHAR + awayName).c_str()); + + // TODO: Inform the recovery daemon +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::AttemptToRecoverFromIOError(bool) +// Purpose: Attempt to recover from an IO error, setting up to read from parity instead. +// Will exception if this isn't possible. +// Created: 2003/07/14 +// +// -------------------------------------------------------------------------- +void RaidFileRead_Raid::AttemptToRecoverFromIOError(bool Stripe1) +{ + BOX_WARNING("Attempting to recover from I/O error: " << mSetNumber << + " " << mFilename << ", on stripe " << (Stripe1?1:2)); + + // Close offending file + if(Stripe1) + { + if(mStripe1Handle != -1) + { + ::close(mStripe1Handle); + mStripe1Handle = -1; + } + } + else + { + if(mStripe2Handle != -1) + { + ::close(mStripe2Handle); + mStripe2Handle = -1; + } + } + + // Check... + ASSERT((Stripe1?mStripe2Handle:mStripe1Handle) != -1); + + // Get rid of the damaged file + MoveDamagedFileAlertDaemon(mSetNumber, mFilename, Stripe1); + + // Get the controller and the disc set we're on + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size()) + { + THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet) + } + // Start disc + int startDisc = rdiscSet.GetSetNumForWriteFiles(mFilename); + + // Mark as nothing in recovery buffer + mRecoveryBufferStart = -1; + + // Seek to zero on the remaining file -- get to nice state + if(::lseek(Stripe1?mStripe2Handle:mStripe1Handle, 0, SEEK_SET) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Open the parity file + std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (2 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + mParityHandle = ::open(parityFilename.c_str(), + O_RDONLY | O_BINARY, 0555); + if(mParityHandle == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Work out whether or not there's a size XORed into the last block + unsigned int bytesInLastTwoBlocks = mFileSize % (mBlockSize * 2); + if(bytesInLastTwoBlocks > mBlockSize && bytesInLastTwoBlocks < ((mBlockSize * 2) - sizeof(FileSizeType))) + { + // Yes, there's something to XOR in the last block + mLastBlockHasSize = true; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::ReadRecovered(const void *, int) +// Purpose: Reads data recreating from the parity stripe +// Created: 2003/07/14 +// +// -------------------------------------------------------------------------- +int RaidFileRead_Raid::ReadRecovered(void *pBuffer, int NBytes) +{ + // Note: NBytes has been adjusted to definately be a range + // inside the given file length. + + // Make sure a buffer is allocated + if(mRecoveryBuffer == 0) + { + mRecoveryBuffer = (char*)::malloc(mBlockSize * 2); + if(mRecoveryBuffer == 0) + { + throw std::bad_alloc(); + } + } + + // Which stripe? + int stripe = (mStripe1Handle != -1)?mStripe1Handle:mStripe2Handle; + if(stripe == -1) + { + // Not enough file handles around + THROW_EXCEPTION(RaidFileException, FileIsDamagedNotRecoverable) + } + + char *outptr = (char*)pBuffer; + int bytesToGo = NBytes; + + pos_type preservedCurrentPosition = mCurrentPosition; + + try + { + // Start offset within buffer + int offset = (mCurrentPosition - mRecoveryBufferStart); + // Let's go! + while(bytesToGo > 0) + { + int bytesLeftInBuffer = 0; + if(mRecoveryBufferStart != -1) + { + bytesLeftInBuffer = (mRecoveryBufferStart + (mBlockSize*2)) - mCurrentPosition; + ASSERT(bytesLeftInBuffer >= 0); + } + + // How many bytes can be copied out? + int toCopy = bytesLeftInBuffer; + if(toCopy > bytesToGo) toCopy = bytesToGo; + //printf("offset = %d, tocopy = %d, bytestogo = %d, leftinbuffer = %d\n", (int)offset, toCopy, bytesToGo, bytesLeftInBuffer); + if(toCopy > 0) + { + for(int l = 0; l < toCopy; ++l) + { + *(outptr++) = mRecoveryBuffer[offset++]; + } + bytesToGo -= toCopy; + mCurrentPosition += toCopy; + } + + // Load in the next buffer? + if(bytesToGo > 0) + { + // Calculate the blocks within the file that are needed to be loaded. + pos_type fileBlock = mCurrentPosition / (mBlockSize * 2); + // Is this the last block + bool isLastBlock = (fileBlock == (mFileSize / (mBlockSize * 2))); + + // Need to reposition file pointers? + if(mRecoveryBufferStart == -1) + { + // Yes! + // And the offset from which to read it + pos_type filePos = fileBlock * mBlockSize; + // Then seek + if(::lseek(stripe, filePos, SEEK_SET) == -1 + || ::lseek(mParityHandle, filePos, SEEK_SET) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + // Load a block from each file, getting the ordering the right way round + int r1 = ::read((mStripe1Handle != -1)?stripe:mParityHandle, mRecoveryBuffer, mBlockSize); + int r2 = ::read((mStripe1Handle != -1)?mParityHandle:stripe, mRecoveryBuffer + mBlockSize, mBlockSize); + if(r1 == -1 || r2 == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // error checking and manipulation + if(isLastBlock) + { + // Allow not full reads, and append zeros if necessary to fill the space. + int r1zeros = mBlockSize - r1; + if(r1zeros > 0) + { + ::memset(mRecoveryBuffer + r1, 0, r1zeros); + } + int r2zeros = mBlockSize - r2; + if(r2zeros > 0) + { + ::memset(mRecoveryBuffer + mBlockSize + r2, 0, r2zeros); + } + + // if it's got the file size in it, XOR it off + if(mLastBlockHasSize) + { + int sizeXorOffset = (mBlockSize - sizeof(FileSizeType)) + ((mStripe1Handle != -1)?mBlockSize:0); + *((FileSizeType*)(mRecoveryBuffer + sizeXorOffset)) ^= box_ntoh64(mFileSize); + } + } + else + { + // Must have got a full block, otherwise things are a bit bad here. + if(r1 != (int)mBlockSize || r2 != (int)mBlockSize) + { + THROW_EXCEPTION(RaidFileException, InvalidRaidFile) + } + } + + // Go XORing! + unsigned int *b1 = (unsigned int*)mRecoveryBuffer; + unsigned int *b2 = (unsigned int *)(mRecoveryBuffer + mBlockSize); + if((mStripe1Handle == -1)) + { + b1 = b2; + b2 = (unsigned int*)mRecoveryBuffer; + } + for(int x = ((mBlockSize/sizeof(unsigned int)) - 1); x >= 0; --x) + { + *b2 = (*b1) ^ (*b2); + ++b1; + ++b2; + } + + // New block location + mRecoveryBufferStart = fileBlock * (mBlockSize * 2); + + // New offset withing block + offset = (mCurrentPosition - mRecoveryBufferStart); + ASSERT(offset >= 0); + } + } + } + catch(...) + { + // Change variables so 1) buffer is invalidated and 2) the file will be seeked properly the next time round + mRecoveryBufferStart = -1; + mCurrentPosition = preservedCurrentPosition; + throw; + } + + return NBytes; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::GetPosition() +// Purpose: Returns current position +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileRead_Raid::GetPosition() const +{ + return mCurrentPosition; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::Seek(RaidFileRead::pos_type, bool) +// Purpose: Seek within the file +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +void RaidFileRead_Raid::Seek(IOStream::pos_type Offset, int SeekType) +{ + pos_type newpos = mCurrentPosition; + switch(SeekType) + { + case IOStream::SeekType_Absolute: + newpos = Offset; + break; + + case IOStream::SeekType_Relative: + newpos += Offset; + break; + + case IOStream::SeekType_End: + newpos = mFileSize + Offset; + break; + + default: + THROW_EXCEPTION(CommonException, IOStreamBadSeekType) + } + + if(newpos != mCurrentPosition) + { + SetPosition(newpos); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::SetPosition(pos_type) +// Purpose: Move the file pointers +// Created: 2003/07/14 +// +// -------------------------------------------------------------------------- +void RaidFileRead_Raid::SetPosition(pos_type FilePosition) +{ + if(FilePosition > mFileSize) + { + FilePosition = mFileSize; + } + + if(mStripe1Handle != -1 && mStripe2Handle != -1) + { + // right then... which block is it in? + pos_type block = FilePosition / mBlockSize; + pos_type offset = FilePosition % mBlockSize; + + // Calculate offsets for each file + pos_type basepos = (block / 2) * mBlockSize; + pos_type s1p, s2p; + if((block & 1) == 0) + { + s1p = basepos + offset; + s2p = basepos; + } + else + { + s1p = basepos + mBlockSize; + s2p = basepos + offset; + } + // Note: lseek isn't in the man pages to return EIO, but assuming that it can return this, + // as it calls various OS bits and returns their error codes, and those fns look like they might. + if(::lseek(mStripe1Handle, s1p, SEEK_SET) == -1) + { + if(errno == EIO) + { + BOX_ERROR("I/O error when seeking in " << + mSetNumber << " " << mFilename << + " (to " << FilePosition << "), " << + "stripe 1"); + // Attempt to recover + AttemptToRecoverFromIOError(true /* is stripe 1 */); + ASSERT(mStripe1Handle == -1); + // Retry + SetPosition(FilePosition); + return; + } + else + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + if(::lseek(mStripe2Handle, s2p, SEEK_SET) == -1) + { + if(errno == EIO) + { + BOX_ERROR("I/O error when seeking in " << + mSetNumber << " " << mFilename << + " (to " << FilePosition << "), " << + "stripe 2"); + // Attempt to recover + AttemptToRecoverFromIOError(false /* is stripe 2 */); + ASSERT(mStripe2Handle == -1); + // Retry + SetPosition(FilePosition); + return; + } + else + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + // Store position + mCurrentPosition = FilePosition; + } + else + { + // Simply store, and mark the recovery buffer invalid + mCurrentPosition = FilePosition; + mRecoveryBufferStart = -1; + } + + // not EOF any more + mEOF = false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::Close() +// Purpose: Close the file (automatically done by destructor) +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +void RaidFileRead_Raid::Close() +{ + if(mStripe1Handle != -1) + { + ::close(mStripe1Handle); + mStripe1Handle = -1; + } + if(mStripe2Handle != -1) + { + ::close(mStripe2Handle); + mStripe2Handle = -1; + } + if(mParityHandle != -1) + { + ::close(mParityHandle); + mParityHandle = -1; + } + + mEOF = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_NonRaid::StreamDataLeft() +// Purpose: Any data left? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool RaidFileRead_Raid::StreamDataLeft() +{ + return !mEOF; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead_Raid::GetFileSize() +// Purpose: Returns file size. +// Created: 2003/07/14 +// +// -------------------------------------------------------------------------- +RaidFileRead::pos_type RaidFileRead_Raid::GetFileSize() const +{ + return mFileSize; +} + + +// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::RaidFileRead(int, const std::string &) +// Purpose: Constructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead::RaidFileRead(int SetNumber, const std::string &Filename) + : mSetNumber(SetNumber), + mFilename(Filename) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::~RaidFileRead() +// Purpose: Destructor +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +RaidFileRead::~RaidFileRead() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::Open(int, const std::string &, int) +// Purpose: Opens a RaidFile for reading. +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string &Filename, int64_t *pRevisionID, int BufferSizeHint) +{ + // See what's available... + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size() && 1 != rdiscSet.size()) // allow non-RAID configurations + { + THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet) + } + + // See if the file exists + int startDisc = 0, existingFiles = 0; + RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, Filename, &startDisc, &existingFiles, pRevisionID); + if(existance == RaidFileUtil::NoFile) + { + BOX_ERROR("Expected raidfile " << Filename << " does not exist"); + THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist) + } + else if(existance == RaidFileUtil::NonRaid) + { + // Simple non-RAID file so far... + + // Get the filename for the write file + std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, Filename)); + + // Attempt to open + int osFileHandle = ::open(writeFilename.c_str(), + O_RDONLY | O_BINARY, 0); + if(osFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, ErrorOpeningFileForRead) + } + + // Return a read object for this file + try + { + return std::auto_ptr<RaidFileRead>(new RaidFileRead_NonRaid(SetNumber, Filename, osFileHandle)); + } + catch(...) + { + ::close(osFileHandle); + throw; + } + } + else if(existance == RaidFileUtil::AsRaid + || ((existingFiles & RaidFileUtil::Stripe1Exists) && (existingFiles & RaidFileUtil::Stripe2Exists))) + { + if(existance != RaidFileUtil::AsRaid) + { + BOX_ERROR("Opening " << SetNumber << " " << + Filename << " in normal mode, but " + "parity file doesn't exist"); + // TODO: Alert recovery daemon + } + + // Open the two stripe files + std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (0 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (1 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + int stripe1 = -1; + int stripe1errno = 0; + int stripe2 = -1; + int stripe2errno = 0; + + try + { + // Open stripe1 + stripe1 = ::open(stripe1Filename.c_str(), + O_RDONLY | O_BINARY, 0555); + if(stripe1 == -1) + { + stripe1errno = errno; + } + // Open stripe2 + stripe2 = ::open(stripe2Filename.c_str(), + O_RDONLY | O_BINARY, 0555); + if(stripe2 == -1) + { + stripe2errno = errno; + } + if(stripe1errno != 0 || stripe2errno != 0) + { + THROW_EXCEPTION(RaidFileException, ErrorOpeningFileForRead) + } + + // stat stripe 1 to find ('half' of) length... + struct stat st; + if(::fstat(stripe1, &st) != 0) + { + stripe1errno = errno; + } + pos_type length = st.st_size; + + // stat stripe2 to find (other 'half' of) length... + if(::fstat(stripe2, &st) != 0) + { + stripe2errno = errno; + } + length += st.st_size; + + // Handle errors + if(stripe1errno != 0 || stripe2errno != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Make a nice object to represent this file + return std::auto_ptr<RaidFileRead>(new RaidFileRead_Raid(SetNumber, Filename, stripe1, stripe2, -1, length, rdiscSet.GetBlockSize(), false /* actually we don't know */)); + } + catch(...) + { + // Close open files + if(stripe1 != -1) + { + ::close(stripe1); + stripe1 = -1; + } + if(stripe2 != -1) + { + ::close(stripe2); + stripe2 = -1; + } + + // Now... maybe we can try again with one less file? + bool oktotryagain = true; + if(stripe1errno == EIO) + { + BOX_ERROR("I/O error on opening " << + SetNumber << " " << Filename << + " stripe 1, trying recovery mode"); + RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, true /* is stripe 1 */); + + existingFiles = existingFiles & ~RaidFileUtil::Stripe1Exists; + existance = (existance == RaidFileUtil::AsRaidWithMissingReadable) + ?RaidFileUtil::AsRaidWithMissingNotRecoverable + :RaidFileUtil::AsRaidWithMissingReadable; + } + else if(stripe1errno != 0) + { + oktotryagain = false; + } + + if(stripe2errno == EIO) + { + BOX_ERROR("I/O error on opening " << + SetNumber << " " << Filename << + " stripe 2, trying recovery mode"); + RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, false /* is stripe 2 */); + + existingFiles = existingFiles & ~RaidFileUtil::Stripe2Exists; + existance = (existance == RaidFileUtil::AsRaidWithMissingReadable) + ?RaidFileUtil::AsRaidWithMissingNotRecoverable + :RaidFileUtil::AsRaidWithMissingReadable; + } + else if(stripe2errno != 0) + { + oktotryagain = false; + } + + if(!oktotryagain) + { + throw; + } + } + } + + if(existance == RaidFileUtil::AsRaidWithMissingReadable) + { + BOX_ERROR("Attempting to open RAID file " << SetNumber << + " " << Filename << " in recovery mode (stripe " << + ((existingFiles & RaidFileUtil::Stripe1Exists)?1:2) << + " present)"); + + // Generate the filenames of all the lovely files + std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (0 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (1 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (2 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); + + int stripe1 = -1; + int stripe2 = -1; + int parity = -1; + + try + { + // Open stripe1? + if(existingFiles & RaidFileUtil::Stripe1Exists) + { + stripe1 = ::open(stripe1Filename.c_str(), + O_RDONLY | O_BINARY, 0555); + if(stripe1 == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + // Open stripe2? + if(existingFiles & RaidFileUtil::Stripe2Exists) + { + stripe2 = ::open(stripe2Filename.c_str(), + O_RDONLY | O_BINARY, 0555); + if(stripe2 == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + // Open parity + parity = ::open(parityFilename.c_str(), + O_RDONLY | O_BINARY, 0555); + if(parity == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Find the length. This is slightly complex. + unsigned int blockSize = rdiscSet.GetBlockSize(); + pos_type length = 0; + + // The easy one... if the parity file is of an integral block size + sizeof(FileSizeType) + // then it's stored at the end of the parity file + struct stat st; + if(::fstat(parity, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + pos_type paritySize = st.st_size; + FileSizeType parityLastData = 0; + bool parityIntegralPlusOffT = ((paritySize % blockSize) == sizeof(FileSizeType)); + if(paritySize >= static_cast<pos_type>(sizeof(parityLastData)) && (parityIntegralPlusOffT || stripe1 != -1)) + { + // Seek to near the end + ASSERT(sizeof(FileSizeType) == 8); // compiler bug (I think) prevents from using 0 - sizeof(FileSizeType)... + if(::lseek(parity, -8 /*(0 - sizeof(FileSizeType))*/, SEEK_END) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Read it in + if(::read(parity, &parityLastData, sizeof(parityLastData)) != sizeof(parityLastData)) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Set back to beginning of file + if(::lseek(parity, 0, SEEK_SET) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + bool lastBlockHasSize = false; + if(parityIntegralPlusOffT) + { + // Wonderful! Have the value + length = box_ntoh64(parityLastData); + } + else + { + // Have to resort to more devious means. + if(existingFiles & RaidFileUtil::Stripe1Exists) + { + // Procedure for stripe 1 existence... + // Get size of stripe1. + // If this is not an integral block size, then size can use this + // to work out the size of the file. + // Otherwise, read in the end of the last block, and use a bit of XORing + // to get the size from the FileSizeType value at end of the file. + if(::fstat(stripe1, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + pos_type stripe1Size = st.st_size; + // Is size integral? + if((stripe1Size % ((pos_type)blockSize)) != 0) + { + // No, so know the size. + length = stripe1Size + ((stripe1Size / blockSize) * blockSize); + } + else + { + // Must read the last bit of data from the block and XOR. + FileSizeType stripe1LastData = 0; // initialise to zero, as we may not read everything from it + + // Work out how many bytes to read + int btr = 0; // bytes to read off end + unsigned int lbs = stripe1Size % blockSize; + if(lbs == 0 && stripe1Size > 0) + { + // integral size, need the entire bit + btr = sizeof(FileSizeType); + } + else if(lbs > (blockSize - sizeof(FileSizeType))) + { + btr = lbs - (blockSize - sizeof(FileSizeType)); + } + + // Seek to near the end + if(btr > 0) + { + ASSERT(sizeof(FileSizeType) == 8); // compiler bug (I think) prevents from using 0 - sizeof(FileSizeType)... + ASSERT(btr <= (int)sizeof(FileSizeType)); + if(::lseek(stripe1, 0 - btr, SEEK_END) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Read it in + if(::read(stripe1, &stripe1LastData, btr) != btr) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + // Set back to beginning of file + if(::lseek(stripe1, 0, SEEK_SET) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + // Lovely! + length = stripe1LastData ^ parityLastData; + // Convert to host byte order + length = box_ntoh64(length); + ASSERT(length <= (paritySize + stripe1Size)); + // Mark is as having this to aid code later + lastBlockHasSize = true; + } + } + else + { + ASSERT(existingFiles & RaidFileUtil::Stripe2Exists); + } + + if(existingFiles & RaidFileUtil::Stripe2Exists) + { + // Get size of stripe2 file + if(::fstat(stripe2, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + pos_type stripe2Size = st.st_size; + + // Is it an integral size? + if(stripe2Size % blockSize != 0) + { + // No. Working out the size is easy. + length = stripe2Size + (((stripe2Size / blockSize)+1) * blockSize); + // Got last block size in there? + if((stripe2Size % blockSize) <= static_cast<pos_type>((blockSize - sizeof(pos_type)))) + { + // Yes... + lastBlockHasSize = true; + } + } + else + { + // Yes. So we need to compare with the parity file to get a clue... + pos_type stripe2Blocks = stripe2Size / blockSize; + pos_type parityBlocks = paritySize / blockSize; + if(stripe2Blocks == parityBlocks) + { + // Same size, so stripe1 must be the same size + length = (stripe2Blocks * 2) * blockSize; + } + else + { + // Different size, so stripe1 must be one block bigger + ASSERT(stripe2Blocks < parityBlocks); + length = ((stripe2Blocks * 2)+1) * blockSize; + } + + // Then... add in the extra bit of the parity length + unsigned int lastBlockSize = paritySize % blockSize; + length += lastBlockSize; + } + } + else + { + ASSERT(existingFiles & RaidFileUtil::Stripe1Exists); + } + } + + // Create a lovely object to return + return std::auto_ptr<RaidFileRead>(new RaidFileRead_Raid(SetNumber, Filename, stripe1, stripe2, parity, length, blockSize, lastBlockHasSize)); + } + catch(...) + { + // Close open files + if(stripe1 != -1) + { + ::close(stripe1); + stripe1 = -1; + } + if(stripe2 != -1) + { + ::close(stripe2); + stripe2 = -1; + } + if(parity != -1) + { + ::close(parity); + parity = -1; + } + throw; + } + } + + THROW_EXCEPTION(RaidFileException, FileIsDamagedNotRecoverable) + + // Avoid compiler warning -- it'll never get here... + return std::auto_ptr<RaidFileRead>(); +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::DirectoryExists(int, const std::string &) +// Purpose: Returns true if the directory exists. Throws exception if it's partially in existence. +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +bool RaidFileRead::DirectoryExists(int SetNumber, const std::string &rDirName) +{ + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + + return DirectoryExists(rdiscSet, rDirName); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::DirectoryExists(const RaidFileDiscSet &, const std::string &) +// Purpose: Returns true if the directory exists. Throws exception if it's partially in existence. +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +bool RaidFileRead::DirectoryExists(const RaidFileDiscSet &rSet, const std::string &rDirName) +{ + // For each directory, test to see if it exists + unsigned int nexist = 0; + for(unsigned int l = 0; l < rSet.size(); ++l) + { + // build name + std::string dn(rSet[l] + DIRECTORY_SEPARATOR + rDirName); + + // check for existence + struct stat st; + if(::stat(dn.c_str(), &st) == 0) + { + // Directory? + if(st.st_mode & S_IFDIR) + { + // yes + nexist++; + } + else + { + // No. It's a file. Bad! + THROW_EXCEPTION(RaidFileException, UnexpectedFileInDirPlace) + } + } + else + { + // Was it a non-exist error? + if(errno != ENOENT) + { + // No. Bad things. + THROW_EXCEPTION(RaidFileException, OSError) + } + } + } + + // Were all of them found? + if(nexist == 0) + { + // None. + return false; + } + else if(nexist == rSet.size()) + { + // All + return true; + } + + // Some exist. We don't like this -- it shows something bad happened before + // TODO: notify recovery daemon + THROW_EXCEPTION(RaidFileException, DirectoryIncomplete) + return false; // avoid compiler warning +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::FileExists(int, const std::string &, int64_t *) +// Purpose: Does a Raid file exist? Optionally return a revision number, which is +// unique to this saving of the file. (revision number may change +// after transformation to RAID -- so only use for cache control, +// not detecting changes to content). +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +bool RaidFileRead::FileExists(int SetNumber, const std::string &rFilename, int64_t *pRevisionID) +{ + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + + return RaidFileUtil::RaidFileExists(rdiscSet, rFilename, 0, 0, pRevisionID) != RaidFileUtil::NoFile; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadDirectoryContents(int, const std::string &, int, std::vector<std::string> &) +// Purpose: Read directory contents, returning whether or not all entries are likely to be readable or not +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector<std::string> &rOutput) +{ + // Remove anything in the vector to begin with. + rOutput.clear(); + + // Controller and set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + + // Collect the directory listings + std::map<std::string, unsigned int> counts; + + unsigned int numDiscs = rdiscSet.size(); + + for(unsigned int l = 0; l < numDiscs; ++l) + { + // build name + std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName); + + // read the contents... + DIR *dirHandle = 0; + try + { + dirHandle = ::opendir(dn.c_str()); + if(dirHandle == 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + struct dirent *en = 0; + while((en = ::readdir(dirHandle)) != 0) + { + if(en->d_name[0] == '.' && + (en->d_name[1] == '\0' || (en->d_name[1] == '.' && en->d_name[2] == '\0'))) + { + // ignore, it's . or .. + continue; + } + + // Entry... + std::string name; + unsigned int countToAdd = 1; + + // stat the file to find out what type it is +#ifdef HAVE_VALID_DIRENT_D_TYPE + if(DirReadType == DirReadType_FilesOnly && en->d_type == DT_REG) +#else + struct stat st; + std::string fullName(dn + DIRECTORY_SEPARATOR + en->d_name); + if(::lstat(fullName.c_str(), &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + if(DirReadType == DirReadType_FilesOnly && (st.st_mode & S_IFDIR) == 0) +#endif + { + // File. Complex, need to check the extension + int dot = -1; + int p = 0; + while(en->d_name[p] != '\0') + { + if(en->d_name[p] == '.') + { + // store location of dot + dot = p; + } + ++p; + } + // p is length of string + if(dot != -1 && ((p - dot) == 3 || (p - dot) == 4) + && en->d_name[dot+1] == 'r' && en->d_name[dot+2] == 'f' + && (en->d_name[dot+3] == 'w' || en->d_name[dot+3] == '\0')) + { + // so has right extension + name.assign(en->d_name, dot); /* get name up to last . */ + // Was it a write file (which counts as everything) + if(en->d_name[dot+3] == 'w') + { + countToAdd = numDiscs; + } + } + } +#ifdef HAVE_VALID_DIRENT_D_TYPE + if(DirReadType == DirReadType_DirsOnly && en->d_type == DT_DIR) +#else + if(DirReadType == DirReadType_DirsOnly && (st.st_mode & S_IFDIR)) +#endif + { + // Directory, and we want directories + name = en->d_name; + } + // Eligable for entry? + if(!name.empty()) + { + // add to map... + std::map<std::string, unsigned int>::iterator i = counts.find(name); + if(i != counts.end()) + { + // add to count + i->second += countToAdd; + } + else + { + // insert into map + counts[name] = countToAdd; + } + } + } + + if(::closedir(dirHandle) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + dirHandle = 0; + } + catch(...) + { + if(dirHandle != 0) + { + ::closedir(dirHandle); + } + throw; + } + } + + // Now go through the map, adding in entries + bool everythingReadable = true; + + for(std::map<std::string, unsigned int>::const_iterator i = counts.begin(); i != counts.end(); ++i) + { + if(i->second < (numDiscs - 1)) + { + // Too few discs to be confident of reading everything + everythingReadable = false; + } + + // Add name to vector + rOutput.push_back(i->first); + } + + return everythingReadable; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::Write(const void *, int) +// Purpose: Not support, throws exception +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void RaidFileRead::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(RaidFileException, UnsupportedReadWriteOrClose) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::StreamClosed() +// Purpose: Never any data to write +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool RaidFileRead::StreamClosed() +{ + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::BytesLeftToRead() +// Purpose: Can tell how many bytes there are to go +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileRead::BytesLeftToRead() +{ + return GetFileSize() - GetPosition(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileRead::GetDiscUsageInBlocks() +// Purpose: Return how many blocks are used. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileRead::GetDiscUsageInBlocks() +{ + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + return RaidFileUtil::DiscUsageInBlocks(GetFileSize(), rdiscSet); +} + + + + diff --git a/lib/raidfile/RaidFileRead.h b/lib/raidfile/RaidFileRead.h new file mode 100644 index 00000000..8a04409d --- /dev/null +++ b/lib/raidfile/RaidFileRead.h @@ -0,0 +1,73 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileRead.h +// Purpose: Read Raid like Files +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- + +#ifndef RAIDFILEREAD__H +#define RAIDFILEREAD__H + +#include <cstring> +#include <cstdlib> +#include <memory> +#include <vector> + +#include "IOStream.h" + +class RaidFileDiscSet; + + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileRead +// Purpose: Read RAID like files +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +class RaidFileRead : public IOStream +{ +protected: + RaidFileRead(int SetNumber, const std::string &Filename); +public: + virtual ~RaidFileRead(); +private: + RaidFileRead(const RaidFileRead &rToCopy); + +public: + // Open a raid file + static std::auto_ptr<RaidFileRead> Open(int SetNumber, const std::string &Filename, int64_t *pRevisionID = 0, int BufferSizeHint = 4096); + + // Extra info + virtual pos_type GetFileSize() const = 0; + + // Utility functions + static bool FileExists(int SetNumber, const std::string &rFilename, int64_t *pRevisionID = 0); + static bool DirectoryExists(const RaidFileDiscSet &rSet, const std::string &rDirName); + static bool DirectoryExists(int SetNumber, const std::string &rDirName); + enum + { + DirReadType_FilesOnly = 0, + DirReadType_DirsOnly = 1 + }; + static bool ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector<std::string> &rOutput); + + // Common IOStream interface implementation + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamClosed(); + virtual pos_type BytesLeftToRead(); + + pos_type GetDiscUsageInBlocks(); + + typedef int64_t FileSizeType; + +protected: + int mSetNumber; + std::string mFilename; +}; + +#endif // RAIDFILEREAD__H + diff --git a/lib/raidfile/RaidFileUtil.cpp b/lib/raidfile/RaidFileUtil.cpp new file mode 100644 index 00000000..7c6299ec --- /dev/null +++ b/lib/raidfile/RaidFileUtil.cpp @@ -0,0 +1,210 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileUtil.cpp +// Purpose: Utilities for raid files +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> +#include <sys/stat.h> + +#include "RaidFileUtil.h" +#include "FileModificationTime.h" +#include "RaidFileRead.h" // for type definition + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileUtil::RaidFileExists(RaidFileDiscSet &, +// const std::string &, int *, int *, int64_t *) +// Purpose: Check to see the state of a RaidFile on disc +// (doesn't look at contents, just at existence of +// files) +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- +RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, + const std::string &rFilename, int *pStartDisc, int *pExistingFiles, + int64_t *pRevisionID) +{ + if(pExistingFiles) + { + *pExistingFiles = 0; + } + + // For stat call, although the results are not examined + struct stat st; + + // check various files + int startDisc = 0; + { + std::string writeFile(RaidFileUtil::MakeWriteFileName(rDiscSet, rFilename, &startDisc)); + if(pStartDisc) + { + *pStartDisc = startDisc; + } + if(::stat(writeFile.c_str(), &st) == 0) + { + // write file exists, use that + + // Get unique ID + if(pRevisionID != 0) + { + #ifdef WIN32 + *pRevisionID = st.st_mtime; + #else + *pRevisionID = FileModificationTime(st); + #endif + +#ifdef BOX_RELEASE_BUILD + // The resolution of timestamps may be very + // low, e.g. 1 second. So add the size to it + // to give a bit more chance of it changing. + // TODO: Make this better. + // Disabled in debug mode, to simulate + // filesystem with 1-second timestamp + // resolution, e.g. MacOS X HFS, Linux. + (*pRevisionID) += st.st_size; +#endif + } + + // return non-raid file + return NonRaid; + } + } + + // Now see how many of the raid components exist + int64_t revisionID = 0; + int setSize = rDiscSet.size(); + int rfCount = 0; + + // TODO: replace this with better linux revision ID detection + int64_t revisionIDplus = 0; + + for(int f = 0; f < setSize; ++f) + { + std::string componentFile(RaidFileUtil::MakeRaidComponentName(rDiscSet, rFilename, (f + startDisc) % setSize)); + if(::stat(componentFile.c_str(), &st) == 0) + { + // Component file exists, add to count + rfCount++; + // Set flags for existance? + if(pExistingFiles) + { + (*pExistingFiles) |= (1 << f); + } + // Revision ID + if(pRevisionID != 0) + { + #ifdef WIN32 + int64_t rid = st.st_mtime; + #else + int64_t rid = FileModificationTime(st); + #endif + + if(rid > revisionID) revisionID = rid; + revisionIDplus += st.st_size; + } + } + } + if(pRevisionID != 0) + { + (*pRevisionID) = revisionID; +#ifdef BOX_RELEASE_BUILD + // The resolution of timestamps may be very low, e.g. + // 1 second. So add the size to it to give a bit more + // chance of it changing. + // TODO: Make this better. + // Disabled in debug mode, to simulate filesystem with + // 1-second timestamp resolution, e.g. MacOS X HFS, Linux. + (*pRevisionID) += revisionIDplus; +#endif + } + + // Return a status based on how many parts are available + if(rfCount == setSize) + { + return AsRaid; + } + else if((setSize > 1) && rfCount == (setSize - 1)) + { + return AsRaidWithMissingReadable; + } + else if(rfCount > 0) + { + return AsRaidWithMissingNotRecoverable; + } + + return NoFile; // Obviously doesn't exist +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileUtil::DiscUsageInBlocks(int64_t, const RaidFileDiscSet &) +// Purpose: Returns the size of the file in blocks, given the file size and disc set +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +int64_t RaidFileUtil::DiscUsageInBlocks(int64_t FileSize, const RaidFileDiscSet &rDiscSet) +{ + // Get block size + int blockSize = rDiscSet.GetBlockSize(); + + // OK... so as the size of the file is always sizes of stripe1 + stripe2, we can + // do a very simple calculation for the main data. + int64_t blocks = (FileSize + (((int64_t)blockSize) - 1)) / ((int64_t)blockSize); + + // It's just that simple calculation for non-RAID disc sets + if(rDiscSet.IsNonRaidSet()) + { + return blocks; + } + + // It's the parity which is mildly complex. + // First of all, add in size for all but the last two blocks. + int64_t parityblocks = (FileSize / ((int64_t)blockSize)) / 2; + blocks += parityblocks; + + // Work out how many bytes are left + int bytesOver = (int)(FileSize - (parityblocks * ((int64_t)(blockSize*2)))); + + // Then... (let compiler optimise this out) + if(bytesOver == 0) + { + // Extra block for the size info + blocks++; + } + else if(bytesOver == sizeof(RaidFileRead::FileSizeType)) + { + // For last block of parity, plus the size info + blocks += 2; + } + else if(bytesOver < blockSize) + { + // Just want the parity block + blocks += 1; + } + else if(bytesOver == blockSize || bytesOver >= ((blockSize*2)-((int)sizeof(RaidFileRead::FileSizeType)))) + { + // Last block, plus size info + blocks += 2; + } + else + { + // Just want parity block + blocks += 1; + } + + return blocks; +} + + diff --git a/lib/raidfile/RaidFileUtil.h b/lib/raidfile/RaidFileUtil.h new file mode 100644 index 00000000..a581047c --- /dev/null +++ b/lib/raidfile/RaidFileUtil.h @@ -0,0 +1,97 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileUtil.h +// Purpose: Utilities for the raid file classes +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- + +#ifndef RAIDFILEUTIL__H +#define RAIDFILEUTIL__H + +#include "RaidFileController.h" +#include "RaidFileException.h" + +// note: these are hardcoded into the directory searching code +#define RAIDFILE_EXTENSION ".rf" +#define RAIDFILE_WRITE_EXTENSION ".rfw" + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileUtil +// Purpose: Utility functions for RaidFile classes +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- +class RaidFileUtil +{ +public: + typedef enum + { + NoFile = 0, + NonRaid = 1, + AsRaid = 2, + AsRaidWithMissingReadable = 3, + AsRaidWithMissingNotRecoverable = 4 + } ExistType; + + enum + { + Stripe1Exists = 1, + Stripe2Exists = 2, + ParityExists = 4 + }; + + static ExistType RaidFileExists(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pStartDisc = 0, int *pExisitingFiles = 0, int64_t *pRevisionID = 0); + + static int64_t DiscUsageInBlocks(int64_t FileSize, const RaidFileDiscSet &rDiscSet); + + // -------------------------------------------------------------------------- + // + // Function + // Name: std::string MakeRaidComponentName(RaidFileDiscSet &, const std::string &, int) + // Purpose: Returns the OS filename for a file of part of a disc set + // Created: 2003/07/11 + // + // -------------------------------------------------------------------------- + static inline std::string MakeRaidComponentName(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int Disc) + { + if(Disc < 0 || Disc >= (int)rDiscSet.size()) + { + THROW_EXCEPTION(RaidFileException, NoSuchDiscSet) + } + std::string r(rDiscSet[Disc]); + r += DIRECTORY_SEPARATOR_ASCHAR; + r += rFilename; + r += RAIDFILE_EXTENSION; + return r; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: std::string MakeWriteFileName(RaidFileDiscSet &, const std::string &) + // Purpose: Returns the OS filename for the temporary write file + // Created: 2003/07/11 + // + // -------------------------------------------------------------------------- + static inline std::string MakeWriteFileName(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pOnDiscSet = 0) + { + int livesOnSet = rDiscSet.GetSetNumForWriteFiles(rFilename); + + // does the caller want to know which set it's on? + if(pOnDiscSet) *pOnDiscSet = livesOnSet; + + // Make the string + std::string r(rDiscSet[livesOnSet]); + r += DIRECTORY_SEPARATOR_ASCHAR; + r += rFilename; + r += RAIDFILE_WRITE_EXTENSION; + return r; + } +}; + +#endif // RAIDFILEUTIL__H + diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp new file mode 100644 index 00000000..f24c2422 --- /dev/null +++ b/lib/raidfile/RaidFileWrite.cpp @@ -0,0 +1,930 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileWrite.cpp +// Purpose: Writing RAID like files +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <errno.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <sys/file.h> + +#include <stdio.h> +#include <string.h> + +#include "Guards.h" +#include "RaidFileWrite.h" +#include "RaidFileController.h" +#include "RaidFileException.h" +#include "RaidFileUtil.h" +#include "Utils.h" +// For DirectoryExists fn +#include "RaidFileRead.h" + +#include "MemLeakFindOn.h" + +// should be a multiple of 2 +#define TRANSFORM_BLOCKS_TO_LOAD 4 +// Must have this number of discs in the set +#define TRANSFORM_NUMBER_DISCS_REQUIRED 3 + +// we want to use POSIX fstat() for now, not the emulated one +#undef fstat + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::RaidFileWrite(int, const std::string &) +// Purpose: Simple constructor, just stores required details +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +RaidFileWrite::RaidFileWrite(int SetNumber, const std::string &Filename) + : mSetNumber(SetNumber), + mFilename(Filename), + mOSFileHandle(-1), // not valid file handle + mRefCount(-1) // unknown refcount +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::RaidFileWrite(int, +// const std::string &, int refcount) +// Purpose: Constructor with check for overwriting file +// with multiple references +// Created: 2009/07/05 +// +// -------------------------------------------------------------------------- +RaidFileWrite::RaidFileWrite(int SetNumber, const std::string &Filename, + int refcount) + : mSetNumber(SetNumber), + mFilename(Filename), + mOSFileHandle(-1), // not valid file handle + mRefCount(refcount) +{ + // Can't check for zero refcount here, because it's legal + // to create a RaidFileWrite to delete an object with zero refcount. + // Check in Commit() and Delete() instead. + if (refcount > 1) + { + BOX_ERROR("Attempted to modify object " << mFilename << + ", which has " << refcount << " references"); + THROW_EXCEPTION(RaidFileException, + RequestedModifyMultiplyReferencedFile); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::~RaidFileWrite() +// Purpose: Destructor (will discard written file if not commited) +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +RaidFileWrite::~RaidFileWrite() +{ + if(mOSFileHandle != -1) + { + Discard(); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Open() +// Purpose: Opens the file for writing +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Open(bool AllowOverwrite) +{ + if(mOSFileHandle != -1) + { + THROW_EXCEPTION(RaidFileException, AlreadyOpen) + } + + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + + // Check for overwriting? (step 1) + if(!AllowOverwrite) + { + // See if the file exists already -- can't overwrite existing files + RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename); + if(existance != RaidFileUtil::NoFile) + { + BOX_ERROR("Attempted to overwrite raidfile " << + mSetNumber << " " << mFilename); + THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile) + } + } + + // Get the filename for the write file + std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename)); + // Add on a temporary extension + writeFilename += 'X'; + + // Attempt to open + mOSFileHandle = ::open(writeFilename.c_str(), + O_WRONLY | O_CREAT | O_BINARY, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + if(mOSFileHandle == -1) + { + BOX_LOG_SYS_ERROR("Failed to open file: " << writeFilename); + THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFile) + } + + // Get a lock on the write file +#ifdef HAVE_FLOCK + int errnoBlock = EWOULDBLOCK; + if(::flock(mOSFileHandle, LOCK_EX | LOCK_NB) != 0) +#elif HAVE_DECL_F_SETLK + int errnoBlock = EAGAIN; + struct flock desc; + desc.l_type = F_WRLCK; + desc.l_whence = SEEK_SET; + desc.l_start = 0; + desc.l_len = 0; + if(::fcntl(mOSFileHandle, F_SETLK, &desc) != 0) +#else + int errnoBlock = ENOSYS; + if (0) +#endif + { + // Lock was not obtained. + bool wasLocked = (errno == errnoBlock); + // Close the file + ::close(mOSFileHandle); + mOSFileHandle = -1; + // Report an exception? + if(wasLocked) + { + THROW_EXCEPTION(RaidFileException, FileIsCurrentlyOpenForWriting) + } + else + { + // Random error occured + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + // Truncate it to size zero + if(::ftruncate(mOSFileHandle, 0) != 0) + { + THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFileOnTruncate) + } + + // Done! +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Write(const void *, int) +// Purpose: Writes a block of data +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Write(const void *pBuffer, int Length) +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Write data + int written = ::write(mOSFileHandle, pBuffer, Length); + if(written != Length) + { + BOX_LOG_SYS_ERROR("RaidFileWrite failed, Length = " << + Length << ", written = " << written); + THROW_EXCEPTION(RaidFileException, OSError) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::GetPosition() +// Purpose: Returns current position in file +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileWrite::GetPosition() const +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Use lseek to find the current file position + off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR); + if(p == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + return p; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Seek(RaidFileWrite::pos_type, bool) +// Purpose: Seeks in the file, relative to current position if Relative is true. +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Seek(IOStream::pos_type SeekTo, int SeekType) +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Seek... + if(::lseek(mOSFileHandle, SeekTo, ConvertSeekTypeToOSWhence(SeekType)) == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Commit(bool) +// Purpose: Closes, and commits the written file +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Commit(bool ConvertToRaidNow) +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + if (mRefCount == 0) + { + BOX_ERROR("Attempted to modify object " << mFilename << + ", which has no references"); + THROW_EXCEPTION(RaidFileException, + RequestedModifyUnreferencedFile); + } + + // Rename it into place -- BEFORE it's closed so lock remains + +#ifdef WIN32 + // Except on Win32 which doesn't allow renaming open files + // Close file... + if(::close(mOSFileHandle) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + mOSFileHandle = -1; +#endif // WIN32 + + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + // Get the filename for the write file + std::string renameTo(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename)); + // And the current name + std::string renameFrom(renameTo + 'X'); + +#ifdef WIN32 + // need to delete the target first + if(::unlink(renameTo.c_str()) != 0 && + GetLastError() != ERROR_FILE_NOT_FOUND) + { + BOX_LOG_WIN_ERROR("Failed to delete file: " << renameTo); + THROW_EXCEPTION(RaidFileException, OSError) + } +#endif + + if(::rename(renameFrom.c_str(), renameTo.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("Failed to rename file: " << renameFrom << + " to " << renameTo); + THROW_EXCEPTION(RaidFileException, OSError) + } + +#ifndef WIN32 + // Close file... + if(::close(mOSFileHandle) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + mOSFileHandle = -1; +#endif // !WIN32 + + // Raid it? + if(ConvertToRaidNow) + { + TransformToRaidStorage(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Discard() +// Purpose: Closes, discarding the data written. +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Discard() +{ + // open? + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, NotOpen) + } + + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + + // Get the filename for the write file (temporary) + std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename)); + writeFilename += 'X'; + + // Unlink and close it + +#ifdef WIN32 + // On Win32 we must close it first + if (::close(mOSFileHandle) != 0 || + ::unlink(writeFilename.c_str()) != 0) +#else // !WIN32 + if (::unlink(writeFilename.c_str()) != 0 || + ::close(mOSFileHandle) != 0) +#endif // !WIN32 + { + BOX_LOG_SYS_ERROR("Failed to delete file: " << writeFilename); + THROW_EXCEPTION(RaidFileException, OSError) + } + + // reset file handle + mOSFileHandle = -1; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::TransformToRaidStorage() +// Purpose: Turns the file into the RAID storage form +// Created: 2003/07/11 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::TransformToRaidStorage() +{ + // open? + if(mOSFileHandle != -1) + { + THROW_EXCEPTION(RaidFileException, WriteFileOpenOnTransform) + } + + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + if(rdiscSet.IsNonRaidSet()) + { + // Not in RAID mode -- do nothing + return; + } + // Otherwise check that it's the right sized set + if(TRANSFORM_NUMBER_DISCS_REQUIRED != rdiscSet.size()) + { + THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet) + } + unsigned int blockSize = rdiscSet.GetBlockSize(); + + // Get the filename for the write file (and get the disc set name for the start disc) + int startDisc = 0; + std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename, &startDisc)); + + // Open it + FileHandleGuard<> writeFile(writeFilename.c_str()); + + // Get file information for write file + struct stat writeFileStat; + if(::fstat(writeFile, &writeFileStat) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } +// // DEBUG MODE -- check file system size block size is same as block size for files +// // doesn't really apply, as space benefits of using fragment size are worth efficiency, +// // and anyway, it'll be buffered eventually so it won't matter. +// #ifndef BOX_RELEASE_BUILD +// { +// if(writeFileStat.st_blksize != blockSize) +// { +// TRACE2("TransformToRaidStorage: optimal block size of file = %d, of set = %d, MISMATCH\n", +// writeFileStat.st_blksize, blockSize); +// } +// } +// #endif + + // How many blocks is the file? (rounding up) + int writeFileSizeInBlocks = (writeFileStat.st_size + (blockSize - 1)) / blockSize; + // And how big should the buffer be? (round up to multiple of 2, and no bigger than the preset limit) + int bufferSizeBlocks = (writeFileSizeInBlocks + 1) & ~1; + if(bufferSizeBlocks > TRANSFORM_BLOCKS_TO_LOAD) bufferSizeBlocks = TRANSFORM_BLOCKS_TO_LOAD; + // How big should the buffer be? + int bufferSize = (TRANSFORM_BLOCKS_TO_LOAD * blockSize); + + // Allocate buffer... + MemoryBlockGuard<char*> buffer(bufferSize); + + // Allocate buffer for parity file + MemoryBlockGuard<char*> parityBuffer(blockSize); + + // Get filenames of eventual files + std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 0) % TRANSFORM_NUMBER_DISCS_REQUIRED)); + std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 1) % TRANSFORM_NUMBER_DISCS_REQUIRED)); + std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 2) % TRANSFORM_NUMBER_DISCS_REQUIRED)); + // Make write equivalents + std::string stripe1FilenameW(stripe1Filename + 'P'); + std::string stripe2FilenameW(stripe2Filename + 'P'); + std::string parityFilenameW(parityFilename + 'P'); + + + // Then open them all for writing (in strict order) + try + { +#if HAVE_DECL_O_EXLOCK + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK | O_BINARY)> stripe1(stripe1FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK | O_BINARY)> stripe2(stripe2FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK | O_BINARY)> parity(parityFilenameW.c_str()); +#else + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_BINARY)> stripe1(stripe1FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_BINARY)> stripe2(stripe2FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_BINARY)> parity(parityFilenameW.c_str()); +#endif + + // Then... read in data... + int bytesRead = -1; + bool sizeRecordRequired = false; + int blocksDone = 0; + while((bytesRead = ::read(writeFile, buffer, bufferSize)) > 0) + { + // Blocks to do... + int blocksToDo = (bytesRead + (blockSize - 1)) / blockSize; + + // Need to add zeros to end? + int blocksRoundUp = (blocksToDo + 1) & ~1; + int zerosEnd = (blocksRoundUp * blockSize); + if(bytesRead != zerosEnd) + { + // Set the end of the blocks to zero + ::memset(buffer + bytesRead, 0, zerosEnd - bytesRead); + } + + // number of int's to XOR + unsigned int num = blockSize / sizeof(unsigned int); + + // Then... calculate and write parity data + for(int b = 0; b < blocksToDo; b += 2) + { + // Calculate int pointers + unsigned int *pstripe1 = (unsigned int *)(buffer + (b * blockSize)); + unsigned int *pstripe2 = (unsigned int *)(buffer + ((b+1) * blockSize)); + unsigned int *pparity = (unsigned int *)((char*)parityBuffer); + + // Do XOR + for(unsigned int n = 0; n < num; ++n) + { + pparity[n] = pstripe1[n] ^ pstripe2[n]; + } + + // Size of parity to write... + int parityWriteSize = blockSize; + + // Adjust if it's the last block + if((blocksDone + (b + 2)) >= writeFileSizeInBlocks) + { + // Yes... + unsigned int bytesInLastTwoBlocks = bytesRead - (b * blockSize); + + // Some special cases... + // Zero will never happen... but in the (imaginary) case it does, the file size will be appended + // by the test at the end. + if(bytesInLastTwoBlocks == sizeof(RaidFileRead::FileSizeType) + || bytesInLastTwoBlocks == blockSize) + { + // Write the entire block, and put the file size at end + sizeRecordRequired = true; + } + else if(bytesInLastTwoBlocks < blockSize) + { + // write only these bits + parityWriteSize = bytesInLastTwoBlocks; + } + else if(bytesInLastTwoBlocks < ((blockSize * 2) - sizeof(RaidFileRead::FileSizeType))) + { + // XOR in the size at the end of the parity block + ASSERT(sizeof(RaidFileRead::FileSizeType) == (2*sizeof(unsigned int))); + ASSERT(sizeof(RaidFileRead::FileSizeType) >= sizeof(off_t)); + int sizePos = (blockSize/sizeof(unsigned int)) - 2; + union { RaidFileRead::FileSizeType l; unsigned int i[2]; } sw; + + sw.l = box_hton64(writeFileStat.st_size); + pparity[sizePos+0] = pstripe1[sizePos+0] ^ sw.i[0]; + pparity[sizePos+1] = pstripe1[sizePos+1] ^ sw.i[1]; + } + else + { + // Write the entire block, and put the file size at end + sizeRecordRequired = true; + } + } + + // Write block + if(::write(parity, parityBuffer, parityWriteSize) != parityWriteSize) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + // Write stripes + char *writeFrom = buffer; + for(int l = 0; l < blocksToDo; ++l) + { + // Write the block + int toWrite = (l == (blocksToDo - 1)) + ?(bytesRead - ((blocksToDo-1)*blockSize)) + :blockSize; + if(::write(((l&1)==0)?stripe1:stripe2, writeFrom, toWrite) != toWrite) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Next block + writeFrom += blockSize; + } + + // Count of blocks done + blocksDone += blocksToDo; + } + // Error on read? + if(bytesRead == -1) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Special case for zero length files + if(writeFileStat.st_size == 0) + { + sizeRecordRequired = true; + } + + // Might need to write the file size to the end of the parity file + // if it can't be worked out some other means -- size is required to rebuild the file if one of the stripe files is missing + if(sizeRecordRequired) + { + ASSERT(sizeof(writeFileStat.st_size) <= sizeof(RaidFileRead::FileSizeType)); + RaidFileRead::FileSizeType sw = box_hton64(writeFileStat.st_size); + ASSERT((::lseek(parity, 0, SEEK_CUR) % blockSize) == 0); + if(::write(parity, &sw, sizeof(sw)) != sizeof(sw)) + { + BOX_LOG_SYS_ERROR("Failed to write to file: " << + writeFilename); + THROW_EXCEPTION(RaidFileException, OSError) + } + } + + // Then close the written files (note in reverse order of opening) + parity.Close(); + stripe2.Close(); + stripe1.Close(); + +#ifdef WIN32 + // Must delete before renaming + #define CHECK_UNLINK(file) \ + { \ + if (::unlink(file) != 0 && errno != ENOENT) \ + { \ + THROW_EXCEPTION(RaidFileException, OSError); \ + } \ + } + CHECK_UNLINK(stripe1Filename.c_str()); + CHECK_UNLINK(stripe2Filename.c_str()); + CHECK_UNLINK(parityFilename.c_str()); + #undef CHECK_UNLINK +#endif + + // Rename them into place + if(::rename(stripe1FilenameW.c_str(), stripe1Filename.c_str()) != 0 + || ::rename(stripe2FilenameW.c_str(), stripe2Filename.c_str()) != 0 + || ::rename(parityFilenameW.c_str(), parityFilename.c_str()) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Close the write file + writeFile.Close(); + + // Finally delete the write file + if(::unlink(writeFilename.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("Failed to delete file: " << + writeFilename); + THROW_EXCEPTION(RaidFileException, OSError) + } + } + catch(...) + { + // Unlink all the dodgy files + ::unlink(stripe1Filename.c_str()); + ::unlink(stripe2Filename.c_str()); + ::unlink(parityFilename.c_str()); + ::unlink(stripe1FilenameW.c_str()); + ::unlink(stripe2FilenameW.c_str()); + ::unlink(parityFilenameW.c_str()); + + // and send the error on its way + throw; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Delete() +// Purpose: Deletes a RAID file +// Created: 2003/07/13 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Delete() +{ + if (mRefCount != 0 && mRefCount != -1) + { + BOX_ERROR("Attempted to delete object " << mFilename << + " which has " << mRefCount << " references"); + THROW_EXCEPTION(RaidFileException, + RequestedDeleteReferencedFile); + } + + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + + // See if the file exists already -- can't delete files which don't exist + RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename); + if(existance == RaidFileUtil::NoFile) + { + THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist) + } + + // Get the filename for the write file + std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename)); + + // Attempt to delete it + bool deletedSomething = false; + if(::unlink(writeFilename.c_str()) == 0) + { + deletedSomething = true; + } + + // If we're not running in RAID mode, stop now + if(rdiscSet.size() == 1) + { + return; + } + + // Now the other files + std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 0 % TRANSFORM_NUMBER_DISCS_REQUIRED)); + std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 1 % TRANSFORM_NUMBER_DISCS_REQUIRED)); + std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 2 % TRANSFORM_NUMBER_DISCS_REQUIRED)); + if(::unlink(stripe1Filename.c_str()) == 0) + { + deletedSomething = true; + } + if(::unlink(stripe2Filename.c_str()) == 0) + { + deletedSomething = true; + } + if(::unlink(parityFilename.c_str()) == 0) + { + deletedSomething = true; + } + + // Check something happened + if(!deletedSomething) + { + THROW_EXCEPTION(RaidFileException, OSError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::CreateDirectory(int, const std::string &, bool, int) +// Purpose: Creates a directory within the raid file directories with the given name. +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::CreateDirectory(int SetNumber, const std::string &rDirName, bool Recursive, int mode) +{ + // Get disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber)); + // Pass on... + CreateDirectory(rdiscSet, rDirName, Recursive, mode); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::CreateDirectory(const RaidFileDiscSet &, const std::string &, bool, int) +// Purpose: Creates a directory within the raid file directories with the given name. +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::CreateDirectory(const RaidFileDiscSet &rSet, const std::string &rDirName, bool Recursive, int mode) +{ + if(Recursive) + { + // split up string + std::vector<std::string> elements; + SplitString(rDirName, DIRECTORY_SEPARATOR_ASCHAR, elements); + + // Do each element in turn + std::string pn; + for(unsigned int e = 0; e < elements.size(); ++e) + { + // Only do this if the element has some text in it + if(elements[e].size() > 0) + { + pn += elements[e]; + if(!RaidFileRead::DirectoryExists(rSet, pn)) + { + CreateDirectory(rSet, pn, false, mode); + } + + // add separator + pn += DIRECTORY_SEPARATOR_ASCHAR; + } + } + + return; + } + + // Create a directory in every disc of the set + for(unsigned int l = 0; l < rSet.size(); ++l) + { + // build name + std::string dn(rSet[l] + DIRECTORY_SEPARATOR + rDirName); + + // attempt to create + if(::mkdir(dn.c_str(), mode) != 0) + { + if(errno == EEXIST) + { + // No. Bad things. + THROW_EXCEPTION(RaidFileException, FileExistsInDirectoryCreation) + } + else + { + THROW_EXCEPTION(RaidFileException, OSError) + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Read(void *, int, int) +// Purpose: Unsupported, will exception +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +int RaidFileWrite::Read(void *pBuffer, int NBytes, int Timeout) +{ + THROW_EXCEPTION(RaidFileException, UnsupportedReadWriteOrClose) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::Close() +// Purpose: Close, discarding file. +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void RaidFileWrite::Close() +{ + if(mOSFileHandle != -1) + { + BOX_WARNING("RaidFileWrite::Close() called, discarding file"); + Discard(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::StreamDataLeft() +// Purpose: Never any data left to read! +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool RaidFileWrite::StreamDataLeft() +{ + return false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::StreamClosed() +// Purpose: Is stream closed for writing? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool RaidFileWrite::StreamClosed() +{ + return mOSFileHandle == -1; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::GetFileSize() +// Purpose: Returns the size of the file written. +// Can only be used before the file is commited. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileWrite::GetFileSize() +{ + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, CanOnlyGetFileSizeBeforeCommit) + } + + // Stat to get size + struct stat st; + if(fstat(mOSFileHandle, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + return st.st_size; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidFileWrite::GetDiscUsageInBlocks() +// Purpose: Returns the amount of disc space used, in blocks. +// Can only be used before the file is commited. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +IOStream::pos_type RaidFileWrite::GetDiscUsageInBlocks() +{ + if(mOSFileHandle == -1) + { + THROW_EXCEPTION(RaidFileException, CanOnlyGetUsageBeforeCommit) + } + + // Stat to get size + struct stat st; + if(fstat(mOSFileHandle, &st) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + + // Then return calculation + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); + return RaidFileUtil::DiscUsageInBlocks(st.st_size, rdiscSet); +} + + diff --git a/lib/raidfile/RaidFileWrite.h b/lib/raidfile/RaidFileWrite.h new file mode 100644 index 00000000..418f90ee --- /dev/null +++ b/lib/raidfile/RaidFileWrite.h @@ -0,0 +1,68 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RaidFileWrite.h +// Purpose: Writing RAID like files +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- + +#ifndef RAIDFILEWRITE__H +#define RAIDFILEWRITE__H + +#include <string> + +#include "IOStream.h" + +class RaidFileDiscSet; + +// -------------------------------------------------------------------------- +// +// Class +// Name: RaidFileWrite +// Purpose: Writing RAID like files +// Created: 2003/07/10 +// +// -------------------------------------------------------------------------- +class RaidFileWrite : public IOStream +{ +public: + RaidFileWrite(int SetNumber, const std::string &Filename); + RaidFileWrite(int SetNumber, const std::string &Filename, int refcount); + ~RaidFileWrite(); +private: + RaidFileWrite(const RaidFileWrite &rToCopy); + +public: + // IOStream interface + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); // will exception + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(pos_type Offset, int SeekType); + virtual void Close(); // will discard the file! Use commit instead. + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + // Extra bits + void Open(bool AllowOverwrite = false); + void Commit(bool ConvertToRaidNow = false); + void Discard(); + void TransformToRaidStorage(); + void Delete(); + pos_type GetFileSize(); + pos_type GetDiscUsageInBlocks(); + + static void CreateDirectory(int SetNumber, const std::string &rDirName, bool Recursive = false, int mode = 0777); + static void CreateDirectory(const RaidFileDiscSet &rSet, const std::string &rDirName, bool Recursive = false, int mode = 0777); + +private: + +private: + int mSetNumber; + std::string mFilename; + int mOSFileHandle; + int mRefCount; +}; + +#endif // RAIDFILEWRITE__H + diff --git a/lib/raidfile/raidfile-config.in b/lib/raidfile/raidfile-config.in new file mode 100755 index 00000000..b8ea73a5 --- /dev/null +++ b/lib/raidfile/raidfile-config.in @@ -0,0 +1,100 @@ +#!@PERL@ +use strict; + +# should be running as root +if($> != 0) +{ + printf "\nWARNING: this should be run as root\n\n" +} + +# check and get command line parameters +if($#ARGV != 4 && $#ARGV != 2) +{ + print <<__E; + +Setup raidfile config utility. + +Bad command line parameters. +Usage: + raidfile-config config-dir block-size dir0 [dir1 dir2] + +Parameters: + config-dir is usually @sysconfdir_expanded@/boxbackup + block-size must be a power of two, and usually the block or + fragment size of your file system + dir0, dir1, dir2 are the directories used as the root of the raid + file system + +If only one directory is specified, then userland RAID is disabled. +Specifying three directories enables it. + +__E + exit(1); +} + +my ($config_dir,$block_size,@dirs) = @ARGV; + +my $conf = $config_dir . '/raidfile.conf'; + +# check dirs are unique, and exist +my %d; +for(@dirs) +{ + die "$_ is used twice" if exists $d{$_}; + die "$_ is not a directory" unless -d $_; + die "$_ should be an absolute path" unless m/\A\//; + $d{$_} = 1; +} + +# check block size is OK +$block_size = int($block_size); +die "Bad block size" if $block_size <= 0; +my $c = 1; +while(1) +{ + last if $c == $block_size; + die "Block size $block_size is not a power of two" if $c > $block_size; + $c = $c * 2; +} + +# check that it doesn't already exist +if(-f $conf) +{ + die "$conf already exists. Delete and try again"; +} + +# create directory +if(!-d $config_dir) +{ + print "Creating $config_dir...\n"; + mkdir $config_dir,0755 or die "Can't create $config_dir"; +} + +# adjust if userland RAID is disabled +if($#dirs == 0) +{ + $dirs[1] = $dirs[0]; + $dirs[2] = $dirs[0]; + print "WARNING: userland RAID is disabled.\n"; +} + +# write the file +open CONFIG,">$conf" or die "Can't open $conf for writing"; + +print CONFIG <<__E; + +disc0 +{ + SetNumber = 0 + BlockSize = $block_size + Dir0 = $dirs[0] + Dir1 = $dirs[1] + Dir2 = $dirs[2] +} + +__E + +close CONFIG; + +print "Config file written.\n"; + diff --git a/lib/server/ConnectionException.txt b/lib/server/ConnectionException.txt new file mode 100644 index 00000000..c3429116 --- /dev/null +++ b/lib/server/ConnectionException.txt @@ -0,0 +1,27 @@ +EXCEPTION Connection 7 + +# for historic reasons not all numbers are used + +SocketWriteError 6 Probably a network issue between client and server. +SocketReadError 7 Probably a network issue between client and server. +SocketNameLookupError 9 Check hostname specified. +SocketShutdownError 12 +SocketConnectError 15 Probably a network issue between client and server, bad hostname, or server not running. +TLSHandshakeFailed 30 +TLSShutdownFailed 32 +TLSWriteFailed 33 Probably a network issue between client and server. +TLSReadFailed 34 Probably a network issue between client and server, or a problem with the server. +TLSNoPeerCertificate 36 +TLSPeerCertificateInvalid 37 Check certification process +TLSClosedWhenWriting 38 +TLSHandshakeTimedOut 39 +Protocol_Timeout 41 Probably a network issue between client and server. +Protocol_ObjTooBig 42 +Protocol_BadCommandRecieved 44 +Protocol_UnknownCommandRecieved 45 +Protocol_TriedToExecuteReplyCommand 46 +Protocol_UnexpectedReply 47 Server probably reported an error. +Protocol_HandshakeFailed 48 +Protocol_StreamWhenObjExpected 49 +Protocol_ObjWhenStreamExpected 50 +Protocol_TimeOutWhenSendingStream 52 Probably a network issue between client and server. diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp new file mode 100644 index 00000000..8b4f1d0c --- /dev/null +++ b/lib/server/Daemon.cpp @@ -0,0 +1,1024 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Daemon.cpp +// Purpose: Basic daemon functionality +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <errno.h> +#include <stdio.h> +#include <signal.h> +#include <string.h> +#include <stdarg.h> + +#ifdef HAVE_BSD_UNISTD_H + #include <bsd/unistd.h> +#endif + +#ifdef WIN32 + #include <ws2tcpip.h> +#endif + +#include <iostream> + +#include "Daemon.h" +#include "Configuration.h" +#include "ServerException.h" +#include "Guards.h" +#include "UnixUser.h" +#include "FileModificationTime.h" +#include "Logging.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +Daemon *Daemon::spDaemon = 0; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Daemon() +// Purpose: Constructor +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +Daemon::Daemon() + : mReloadConfigWanted(false), + mTerminateWanted(false), + #ifdef WIN32 + mSingleProcess(true), + mRunInForeground(true), + mKeepConsoleOpenAfterFork(true), + #else + mSingleProcess(false), + mRunInForeground(false), + mKeepConsoleOpenAfterFork(false), + #endif + mHaveConfigFile(false), + mAppName(DaemonName()) +{ + // In debug builds, switch on assert failure logging to syslog + ASSERT_FAILS_TO_SYSLOG_ON + // And trace goes to syslog too + TRACE_TO_SYSLOG(true) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::~Daemon() +// Purpose: Destructor +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +Daemon::~Daemon() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::GetOptionString() +// Purpose: Returns the valid Getopt command-line options. +// This should be overridden by subclasses to add +// their own options, which should override +// ProcessOption, handle their own, and delegate to +// ProcessOption for the standard options. +// Created: 2007/09/18 +// +// -------------------------------------------------------------------------- +std::string Daemon::GetOptionString() +{ + return "c:" + #ifndef WIN32 + "DFK" + #endif + "hkPqQt:TUvVW:"; +} + +void Daemon::Usage() +{ + std::cout << + DaemonBanner() << "\n" + "\n" + "Usage: " << mAppName << " [options] [config file]\n" + "\n" + "Options:\n" + " -c <file> Use the specified configuration file. If -c is omitted, the last\n" + " argument is the configuration file, or else the default \n" + " [" << GetConfigFileName() << "]\n" +#ifndef WIN32 + " -D Debugging mode, do not fork, one process only, one client only\n" + " -F Do not fork into background, but fork to serve multiple clients\n" +#endif + " -k Keep console open after fork, keep writing log messages to it\n" +#ifndef WIN32 + " -K Stop writing log messages to console while daemon is running\n" + " -P Show process ID (PID) in console output\n" +#endif + " -q Run more quietly, reduce verbosity level by one, can repeat\n" + " -Q Run at minimum verbosity, log nothing\n" + " -v Run more verbosely, increase verbosity level by one, can repeat\n" + " -V Run at maximum verbosity, log everything\n" + " -W <level> Set verbosity to error/warning/notice/info/trace/everything\n" + " -t <tag> Tag console output with specified marker\n" + " -T Timestamp console output\n" + " -U Timestamp console output with microseconds\n"; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::ProcessOption(int option) +// Purpose: Processes the supplied option (equivalent to the +// return code from getopt()). Return zero if the +// option was handled successfully, or nonzero to +// abort the program with that return value. +// Created: 2007/09/18 +// +// -------------------------------------------------------------------------- +int Daemon::ProcessOption(signed int option) +{ + switch(option) + { + case 'c': + { + mConfigFileName = optarg; + mHaveConfigFile = true; + } + break; + +#ifndef WIN32 + case 'D': + { + mSingleProcess = true; + } + break; + + case 'F': + { + mRunInForeground = true; + } + break; +#endif // !WIN32 + + case 'k': + { + mKeepConsoleOpenAfterFork = true; + } + break; + + case 'K': + { + mKeepConsoleOpenAfterFork = false; + } + break; + + case 'h': + { + Usage(); + return 2; + } + break; + + case 'P': + { + Console::SetShowPID(true); + } + break; + + case 'q': + { + if(mLogLevel == Log::NOTHING) + { + BOX_FATAL("Too many '-q': " + "Cannot reduce logging " + "level any more"); + return 2; + } + mLogLevel--; + } + break; + + case 'Q': + { + mLogLevel = Log::NOTHING; + } + break; + + + case 'v': + { + if(mLogLevel == Log::EVERYTHING) + { + BOX_FATAL("Too many '-v': " + "Cannot increase logging " + "level any more"); + return 2; + } + mLogLevel++; + } + break; + + case 'V': + { + mLogLevel = Log::EVERYTHING; + } + break; + + case 'W': + { + mLogLevel = Logging::GetNamedLevel(optarg); + if (mLogLevel == Log::INVALID) + { + BOX_FATAL("Invalid logging level"); + return 2; + } + } + break; + + case 't': + { + Logging::SetProgramName(optarg); + Console::SetShowTag(true); + } + break; + + case 'T': + { + Console::SetShowTime(true); + } + break; + + case 'U': + { + Console::SetShowTime(true); + Console::SetShowTimeMicros(true); + } + break; + + case '?': + { + BOX_FATAL("Unknown option on command line: " + << "'" << (char)optopt << "'"); + return 2; + } + break; + + default: + { + BOX_FATAL("Unknown error in getopt: returned " + << "'" << option << "'"); + return 1; + } + } + + return 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Main(const char *, int, const char *[]) +// Purpose: Parses command-line options, and then calls +// Main(std::string& configFile, bool singleProcess) +// to start the daemon. +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) +{ + // Find filename of config file + mConfigFileName = DefaultConfigFile; + mAppName = argv[0]; + + #ifdef BOX_RELEASE_BUILD + mLogLevel = Log::NOTICE; // need an int to do math with + #else + mLogLevel = Log::INFO; // need an int to do math with + #endif + + if (argc == 2 && strcmp(argv[1], "/?") == 0) + { + Usage(); + return 2; + } + + signed int c; + + // reset getopt, just in case anybody used it before. + // unfortunately glibc and BSD differ on this point! + // http://www.ussg.iu.edu/hypermail/linux/kernel/0305.3/0262.html + #if HAVE_DECL_OPTRESET == 1 || defined WIN32 + optind = 1; + optreset = 1; + #elif defined __GLIBC__ + optind = 0; + #else // Solaris, any others? + optind = 1; + #endif + + while((c = getopt(argc, (char * const *)argv, + GetOptionString().c_str())) != -1) + { + int returnCode = ProcessOption(c); + + if (returnCode != 0) + { + return returnCode; + } + } + + if (argc > optind && !mHaveConfigFile) + { + mConfigFileName = argv[optind]; optind++; + mHaveConfigFile = true; + } + + if (argc > optind && ::strcmp(argv[optind], "SINGLEPROCESS") == 0) + { + mSingleProcess = true; optind++; + } + + if (argc > optind) + { + BOX_FATAL("Unknown parameter on command line: " + << "'" << std::string(argv[optind]) << "'"); + return 2; + } + + Logging::FilterConsole((Log::Level)mLogLevel); + Logging::FilterSyslog ((Log::Level)mLogLevel); + + return Main(mConfigFileName); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Configure(const std::string& rConfigFileName) +// Purpose: Loads daemon configuration. Useful when you have +// a local Daemon object and don't intend to fork() +// or call Main(). +// Created: 2008/04/19 +// +// -------------------------------------------------------------------------- + +bool Daemon::Configure(const std::string& rConfigFileName) +{ + // Load the configuration file. + std::string errors; + std::auto_ptr<Configuration> apConfig; + + try + { + if (!FileExists(rConfigFileName.c_str())) + { + BOX_FATAL("The main configuration file for " << + DaemonName() << " was not found: " << + rConfigFileName); + if (!mHaveConfigFile) + { + BOX_WARNING("The default configuration " + "directory has changed from /etc/box " + "to /etc/boxbackup"); + } + return false; + } + + apConfig = Configuration::LoadAndVerify(rConfigFileName, + GetConfigVerify(), errors); + } + catch(BoxException &e) + { + if(e.GetType() == CommonException::ExceptionType && + e.GetSubType() == CommonException::OSFileOpenError) + { + BOX_ERROR("Failed to open configuration file: " << + rConfigFileName); + return false; + } + + throw; + } + + // Got errors? + if(apConfig.get() == 0) + { + BOX_ERROR("Failed to load or verify configuration file"); + return false; + } + + if(!Configure(*apConfig)) + { + BOX_ERROR("Failed to verify configuration file"); + return false; + } + + // Store configuration + mConfigFileName = rConfigFileName; + mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Configure(const Configuration& rConfig) +// Purpose: Loads daemon configuration. Useful when you have +// a local Daemon object and don't intend to fork() +// or call Main(). +// Created: 2008/08/12 +// +// -------------------------------------------------------------------------- + +bool Daemon::Configure(const Configuration& rConfig) +{ + std::string errors; + + // Verify() may modify the configuration, e.g. adding default values + // for required keys, so need to make a copy here + std::auto_ptr<Configuration> apConf(new Configuration(rConfig)); + apConf->Verify(*GetConfigVerify(), errors); + + // Got errors? + if(!errors.empty()) + { + BOX_ERROR("Configuration errors: " << errors); + return false; + } + + // Store configuration + mapConfiguration = apConf; + + // Let the derived class have a go at setting up stuff + // in the initial process + SetupInInitialProcess(); + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Main(const std::string& rConfigFileName) +// Purpose: Starts the daemon off -- equivalent of C main() function +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +int Daemon::Main(const std::string &rConfigFileName) +{ + // Banner (optional) + { + BOX_SYSLOG(Log::NOTICE, DaemonBanner()); + } + + std::string pidFileName; + + bool asDaemon = !mSingleProcess && !mRunInForeground; + + try + { + if (!Configure(rConfigFileName)) + { + BOX_FATAL("Failed to start: failed to load " + "configuration file: " << rConfigFileName); + return 1; + } + + // Server configuration + const Configuration &serverConfig( + mapConfiguration->GetSubConfiguration("Server")); + + if(serverConfig.KeyExists("LogFacility")) + { + std::string facility = + serverConfig.GetKeyValue("LogFacility"); + Logging::SetFacility(Syslog::GetNamedFacility(facility)); + } + + // Open PID file for writing + pidFileName = serverConfig.GetKeyValue("PidFile"); + FileHandleGuard<(O_WRONLY | O_CREAT | O_TRUNC), (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)> pidFile(pidFileName.c_str()); + +#ifndef WIN32 + // Handle changing to a different user + if(serverConfig.KeyExists("User")) + { + // Config file specifies an user -- look up + UnixUser daemonUser(serverConfig.GetKeyValue("User").c_str()); + + // Change the owner on the PID file, so it can be deleted properly on termination + if(::fchown(pidFile, daemonUser.GetUID(), daemonUser.GetGID()) != 0) + { + THROW_EXCEPTION(ServerException, CouldNotChangePIDFileOwner) + } + + // Change the process ID + daemonUser.ChangeProcessUser(); + } + + if(asDaemon) + { + // Let's go... Daemonise... + switch(::fork()) + { + case -1: + // error + THROW_EXCEPTION(ServerException, DaemoniseFailed) + break; + + default: + // parent + // _exit(0); + return 0; + break; + + case 0: + // child + break; + } + + // In child + + // Set new session + if(::setsid() == -1) + { + BOX_LOG_SYS_ERROR("Failed to setsid()"); + THROW_EXCEPTION(ServerException, DaemoniseFailed) + } + + // Fork again... + switch(::fork()) + { + case -1: + // error + BOX_LOG_SYS_ERROR("Failed to fork() a child"); + THROW_EXCEPTION(ServerException, DaemoniseFailed) + break; + + default: + // parent + _exit(0); + return 0; + break; + + case 0: + // child + break; + } + } +#endif // !WIN32 + + // Must set spDaemon before installing signal handler, + // otherwise the handler will crash if invoked too soon. + if(spDaemon != NULL) + { + THROW_EXCEPTION(ServerException, AlreadyDaemonConstructed) + } + spDaemon = this; + +#ifndef WIN32 + // Set signal handler + // Don't do this in the parent, since it might be anything + // (e.g. test/bbackupd) + + struct sigaction sa; + sa.sa_handler = SignalHandler; + sa.sa_flags = 0; + sigemptyset(&sa.sa_mask); // macro + if(::sigaction(SIGHUP, &sa, NULL) != 0 || + ::sigaction(SIGTERM, &sa, NULL) != 0) + { + BOX_LOG_SYS_ERROR("Failed to set signal handlers"); + THROW_EXCEPTION(ServerException, DaemoniseFailed) + } +#endif // !WIN32 + + // Write PID to file + char pid[32]; + + int pidsize = sprintf(pid, "%d", (int)getpid()); + + if(::write(pidFile, pid, pidsize) != pidsize) + { + BOX_LOG_SYS_FATAL("Failed to write PID file: " << + pidFileName); + THROW_EXCEPTION(ServerException, DaemoniseFailed) + } + + // Set up memory leak reporting + #ifdef BOX_MEMORY_LEAK_TESTING + { + char filename[256]; + sprintf(filename, "%s.memleaks", DaemonName()); + memleakfinder_setup_exit_report(filename, DaemonName()); + } + #endif // BOX_MEMORY_LEAK_TESTING + + if(asDaemon && !mKeepConsoleOpenAfterFork) + { +#ifndef WIN32 + // Close standard streams + ::close(0); + ::close(1); + ::close(2); + + // Open and redirect them into /dev/null + int devnull = ::open(PLATFORM_DEV_NULL, O_RDWR, 0); + if(devnull == -1) + { + BOX_LOG_SYS_ERROR("Failed to open /dev/null"); + THROW_EXCEPTION(CommonException, OSFileError); + } + // Then duplicate them to all three handles + if(devnull != 0) dup2(devnull, 0); + if(devnull != 1) dup2(devnull, 1); + if(devnull != 2) dup2(devnull, 2); + // Close the original handle if it was opened above the std* range + if(devnull > 2) + { + ::close(devnull); + } + + // And definitely don't try and send anything to those file descriptors + // -- this has in the past sent text to something which isn't expecting it. + TRACE_TO_STDOUT(false); +#endif // ! WIN32 + Logging::ToConsole(false); + } + + // Log the start message + BOX_NOTICE("Starting daemon, version: " << BOX_VERSION); + BOX_NOTICE("Using configuration file: " << mConfigFileName); + } + catch(BoxException &e) + { + BOX_FATAL("Failed to start: exception " << e.what() + << " (" << e.GetType() + << "/" << e.GetSubType() << ")"); + return 1; + } + catch(std::exception &e) + { + BOX_FATAL("Failed to start: exception " << e.what()); + return 1; + } + catch(...) + { + BOX_FATAL("Failed to start: unknown error"); + return 1; + } + +#ifdef WIN32 + // Under win32 we must initialise the Winsock library + // before using sockets + + WSADATA info; + + if (WSAStartup(0x0101, &info) == SOCKET_ERROR) + { + // will not run without sockets + BOX_FATAL("Failed to initialise Windows Sockets"); + THROW_EXCEPTION(CommonException, Internal) + } +#endif + + int retcode = 0; + + // Main Daemon running + try + { + while(!mTerminateWanted) + { + Run(); + + if(mReloadConfigWanted && !mTerminateWanted) + { + // Need to reload that config file... + BOX_NOTICE("Reloading configuration file: " + << mConfigFileName); + std::string errors; + std::auto_ptr<Configuration> pconfig( + Configuration::LoadAndVerify( + mConfigFileName.c_str(), + GetConfigVerify(), errors)); + + // Got errors? + if(pconfig.get() == 0 || !errors.empty()) + { + // Tell user about errors + BOX_FATAL("Error in configuration " + << "file: " << mConfigFileName + << ": " << errors); + // And give up + retcode = 1; + break; + } + + // Store configuration + mapConfiguration = pconfig; + mLoadedConfigModifiedTime = + GetConfigFileModifiedTime(); + + // Stop being marked for loading config again + mReloadConfigWanted = false; + } + } + + // Delete the PID file + ::unlink(pidFileName.c_str()); + + // Log + BOX_NOTICE("Terminating daemon"); + } + catch(BoxException &e) + { + BOX_FATAL("Terminating due to exception " << e.what() + << " (" << e.GetType() + << "/" << e.GetSubType() << ")"); + retcode = 1; + } + catch(std::exception &e) + { + BOX_FATAL("Terminating due to exception " << e.what()); + retcode = 1; + } + catch(...) + { + BOX_FATAL("Terminating due to unknown exception"); + retcode = 1; + } + +#ifdef WIN32 + WSACleanup(); +#else + // Should clean up here, but it breaks memory leak tests. + /* + if(asDaemon) + { + // we are running in the child by now, and should not return + mapConfiguration.reset(); + exit(0); + } + */ +#endif + + ASSERT(spDaemon == this); + spDaemon = NULL; + + return retcode; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::EnterChild() +// Purpose: Sets up for a child task of the main server. Call +// just after fork(). +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void Daemon::EnterChild() +{ +#ifndef WIN32 + // Unset signal handlers + struct sigaction sa; + sa.sa_handler = SIG_DFL; + sa.sa_flags = 0; + sigemptyset(&sa.sa_mask); // macro + ::sigaction(SIGHUP, &sa, NULL); + ::sigaction(SIGTERM, &sa, NULL); +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::SignalHandler(int) +// Purpose: Signal handler +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +void Daemon::SignalHandler(int sigraised) +{ +#ifndef WIN32 + if(spDaemon != 0) + { + switch(sigraised) + { + case SIGHUP: + spDaemon->mReloadConfigWanted = true; + break; + + case SIGTERM: + spDaemon->mTerminateWanted = true; + break; + + default: + break; + } + } +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::DaemonName() +// Purpose: Returns name of the daemon +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +const char *Daemon::DaemonName() const +{ + return "generic-daemon"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::DaemonBanner() +// Purpose: Returns the text banner for this daemon's startup +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +std::string Daemon::DaemonBanner() const +{ + return "Generic daemon using the Box Application Framework"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Run() +// Purpose: Main run function after basic Daemon initialisation +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +void Daemon::Run() +{ + while(!StopRun()) + { + ::sleep(10); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::GetConfigVerify() +// Purpose: Returns the configuration file verification structure for this daemon +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify *Daemon::GetConfigVerify() const +{ + static ConfigurationVerifyKey verifyserverkeys[] = + { + DAEMON_VERIFY_SERVER_KEYS + }; + + static ConfigurationVerify verifyserver[] = + { + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } + }; + + static ConfigurationVerify verify = + { + "root", + verifyserver, + 0, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + }; + + return &verify; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::GetConfiguration() +// Purpose: Returns the daemon configuration object +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +const Configuration &Daemon::GetConfiguration() const +{ + if(mapConfiguration.get() == 0) + { + // Shouldn't get anywhere near this if a configuration file can't be loaded + THROW_EXCEPTION(ServerException, Internal) + } + + return *mapConfiguration; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::SetupInInitialProcess() +// Purpose: A chance for the daemon to do something initial +// setting up in the process which initiates +// everything, and after the configuration file has +// been read and verified. +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +void Daemon::SetupInInitialProcess() +{ + // Base class doesn't do anything. +} + + +void Daemon::SetProcessTitle(const char *format, ...) +{ + // On OpenBSD, setproctitle() sets the process title to imagename: <text> (imagename) + // -- make sure other platforms include the image name somewhere so ps listings give + // useful information. + +#ifdef HAVE_SETPROCTITLE + // optional arguments + va_list args; + va_start(args, format); + + // Make the string + char title[256]; + ::vsnprintf(title, sizeof(title), format, args); + + // Set process title + ::setproctitle("%s", title); + +#endif // HAVE_SETPROCTITLE +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::GetConfigFileModifiedTime() +// Purpose: Returns the timestamp when the configuration file +// was last modified +// +// Created: 2006/01/29 +// +// -------------------------------------------------------------------------- + +box_time_t Daemon::GetConfigFileModifiedTime() const +{ + EMU_STRUCT_STAT st; + + if(EMU_STAT(GetConfigFileName().c_str(), &st) != 0) + { + if (errno == ENOENT) + { + return 0; + } + BOX_LOG_SYS_ERROR("Failed to stat configuration file: " << + GetConfigFileName()); + THROW_EXCEPTION(CommonException, OSFileError) + } + + return FileModificationTime(st); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::GetLoadedConfigModifiedTime() +// Purpose: Returns the timestamp when the configuration file +// had been last modified, at the time when it was +// loaded +// +// Created: 2006/01/29 +// +// -------------------------------------------------------------------------- + +box_time_t Daemon::GetLoadedConfigModifiedTime() const +{ + return mLoadedConfigModifiedTime; +} + diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h new file mode 100644 index 00000000..a3212a00 --- /dev/null +++ b/lib/server/Daemon.h @@ -0,0 +1,112 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Daemon.h +// Purpose: Basic daemon functionality +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- + +/* NOTE: will log to local6: include a line like + local6.info /var/log/box + in /etc/syslog.conf +*/ + + +#ifndef DAEMON__H +#define DAEMON__H + +#include <string> + +#include "BoxTime.h" +#include "Configuration.h" + +class ConfigurationVerify; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Daemon +// Purpose: Basic daemon functionality +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +class Daemon +{ +public: + Daemon(); + virtual ~Daemon(); +private: + Daemon(const Daemon &rToCopy); +public: + + virtual int Main(const char *DefaultConfigFile, int argc, + const char *argv[]); + + /* override this Main() if you want custom option processing: */ + virtual int Main(const std::string &rConfigFile); + + virtual void Run(); + const Configuration &GetConfiguration() const; + const std::string &GetConfigFileName() const {return mConfigFileName;} + + virtual const char *DaemonName() const; + virtual std::string DaemonBanner() const; + virtual const ConfigurationVerify *GetConfigVerify() const; + virtual void Usage(); + + virtual bool Configure(const std::string& rConfigFileName); + virtual bool Configure(const Configuration& rConfig); + + bool StopRun() {return mReloadConfigWanted | mTerminateWanted;} + bool IsReloadConfigWanted() {return mReloadConfigWanted;} + bool IsTerminateWanted() {return mTerminateWanted;} + + // To allow derived classes to get these signals in other ways + void SetReloadConfigWanted() {mReloadConfigWanted = true;} + void SetTerminateWanted() {mTerminateWanted = true;} + + virtual void EnterChild(); + + static void SetProcessTitle(const char *format, ...); + void SetRunInForeground(bool foreground) + { + mRunInForeground = foreground; + } + void SetSingleProcess(bool value) + { + mSingleProcess = value; + } + +protected: + virtual void SetupInInitialProcess(); + box_time_t GetLoadedConfigModifiedTime() const; + bool IsSingleProcess() { return mSingleProcess; } + virtual std::string GetOptionString(); + virtual int ProcessOption(signed int option); + +private: + static void SignalHandler(int sigraised); + box_time_t GetConfigFileModifiedTime() const; + + std::string mConfigFileName; + std::auto_ptr<Configuration> mapConfiguration; + box_time_t mLoadedConfigModifiedTime; + bool mReloadConfigWanted; + bool mTerminateWanted; + bool mSingleProcess; + bool mRunInForeground; + bool mKeepConsoleOpenAfterFork; + bool mHaveConfigFile; + int mLogLevel; // need an int to do math with + static Daemon *spDaemon; + std::string mAppName; +}; + +#define DAEMON_VERIFY_SERVER_KEYS \ + ConfigurationVerifyKey("PidFile", ConfigTest_Exists), \ + ConfigurationVerifyKey("LogFacility", 0), \ + ConfigurationVerifyKey("User", ConfigTest_LastEntry) + +#endif // DAEMON__H + diff --git a/lib/server/LocalProcessStream.cpp b/lib/server/LocalProcessStream.cpp new file mode 100644 index 00000000..c331a135 --- /dev/null +++ b/lib/server/LocalProcessStream.cpp @@ -0,0 +1,180 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: LocalProcessStream.cpp +// Purpose: Opens a process, and presents stdin/stdout as a stream. +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_SYS_SOCKET_H + #include <sys/socket.h> +#endif + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include "LocalProcessStream.h" +#include "autogen_ServerException.h" +#include "Utils.h" + +#ifdef WIN32 + #include "FileStream.h" +#else + #include "SocketStream.h" +#endif + +#include "MemLeakFindOn.h" + +#define MAX_ARGUMENTS 64 + +// -------------------------------------------------------------------------- +// +// Function +// Name: LocalProcessStream(const char *, pid_t &) +// Purpose: Run a new process, and return a stream giving access +// to its stdin and stdout (stdout and stderr on +// Win32). Returns the PID of the new process -- this +// must be waited on at some point to avoid zombies +// (except on Win32). +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> LocalProcessStream(const std::string& rCommandLine, + pid_t &rPidOut) +{ +#ifndef WIN32 + + // Split up command + std::vector<std::string> command; + SplitString(rCommandLine, ' ', command); + + // Build arguments + char *args[MAX_ARGUMENTS + 4]; + { + int a = 0; + std::vector<std::string>::const_iterator i(command.begin()); + while(a < MAX_ARGUMENTS && i != command.end()) + { + args[a++] = (char*)(*(i++)).c_str(); + } + args[a] = NULL; + } + + // Create a socket pair to communicate over. + int sv[2] = {-1,-1}; + if(::socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, sv) != 0) + { + THROW_EXCEPTION(ServerException, SocketPairFailed) + } + + std::auto_ptr<IOStream> stream(new SocketStream(sv[0])); + + // Fork + pid_t pid = 0; + switch(pid = vfork()) + { + case -1: // error + ::close(sv[0]); + ::close(sv[1]); + THROW_EXCEPTION(ServerException, ServerForkError) + break; + + case 0: // child + // Close end of the socket not being used + ::close(sv[0]); + // Duplicate the file handles to stdin and stdout + if(sv[1] != 0) ::dup2(sv[1], 0); + if(sv[1] != 1) ::dup2(sv[1], 1); + // Close the now redundant socket + if(sv[1] != 0 && sv[1] != 1) + { + ::close(sv[1]); + } + // Execute command! + ::execv(args[0], args); + ::_exit(127); // report error + break; + + default: + // just continue... + break; + } + + // Close the file descriptor not being used + ::close(sv[1]); + + // Return the stream object and PID + rPidOut = pid; + return stream; + +#else // WIN32 + + SECURITY_ATTRIBUTES secAttr; + secAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + secAttr.bInheritHandle = TRUE; + secAttr.lpSecurityDescriptor = NULL; + + HANDLE writeInChild, readFromChild; + if(!CreatePipe(&readFromChild, &writeInChild, &secAttr, 0)) + { + BOX_ERROR("Failed to CreatePipe for child process: " << + GetErrorMessage(GetLastError())); + THROW_EXCEPTION(ServerException, SocketPairFailed) + } + SetHandleInformation(readFromChild, HANDLE_FLAG_INHERIT, 0); + + PROCESS_INFORMATION procInfo; + STARTUPINFO startupInfo; + + ZeroMemory(&procInfo, sizeof(procInfo)); + ZeroMemory(&startupInfo, sizeof(startupInfo)); + startupInfo.cb = sizeof(startupInfo); + startupInfo.hStdError = writeInChild; + startupInfo.hStdOutput = writeInChild; + startupInfo.hStdInput = INVALID_HANDLE_VALUE; + startupInfo.dwFlags |= STARTF_USESTDHANDLES; + + CHAR* commandLineCopy = (CHAR*)malloc(rCommandLine.size() + 1); + strcpy(commandLineCopy, rCommandLine.c_str()); + + BOOL result = CreateProcess(NULL, + commandLineCopy, // command line + NULL, // process security attributes + NULL, // primary thread security attributes + TRUE, // handles are inherited + 0, // creation flags + NULL, // use parent's environment + NULL, // use parent's current directory + &startupInfo, // STARTUPINFO pointer + &procInfo); // receives PROCESS_INFORMATION + + free(commandLineCopy); + + if(!result) + { + BOX_ERROR("Failed to CreateProcess: '" << rCommandLine << + "': " << GetErrorMessage(GetLastError())); + CloseHandle(writeInChild); + CloseHandle(readFromChild); + THROW_EXCEPTION(ServerException, ServerForkError) + } + + CloseHandle(procInfo.hProcess); + CloseHandle(procInfo.hThread); + CloseHandle(writeInChild); + + rPidOut = (int)(procInfo.dwProcessId); + + std::auto_ptr<IOStream> stream(new FileStream(readFromChild)); + return stream; + +#endif // ! WIN32 +} + + + + diff --git a/lib/server/LocalProcessStream.h b/lib/server/LocalProcessStream.h new file mode 100644 index 00000000..51e51f8a --- /dev/null +++ b/lib/server/LocalProcessStream.h @@ -0,0 +1,20 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: LocalProcessStream.h +// Purpose: Opens a process, and presents stdin/stdout as a stream. +// Created: 12/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef LOCALPROCESSSTREAM__H +#define LOCALPROCESSSTREAM__H + +#include <memory> +#include "IOStream.h" + +std::auto_ptr<IOStream> LocalProcessStream(const std::string& rCommandLine, + pid_t &rPidOut); + +#endif // LOCALPROCESSSTREAM__H + diff --git a/lib/server/Makefile.extra b/lib/server/Makefile.extra new file mode 100644 index 00000000..7fc6baf9 --- /dev/null +++ b/lib/server/Makefile.extra @@ -0,0 +1,11 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_ServerException.h autogen_ServerException.cpp: $(MAKEEXCEPTION) ServerException.txt + $(_PERL) $(MAKEEXCEPTION) ServerException.txt + +# AUTOGEN SEEDING +autogen_ConnectionException.h autogen_ConnectionException.cpp: $(MAKEEXCEPTION) ConnectionException.txt + $(_PERL) $(MAKEEXCEPTION) ConnectionException.txt + diff --git a/lib/server/OverlappedIO.h b/lib/server/OverlappedIO.h new file mode 100644 index 00000000..12495053 --- /dev/null +++ b/lib/server/OverlappedIO.h @@ -0,0 +1,42 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: OverlappedIO.h +// Purpose: Windows overlapped IO handle guard +// Created: 2008/09/30 +// +// -------------------------------------------------------------------------- + +#ifndef OVERLAPPEDIO__H +#define OVERLAPPEDIO__H + +class OverlappedIO +{ +public: + OVERLAPPED mOverlapped; + + OverlappedIO() + { + ZeroMemory(&mOverlapped, sizeof(mOverlapped)); + mOverlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, + NULL); + if (mOverlapped.hEvent == INVALID_HANDLE_VALUE) + { + BOX_LOG_WIN_ERROR("Failed to create event for " + "overlapped I/O"); + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + } + + ~OverlappedIO() + { + if (CloseHandle(mOverlapped.hEvent) != TRUE) + { + BOX_LOG_WIN_ERROR("Failed to delete event for " + "overlapped I/O"); + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + } +}; + +#endif // !OVERLAPPEDIO__H diff --git a/lib/server/Protocol.cpp b/lib/server/Protocol.cpp new file mode 100644 index 00000000..5dc5d0b1 --- /dev/null +++ b/lib/server/Protocol.cpp @@ -0,0 +1,1160 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Protocol.cpp +// Purpose: Generic protocol support +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/types.h> + +#include <stdlib.h> +#include <string.h> + +#include <new> + +#include "Protocol.h" +#include "ProtocolWire.h" +#include "IOStream.h" +#include "ServerException.h" +#include "PartialReadStream.h" +#include "ProtocolUncertainStream.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +#ifdef BOX_RELEASE_BUILD + #define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK 1024 +#else +// #define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK 1024 + #define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK 4 +#endif + +#define UNCERTAIN_STREAM_SIZE_BLOCK (64*1024) + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Protocol(IOStream &rStream) +// Purpose: Constructor +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +Protocol::Protocol(IOStream &rStream) + : mrStream(rStream), + mHandshakeDone(false), + mMaxObjectSize(PROTOCOL_DEFAULT_MAXOBJSIZE), + mTimeout(PROTOCOL_DEFAULT_TIMEOUT), + mpBuffer(0), + mBufferSize(0), + mReadOffset(-1), + mWriteOffset(-1), + mValidDataSize(-1), + mLastErrorType(NoError), + mLastErrorSubType(NoError) +{ + BOX_TRACE("Send block allocation size is " << + PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::~Protocol() +// Purpose: Destructor +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +Protocol::~Protocol() +{ + // Free buffer? + if(mpBuffer != 0) + { + free(mpBuffer); + mpBuffer = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::GetLastError(int &, int &) +// Purpose: Returns true if there was an error, and type and subtype if there was. +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +bool Protocol::GetLastError(int &rTypeOut, int &rSubTypeOut) +{ + if(mLastErrorType == NoError) + { + // no error. + return false; + } + + // Return type and subtype in args + rTypeOut = mLastErrorType; + rSubTypeOut = mLastErrorSubType; + + // and unset them + mLastErrorType = NoError; + mLastErrorSubType = NoError; + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Handshake() +// Purpose: Handshake with peer (exchange ident strings) +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +void Protocol::Handshake() +{ + // Already done? + if(mHandshakeDone) + { + THROW_EXCEPTION(CommonException, Internal) + } + + // Make handshake block + PW_Handshake hsSend; + ::memset(&hsSend, 0, sizeof(hsSend)); + // Copy in ident string + ::strncpy(hsSend.mIdent, GetIdentString(), sizeof(hsSend.mIdent)); + + // Send it + mrStream.Write(&hsSend, sizeof(hsSend)); + mrStream.WriteAllBuffered(); + + // Receive a handshake from the peer + PW_Handshake hsReceive; + ::memset(&hsReceive, 0, sizeof(hsReceive)); + char *readInto = (char*)&hsReceive; + int bytesToRead = sizeof(hsReceive); + while(bytesToRead > 0) + { + // Get some data from the stream + int bytesRead = mrStream.Read(readInto, bytesToRead, mTimeout); + if(bytesRead == 0) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + } + readInto += bytesRead; + bytesToRead -= bytesRead; + } + ASSERT(bytesToRead == 0); + + // Are they the same? + if(::memcmp(&hsSend, &hsReceive, sizeof(hsSend)) != 0) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_HandshakeFailed) + } + + // Mark as done + mHandshakeDone = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::CheckAndReadHdr(void *) +// Purpose: Check read for recieve call and get object header from stream. +// Don't use type here to avoid dependency in .h file. +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void Protocol::CheckAndReadHdr(void *hdr) +{ + // Check usage + if(mValidDataSize != -1 || mWriteOffset != -1 || mReadOffset != -1) + { + THROW_EXCEPTION(ServerException, Protocol_BadUsage) + } + + // Handshake done? + if(!mHandshakeDone) + { + Handshake(); + } + + // Get some data into this header + if(!mrStream.ReadFullBuffer(hdr, sizeof(PW_ObjectHeader), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Recieve() +// Purpose: Recieves an object from the stream, creating it from the factory object type +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> Protocol::Receive() +{ + // Get object header + PW_ObjectHeader objHeader; + CheckAndReadHdr(&objHeader); + + // Hope it's not a stream + if(ntohl(objHeader.mObjType) == SPECIAL_STREAM_OBJECT_TYPE) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_StreamWhenObjExpected) + } + + // Check the object size + u_int32_t objSize = ntohl(objHeader.mObjSize); + if(objSize < sizeof(objHeader) || objSize > mMaxObjectSize) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_ObjTooBig) + } + + // Create a blank object + std::auto_ptr<ProtocolObject> obj(MakeProtocolObject(ntohl(objHeader.mObjType))); + + // Make sure memory is allocated to read it into + EnsureBufferAllocated(objSize); + + // Read data + if(!mrStream.ReadFullBuffer(mpBuffer, objSize - sizeof(objHeader), 0 /* not interested in bytes read if this fails */, mTimeout)) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + } + + // Setup ready to read out data from the buffer + mValidDataSize = objSize - sizeof(objHeader); + mReadOffset = 0; + + // Get the object to read its properties from the data recieved + try + { + obj->SetPropertiesFromStreamData(*this); + } + catch(...) + { + // Make sure state is reset! + mValidDataSize = -1; + mReadOffset = -1; + throw; + } + + // Any data left over? + bool dataLeftOver = (mValidDataSize != mReadOffset); + + // Unset read state, so future read calls don't fail + mValidDataSize = -1; + mReadOffset = -1; + + // Exception if not all the data was consumed + if(dataLeftOver) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_BadCommandRecieved) + } + + return obj; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Send() +// Purpose: Send an object to the other side of the connection. +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Send(const ProtocolObject &rObject) +{ + // Check usage + if(mValidDataSize != -1 || mWriteOffset != -1 || mReadOffset != -1) + { + THROW_EXCEPTION(ServerException, Protocol_BadUsage) + } + + // Handshake done? + if(!mHandshakeDone) + { + Handshake(); + } + + // Make sure there's a little bit of space allocated + EnsureBufferAllocated(((sizeof(PW_ObjectHeader) + PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK - 1) / PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK) * PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK); + ASSERT(mBufferSize >= (int)sizeof(PW_ObjectHeader)); + + // Setup for write operation + mValidDataSize = 0; // Not used, but must not be -1 + mWriteOffset = sizeof(PW_ObjectHeader); + + try + { + rObject.WritePropertiesToStreamData(*this); + } + catch(...) + { + // Make sure state is reset! + mValidDataSize = -1; + mWriteOffset = -1; + throw; + } + + // How big? + int writtenSize = mWriteOffset; + + // Reset write state + mValidDataSize = -1; + mWriteOffset = -1; + + // Make header in the existing block + PW_ObjectHeader *pobjHeader = (PW_ObjectHeader*)(mpBuffer); + pobjHeader->mObjSize = htonl(writtenSize); + pobjHeader->mObjType = htonl(rObject.GetType()); + + // Write data + mrStream.Write(mpBuffer, writtenSize); + mrStream.WriteAllBuffered(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::EnsureBufferAllocated(int) +// Purpose: Private. Ensures the buffer is at least the size requested. +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::EnsureBufferAllocated(int Size) +{ + if(mpBuffer != 0 && mBufferSize >= Size) + { + // Nothing to do! + return; + } + + // Need to allocate, or reallocate, the block + if(mpBuffer != 0) + { + // Reallocate + void *b = realloc(mpBuffer, Size); + if(b == 0) + { + throw std::bad_alloc(); + } + mpBuffer = (char*)b; + mBufferSize = Size; + } + else + { + // Just allocate + mpBuffer = (char*)malloc(Size); + if(mpBuffer == 0) + { + throw std::bad_alloc(); + } + mBufferSize = Size; + } +} + + +#define READ_START_CHECK \ + if(mValidDataSize == -1 || mWriteOffset != -1 || mReadOffset == -1) \ + { \ + THROW_EXCEPTION(ServerException, Protocol_BadUsage) \ + } + +#define READ_CHECK_BYTES_AVAILABLE(bytesRequired) \ + if((mReadOffset + (int)(bytesRequired)) > mValidDataSize) \ + { \ + THROW_EXCEPTION(ConnectionException, Conn_Protocol_BadCommandRecieved) \ + } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(void *, int) +// Purpose: Read raw data from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(void *Buffer, int Size) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(Size) + + // Copy data out + ::memmove(Buffer, mpBuffer + mReadOffset, Size); + mReadOffset += Size; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(std::string &, int) +// Purpose: Read raw data from the stream (buffered), into a std::string +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void Protocol::Read(std::string &rOut, int Size) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(Size) + + rOut.assign(mpBuffer + mReadOffset, Size); + mReadOffset += Size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(int64_t &) +// Purpose: Read a value from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(int64_t &rOut) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(sizeof(int64_t)) + +#ifdef HAVE_ALIGNED_ONLY_INT64 + int64_t nvalue; + memcpy(&nvalue, mpBuffer + mReadOffset, sizeof(int64_t)); +#else + int64_t nvalue = *((int64_t*)(mpBuffer + mReadOffset)); +#endif + rOut = box_ntoh64(nvalue); + + mReadOffset += sizeof(int64_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(int32_t &) +// Purpose: Read a value from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(int32_t &rOut) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(sizeof(int32_t)) + +#ifdef HAVE_ALIGNED_ONLY_INT32 + int32_t nvalue; + memcpy(&nvalue, mpBuffer + mReadOffset, sizeof(int32_t)); +#else + int32_t nvalue = *((int32_t*)(mpBuffer + mReadOffset)); +#endif + rOut = ntohl(nvalue); + mReadOffset += sizeof(int32_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(int16_t &) +// Purpose: Read a value from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(int16_t &rOut) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(sizeof(int16_t)) + + rOut = ntohs(*((int16_t*)(mpBuffer + mReadOffset))); + mReadOffset += sizeof(int16_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(int8_t &) +// Purpose: Read a value from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(int8_t &rOut) +{ + READ_START_CHECK + READ_CHECK_BYTES_AVAILABLE(sizeof(int8_t)) + + rOut = *((int8_t*)(mpBuffer + mReadOffset)); + mReadOffset += sizeof(int8_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Read(std::string &) +// Purpose: Read a value from the stream (buffered) +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Read(std::string &rOut) +{ + // READ_START_CHECK implied + int32_t size; + Read(size); + + READ_CHECK_BYTES_AVAILABLE(size) + + // initialise string + rOut.assign(mpBuffer + mReadOffset, size); + mReadOffset += size; +} + + + + +#define WRITE_START_CHECK \ + if(mValidDataSize == -1 || mWriteOffset == -1 || mReadOffset != -1) \ + { \ + THROW_EXCEPTION(ServerException, Protocol_BadUsage) \ + } + +#define WRITE_ENSURE_BYTES_AVAILABLE(bytesToWrite) \ + if(mWriteOffset + (int)(bytesToWrite) > mBufferSize) \ + { \ + EnsureBufferAllocated((((mWriteOffset + (int)(bytesToWrite)) + PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK - 1) / PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK) * PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK); \ + ASSERT(mWriteOffset + (int)(bytesToWrite) <= mBufferSize); \ + } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(const void *, int) +// Purpose: Writes the contents of a buffer to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(const void *Buffer, int Size) +{ + WRITE_START_CHECK + WRITE_ENSURE_BYTES_AVAILABLE(Size) + + ::memmove(mpBuffer + mWriteOffset, Buffer, Size); + mWriteOffset += Size; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(int64_t) +// Purpose: Writes a value to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(int64_t Value) +{ + WRITE_START_CHECK + WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int64_t)) + + int64_t nvalue = box_hton64(Value); +#ifdef HAVE_ALIGNED_ONLY_INT64 + memcpy(mpBuffer + mWriteOffset, &nvalue, sizeof(int64_t)); +#else + *((int64_t*)(mpBuffer + mWriteOffset)) = nvalue; +#endif + mWriteOffset += sizeof(int64_t); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(int32_t) +// Purpose: Writes a value to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(int32_t Value) +{ + WRITE_START_CHECK + WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int32_t)) + + int32_t nvalue = htonl(Value); +#ifdef HAVE_ALIGNED_ONLY_INT32 + memcpy(mpBuffer + mWriteOffset, &nvalue, sizeof(int32_t)); +#else + *((int32_t*)(mpBuffer + mWriteOffset)) = nvalue; +#endif + mWriteOffset += sizeof(int32_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(int16_t) +// Purpose: Writes a value to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(int16_t Value) +{ + WRITE_START_CHECK + WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int16_t)) + + *((int16_t*)(mpBuffer + mWriteOffset)) = htons(Value); + mWriteOffset += sizeof(int16_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(int8_t) +// Purpose: Writes a value to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(int8_t Value) +{ + WRITE_START_CHECK + WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int8_t)) + + *((int8_t*)(mpBuffer + mWriteOffset)) = Value; + mWriteOffset += sizeof(int8_t); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::Write(const std::string &) +// Purpose: Writes a value to the stream +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void Protocol::Write(const std::string &rValue) +{ + // WRITE_START_CHECK implied + Write((int32_t)(rValue.size())); + + WRITE_ENSURE_BYTES_AVAILABLE(rValue.size()) + Write(rValue.c_str(), rValue.size()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::ReceieveStream() +// Purpose: Receive a stream from the remote side +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +std::auto_ptr<IOStream> Protocol::ReceiveStream() +{ + // Get object header + PW_ObjectHeader objHeader; + CheckAndReadHdr(&objHeader); + + // Hope it's not an object + if(ntohl(objHeader.mObjType) != SPECIAL_STREAM_OBJECT_TYPE) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_ObjWhenStreamExpected) + } + + // Get the stream size + u_int32_t streamSize = ntohl(objHeader.mObjSize); + + // Inform sub class + InformStreamReceiving(streamSize); + + // Return a stream object + if(streamSize == ProtocolStream_SizeUncertain) + { + BOX_TRACE("Receiving stream, size uncertain"); + return std::auto_ptr<IOStream>( + new ProtocolUncertainStream(mrStream)); + } + else + { + BOX_TRACE("Receiving stream, size " << streamSize << " bytes"); + return std::auto_ptr<IOStream>( + new PartialReadStream(mrStream, streamSize)); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::SendStream(IOStream &) +// Purpose: Send a stream to the remote side +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void Protocol::SendStream(IOStream &rStream) +{ + // Check usage + if(mValidDataSize != -1 || mWriteOffset != -1 || mReadOffset != -1) + { + THROW_EXCEPTION(ServerException, Protocol_BadUsage) + } + + // Handshake done? + if(!mHandshakeDone) + { + Handshake(); + } + + // How should this be streamed? + bool uncertainSize = false; + IOStream::pos_type streamSize = rStream.BytesLeftToRead(); + if(streamSize == IOStream::SizeOfStreamUnknown + || streamSize > 0x7fffffff) + { + // Can't send this using the fixed size header + uncertainSize = true; + } + + // Inform sub class + InformStreamSending(streamSize); + + // Make header + PW_ObjectHeader objHeader; + objHeader.mObjSize = htonl(uncertainSize?(ProtocolStream_SizeUncertain):streamSize); + objHeader.mObjType = htonl(SPECIAL_STREAM_OBJECT_TYPE); + + // Write header + mrStream.Write(&objHeader, sizeof(objHeader)); + // Could be sent in one of two ways + if(uncertainSize) + { + // Don't know how big this is going to be -- so send it in chunks + + // Allocate memory + uint8_t *blockA = (uint8_t *)malloc(UNCERTAIN_STREAM_SIZE_BLOCK + sizeof(int)); + if(blockA == 0) + { + throw std::bad_alloc(); + } + uint8_t *block = blockA + sizeof(int); // so that everything is word aligned for reading, but can put the one byte header before it + + try + { + int bytesInBlock = 0; + while(rStream.StreamDataLeft()) + { + // Read some of it + bytesInBlock += rStream.Read(block + bytesInBlock, UNCERTAIN_STREAM_SIZE_BLOCK - bytesInBlock); + + // Send as much as we can out + bytesInBlock -= SendStreamSendBlock(block, bytesInBlock); + } + + // Everything recieved from stream, but need to send whatevers left in the block + while(bytesInBlock > 0) + { + bytesInBlock -= SendStreamSendBlock(block, bytesInBlock); + } + + // Send final byte to finish the stream + BOX_TRACE("Sending end of stream byte"); + uint8_t endOfStream = ProtocolStreamHeader_EndOfStream; + mrStream.Write(&endOfStream, 1); + BOX_TRACE("Sent end of stream byte"); + } + catch(...) + { + free(blockA); + throw; + } + + // Clean up + free(blockA); + } + else + { + // Fixed size stream, send it all in one go + if(!rStream.CopyStreamTo(mrStream, mTimeout, 4096 /* slightly larger buffer */)) + { + THROW_EXCEPTION(ConnectionException, Conn_Protocol_TimeOutWhenSendingStream) + } + } + // Make sure everything is written + mrStream.WriteAllBuffered(); + +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::SendStreamSendBlock(uint8_t *, int) +// Purpose: Sends as much of the block as can be sent, moves the remainer down to the beginning, +// and returns the number of bytes sent. WARNING: Will write to Block[-1] +// Created: 5/12/03 +// +// -------------------------------------------------------------------------- +int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) +{ + // Quick sanity check + if(BytesInBlock == 0) + { + BOX_TRACE("Zero size block, not sending anything"); + return 0; + } + + // Work out the header byte + uint8_t header = 0; + int writeSize = 0; + if(BytesInBlock >= (64*1024)) + { + header = ProtocolStreamHeader_SizeIs64k; + writeSize = (64*1024); + } + else + { + // Scan the table to find the most that can be written + for(int s = ProtocolStreamHeader_MaxEncodedSizeValue; s > 0; --s) + { + if(sProtocolStreamHeaderLengths[s] <= BytesInBlock) + { + header = s; + writeSize = sProtocolStreamHeaderLengths[s]; + break; + } + } + } + ASSERT(header > 0); + BOX_TRACE("Sending header byte " << (int)header << " plus " << + writeSize << " bytes to stream"); + + // Store the header + Block[-1] = header; + + // Write everything out + mrStream.Write(Block - 1, writeSize + 1); + + BOX_TRACE("Sent " << (writeSize+1) << " bytes to stream"); + // move the remainer to the beginning of the block for the next time round + if(writeSize != BytesInBlock) + { + ::memmove(Block, Block + writeSize, BytesInBlock - writeSize); + } + + return writeSize; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::InformStreamReceiving(u_int32_t) +// Purpose: Informs sub classes about streams being received +// Created: 2003/10/27 +// +// -------------------------------------------------------------------------- +void Protocol::InformStreamReceiving(u_int32_t Size) +{ + // Do nothing +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Protocol::InformStreamSending(u_int32_t) +// Purpose: Informs sub classes about streams being sent +// Created: 2003/10/27 +// +// -------------------------------------------------------------------------- +void Protocol::InformStreamSending(u_int32_t Size) +{ + // Do nothing +} + + +/* +perl code to generate the table below + +#!/usr/bin/perl +use strict; +open OUT,">protolengths.txt"; +my $len = 0; +for(0 .. 255) +{ + print OUT "\t$len,\t// $_\n"; + my $inc = 1; + $inc = 8 if $_ >= 64; + $inc = 16 if $_ >= 96; + $inc = 32 if $_ >= 112; + $inc = 64 if $_ >= 128; + $inc = 128 if $_ >= 135; + $inc = 256 if $_ >= 147; + $inc = 512 if $_ >= 159; + $inc = 1024 if $_ >= 231; + $len += $inc; +} +close OUT; + +*/ +const uint16_t Protocol::sProtocolStreamHeaderLengths[256] = +{ + 0, // 0 + 1, // 1 + 2, // 2 + 3, // 3 + 4, // 4 + 5, // 5 + 6, // 6 + 7, // 7 + 8, // 8 + 9, // 9 + 10, // 10 + 11, // 11 + 12, // 12 + 13, // 13 + 14, // 14 + 15, // 15 + 16, // 16 + 17, // 17 + 18, // 18 + 19, // 19 + 20, // 20 + 21, // 21 + 22, // 22 + 23, // 23 + 24, // 24 + 25, // 25 + 26, // 26 + 27, // 27 + 28, // 28 + 29, // 29 + 30, // 30 + 31, // 31 + 32, // 32 + 33, // 33 + 34, // 34 + 35, // 35 + 36, // 36 + 37, // 37 + 38, // 38 + 39, // 39 + 40, // 40 + 41, // 41 + 42, // 42 + 43, // 43 + 44, // 44 + 45, // 45 + 46, // 46 + 47, // 47 + 48, // 48 + 49, // 49 + 50, // 50 + 51, // 51 + 52, // 52 + 53, // 53 + 54, // 54 + 55, // 55 + 56, // 56 + 57, // 57 + 58, // 58 + 59, // 59 + 60, // 60 + 61, // 61 + 62, // 62 + 63, // 63 + 64, // 64 + 72, // 65 + 80, // 66 + 88, // 67 + 96, // 68 + 104, // 69 + 112, // 70 + 120, // 71 + 128, // 72 + 136, // 73 + 144, // 74 + 152, // 75 + 160, // 76 + 168, // 77 + 176, // 78 + 184, // 79 + 192, // 80 + 200, // 81 + 208, // 82 + 216, // 83 + 224, // 84 + 232, // 85 + 240, // 86 + 248, // 87 + 256, // 88 + 264, // 89 + 272, // 90 + 280, // 91 + 288, // 92 + 296, // 93 + 304, // 94 + 312, // 95 + 320, // 96 + 336, // 97 + 352, // 98 + 368, // 99 + 384, // 100 + 400, // 101 + 416, // 102 + 432, // 103 + 448, // 104 + 464, // 105 + 480, // 106 + 496, // 107 + 512, // 108 + 528, // 109 + 544, // 110 + 560, // 111 + 576, // 112 + 608, // 113 + 640, // 114 + 672, // 115 + 704, // 116 + 736, // 117 + 768, // 118 + 800, // 119 + 832, // 120 + 864, // 121 + 896, // 122 + 928, // 123 + 960, // 124 + 992, // 125 + 1024, // 126 + 1056, // 127 + 1088, // 128 + 1152, // 129 + 1216, // 130 + 1280, // 131 + 1344, // 132 + 1408, // 133 + 1472, // 134 + 1536, // 135 + 1664, // 136 + 1792, // 137 + 1920, // 138 + 2048, // 139 + 2176, // 140 + 2304, // 141 + 2432, // 142 + 2560, // 143 + 2688, // 144 + 2816, // 145 + 2944, // 146 + 3072, // 147 + 3328, // 148 + 3584, // 149 + 3840, // 150 + 4096, // 151 + 4352, // 152 + 4608, // 153 + 4864, // 154 + 5120, // 155 + 5376, // 156 + 5632, // 157 + 5888, // 158 + 6144, // 159 + 6656, // 160 + 7168, // 161 + 7680, // 162 + 8192, // 163 + 8704, // 164 + 9216, // 165 + 9728, // 166 + 10240, // 167 + 10752, // 168 + 11264, // 169 + 11776, // 170 + 12288, // 171 + 12800, // 172 + 13312, // 173 + 13824, // 174 + 14336, // 175 + 14848, // 176 + 15360, // 177 + 15872, // 178 + 16384, // 179 + 16896, // 180 + 17408, // 181 + 17920, // 182 + 18432, // 183 + 18944, // 184 + 19456, // 185 + 19968, // 186 + 20480, // 187 + 20992, // 188 + 21504, // 189 + 22016, // 190 + 22528, // 191 + 23040, // 192 + 23552, // 193 + 24064, // 194 + 24576, // 195 + 25088, // 196 + 25600, // 197 + 26112, // 198 + 26624, // 199 + 27136, // 200 + 27648, // 201 + 28160, // 202 + 28672, // 203 + 29184, // 204 + 29696, // 205 + 30208, // 206 + 30720, // 207 + 31232, // 208 + 31744, // 209 + 32256, // 210 + 32768, // 211 + 33280, // 212 + 33792, // 213 + 34304, // 214 + 34816, // 215 + 35328, // 216 + 35840, // 217 + 36352, // 218 + 36864, // 219 + 37376, // 220 + 37888, // 221 + 38400, // 222 + 38912, // 223 + 39424, // 224 + 39936, // 225 + 40448, // 226 + 40960, // 227 + 41472, // 228 + 41984, // 229 + 42496, // 230 + 43008, // 231 + 44032, // 232 + 45056, // 233 + 46080, // 234 + 47104, // 235 + 48128, // 236 + 49152, // 237 + 50176, // 238 + 51200, // 239 + 52224, // 240 + 53248, // 241 + 54272, // 242 + 55296, // 243 + 56320, // 244 + 57344, // 245 + 58368, // 246 + 59392, // 247 + 60416, // 248 + 61440, // 249 + 62464, // 250 + 63488, // 251 + 64512, // 252 + 0, // 253 = 65536 / 64k + 0, // 254 = special (reserved) + 0 // 255 = special (reserved) +}; + + + + diff --git a/lib/server/Protocol.h b/lib/server/Protocol.h new file mode 100644 index 00000000..e037e33c --- /dev/null +++ b/lib/server/Protocol.h @@ -0,0 +1,201 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Protocol.h +// Purpose: Generic protocol support +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- + +#ifndef PROTOCOL__H +#define PROTOCOL__H + +#include <sys/types.h> + +class IOStream; +#include "ProtocolObject.h" +#include <memory> +#include <vector> +#include <string> + +// default timeout is 15 minutes +#define PROTOCOL_DEFAULT_TIMEOUT (15*60*1000) +// 16 default maximum object size -- should be enough +#define PROTOCOL_DEFAULT_MAXOBJSIZE (16*1024) + +// -------------------------------------------------------------------------- +// +// Class +// Name: Protocol +// Purpose: Generic command / response protocol support +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +class Protocol +{ +public: + Protocol(IOStream &rStream); + virtual ~Protocol(); + +private: + Protocol(const Protocol &rToCopy); + +public: + void Handshake(); + std::auto_ptr<ProtocolObject> Receive(); + void Send(const ProtocolObject &rObject); + + std::auto_ptr<IOStream> ReceiveStream(); + void SendStream(IOStream &rStream); + + enum + { + NoError = -1, + UnknownError = 0 + }; + + bool GetLastError(int &rTypeOut, int &rSubTypeOut); + + // -------------------------------------------------------------------------- + // + // Function + // Name: Protocol::SetTimeout(int) + // Purpose: Sets the timeout for sending and reciving + // Created: 2003/08/19 + // + // -------------------------------------------------------------------------- + void SetTimeout(int NewTimeout) {mTimeout = NewTimeout;} + + + // -------------------------------------------------------------------------- + // + // Function + // Name: Protocol::GetTimeout() + // Purpose: Get current timeout for sending and receiving + // Created: 2003/09/06 + // + // -------------------------------------------------------------------------- + int GetTimeout() {return mTimeout;} + + // -------------------------------------------------------------------------- + // + // Function + // Name: Protocol::SetMaxObjectSize(int) + // Purpose: Sets the maximum size of an object which will be accepted + // Created: 2003/08/19 + // + // -------------------------------------------------------------------------- + void SetMaxObjectSize(unsigned int NewMaxObjSize) {mMaxObjectSize = NewMaxObjSize;} + + // For ProtocolObject derived classes + void Read(void *Buffer, int Size); + void Read(std::string &rOut, int Size); + void Read(int64_t &rOut); + void Read(int32_t &rOut); + void Read(int16_t &rOut); + void Read(int8_t &rOut); + void Read(bool &rOut) {int8_t read; Read(read); rOut = (read == true);} + void Read(std::string &rOut); + template<typename type> + void Read(type &rOut) + { + rOut.ReadFromProtocol(*this); + } + // -------------------------------------------------------------------------- + // + // Function + // Name: Protocol::ReadVector(std::vector<> &) + // Purpose: Reads a vector/list of items from the stream + // Created: 2003/08/19 + // + // -------------------------------------------------------------------------- + template<typename type> + void ReadVector(std::vector<type> &rOut) + { + rOut.clear(); + int16_t num = 0; + Read(num); + for(int16_t n = 0; n < num; ++n) + { + type v; + Read(v); + rOut.push_back(v); + } + } + + void Write(const void *Buffer, int Size); + void Write(int64_t Value); + void Write(int32_t Value); + void Write(int16_t Value); + void Write(int8_t Value); + void Write(bool Value) {int8_t write = Value; Write(write);} + void Write(const std::string &rValue); + template<typename type> + void Write(const type &rValue) + { + rValue.WriteToProtocol(*this); + } + template<typename type> + // -------------------------------------------------------------------------- + // + // Function + // Name: Protocol::WriteVector(const std::vector<> &) + // Purpose: Writes a vector/list of items from the stream + // Created: 2003/08/19 + // + // -------------------------------------------------------------------------- + void WriteVector(const std::vector<type> &rValue) + { + int16_t num = rValue.size(); + Write(num); + for(int16_t n = 0; n < num; ++n) + { + Write(rValue[n]); + } + } + +public: + static const uint16_t sProtocolStreamHeaderLengths[256]; + enum + { + ProtocolStreamHeader_EndOfStream = 0, + ProtocolStreamHeader_MaxEncodedSizeValue = 252, + ProtocolStreamHeader_SizeIs64k = 253, + ProtocolStreamHeader_Reserved1 = 254, + ProtocolStreamHeader_Reserved2 = 255 + }; + enum + { + ProtocolStream_SizeUncertain = 0xffffffff + }; + +protected: + virtual std::auto_ptr<ProtocolObject> MakeProtocolObject(int ObjType) = 0; + virtual const char *GetIdentString() = 0; + void SetError(int Type, int SubType) {mLastErrorType = Type; mLastErrorSubType = SubType;} + void CheckAndReadHdr(void *hdr); // don't use type here to avoid dependency + + // Will be used for logging + virtual void InformStreamReceiving(u_int32_t Size); + virtual void InformStreamSending(u_int32_t Size); + +private: + void EnsureBufferAllocated(int Size); + int SendStreamSendBlock(uint8_t *Block, int BytesInBlock); + +private: + IOStream &mrStream; + bool mHandshakeDone; + unsigned int mMaxObjectSize; + int mTimeout; + char *mpBuffer; + int mBufferSize; + int mReadOffset; + int mWriteOffset; + int mValidDataSize; + int mLastErrorType; + int mLastErrorSubType; +}; + +#endif // PROTOCOL__H + diff --git a/lib/server/ProtocolObject.cpp b/lib/server/ProtocolObject.cpp new file mode 100644 index 00000000..fb09f820 --- /dev/null +++ b/lib/server/ProtocolObject.cpp @@ -0,0 +1,125 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ProtocolObject.h +// Purpose: Protocol object base class +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "ProtocolObject.h" +#include "CommonException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::ProtocolObject() +// Purpose: Default constructor +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +ProtocolObject::ProtocolObject() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::ProtocolObject() +// Purpose: Destructor +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +ProtocolObject::~ProtocolObject() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::ProtocolObject() +// Purpose: Copy constructor +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +ProtocolObject::ProtocolObject(const ProtocolObject &rToCopy) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::IsError(int &, int &) +// Purpose: Does this represent an error, and if so, what is the type and subtype? +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +bool ProtocolObject::IsError(int &rTypeOut, int &rSubTypeOut) const +{ + return false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::IsConversationEnd() +// Purpose: Does this command end the conversation? +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +bool ProtocolObject::IsConversationEnd() const +{ + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::GetType() +// Purpose: Return type of the object +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +int ProtocolObject::GetType() const +{ + // This isn't implemented in the base class! + THROW_EXCEPTION(CommonException, Internal) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::SetPropertiesFromStreamData(Protocol &) +// Purpose: Set the properties of the object from the stream data ready in the Protocol object +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void ProtocolObject::SetPropertiesFromStreamData(Protocol &rProtocol) +{ + // This isn't implemented in the base class! + THROW_EXCEPTION(CommonException, Internal) +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolObject::WritePropertiesToStreamData(Protocol &) +// Purpose: Write the properties of the object into the stream data in the Protocol object +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +void ProtocolObject::WritePropertiesToStreamData(Protocol &rProtocol) const +{ + // This isn't implemented in the base class! + THROW_EXCEPTION(CommonException, Internal) +} + + + diff --git a/lib/server/ProtocolObject.h b/lib/server/ProtocolObject.h new file mode 100644 index 00000000..0a127ab5 --- /dev/null +++ b/lib/server/ProtocolObject.h @@ -0,0 +1,41 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ProtocolObject.h +// Purpose: Protocol object base class +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- + +#ifndef PROTOCOLOBJECT__H +#define PROTOCOLOBJECT__H + +class Protocol; + +// -------------------------------------------------------------------------- +// +// Class +// Name: ProtocolObject +// Purpose: Basic object representation of objects to pass through a Protocol session +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- +class ProtocolObject +{ +public: + ProtocolObject(); + virtual ~ProtocolObject(); + ProtocolObject(const ProtocolObject &rToCopy); + + // Info about this object + virtual int GetType() const; + virtual bool IsError(int &rTypeOut, int &rSubTypeOut) const; + virtual bool IsConversationEnd() const; + + // reading and writing with Protocol objects + virtual void SetPropertiesFromStreamData(Protocol &rProtocol); + virtual void WritePropertiesToStreamData(Protocol &rProtocol) const; +}; + +#endif // PROTOCOLOBJECT__H + diff --git a/lib/server/ProtocolUncertainStream.cpp b/lib/server/ProtocolUncertainStream.cpp new file mode 100644 index 00000000..84a213a8 --- /dev/null +++ b/lib/server/ProtocolUncertainStream.cpp @@ -0,0 +1,206 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ProtocolUncertainStream.h +// Purpose: Read part of another stream +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "ProtocolUncertainStream.h" +#include "ServerException.h" +#include "Protocol.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::ProtocolUncertainStream(IOStream &, int) +// Purpose: Constructor, taking another stream. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +ProtocolUncertainStream::ProtocolUncertainStream(IOStream &rSource) + : mrSource(rSource), + mBytesLeftInCurrentBlock(0), + mFinished(false) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::~ProtocolUncertainStream() +// Purpose: Destructor. Won't absorb any unread bytes. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +ProtocolUncertainStream::~ProtocolUncertainStream() +{ + if(!mFinished) + { + BOX_WARNING("ProtocolUncertainStream destroyed before " + "stream finished"); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::Read(void *, int, int) +// Purpose: As interface. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // Finished? + if(mFinished) + { + return 0; + } + + int read = 0; + while(read < NBytes) + { + // Anything we can get from the current block? + ASSERT(mBytesLeftInCurrentBlock >= 0); + if(mBytesLeftInCurrentBlock > 0) + { + // Yes, let's use some of these up + int toRead = (NBytes - read); + if(toRead > mBytesLeftInCurrentBlock) + { + // Adjust downwards to only read stuff out of the current block + toRead = mBytesLeftInCurrentBlock; + } + + BOX_TRACE("Reading " << toRead << " bytes from stream"); + + // Read it + int r = mrSource.Read(((uint8_t*)pBuffer) + read, toRead, Timeout); + // Give up now if it didn't return anything + if(r == 0) + { + BOX_TRACE("Read " << r << " bytes from " + "stream, returning"); + return read; + } + + // Adjust counts of bytes by the bytes recieved + read += r; + mBytesLeftInCurrentBlock -= r; + + // stop now if the stream returned less than we asked for -- avoid blocking + if(r != toRead) + { + BOX_TRACE("Read " << r << " bytes from " + "stream, returning"); + return read; + } + } + else + { + // Read the header byte to find out how much there is + // in the next block + uint8_t header; + if(mrSource.Read(&header, 1, Timeout) == 0) + { + // Didn't get the byte, return now + BOX_TRACE("Read 0 bytes of block header, " + "returning with " << read << " bytes " + "read this time"); + return read; + } + + // Interpret the byte... + if(header == Protocol::ProtocolStreamHeader_EndOfStream) + { + // All done. + mFinished = true; + BOX_TRACE("Stream finished, returning with " << + read << " bytes read this time"); + return read; + } + else if(header <= Protocol::ProtocolStreamHeader_MaxEncodedSizeValue) + { + // get size of the block from the Protocol's lovely list + mBytesLeftInCurrentBlock = Protocol::sProtocolStreamHeaderLengths[header]; + } + else if(header == Protocol::ProtocolStreamHeader_SizeIs64k) + { + // 64k + mBytesLeftInCurrentBlock = (64*1024); + } + else + { + // Bad. It used the reserved values. + THROW_EXCEPTION(ServerException, ProtocolUncertainStreamBadBlockHeader) + } + + BOX_TRACE("Read header byte " << (int)header << ", " + "next block has " << + mBytesLeftInCurrentBlock << " bytes"); + } + } + + // Return the number read + return read; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::BytesLeftToRead() +// Purpose: As interface. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ProtocolUncertainStream::BytesLeftToRead() +{ + // Only know how much is left if everything is finished + return mFinished?(0):(IOStream::SizeOfStreamUnknown); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::Write(const void *, int) +// Purpose: As interface. But will exception. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +void ProtocolUncertainStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(ServerException, CantWriteToProtocolUncertainStream) +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::StreamDataLeft() +// Purpose: As interface. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +bool ProtocolUncertainStream::StreamDataLeft() +{ + return !mFinished; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ProtocolUncertainStream::StreamClosed() +// Purpose: As interface. +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +bool ProtocolUncertainStream::StreamClosed() +{ + // always closed + return true; +} + diff --git a/lib/server/ProtocolUncertainStream.h b/lib/server/ProtocolUncertainStream.h new file mode 100644 index 00000000..4954cf88 --- /dev/null +++ b/lib/server/ProtocolUncertainStream.h @@ -0,0 +1,47 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ProtocolUncertainStream.h +// Purpose: Read part of another stream +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- + +#ifndef PROTOCOLUNCERTAINSTREAM__H +#define PROTOCOLUNCERTAINSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: ProtocolUncertainStream +// Purpose: Read part of another stream +// Created: 2003/12/05 +// +// -------------------------------------------------------------------------- +class ProtocolUncertainStream : public IOStream +{ +public: + ProtocolUncertainStream(IOStream &rSource); + ~ProtocolUncertainStream(); +private: + // no copying allowed + ProtocolUncertainStream(const IOStream &); + ProtocolUncertainStream(const ProtocolUncertainStream &); + +public: + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + IOStream &mrSource; + int mBytesLeftInCurrentBlock; + bool mFinished; +}; + +#endif // PROTOCOLUNCERTAINSTREAM__H + diff --git a/lib/server/ProtocolWire.h b/lib/server/ProtocolWire.h new file mode 100644 index 00000000..ff62b66e --- /dev/null +++ b/lib/server/ProtocolWire.h @@ -0,0 +1,43 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ProtocolWire.h +// Purpose: On the wire structures for Protocol +// Created: 2003/08/19 +// +// -------------------------------------------------------------------------- + +#ifndef PROTOCOLWIRE__H +#define PROTOCOLWIRE__H + +#include <sys/types.h> + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + char mIdent[32]; +} PW_Handshake; + +typedef struct +{ + u_int32_t mObjSize; + u_int32_t mObjType; +} PW_ObjectHeader; + +#define SPECIAL_STREAM_OBJECT_TYPE 0xffffffff + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +#endif // PROTOCOLWIRE__H + diff --git a/lib/server/SSLLib.cpp b/lib/server/SSLLib.cpp new file mode 100644 index 00000000..de7a941b --- /dev/null +++ b/lib/server/SSLLib.cpp @@ -0,0 +1,111 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SSLLib.cpp +// Purpose: Utility functions for dealing with the OpenSSL library +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#define TLS_CLASS_IMPLEMENTATION_CPP +#include <openssl/ssl.h> +#include <openssl/err.h> +#include <openssl/rand.h> + +#ifdef WIN32 + #include <wincrypt.h> +#endif + +#include "SSLLib.h" +#include "ServerException.h" + +#include "MemLeakFindOn.h" + +#ifndef BOX_RELEASE_BUILD + bool SSLLib__TraceErrors = false; +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: SSLLib::Initialise() +// Purpose: Initialise SSL library +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SSLLib::Initialise() +{ + if(!::SSL_library_init()) + { + LogError("initialising OpenSSL"); + THROW_EXCEPTION(ServerException, SSLLibraryInitialisationError) + } + + // More helpful error messages + ::SSL_load_error_strings(); + + // Extra seeding over and above what's already done by the library +#ifdef WIN32 + HCRYPTPROV provider; + if(!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, + CRYPT_VERIFYCONTEXT)) + { + BOX_LOG_WIN_ERROR("Failed to acquire crypto context"); + BOX_WARNING("No random device -- additional seeding of " + "random number generator not performed."); + } + else + { + // must free provider + BYTE buf[1024]; + + if(!CryptGenRandom(provider, sizeof(buf), buf)) + { + BOX_LOG_WIN_ERROR("Failed to get random data"); + BOX_WARNING("No random device -- additional seeding of " + "random number generator not performed."); + } + else + { + RAND_seed(buf, sizeof(buf)); + } + + if(!CryptReleaseContext(provider, 0)) + { + BOX_LOG_WIN_ERROR("Failed to release crypto context"); + } + } +#elif HAVE_RANDOM_DEVICE + if(::RAND_load_file(RANDOM_DEVICE, 1024) != 1024) + { + THROW_EXCEPTION(ServerException, SSLRandomInitFailed) + } +#else + BOX_WARNING("No random device -- additional seeding of " + "random number generator not performed."); +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SSLLib::LogError(const char *) +// Purpose: Logs an error +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SSLLib::LogError(const std::string& rErrorDuringAction) +{ + unsigned long errcode; + char errname[256]; // SSL docs say at least 120 bytes + while((errcode = ERR_get_error()) != 0) + { + ::ERR_error_string_n(errcode, errname, sizeof(errname)); + BOX_ERROR("SSL error while " << rErrorDuringAction << ": " << + errname); + } +} + diff --git a/lib/server/SSLLib.h b/lib/server/SSLLib.h new file mode 100644 index 00000000..ff4aab19 --- /dev/null +++ b/lib/server/SSLLib.h @@ -0,0 +1,36 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SSLLib.h +// Purpose: Utility functions for dealing with the OpenSSL library +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#ifndef SSLLIB__H +#define SSLLIB__H + +#ifndef BOX_RELEASE_BUILD + extern bool SSLLib__TraceErrors; + #define SET_DEBUG_SSLLIB_TRACE_ERRORS {SSLLib__TraceErrors = true;} +#else + #define SET_DEBUG_SSLLIB_TRACE_ERRORS +#endif + + +// -------------------------------------------------------------------------- +// +// Namespace +// Name: SSLLib +// Purpose: Utility functions for dealing with the OpenSSL library +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +namespace SSLLib +{ + void Initialise(); + void LogError(const std::string& rErrorDuringAction); +}; + +#endif // SSLLIB__H + diff --git a/lib/server/ServerControl.cpp b/lib/server/ServerControl.cpp new file mode 100644 index 00000000..b9650cee --- /dev/null +++ b/lib/server/ServerControl.cpp @@ -0,0 +1,228 @@ +#include "Box.h" + +#include <errno.h> +#include <stdio.h> + +#ifdef HAVE_SYS_TYPES_H + #include <sys/types.h> +#endif + +#ifdef HAVE_SYS_WAIT_H + #include <sys/wait.h> +#endif + +#ifdef HAVE_SIGNAL_H + #include <signal.h> +#endif + +#include "ServerControl.h" +#include "Test.h" + +#ifdef WIN32 + +#include "WinNamedPipeStream.h" +#include "IOStreamGetLine.h" +#include "BoxPortsAndFiles.h" + +static std::string sPipeName; + +void SetNamedPipeName(const std::string& rPipeName) +{ + sPipeName = rPipeName; +} + +bool SendCommands(const std::string& rCmd) +{ + WinNamedPipeStream connection; + + try + { + connection.Connect(sPipeName); + } + catch(...) + { + BOX_ERROR("Failed to connect to daemon control socket"); + return false; + } + + // For receiving data + IOStreamGetLine getLine(connection); + + // Wait for the configuration summary + std::string configSummary; + if(!getLine.GetLine(configSummary)) + { + BOX_ERROR("Failed to receive configuration summary from daemon"); + return false; + } + + // Was the connection rejected by the server? + if(getLine.IsEOF()) + { + BOX_ERROR("Server rejected the connection"); + return false; + } + + // Decode it + int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait; + if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", + &autoBackup, &updateStoreInterval, + &minimumFileAge, &maxUploadWait) != 4) + { + BOX_ERROR("Config summary didn't decode"); + return false; + } + + std::string cmds; + bool expectResponse; + + if (rCmd != "") + { + cmds = rCmd; + cmds += "\nquit\n"; + expectResponse = true; + } + else + { + cmds = "quit\n"; + expectResponse = false; + } + + connection.Write(cmds.c_str(), cmds.size()); + + // Read the response + std::string line; + bool statusOk = !expectResponse; + + while (expectResponse && !getLine.IsEOF() && getLine.GetLine(line)) + { + // Is this an OK or error line? + if (line == "ok") + { + statusOk = true; + } + else if (line == "error") + { + BOX_ERROR(rCmd); + break; + } + else + { + BOX_WARNING("Unexpected response to command '" << + rCmd << "': " << line) + } + } + + return statusOk; +} + +bool HUPServer(int pid) +{ + return SendCommands("reload"); +} + +bool KillServerInternal(int pid) +{ + HANDLE hProcess = OpenProcess(PROCESS_TERMINATE, false, pid); + if (hProcess == NULL) + { + BOX_ERROR("Failed to open process " << pid << ": " << + GetErrorMessage(GetLastError())); + return false; + } + + if (!TerminateProcess(hProcess, 1)) + { + BOX_ERROR("Failed to terminate process " << pid << ": " << + GetErrorMessage(GetLastError())); + CloseHandle(hProcess); + return false; + } + + CloseHandle(hProcess); + return true; +} + +#else // !WIN32 + +bool HUPServer(int pid) +{ + if(pid == 0) return false; + return ::kill(pid, SIGHUP) == 0; +} + +bool KillServerInternal(int pid) +{ + if(pid == 0 || pid == -1) return false; + bool killed = (::kill(pid, SIGTERM) == 0); + if (!killed) + { + BOX_LOG_SYS_ERROR("Failed to kill process " << pid); + } + TEST_THAT(killed); + return killed; +} + +#endif // WIN32 + +bool KillServer(int pid, bool WaitForProcess) +{ + if (!KillServerInternal(pid)) + { + return false; + } + + #ifdef HAVE_WAITPID + if (WaitForProcess) + { + int status, result; + + result = waitpid(pid, &status, 0); + if (result != pid) + { + BOX_LOG_SYS_ERROR("waitpid failed"); + } + TEST_THAT(result == pid); + + TEST_THAT(WIFEXITED(status)); + if (WIFEXITED(status)) + { + if (WEXITSTATUS(status) != 0) + { + BOX_WARNING("process exited with code " << + WEXITSTATUS(status)); + } + TEST_THAT(WEXITSTATUS(status) == 0); + } + } + #endif + + for (int i = 0; i < 30; i++) + { + if (i == 0) + { + printf("Waiting for server to die (pid %d): ", pid); + } + + printf("."); + fflush(stdout); + + if (!ServerIsAlive(pid)) break; + ::sleep(1); + if (!ServerIsAlive(pid)) break; + } + + if (!ServerIsAlive(pid)) + { + printf(" done.\n"); + } + else + { + printf(" failed!\n"); + } + + fflush(stdout); + + return !ServerIsAlive(pid); +} + diff --git a/lib/server/ServerControl.h b/lib/server/ServerControl.h new file mode 100644 index 00000000..b2e51864 --- /dev/null +++ b/lib/server/ServerControl.h @@ -0,0 +1,18 @@ +#ifndef SERVER_CONTROL_H +#define SERVER_CONTROL_H + +#include "Test.h" + +bool HUPServer(int pid); +bool KillServer(int pid, bool WaitForProcess = false); + +#ifdef WIN32 + #include "WinNamedPipeStream.h" + #include "IOStreamGetLine.h" + #include "BoxPortsAndFiles.h" + + void SetNamedPipeName(const std::string& rPipeName); + // bool SendCommands(const std::string& rCmd); +#endif // WIN32 + +#endif // SERVER_CONTROL_H diff --git a/lib/server/ServerException.h b/lib/server/ServerException.h new file mode 100644 index 00000000..8851b90a --- /dev/null +++ b/lib/server/ServerException.h @@ -0,0 +1,46 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ServerException.h +// Purpose: Exception +// Created: 2003/07/08 +// +// -------------------------------------------------------------------------- + +#ifndef SERVEREXCEPTION__H +#define SERVEREXCEPTION__H + +// Compatibility header +#include "autogen_ServerException.h" +#include "autogen_ConnectionException.h" + +// Rename old connection exception names to new names without Conn_ prefix +// This is all because ConnectionException used to be derived from ServerException +// with some funky magic with subtypes. Perhaps a little unreliable, and the +// usefulness of it never really was used. +#define Conn_SocketWriteError SocketWriteError +#define Conn_SocketReadError SocketReadError +#define Conn_SocketNameLookupError SocketNameLookupError +#define Conn_SocketShutdownError SocketShutdownError +#define Conn_SocketConnectError SocketConnectError +#define Conn_TLSHandshakeFailed TLSHandshakeFailed +#define Conn_TLSShutdownFailed TLSShutdownFailed +#define Conn_TLSWriteFailed TLSWriteFailed +#define Conn_TLSReadFailed TLSReadFailed +#define Conn_TLSNoPeerCertificate TLSNoPeerCertificate +#define Conn_TLSPeerCertificateInvalid TLSPeerCertificateInvalid +#define Conn_TLSClosedWhenWriting TLSClosedWhenWriting +#define Conn_TLSHandshakeTimedOut TLSHandshakeTimedOut +#define Conn_Protocol_Timeout Protocol_Timeout +#define Conn_Protocol_ObjTooBig Protocol_ObjTooBig +#define Conn_Protocol_BadCommandRecieved Protocol_BadCommandRecieved +#define Conn_Protocol_UnknownCommandRecieved Protocol_UnknownCommandRecieved +#define Conn_Protocol_TriedToExecuteReplyCommand Protocol_TriedToExecuteReplyCommand +#define Conn_Protocol_UnexpectedReply Protocol_UnexpectedReply +#define Conn_Protocol_HandshakeFailed Protocol_HandshakeFailed +#define Conn_Protocol_StreamWhenObjExpected Protocol_StreamWhenObjExpected +#define Conn_Protocol_ObjWhenStreamExpected Protocol_ObjWhenStreamExpected +#define Conn_Protocol_TimeOutWhenSendingStream Protocol_TimeOutWhenSendingStream + +#endif // SERVEREXCEPTION__H + diff --git a/lib/server/ServerException.txt b/lib/server/ServerException.txt new file mode 100644 index 00000000..ed591b73 --- /dev/null +++ b/lib/server/ServerException.txt @@ -0,0 +1,39 @@ +EXCEPTION Server 3 + +# for historic reasons, some codes are not used + +Internal 0 +FailedToLoadConfiguration 1 +DaemoniseFailed 2 +AlreadyDaemonConstructed 3 +BadSocketHandle 4 +DupError 5 +SocketAlreadyOpen 8 +SocketOpenError 10 +SocketPollError 11 +SocketCloseError 13 +SocketNameUNIXPathTooLong 14 +SocketBindError 16 Check the ListenAddresses directive in your config file -- must refer to local IP addresses only +SocketAcceptError 17 +ServerStreamBadListenAddrs 18 +ServerForkError 19 +ServerWaitOnChildError 20 +TooManySocketsInMultiListen 21 There is a limit on how many addresses you can listen on simulatiously. +ServerStreamTooManyListenAddresses 22 +TLSContextNotInitialised 23 +TLSAllocationFailed 24 +TLSLoadCertificatesFailed 25 +TLSLoadPrivateKeyFailed 26 +TLSLoadTrustedCAsFailed 27 +TLSSetCiphersFailed 28 +SSLLibraryInitialisationError 29 +TLSNoSSLObject 31 +TLSAlreadyHandshaked 35 +SocketSetNonBlockingFailed 40 +Protocol_BadUsage 43 +Protocol_UnsuitableStreamTypeForSending 51 +CantWriteToProtocolUncertainStream 53 +ProtocolUncertainStreamBadBlockHeader 54 +SocketPairFailed 55 +CouldNotChangePIDFileOwner 56 +SSLRandomInitFailed 57 Read from /dev/*random device failed diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h new file mode 100644 index 00000000..e49dbcbe --- /dev/null +++ b/lib/server/ServerStream.h @@ -0,0 +1,418 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ServerStream.h +// Purpose: Stream based server daemons +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef SERVERSTREAM__H +#define SERVERSTREAM__H + +#include <stdlib.h> +#include <errno.h> + +#ifndef WIN32 + #include <sys/wait.h> +#endif + +#include "Daemon.h" +#include "SocketListen.h" +#include "Utils.h" +#include "Configuration.h" +#include "WaitForEvent.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: ServerStream +// Purpose: Stream based server daemon +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +template<typename StreamType, int Port, int ListenBacklog = 128, bool ForkToHandleRequests = true> +class ServerStream : public Daemon +{ +public: + ServerStream() + { + } + ~ServerStream() + { + DeleteSockets(); + } +private: + ServerStream(const ServerStream &rToCopy) + { + } +public: + + virtual const char *DaemonName() const + { + return "generic-stream-server"; + } + + virtual void OnIdle() { } + + virtual void Run() + { + // Set process title as appropriate + SetProcessTitle(ForkToHandleRequests?"server":"idle"); + + // Handle exceptions and child task quitting gracefully. + bool childExit = false; + try + { + Run2(childExit); + } + catch(BoxException &e) + { + if(childExit) + { + BOX_ERROR("Error in child process, " + "terminating connection: exception " << + e.what() << "(" << e.GetType() << + "/" << e.GetSubType() << ")"); + _exit(1); + } + else throw; + } + catch(std::exception &e) + { + if(childExit) + { + BOX_ERROR("Error in child process, " + "terminating connection: exception " << + e.what()); + _exit(1); + } + else throw; + } + catch(...) + { + if(childExit) + { + BOX_ERROR("Error in child process, " + "terminating connection: " + "unknown exception"); + _exit(1); + } + else throw; + } + + // if it's a child fork, exit the process now + if(childExit) + { + // Child task, dump leaks to trace, which we make sure is on + #ifdef BOX_MEMORY_LEAK_TESTING + #ifndef BOX_RELEASE_BUILD + TRACE_TO_SYSLOG(true); + TRACE_TO_STDOUT(true); + #endif + memleakfinder_traceblocksinsection(); + #endif + + // If this is a child quitting, exit now to stop bad things happening + _exit(0); + } + } + +protected: + virtual void NotifyListenerIsReady() { } + +public: + virtual void Run2(bool &rChildExit) + { + try + { + // Wait object with a timeout of 1 second, which + // is a reasonable time to wait before cleaning up + // finished child processes, and allows the daemon + // to terminate reasonably quickly on request. + WaitForEvent connectionWait(1000); + + // BLOCK + { + // Get the address we need to bind to + // this-> in next line required to build under some gcc versions + const Configuration &config(this->GetConfiguration()); + const Configuration &server(config.GetSubConfiguration("Server")); + std::string addrs = server.GetKeyValue("ListenAddresses"); + + // split up the list of addresses + std::vector<std::string> addrlist; + SplitString(addrs, ',', addrlist); + + for(unsigned int a = 0; a < addrlist.size(); ++a) + { + // split the address up into components + std::vector<std::string> c; + SplitString(addrlist[a], ':', c); + + // listen! + SocketListen<StreamType, ListenBacklog> *psocket = new SocketListen<StreamType, ListenBacklog>; + try + { + if(c[0] == "inet") + { + // Check arguments + if(c.size() != 2 && c.size() != 3) + { + THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) + } + + // Which port? + int port = Port; + + if(c.size() == 3) + { + // Convert to number + port = ::atol(c[2].c_str()); + if(port <= 0 || port > ((64*1024)-1)) + { + THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) + } + } + + // Listen + psocket->Listen(Socket::TypeINET, c[1].c_str(), port); + } + else if(c[0] == "unix") + { + #ifdef WIN32 + BOX_WARNING("Ignoring request to listen on a Unix socket on Windows: " << addrlist[a]); + delete psocket; + psocket = NULL; + #else + // Check arguments size + if(c.size() != 2) + { + THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) + } + + // unlink anything there + ::unlink(c[1].c_str()); + + psocket->Listen(Socket::TypeUNIX, c[1].c_str()); + #endif // WIN32 + } + else + { + delete psocket; + THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) + } + + if (psocket != NULL) + { + // Add to list of sockets + mSockets.push_back(psocket); + } + } + catch(...) + { + delete psocket; + throw; + } + + if (psocket != NULL) + { + // Add to the list of things to wait on + connectionWait.Add(psocket); + } + } + } + + NotifyListenerIsReady(); + + while(!StopRun()) + { + // Wait for a connection, or timeout + SocketListen<StreamType, ListenBacklog> *psocket + = (SocketListen<StreamType, ListenBacklog> *)connectionWait.Wait(); + + if(psocket) + { + // Get the incoming connection + // (with zero wait time) + std::string logMessage; + std::auto_ptr<StreamType> connection(psocket->Accept(0, &logMessage)); + + // Was there one (there should be...) + if(connection.get()) + { + // Since this is a template parameter, the if() will be optimised out by the compiler + #ifndef WIN32 // no fork on Win32 + if(ForkToHandleRequests && !IsSingleProcess()) + { + pid_t pid = ::fork(); + switch(pid) + { + case -1: + // Error! + THROW_EXCEPTION(ServerException, ServerForkError) + break; + + case 0: + // Child process + rChildExit = true; + // Close listening sockets + DeleteSockets(); + + // Set up daemon + EnterChild(); + SetProcessTitle("transaction"); + + // Memory leak test the forked process + #ifdef BOX_MEMORY_LEAK_TESTING + memleakfinder_startsectionmonitor(); + #endif + + // The derived class does some server magic with the connection + HandleConnection(*connection); + // Since rChildExit == true, the forked process will call _exit() on return from this fn + return; + + default: + // parent daemon process + break; + } + + // Log it + BOX_NOTICE("Message from child process " << pid << ": " << logMessage); + } + else + { + #endif // !WIN32 + // Just handle in this process + SetProcessTitle("handling"); + HandleConnection(*connection); + SetProcessTitle("idle"); + #ifndef WIN32 + } + #endif // !WIN32 + } + } + + OnIdle(); + + #ifndef WIN32 + // Clean up child processes (if forking daemon) + if(ForkToHandleRequests && !IsSingleProcess()) + { + WaitForChildren(); + } + #endif // !WIN32 + } + } + catch(...) + { + DeleteSockets(); + throw; + } + + // Delete the sockets + DeleteSockets(); + } + + #ifndef WIN32 // no waitpid() on Windows + void WaitForChildren() + { + int p = 0; + do + { + int status = 0; + p = ::waitpid(0 /* any child in process group */, + &status, WNOHANG); + + if(p == -1 && errno != ECHILD && errno != EINTR) + { + THROW_EXCEPTION(ServerException, + ServerWaitOnChildError) + } + else if(p == 0) + { + // no children exited, will return from + // function + } + else if(WIFEXITED(status)) + { + BOX_INFO("child process " << p << " " + "terminated normally"); + } + else if(WIFSIGNALED(status)) + { + int sig = WTERMSIG(status); + BOX_ERROR("child process " << p << " " + "terminated abnormally with " + "signal " << sig); + } + else + { + BOX_WARNING("something unknown happened " + "to child process " << p << ": " + "status = " << status); + } + } + while(p > 0); + } + #endif + + virtual void HandleConnection(StreamType &rStream) + { + Connection(rStream); + } + + virtual void Connection(StreamType &rStream) = 0; + +protected: + // For checking code in derived classes -- use if you have an algorithm which + // depends on the forking model in case someone changes it later. + bool WillForkToHandleRequests() + { + #ifdef WIN32 + return false; + #else + return ForkToHandleRequests && !IsSingleProcess(); + #endif // WIN32 + } + +private: + // -------------------------------------------------------------------------- + // + // Function + // Name: ServerStream::DeleteSockets() + // Purpose: Delete sockets + // Created: 9/3/04 + // + // -------------------------------------------------------------------------- + void DeleteSockets() + { + for(unsigned int l = 0; l < mSockets.size(); ++l) + { + if(mSockets[l]) + { + mSockets[l]->Close(); + delete mSockets[l]; + } + mSockets[l] = 0; + } + mSockets.clear(); + } + +private: + std::vector<SocketListen<StreamType, ListenBacklog> *> mSockets; +}; + +#define SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ + ConfigurationVerifyKey("ListenAddresses", 0, DEFAULT_ADDRESSES), \ + DAEMON_VERIFY_SERVER_KEYS + +#include "MemLeakFindOff.h" + +#endif // SERVERSTREAM__H + + + diff --git a/lib/server/ServerTLS.h b/lib/server/ServerTLS.h new file mode 100644 index 00000000..a74a671e --- /dev/null +++ b/lib/server/ServerTLS.h @@ -0,0 +1,80 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ServerTLS.h +// Purpose: Implementation of a server using TLS streams +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#ifndef SERVERTLS__H +#define SERVERTLS__H + +#include "ServerStream.h" +#include "SocketStreamTLS.h" +#include "SSLLib.h" +#include "TLSContext.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: ServerTLS +// Purpose: Implementation of a server using TLS streams +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +template<int Port, int ListenBacklog = 128, bool ForkToHandleRequests = true> +class ServerTLS : public ServerStream<SocketStreamTLS, Port, ListenBacklog, ForkToHandleRequests> +{ +public: + ServerTLS() + { + // Safe to call this here, as the Daemon class makes sure there is only one instance every of a Daemon. + SSLLib::Initialise(); + } + + ~ServerTLS() + { + } +private: + ServerTLS(const ServerTLS &) + { + } +public: + + virtual void Run2(bool &rChildExit) + { + // First, set up the SSL context. + // Get parameters from the configuration + // this-> in next line required to build under some gcc versions + const Configuration &conf(this->GetConfiguration()); + const Configuration &serverconf(conf.GetSubConfiguration("Server")); + std::string certFile(serverconf.GetKeyValue("CertificateFile")); + std::string keyFile(serverconf.GetKeyValue("PrivateKeyFile")); + std::string caFile(serverconf.GetKeyValue("TrustedCAsFile")); + mContext.Initialise(true /* as server */, certFile.c_str(), keyFile.c_str(), caFile.c_str()); + + // Then do normal stream server stuff + ServerStream<SocketStreamTLS, Port, ListenBacklog, + ForkToHandleRequests>::Run2(rChildExit); + } + + virtual void HandleConnection(SocketStreamTLS &rStream) + { + rStream.Handshake(mContext, true /* is server */); + // this-> in next line required to build under some gcc versions + this->Connection(rStream); + } + +private: + TLSContext mContext; +}; + +#define SERVERTLS_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ + ConfigurationVerifyKey("CertificateFile", ConfigTest_Exists), \ + ConfigurationVerifyKey("PrivateKeyFile", ConfigTest_Exists), \ + ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_Exists), \ + SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) + +#endif // SERVERTLS__H + diff --git a/lib/server/Socket.cpp b/lib/server/Socket.cpp new file mode 100644 index 00000000..4a83bdb0 --- /dev/null +++ b/lib/server/Socket.cpp @@ -0,0 +1,184 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Socket.cpp +// Purpose: Socket related stuff +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <sys/types.h> +#ifndef WIN32 +#include <sys/socket.h> +#include <netdb.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#endif + +#include <string.h> +#include <stdio.h> + +#include "Socket.h" +#include "ServerException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: Socket::NameLookupToSockAddr(SocketAllAddr &, int, +// char *, int) +// Purpose: Sets up a sockaddr structure given a name and type +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, + enum Type Type, const std::string& rName, int Port, + int &rSockAddrLenOut) +{ + int sockAddrLen = 0; + + switch(Type) + { + case TypeINET: + sockDomain = AF_INET; + { + // Lookup hostname + struct hostent *phost = ::gethostbyname(rName.c_str()); + if(phost != NULL) + { + if(phost->h_addr_list[0] != 0) + { + sockAddrLen = sizeof(addr.sa_inet); +#ifdef HAVE_STRUCT_SOCKADDR_IN_SIN_LEN + addr.sa_inet.sin_len = sizeof(addr.sa_inet); +#endif + addr.sa_inet.sin_family = PF_INET; + addr.sa_inet.sin_port = htons(Port); + addr.sa_inet.sin_addr = *((in_addr*)phost->h_addr_list[0]); + for(unsigned int l = 0; l < sizeof(addr.sa_inet.sin_zero); ++l) + { + addr.sa_inet.sin_zero[l] = 0; + } + } + else + { + THROW_EXCEPTION(ConnectionException, Conn_SocketNameLookupError); + } + } + else + { + THROW_EXCEPTION(ConnectionException, Conn_SocketNameLookupError); + } + } + break; + +#ifndef WIN32 + case TypeUNIX: + sockDomain = AF_UNIX; + { + // Check length of name is OK + unsigned int nameLen = rName.length(); + if(nameLen >= (sizeof(addr.sa_unix.sun_path) - 1)) + { + THROW_EXCEPTION(ServerException, SocketNameUNIXPathTooLong); + } + sockAddrLen = nameLen + (((char*)(&(addr.sa_unix.sun_path[0]))) - ((char*)(&addr.sa_unix))); +#ifdef HAVE_STRUCT_SOCKADDR_IN_SIN_LEN + addr.sa_unix.sun_len = sockAddrLen; +#endif + addr.sa_unix.sun_family = PF_UNIX; + ::strncpy(addr.sa_unix.sun_path, rName.c_str(), + sizeof(addr.sa_unix.sun_path) - 1); + addr.sa_unix.sun_path[sizeof(addr.sa_unix.sun_path)-1] = 0; + } + break; +#endif + + default: + THROW_EXCEPTION(CommonException, BadArguments) + break; + } + + // Return size of structure to caller + rSockAddrLenOut = sockAddrLen; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Socket::LogIncomingConnection(const struct sockaddr *, socklen_t) +// Purpose: Writes a message logging the connection to syslog +// Created: 2003/08/01 +// +// -------------------------------------------------------------------------- +void Socket::LogIncomingConnection(const struct sockaddr *addr, socklen_t addrlen) +{ + if(addr == NULL) {THROW_EXCEPTION(CommonException, BadArguments)} + + switch(addr->sa_family) + { + case AF_UNIX: + BOX_INFO("Incoming connection from local (UNIX socket)"); + break; + + case AF_INET: + { + sockaddr_in *a = (sockaddr_in*)addr; + BOX_INFO("Incoming connection from " << + inet_ntoa(a->sin_addr) << " port " << + ntohs(a->sin_port)); + } + break; + + default: + BOX_WARNING("Incoming connection of unknown type"); + break; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Socket::IncomingConnectionLogMessage(const struct sockaddr *, socklen_t) +// Purpose: Returns a string for use in log messages +// Created: 2003/08/01 +// +// -------------------------------------------------------------------------- +std::string Socket::IncomingConnectionLogMessage(const struct sockaddr *addr, socklen_t addrlen) +{ + if(addr == NULL) {THROW_EXCEPTION(CommonException, BadArguments)} + + switch(addr->sa_family) + { + case AF_UNIX: + return std::string("Incoming connection from local (UNIX socket)"); + break; + + case AF_INET: + { + char msg[256]; // more than enough + sockaddr_in *a = (sockaddr_in*)addr; + sprintf(msg, "Incoming connection from %s port %d", inet_ntoa(a->sin_addr), ntohs(a->sin_port)); + return std::string(msg); + } + break; + + default: + return std::string("Incoming connection of unknown type"); + break; + } + + // Dummy. + return std::string(); +} + diff --git a/lib/server/Socket.h b/lib/server/Socket.h new file mode 100644 index 00000000..5034dbd8 --- /dev/null +++ b/lib/server/Socket.h @@ -0,0 +1,56 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Socket.h +// Purpose: Socket related stuff +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef SOCKET__H +#define SOCKET__H + +#ifdef WIN32 +#include "emu.h" +typedef int socklen_t; +#else +#include <sys/socket.h> +#include <netinet/in.h> +#include <sys/un.h> +#endif + +#include <string> + +typedef union { + struct sockaddr sa_generic; + struct sockaddr_in sa_inet; +#ifndef WIN32 + struct sockaddr_un sa_unix; +#endif +} SocketAllAddr; + +// -------------------------------------------------------------------------- +// +// Namespace +// Name: Socket +// Purpose: Socket utilities +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +namespace Socket +{ + enum Type + { + TypeINET = 1, + TypeUNIX = 2 + }; + + void NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, + enum Type type, const std::string& rName, int Port, + int &rSockAddrLenOut); + void LogIncomingConnection(const struct sockaddr *addr, socklen_t addrlen); + std::string IncomingConnectionLogMessage(const struct sockaddr *addr, socklen_t addrlen); +}; + +#endif // SOCKET__H + diff --git a/lib/server/SocketListen.h b/lib/server/SocketListen.h new file mode 100644 index 00000000..586adf22 --- /dev/null +++ b/lib/server/SocketListen.h @@ -0,0 +1,312 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SocketListen.h +// Purpose: Stream based sockets for servers +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef SOCKETLISTEN__H +#define SOCKETLISTEN__H + +#include <errno.h> + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#ifdef HAVE_KQUEUE + #include <sys/event.h> + #include <sys/time.h> +#endif + +#ifndef WIN32 + #include <poll.h> +#endif + +#include <new> +#include <memory> +#include <string> + +#include "Socket.h" +#include "ServerException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: _NoSocketLocking +// Purpose: Default locking class for SocketListen +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +class _NoSocketLocking +{ +public: + _NoSocketLocking(int sock) + { + } + + ~_NoSocketLocking() + { + } + + bool HaveLock() + { + return true; + } + +private: + _NoSocketLocking(const _NoSocketLocking &rToCopy) + { + } +}; + + +// -------------------------------------------------------------------------- +// +// Class +// Name: SocketListen +// Purpose: +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +template<typename SocketType, int ListenBacklog = 128, typename SocketLockingType = _NoSocketLocking, int MaxMultiListenSockets = 16> +class SocketListen +{ +public: + // Initialise + SocketListen() + : mSocketHandle(-1) + { + } + // Close socket nicely + ~SocketListen() + { + Close(); + } +private: + SocketListen(const SocketListen &rToCopy) + { + } +public: + + enum + { + MaxMultipleListenSockets = MaxMultiListenSockets + }; + + void Close() + { + if(mSocketHandle != -1) + { +#ifdef WIN32 + if(::closesocket(mSocketHandle) == -1) +#else + if(::close(mSocketHandle) == -1) +#endif + { + BOX_LOG_SYS_ERROR("Failed to close network " + "socket"); + THROW_EXCEPTION(ServerException, + SocketCloseError) + } + } + mSocketHandle = -1; + } + + // ------------------------------------------------------------------ + // + // Function + // Name: SocketListen::Listen(int, char*, int) + // Purpose: Initialises, starts the socket listening. + // Created: 2003/07/31 + // + // ------------------------------------------------------------------ + void Listen(Socket::Type Type, const char *Name, int Port = 0) + { + if(mSocketHandle != -1) + { + THROW_EXCEPTION(ServerException, SocketAlreadyOpen); + } + + // Setup parameters based on type, looking up names if required + int sockDomain = 0; + SocketAllAddr addr; + int addrLen = 0; + Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, + Port, addrLen); + + // Create the socket + mSocketHandle = ::socket(sockDomain, SOCK_STREAM, + 0 /* let OS choose protocol */); + if(mSocketHandle == -1) + { + BOX_LOG_SYS_ERROR("Failed to create a network socket"); + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + // Set an option to allow reuse (useful for -HUP situations!) +#ifdef WIN32 + if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, "", + 0) == -1) +#else + int option = true; + if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, + &option, sizeof(option)) == -1) +#endif + { + BOX_LOG_SYS_ERROR("Failed to set socket options"); + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + // Bind it to the right port, and start listening + if(::bind(mSocketHandle, &addr.sa_generic, addrLen) == -1 + || ::listen(mSocketHandle, ListenBacklog) == -1) + { + // Dispose of the socket + ::close(mSocketHandle); + mSocketHandle = -1; + THROW_EXCEPTION(ServerException, SocketBindError) + } + } + + // ------------------------------------------------------------------ + // + // Function + // Name: SocketListen::Accept(int) + // Purpose: Accepts a connection, returning a pointer to + // a class of the specified type. May return a + // null pointer if a signal happens, or there's + // a timeout. Timeout specified in + // milliseconds, defaults to infinite time. + // Created: 2003/07/31 + // + // ------------------------------------------------------------------ + std::auto_ptr<SocketType> Accept(int Timeout = INFTIM, + std::string *pLogMsg = 0) + { + if(mSocketHandle == -1) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + + // Do the accept, using the supplied locking type + int sock; + struct sockaddr addr; + socklen_t addrlen = sizeof(addr); + // BLOCK + { + SocketLockingType socklock(mSocketHandle); + + if(!socklock.HaveLock()) + { + // Didn't get the lock for some reason. + // Wait a while, then return nothing. + BOX_ERROR("Failed to get a lock on incoming " + "connection"); + ::sleep(1); + return std::auto_ptr<SocketType>(); + } + + // poll this socket + struct pollfd p; + p.fd = mSocketHandle; + p.events = POLLIN; + p.revents = 0; + switch(::poll(&p, 1, Timeout)) + { + case -1: + // signal? + if(errno == EINTR) + { + BOX_ERROR("Failed to accept " + "connection: interrupted by " + "signal"); + // return nothing + return std::auto_ptr<SocketType>(); + } + else + { + BOX_LOG_SYS_ERROR("Failed to poll " + "connection"); + THROW_EXCEPTION(ServerException, + SocketPollError) + } + break; + case 0: // timed out + return std::auto_ptr<SocketType>(); + break; + default: // got some thing... + // control flows on... + break; + } + + sock = ::accept(mSocketHandle, &addr, &addrlen); + } + + // Got socket (or error), unlock (implicit in destruction) + if(sock == -1) + { + BOX_LOG_SYS_ERROR("Failed to accept connection"); + THROW_EXCEPTION(ServerException, SocketAcceptError) + } + + // Log it + if(pLogMsg) + { + *pLogMsg = Socket::IncomingConnectionLogMessage(&addr, + addrlen); + } + else + { + // Do logging ourselves + Socket::LogIncomingConnection(&addr, addrlen); + } + + return std::auto_ptr<SocketType>(new SocketType(sock)); + } + + // Functions to allow adding to WaitForEvent class, for efficient waiting + // on multiple sockets. +#ifdef HAVE_KQUEUE + // ------------------------------------------------------------------ + // + // Function + // Name: SocketListen::FillInKEevent + // Purpose: Fills in a kevent structure for this socket + // Created: 9/3/04 + // + // ------------------------------------------------------------------ + void FillInKEvent(struct kevent &rEvent, int Flags = 0) const + { + EV_SET(&rEvent, mSocketHandle, EVFILT_READ, 0, 0, 0, + (void*)this); + } +#else + // ------------------------------------------------------------------ + // + // Function + // Name: SocketListen::FillInPoll + // Purpose: Fills in the data necessary for a poll + // operation + // Created: 9/3/04 + // + // ------------------------------------------------------------------ + void FillInPoll(int &fd, short &events, int Flags = 0) const + { + fd = mSocketHandle; + events = POLLIN; + } +#endif + +private: + int mSocketHandle; +}; + +#include "MemLeakFindOff.h" + +#endif // SOCKETLISTEN__H + diff --git a/lib/server/SocketStream.cpp b/lib/server/SocketStream.cpp new file mode 100644 index 00000000..95b4b4f4 --- /dev/null +++ b/lib/server/SocketStream.cpp @@ -0,0 +1,514 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SocketStream.cpp +// Purpose: I/O stream interface for sockets +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <sys/types.h> +#include <errno.h> +#include <string.h> + +#ifndef WIN32 + #include <poll.h> +#endif + +#ifdef HAVE_UCRED_H + #include <ucred.h> +#endif + +#include "SocketStream.h" +#include "ServerException.h" +#include "CommonException.h" +#include "Socket.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::SocketStream() +// Purpose: Constructor (create stream ready for Open() call) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +SocketStream::SocketStream() + : mSocketHandle(INVALID_SOCKET_VALUE), + mReadClosed(false), + mWriteClosed(false), + mBytesRead(0), + mBytesWritten(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::SocketStream(int) +// Purpose: Create stream from existing socket handle +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +SocketStream::SocketStream(int socket) + : mSocketHandle(socket), + mReadClosed(false), + mWriteClosed(false), + mBytesRead(0), + mBytesWritten(0) +{ + if(socket < 0) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::SocketStream(const SocketStream &) +// Purpose: Copy constructor (dup()s socket) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +SocketStream::SocketStream(const SocketStream &rToCopy) + : mSocketHandle(::dup(rToCopy.mSocketHandle)), + mReadClosed(rToCopy.mReadClosed), + mWriteClosed(rToCopy.mWriteClosed), + mBytesRead(rToCopy.mBytesRead), + mBytesWritten(rToCopy.mBytesWritten) + +{ + if(rToCopy.mSocketHandle < 0) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + if(mSocketHandle == INVALID_SOCKET_VALUE) + { + THROW_EXCEPTION(ServerException, DupError); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::~SocketStream() +// Purpose: Destructor, closes stream if open +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +SocketStream::~SocketStream() +{ + if(mSocketHandle != INVALID_SOCKET_VALUE) + { + Close(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Attach(int) +// Purpose: Attach a socket handle to this stream +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void SocketStream::Attach(int socket) +{ + if(mSocketHandle != INVALID_SOCKET_VALUE) + { + THROW_EXCEPTION(ServerException, SocketAlreadyOpen) + } + + ResetCounters(); + + mSocketHandle = socket; + mReadClosed = false; + mWriteClosed = false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Open(Socket::Type, char *, int) +// Purpose: Opens a connection to a listening socket (INET or UNIX) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void SocketStream::Open(Socket::Type Type, const std::string& rName, int Port) +{ + if(mSocketHandle != INVALID_SOCKET_VALUE) + { + THROW_EXCEPTION(ServerException, SocketAlreadyOpen) + } + + // Setup parameters based on type, looking up names if required + int sockDomain = 0; + SocketAllAddr addr; + int addrLen = 0; + Socket::NameLookupToSockAddr(addr, sockDomain, Type, rName, Port, addrLen); + + // Create the socket + mSocketHandle = ::socket(sockDomain, SOCK_STREAM, + 0 /* let OS choose protocol */); + if(mSocketHandle == INVALID_SOCKET_VALUE) + { + BOX_LOG_SYS_ERROR("Failed to create a network socket"); + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + // Connect it + if(::connect(mSocketHandle, &addr.sa_generic, addrLen) == -1) + { + // Dispose of the socket +#ifdef WIN32 + DWORD err = WSAGetLastError(); + ::closesocket(mSocketHandle); + BOX_LOG_WIN_ERROR_NUMBER("Failed to connect to socket " + "(type " << Type << ", name " << rName << + ", port " << Port << ")", err); +#else // !WIN32 + BOX_LOG_SYS_ERROR("Failed to connect to socket (type " << + Type << ", name " << rName << ", port " << Port << + ")"); + ::close(mSocketHandle); +#endif // WIN32 + + mSocketHandle = INVALID_SOCKET_VALUE; + THROW_EXCEPTION(ConnectionException, Conn_SocketConnectError) + } + + ResetCounters(); + + mReadClosed = false; + mWriteClosed = false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Read(void *pBuffer, int NBytes) +// Purpose: Reads data from stream. Maybe returns less than asked for. +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + if(mSocketHandle == INVALID_SOCKET_VALUE) + { + THROW_EXCEPTION(ServerException, BadSocketHandle) + } + + if(Timeout != IOStream::TimeOutInfinite) + { + struct pollfd p; + p.fd = mSocketHandle; + p.events = POLLIN; + p.revents = 0; + switch(::poll(&p, 1, (Timeout == IOStream::TimeOutInfinite)?INFTIM:Timeout)) + { + case -1: + // error + if(errno == EINTR) + { + // Signal. Just return 0 bytes + return 0; + } + else + { + // Bad! + BOX_LOG_SYS_ERROR("Failed to poll socket"); + THROW_EXCEPTION(ServerException, + SocketPollError) + } + break; + + case 0: + // no data + return 0; + break; + + default: + // good to go! + break; + } + } + +#ifdef WIN32 + int r = ::recv(mSocketHandle, (char*)pBuffer, NBytes, 0); +#else + int r = ::read(mSocketHandle, pBuffer, NBytes); +#endif + if(r == -1) + { + if(errno == EINTR) + { + // Nothing could be read + return 0; + } + else + { + // Other error + BOX_LOG_SYS_ERROR("Failed to read from socket"); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError); + } + } + + // Closed for reading? + if(r == 0) + { + mReadClosed = true; + } + + mBytesRead += r; + return r; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Write(void *pBuffer, int NBytes) +// Purpose: Writes data, blocking until it's all done. +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void SocketStream::Write(const void *pBuffer, int NBytes) +{ + if(mSocketHandle == INVALID_SOCKET_VALUE) + { + THROW_EXCEPTION(ServerException, BadSocketHandle) + } + + // Buffer in byte sized type. + ASSERT(sizeof(char) == 1); + const char *buffer = (char *)pBuffer; + + // Bytes left to send + int bytesLeft = NBytes; + + while(bytesLeft > 0) + { + // Try to send. +#ifdef WIN32 + int sent = ::send(mSocketHandle, buffer, bytesLeft, 0); +#else + int sent = ::write(mSocketHandle, buffer, bytesLeft); +#endif + if(sent == -1) + { + // Error. + mWriteClosed = true; // assume can't write again + BOX_LOG_SYS_ERROR("Failed to write to socket"); + THROW_EXCEPTION(ConnectionException, + Conn_SocketWriteError); + } + + // Knock off bytes sent + bytesLeft -= sent; + // Move buffer pointer + buffer += sent; + + mBytesWritten += sent; + + // Need to wait until it can send again? + if(bytesLeft > 0) + { + BOX_TRACE("Waiting to send data on socket " << + mSocketHandle << " (" << bytesLeft << + " of " << NBytes << " bytes left)"); + + // Wait for data to send. + struct pollfd p; + p.fd = mSocketHandle; + p.events = POLLOUT; + p.revents = 0; + + if(::poll(&p, 1, 16000 /* 16 seconds */) == -1) + { + // Don't exception if it's just a signal + if(errno != EINTR) + { + BOX_LOG_SYS_ERROR("Failed to poll " + "socket"); + THROW_EXCEPTION(ServerException, + SocketPollError) + } + } + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Close() +// Purpose: Closes connection to remote socket +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void SocketStream::Close() +{ + if(mSocketHandle == INVALID_SOCKET_VALUE) + { + THROW_EXCEPTION(ServerException, BadSocketHandle) + } +#ifdef WIN32 + if(::closesocket(mSocketHandle) == -1) +#else + if(::close(mSocketHandle) == -1) +#endif + { + BOX_LOG_SYS_ERROR("Failed to close socket"); + // don't throw an exception here, assume that the socket was + // already closed or closing. + } + mSocketHandle = INVALID_SOCKET_VALUE; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::Shutdown(bool, bool) +// Purpose: Shuts down a socket for further reading and/or writing +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void SocketStream::Shutdown(bool Read, bool Write) +{ + if(mSocketHandle == INVALID_SOCKET_VALUE) + { + THROW_EXCEPTION(ServerException, BadSocketHandle) + } + + // Do anything? + if(!Read && !Write) return; + + int how = SHUT_RDWR; + if(Read && !Write) how = SHUT_RD; + if(!Read && Write) how = SHUT_WR; + + // Shut it down! + if(::shutdown(mSocketHandle, how) == -1) + { + BOX_LOG_SYS_ERROR("Failed to shutdown socket"); + THROW_EXCEPTION(ConnectionException, Conn_SocketShutdownError) + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::StreamDataLeft() +// Purpose: Still capable of reading data? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool SocketStream::StreamDataLeft() +{ + return !mReadClosed; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::StreamClosed() +// Purpose: Connection been closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool SocketStream::StreamClosed() +{ + return mWriteClosed; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::GetSocketHandle() +// Purpose: Returns socket handle for this stream (derived classes only). +// Will exception if there's no valid socket. +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +tOSSocketHandle SocketStream::GetSocketHandle() +{ + if(mSocketHandle == INVALID_SOCKET_VALUE) + { + THROW_EXCEPTION(ServerException, BadSocketHandle) + } + return mSocketHandle; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStream::GetPeerCredentials(uid_t &, gid_t &) +// Purpose: Returns true if the peer credientials are available. +// (will work on UNIX domain sockets only) +// Created: 19/2/04 +// +// -------------------------------------------------------------------------- +bool SocketStream::GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut) +{ +#ifdef HAVE_GETPEEREID + uid_t remoteEUID = 0xffff; + gid_t remoteEGID = 0xffff; + + if(::getpeereid(mSocketHandle, &remoteEUID, &remoteEGID) == 0) + { + rUidOut = remoteEUID; + rGidOut = remoteEGID; + return true; + } +#endif + +#if HAVE_DECL_SO_PEERCRED + struct ucred cred; + socklen_t credLen = sizeof(cred); + + if(::getsockopt(mSocketHandle, SOL_SOCKET, SO_PEERCRED, &cred, + &credLen) == 0) + { + rUidOut = cred.uid; + rGidOut = cred.gid; + return true; + } + + BOX_LOG_SYS_ERROR("Failed to get peer credentials on socket"); +#endif + +#if defined HAVE_UCRED_H && HAVE_GETPEERUCRED + ucred_t *pucred = NULL; + if(::getpeerucred(mSocketHandle, &pucred) == 0) + { + rUidOut = ucred_geteuid(pucred); + rGidOut = ucred_getegid(pucred); + ucred_free(pucred); + if (rUidOut == -1 || rGidOut == -1) + { + BOX_ERROR("Failed to get peer credentials on " + "socket: insufficient information"); + return false; + } + return true; + } + + BOX_LOG_SYS_ERROR("Failed to get peer credentials on socket"); +#endif + + // Not available + return false; +} + diff --git a/lib/server/SocketStream.h b/lib/server/SocketStream.h new file mode 100644 index 00000000..2b582f21 --- /dev/null +++ b/lib/server/SocketStream.h @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SocketStream.h +// Purpose: I/O stream interface for sockets +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- + +#ifndef SOCKETSTREAM__H +#define SOCKETSTREAM__H + +#include "IOStream.h" +#include "Socket.h" + +#ifdef WIN32 + typedef SOCKET tOSSocketHandle; + #define INVALID_SOCKET_VALUE (tOSSocketHandle)(-1) +#else + typedef int tOSSocketHandle; + #define INVALID_SOCKET_VALUE -1 +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: SocketStream +// Purpose: Stream interface for sockets +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +class SocketStream : public IOStream +{ +public: + SocketStream(); + SocketStream(int socket); + SocketStream(const SocketStream &rToCopy); + ~SocketStream(); + + void Open(Socket::Type Type, const std::string& rName, int Port = 0); + void Attach(int socket); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes); + virtual void Close(); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + + virtual void Shutdown(bool Read = true, bool Write = true); + + virtual bool GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut); + +protected: + tOSSocketHandle GetSocketHandle(); + void MarkAsReadClosed() {mReadClosed = true;} + void MarkAsWriteClosed() {mWriteClosed = true;} + +private: + tOSSocketHandle mSocketHandle; + bool mReadClosed; + bool mWriteClosed; + +protected: + off_t mBytesRead; + off_t mBytesWritten; + +public: + off_t GetBytesRead() const {return mBytesRead;} + off_t GetBytesWritten() const {return mBytesWritten;} + void ResetCounters() {mBytesRead = mBytesWritten = 0;} + bool IsOpened() { return mSocketHandle != INVALID_SOCKET_VALUE; } +}; + +#endif // SOCKETSTREAM__H + diff --git a/lib/server/SocketStreamTLS.cpp b/lib/server/SocketStreamTLS.cpp new file mode 100644 index 00000000..19fdadd4 --- /dev/null +++ b/lib/server/SocketStreamTLS.cpp @@ -0,0 +1,492 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SocketStreamTLS.cpp +// Purpose: Socket stream encrpyted and authenticated by TLS +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#define TLS_CLASS_IMPLEMENTATION_CPP +#include <openssl/ssl.h> +#include <openssl/bio.h> +#include <errno.h> +#include <fcntl.h> + +#ifndef WIN32 +#include <poll.h> +#endif + +#include "SocketStreamTLS.h" +#include "SSLLib.h" +#include "ServerException.h" +#include "TLSContext.h" +#include "BoxTime.h" + +#include "MemLeakFindOn.h" + +// Allow 5 minutes to handshake (in milliseconds) +#define TLS_HANDSHAKE_TIMEOUT (5*60*1000) + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::SocketStreamTLS() +// Purpose: Constructor +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +SocketStreamTLS::SocketStreamTLS() + : mpSSL(0), mpBIO(0) +{ + ResetCounters(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::SocketStreamTLS(int) +// Purpose: Constructor, taking previously connected socket +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +SocketStreamTLS::SocketStreamTLS(int socket) + : SocketStream(socket), + mpSSL(0), mpBIO(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::~SocketStreamTLS() +// Purpose: Destructor +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +SocketStreamTLS::~SocketStreamTLS() +{ + if(mpSSL) + { + // Attempt to close to avoid problems + Close(); + + // And if that didn't work... + if(mpSSL) + { + ::SSL_free(mpSSL); + mpSSL = 0; + mpBIO = 0; // implicity freed by the SSL_free call + } + } + + // If we only got to creating that BIO. + if(mpBIO) + { + ::BIO_free(mpBIO); + mpBIO = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Open(const TLSContext &, int, const char *, int) +// Purpose: Open connection, and perform TLS handshake +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SocketStreamTLS::Open(const TLSContext &rContext, Socket::Type Type, + const std::string& rName, int Port) +{ + SocketStream::Open(Type, rName, Port); + Handshake(rContext); + ResetCounters(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Handshake(const TLSContext &, bool) +// Purpose: Perform TLS handshake +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) +{ + if(mpBIO || mpSSL) {THROW_EXCEPTION(ServerException, TLSAlreadyHandshaked)} + + // Create a BIO for this socket + mpBIO = ::BIO_new(::BIO_s_socket()); + if(mpBIO == 0) + { + SSLLib::LogError("creating socket bio"); + THROW_EXCEPTION(ServerException, TLSAllocationFailed) + } + + tOSSocketHandle socket = GetSocketHandle(); + BIO_set_fd(mpBIO, socket, BIO_NOCLOSE); + + // Then the SSL object + mpSSL = ::SSL_new(rContext.GetRawContext()); + if(mpSSL == 0) + { + SSLLib::LogError("creating SSL object"); + THROW_EXCEPTION(ServerException, TLSAllocationFailed) + } + + // Make the socket non-blocking so timeouts on Read work + +#ifdef WIN32 + u_long nonblocking = 1; + ioctlsocket(socket, FIONBIO, &nonblocking); +#else // !WIN32 + // This is more portable than using ioctl with FIONBIO + int statusFlags = 0; + if(::fcntl(socket, F_GETFL, &statusFlags) < 0 + || ::fcntl(socket, F_SETFL, statusFlags | O_NONBLOCK) == -1) + { + THROW_EXCEPTION(ServerException, SocketSetNonBlockingFailed) + } +#endif + + // FIXME: This is less portable than the above. However, it MAY be needed + // for cygwin, which has/had bugs with fcntl + // + // int nonblocking = true; + // if(::ioctl(socket, FIONBIO, &nonblocking) == -1) + // { + // THROW_EXCEPTION(ServerException, SocketSetNonBlockingFailed) + // } + + // Set the two to know about each other + ::SSL_set_bio(mpSSL, mpBIO, mpBIO); + + bool waitingForHandshake = true; + while(waitingForHandshake) + { + // Attempt to do the handshake + int r = 0; + if(IsServer) + { + r = ::SSL_accept(mpSSL); + } + else + { + r = ::SSL_connect(mpSSL); + } + + // check return code + int se; + switch((se = ::SSL_get_error(mpSSL, r))) + { + case SSL_ERROR_NONE: + // No error, handshake succeeded + waitingForHandshake = false; + break; + + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + // wait for the requried data + if(WaitWhenRetryRequired(se, TLS_HANDSHAKE_TIMEOUT) == false) + { + // timed out + THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeTimedOut) + } + break; + + default: // (and SSL_ERROR_ZERO_RETURN) + // Error occured + if(IsServer) + { + SSLLib::LogError("accepting connection"); + THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) + } + else + { + SSLLib::LogError("connecting"); + THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) + } + } + } + + // And that's it +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WaitWhenRetryRequired(int, int) +// Purpose: Waits until the condition required by the TLS layer is met. +// Returns true if the condition is met, false if timed out. +// Created: 2003/08/15 +// +// -------------------------------------------------------------------------- +bool SocketStreamTLS::WaitWhenRetryRequired(int SSLErrorCode, int Timeout) +{ + struct pollfd p; + p.fd = GetSocketHandle(); + switch(SSLErrorCode) + { + case SSL_ERROR_WANT_READ: + p.events = POLLIN; + break; + + case SSL_ERROR_WANT_WRITE: + p.events = POLLOUT; + break; + + default: + // Not good! + THROW_EXCEPTION(ServerException, Internal) + break; + } + p.revents = 0; + + int64_t start, end; + start = BoxTimeToMilliSeconds(GetCurrentBoxTime()); + end = start + Timeout; + int result; + + do + { + int64_t now = BoxTimeToMilliSeconds(GetCurrentBoxTime()); + int poll_timeout = (int)(end - now); + if (poll_timeout < 0) poll_timeout = 0; + if (Timeout == IOStream::TimeOutInfinite) + { + poll_timeout = INFTIM; + } + result = ::poll(&p, 1, poll_timeout); + } + while(result == -1 && errno == EINTR); + + switch(result) + { + case -1: + // error - Bad! + THROW_EXCEPTION(ServerException, SocketPollError) + break; + + case 0: + // Condition not met, timed out + return false; + break; + + default: + // good to go! + return true; + break; + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Read(void *, int, int Timeout) +// Purpose: See base class +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) +{ + if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} + + // Make sure zero byte reads work as expected + if(NBytes == 0) + { + return 0; + } + + while(true) + { + int r = ::SSL_read(mpSSL, pBuffer, NBytes); + + int se; + switch((se = ::SSL_get_error(mpSSL, r))) + { + case SSL_ERROR_NONE: + // No error, return number of bytes read + mBytesRead += r; + return r; + break; + + case SSL_ERROR_ZERO_RETURN: + // Connection closed + MarkAsReadClosed(); + return 0; + break; + + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + // wait for the required data + // Will only get once around this loop, so don't need to calculate timeout values + if(WaitWhenRetryRequired(se, Timeout) == false) + { + // timed out + return 0; + } + break; + + default: + SSLLib::LogError("reading"); + THROW_EXCEPTION(ConnectionException, Conn_TLSReadFailed) + break; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Write(const void *, int) +// Purpose: See base class +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SocketStreamTLS::Write(const void *pBuffer, int NBytes) +{ + if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} + + // Make sure zero byte writes work as expected + if(NBytes == 0) + { + return; + } + + // from man SSL_write + // + // SSL_write() will only return with success, when the + // complete contents of buf of length num has been written. + // + // So no worries about partial writes and moving the buffer around + + while(true) + { + // try the write + int r = ::SSL_write(mpSSL, pBuffer, NBytes); + + int se; + switch((se = ::SSL_get_error(mpSSL, r))) + { + case SSL_ERROR_NONE: + // No error, data sent, return success + mBytesWritten += r; + return; + break; + + case SSL_ERROR_ZERO_RETURN: + // Connection closed + MarkAsWriteClosed(); + THROW_EXCEPTION(ConnectionException, Conn_TLSClosedWhenWriting) + break; + + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + // wait for the requried data + { + #ifndef BOX_RELEASE_BUILD + bool conditionmet = + #endif + WaitWhenRetryRequired(se, IOStream::TimeOutInfinite); + ASSERT(conditionmet); + } + break; + + default: + SSLLib::LogError("writing"); + THROW_EXCEPTION(ConnectionException, Conn_TLSWriteFailed) + break; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Close() +// Purpose: See base class +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SocketStreamTLS::Close() +{ + if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} + + // Base class to close + SocketStream::Close(); + + // Free resources + ::SSL_free(mpSSL); + mpSSL = 0; + mpBIO = 0; // implicitly freed by SSL_free +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::Shutdown() +// Purpose: See base class +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void SocketStreamTLS::Shutdown(bool Read, bool Write) +{ + if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} + + if(::SSL_shutdown(mpSSL) < 0) + { + SSLLib::LogError("shutting down"); + THROW_EXCEPTION(ConnectionException, Conn_TLSShutdownFailed) + } + + // Don't ask the base class to shutdown -- BIO does this, apparently. +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SocketStreamTLS::GetPeerCommonName() +// Purpose: Returns the common name of the other end of the connection +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +std::string SocketStreamTLS::GetPeerCommonName() +{ + if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} + + // Get certificate + X509 *cert = ::SSL_get_peer_certificate(mpSSL); + if(cert == 0) + { + ::X509_free(cert); + THROW_EXCEPTION(ConnectionException, Conn_TLSNoPeerCertificate) + } + + // Subject details + X509_NAME *subject = ::X509_get_subject_name(cert); + if(subject == 0) + { + ::X509_free(cert); + THROW_EXCEPTION(ConnectionException, Conn_TLSPeerCertificateInvalid) + } + + // Common name + char commonName[256]; + if(::X509_NAME_get_text_by_NID(subject, NID_commonName, commonName, sizeof(commonName)) <= 0) + { + ::X509_free(cert); + THROW_EXCEPTION(ConnectionException, Conn_TLSPeerCertificateInvalid) + } + // Terminate just in case + commonName[sizeof(commonName)-1] = '\0'; + + // Done. + return std::string(commonName); +} diff --git a/lib/server/SocketStreamTLS.h b/lib/server/SocketStreamTLS.h new file mode 100644 index 00000000..bb40ed10 --- /dev/null +++ b/lib/server/SocketStreamTLS.h @@ -0,0 +1,61 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SocketStreamTLS.h +// Purpose: Socket stream encrpyted and authenticated by TLS +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#ifndef SOCKETSTREAMTLS__H +#define SOCKETSTREAMTLS__H + +#include <string> + +#include "SocketStream.h" + +class TLSContext; +#ifndef TLS_CLASS_IMPLEMENTATION_CPP + class SSL; + class BIO; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: SocketStreamTLS +// Purpose: Socket stream encrpyted and authenticated by TLS +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +class SocketStreamTLS : public SocketStream +{ +public: + SocketStreamTLS(); + SocketStreamTLS(int socket); + ~SocketStreamTLS(); +private: + SocketStreamTLS(const SocketStreamTLS &rToCopy); +public: + + void Open(const TLSContext &rContext, Socket::Type Type, + const std::string& rName, int Port = 0); + void Handshake(const TLSContext &rContext, bool IsServer = false); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes); + virtual void Close(); + virtual void Shutdown(bool Read = true, bool Write = true); + + std::string GetPeerCommonName(); + +private: + bool WaitWhenRetryRequired(int SSLErrorCode, int Timeout); + +private: + SSL *mpSSL; + BIO *mpBIO; +}; + +#endif // SOCKETSTREAMTLS__H + diff --git a/lib/server/TLSContext.cpp b/lib/server/TLSContext.cpp new file mode 100644 index 00000000..ebc7384a --- /dev/null +++ b/lib/server/TLSContext.cpp @@ -0,0 +1,131 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: TLSContext.h +// Purpose: TLS (SSL) context for connections +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#define TLS_CLASS_IMPLEMENTATION_CPP +#include <openssl/ssl.h> + +#include "TLSContext.h" +#include "ServerException.h" +#include "SSLLib.h" +#include "TLSContext.h" + +#include "MemLeakFindOn.h" + +#define MAX_VERIFICATION_DEPTH 2 +#define CIPHER_LIST "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH" + +// -------------------------------------------------------------------------- +// +// Function +// Name: TLSContext::TLSContext() +// Purpose: Constructor +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +TLSContext::TLSContext() + : mpContext(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: TLSContext::~TLSContext() +// Purpose: Destructor +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +TLSContext::~TLSContext() +{ + if(mpContext != 0) + { + ::SSL_CTX_free(mpContext); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: TLSContext::Initialise(bool, const char *, const char *, const char *) +// Purpose: Initialise the context, loading in the specified certificate and private key files +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const char *PrivateKeyFile, const char *TrustedCAsFile) +{ + if(mpContext != 0) + { + ::SSL_CTX_free(mpContext); + } + + mpContext = ::SSL_CTX_new(AsServer?TLSv1_server_method():TLSv1_client_method()); + if(mpContext == NULL) + { + THROW_EXCEPTION(ServerException, TLSAllocationFailed) + } + + // Setup our identity + if(::SSL_CTX_use_certificate_chain_file(mpContext, CertificatesFile) != 1) + { + std::string msg = "loading certificates from "; + msg += CertificatesFile; + SSLLib::LogError(msg); + THROW_EXCEPTION(ServerException, TLSLoadCertificatesFailed) + } + if(::SSL_CTX_use_PrivateKey_file(mpContext, PrivateKeyFile, SSL_FILETYPE_PEM) != 1) + { + std::string msg = "loading private key from "; + msg += PrivateKeyFile; + SSLLib::LogError(msg); + THROW_EXCEPTION(ServerException, TLSLoadPrivateKeyFailed) + } + + // Setup the identify of CAs we trust + if(::SSL_CTX_load_verify_locations(mpContext, TrustedCAsFile, NULL) != 1) + { + std::string msg = "loading CA cert from "; + msg += TrustedCAsFile; + SSLLib::LogError(msg); + THROW_EXCEPTION(ServerException, TLSLoadTrustedCAsFailed) + } + + // Setup options to require these certificates + ::SSL_CTX_set_verify(mpContext, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL); + // and a sensible maximum depth + ::SSL_CTX_set_verify_depth(mpContext, MAX_VERIFICATION_DEPTH); + + // Setup allowed ciphers + if(::SSL_CTX_set_cipher_list(mpContext, CIPHER_LIST) != 1) + { + SSLLib::LogError("setting cipher list to " CIPHER_LIST); + THROW_EXCEPTION(ServerException, TLSSetCiphersFailed) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: TLSContext::GetRawContext() +// Purpose: Get the raw context for OpenSSL API +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +SSL_CTX *TLSContext::GetRawContext() const +{ + if(mpContext == 0) + { + THROW_EXCEPTION(ServerException, TLSContextNotInitialised) + } + return mpContext; +} + + + diff --git a/lib/server/TLSContext.h b/lib/server/TLSContext.h new file mode 100644 index 00000000..f52f5457 --- /dev/null +++ b/lib/server/TLSContext.h @@ -0,0 +1,41 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: TLSContext.h +// Purpose: TLS (SSL) context for connections +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- + +#ifndef TLSCONTEXT__H +#define TLSCONTEXT__H + +#ifndef TLS_CLASS_IMPLEMENTATION_CPP + class SSL_CTX; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: TLSContext +// Purpose: TLS (SSL) context for connections +// Created: 2003/08/06 +// +// -------------------------------------------------------------------------- +class TLSContext +{ +public: + TLSContext(); + ~TLSContext(); +private: + TLSContext(const TLSContext &); +public: + void Initialise(bool AsServer, const char *CertificatesFile, const char *PrivateKeyFile, const char *TrustedCAsFile); + SSL_CTX *GetRawContext() const; + +private: + SSL_CTX *mpContext; +}; + +#endif // TLSCONTEXT__H + diff --git a/lib/server/WinNamedPipeListener.h b/lib/server/WinNamedPipeListener.h new file mode 100644 index 00000000..26e76e3d --- /dev/null +++ b/lib/server/WinNamedPipeListener.h @@ -0,0 +1,232 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: WinNamedPipeListener.h +// Purpose: Windows named pipe socket connection listener +// for server +// Created: 2008/09/30 +// +// -------------------------------------------------------------------------- + +#ifndef WINNAMEDPIPELISTENER__H +#define WINNAMEDPIPELISTENER__H + +#include <OverlappedIO.h> +#include <WinNamedPipeStream.h> + +#include "ServerException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: WinNamedPipeListener +// Purpose: +// Created: 2008/09/30 +// +// -------------------------------------------------------------------------- +template<int ListenBacklog = 128> +class WinNamedPipeListener +{ +private: + std::auto_ptr<std::string> mapPipeName; + std::auto_ptr<OverlappedIO> mapOverlapConnect; + HANDLE mPipeHandle; + +public: + // Initialise + WinNamedPipeListener() + : mPipeHandle(INVALID_HANDLE_VALUE) + { } + +private: + WinNamedPipeListener(const WinNamedPipeListener &rToCopy) + { /* forbidden */ } + + HANDLE CreatePipeHandle(const std::string& rName) + { + std::string socket = WinNamedPipeStream::sPipeNamePrefix + + rName; + + HANDLE handle = CreateNamedPipeA( + socket.c_str(), // pipe name + PIPE_ACCESS_DUPLEX | // read/write access + FILE_FLAG_OVERLAPPED, // enabled overlapped I/O + PIPE_TYPE_BYTE | // message type pipe + PIPE_READMODE_BYTE | // message-read mode + PIPE_WAIT, // blocking mode + ListenBacklog + 1, // max. instances + 4096, // output buffer size + 4096, // input buffer size + NMPWAIT_USE_DEFAULT_WAIT, // client time-out + NULL); // default security attribute + + if (handle == INVALID_HANDLE_VALUE) + { + BOX_LOG_WIN_ERROR("Failed to create named pipe " << + socket); + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + return handle; + } + +public: + ~WinNamedPipeListener() + { + Close(); + } + + void Close() + { + if (mPipeHandle != INVALID_HANDLE_VALUE) + { + if (mapOverlapConnect.get()) + { + // outstanding connect in progress + if (CancelIo(mPipeHandle) != TRUE) + { + BOX_LOG_WIN_ERROR("Failed to cancel " + "outstanding connect request " + "on named pipe"); + } + + mapOverlapConnect.reset(); + } + + if (CloseHandle(mPipeHandle) != TRUE) + { + BOX_LOG_WIN_ERROR("Failed to close named pipe " + "handle"); + } + + mPipeHandle = INVALID_HANDLE_VALUE; + } + } + + // ------------------------------------------------------------------ + // + // Function + // Name: WinNamedPipeListener::Listen(std::string name) + // Purpose: Initialises socket name + // Created: 2003/07/31 + // + // ------------------------------------------------------------------ + void Listen(const std::string& rName) + { + Close(); + mapPipeName.reset(new std::string(rName)); + mPipeHandle = CreatePipeHandle(rName); + } + + // ------------------------------------------------------------------ + // + // Function + // Name: WinNamedPipeListener::Accept(int) + // Purpose: Accepts a connection, returning a pointer to + // a class of the specified type. May return a + // null pointer if a signal happens, or there's + // a timeout. Timeout specified in + // milliseconds, defaults to infinite time. + // Created: 2003/07/31 + // + // ------------------------------------------------------------------ + std::auto_ptr<WinNamedPipeStream> Accept(int Timeout = INFTIM, + const char* pLogMsgOut = NULL) + { + if(!mapPipeName.get()) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + + BOOL connected = FALSE; + std::auto_ptr<WinNamedPipeStream> mapStream; + + if (!mapOverlapConnect.get()) + { + // start a new connect operation + mapOverlapConnect.reset(new OverlappedIO()); + connected = ConnectNamedPipe(mPipeHandle, + &mapOverlapConnect->mOverlapped); + + if (connected == FALSE) + { + if (GetLastError() == ERROR_PIPE_CONNECTED) + { + connected = TRUE; + } + else if (GetLastError() != ERROR_IO_PENDING) + { + BOX_LOG_WIN_ERROR("Failed to connect " + "named pipe"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + } + } + + if (connected == FALSE) + { + // wait for connection + DWORD result = WaitForSingleObject( + mapOverlapConnect->mOverlapped.hEvent, + (Timeout == INFTIM) ? INFINITE : Timeout); + + if (result == WAIT_OBJECT_0) + { + DWORD dummy; + + if (!GetOverlappedResult(mPipeHandle, + &mapOverlapConnect->mOverlapped, + &dummy, TRUE)) + { + BOX_LOG_WIN_ERROR("Failed to get " + "overlapped connect result"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + + connected = TRUE; + } + else if (result == WAIT_TIMEOUT) + { + return mapStream; // contains NULL + } + else if (result == WAIT_ABANDONED) + { + BOX_ERROR("Wait for named pipe connection " + "was abandoned by the system"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + else if (result == WAIT_FAILED) + { + BOX_LOG_WIN_ERROR("Failed to wait for named " + "pipe connection"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + else + { + BOX_ERROR("Failed to wait for named pipe " + "connection: unknown return code " << + result); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + } + + ASSERT(connected == TRUE); + + mapStream.reset(new WinNamedPipeStream(mPipeHandle)); + mPipeHandle = CreatePipeHandle(*mapPipeName); + mapOverlapConnect.reset(); + + return mapStream; + } +}; + +#include "MemLeakFindOff.h" + +#endif // WINNAMEDPIPELISTENER__H diff --git a/lib/server/WinNamedPipeStream.cpp b/lib/server/WinNamedPipeStream.cpp new file mode 100644 index 00000000..1179516e --- /dev/null +++ b/lib/server/WinNamedPipeStream.cpp @@ -0,0 +1,620 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: WinNamedPipeStream.cpp +// Purpose: I/O stream interface for Win32 named pipes +// Created: 2005/12/07 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef WIN32 + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <sys/types.h> +#include <errno.h> +#include <windows.h> + +#include "WinNamedPipeStream.h" +#include "ServerException.h" +#include "CommonException.h" +#include "Socket.h" + +#include "MemLeakFindOn.h" + +std::string WinNamedPipeStream::sPipeNamePrefix = "\\\\.\\pipe\\"; + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::WinNamedPipeStream() +// Purpose: Constructor (create stream ready for Open() call) +// Created: 2005/12/07 +// +// -------------------------------------------------------------------------- +WinNamedPipeStream::WinNamedPipeStream() + : mSocketHandle(INVALID_HANDLE_VALUE), + mReadableEvent(INVALID_HANDLE_VALUE), + mBytesInBuffer(0), + mReadClosed(false), + mWriteClosed(false), + mIsServer(false), + mIsConnected(false) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::WinNamedPipeStream(HANDLE) +// Purpose: Constructor (with already-connected pipe handle) +// Created: 2008/10/01 +// +// -------------------------------------------------------------------------- +WinNamedPipeStream::WinNamedPipeStream(HANDLE hNamedPipe) + : mSocketHandle(hNamedPipe), + mReadableEvent(INVALID_HANDLE_VALUE), + mBytesInBuffer(0), + mReadClosed(false), + mWriteClosed(false), + mIsServer(true), + mIsConnected(true) +{ + // create the Readable event + mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + + if (mReadableEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to create the Readable event: " << + GetErrorMessage(GetLastError())); + Close(); + THROW_EXCEPTION(CommonException, Internal) + } + + // initialise the OVERLAPPED structure + memset(&mReadOverlap, 0, sizeof(mReadOverlap)); + mReadOverlap.hEvent = mReadableEvent; + + // start the first overlapped read + if (!ReadFile(mSocketHandle, mReadBuffer, sizeof(mReadBuffer), + NULL, &mReadOverlap)) + { + DWORD err = GetLastError(); + + if (err != ERROR_IO_PENDING) + { + BOX_ERROR("Failed to start overlapped read: " << + GetErrorMessage(err)); + Close(); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::~WinNamedPipeStream() +// Purpose: Destructor, closes stream if open +// Created: 2005/12/07 +// +// -------------------------------------------------------------------------- +WinNamedPipeStream::~WinNamedPipeStream() +{ + if (mSocketHandle != INVALID_HANDLE_VALUE) + { + try + { + Close(); + } + catch (std::exception &e) + { + BOX_ERROR("Caught exception while destroying " + "named pipe, ignored: " << e.what()); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::Accept(const std::string& rName) +// Purpose: Creates a new named pipe with the given name, +// and wait for a connection on it +// Created: 2005/12/07 +// +// -------------------------------------------------------------------------- +/* +void WinNamedPipeStream::Accept() +{ + if (mSocketHandle == INVALID_HANDLE_VALUE) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + + if (mIsConnected) + { + THROW_EXCEPTION(ServerException, SocketAlreadyOpen); + } + + bool connected = ConnectNamedPipe(mSocketHandle, (LPOVERLAPPED) NULL); + + if (!connected) + { + BOX_ERROR("Failed to ConnectNamedPipe(" << socket << "): " << + GetErrorMessage(GetLastError())); + Close(); + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + mBytesInBuffer = 0; + mReadClosed = false; + mWriteClosed = false; + mIsServer = true; // must flush and disconnect before closing + mIsConnected = true; + + // create the Readable event + mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + + if (mReadableEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to create the Readable event: " << + GetErrorMessage(GetLastError())); + Close(); + THROW_EXCEPTION(CommonException, Internal) + } + + // initialise the OVERLAPPED structure + memset(&mReadOverlap, 0, sizeof(mReadOverlap)); + mReadOverlap.hEvent = mReadableEvent; + + // start the first overlapped read + if (!ReadFile(mSocketHandle, mReadBuffer, sizeof(mReadBuffer), + NULL, &mReadOverlap)) + { + DWORD err = GetLastError(); + + if (err != ERROR_IO_PENDING) + { + BOX_ERROR("Failed to start overlapped read: " << + GetErrorMessage(err)); + Close(); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } +} +*/ + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::Connect(const std::string& rName) +// Purpose: Opens a connection to a listening named pipe +// Created: 2005/12/07 +// +// -------------------------------------------------------------------------- +void WinNamedPipeStream::Connect(const std::string& rName) +{ + if (mSocketHandle != INVALID_HANDLE_VALUE || mIsConnected) + { + THROW_EXCEPTION(ServerException, SocketAlreadyOpen) + } + + std::string socket = sPipeNamePrefix + rName; + + mSocketHandle = CreateFileA( + socket.c_str(), // pipe name + GENERIC_READ | // read and write access + GENERIC_WRITE, + 0, // no sharing + NULL, // default security attributes + OPEN_EXISTING, + 0, // default attributes + NULL); // no template file + + if (mSocketHandle == INVALID_HANDLE_VALUE) + { + DWORD err = GetLastError(); + if (err == ERROR_PIPE_BUSY) + { + BOX_ERROR("Failed to connect to backup daemon: " + "it is busy with another connection"); + } + else + { + BOX_ERROR("Failed to connect to backup daemon: " << + GetErrorMessage(err)); + } + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + mReadClosed = false; + mWriteClosed = false; + mIsServer = false; // just close the socket + mIsConnected = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::Read(void *pBuffer, int NBytes) +// Purpose: Reads data from stream. Maybe returns less than asked for. +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + // TODO no support for timeouts yet + if (!mIsServer && Timeout != IOStream::TimeOutInfinite) + { + THROW_EXCEPTION(CommonException, AssertFailed) + } + + if (mSocketHandle == INVALID_HANDLE_VALUE || !mIsConnected) + { + THROW_EXCEPTION(ServerException, BadSocketHandle) + } + + if (mReadClosed) + { + THROW_EXCEPTION(ConnectionException, SocketShutdownError) + } + + // ensure safe to cast NBytes to unsigned + if (NBytes < 0) + { + THROW_EXCEPTION(CommonException, AssertFailed) + } + + DWORD NumBytesRead; + + if (mIsServer) + { + // satisfy from buffer if possible, to avoid + // blocking on read. + bool needAnotherRead = false; + if (mBytesInBuffer == 0) + { + // overlapped I/O completed successfully? + // (wait if needed) + DWORD waitResult = WaitForSingleObject( + mReadOverlap.hEvent, Timeout); + + if (waitResult == WAIT_ABANDONED) + { + BOX_ERROR("Wait for command socket read " + "abandoned by system"); + THROW_EXCEPTION(ServerException, + BadSocketHandle); + } + else if (waitResult == WAIT_TIMEOUT) + { + // wait timed out, nothing to read + NumBytesRead = 0; + } + else if (waitResult != WAIT_OBJECT_0) + { + BOX_ERROR("Failed to wait for command " + "socket read: unknown result " << + waitResult); + } + // object is ready to read from + else if (GetOverlappedResult(mSocketHandle, + &mReadOverlap, &NumBytesRead, TRUE)) + { + needAnotherRead = true; + } + else + { + DWORD err = GetLastError(); + + if (err == ERROR_HANDLE_EOF) + { + mReadClosed = true; + } + else + { + if (err == ERROR_BROKEN_PIPE) + { + BOX_NOTICE("Control client " + "disconnected"); + } + else + { + BOX_ERROR("Failed to wait for " + "ReadFile to complete: " + << GetErrorMessage(err)); + } + + Close(); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } + } + else + { + NumBytesRead = 0; + } + + size_t BytesToCopy = NumBytesRead + mBytesInBuffer; + size_t BytesRemaining = 0; + + if (BytesToCopy > (size_t)NBytes) + { + BytesRemaining = BytesToCopy - NBytes; + BytesToCopy = NBytes; + } + + memcpy(pBuffer, mReadBuffer, BytesToCopy); + memmove(mReadBuffer, mReadBuffer + BytesToCopy, BytesRemaining); + + mBytesInBuffer = BytesRemaining; + NumBytesRead = BytesToCopy; + + if (needAnotherRead) + { + // reinitialise the OVERLAPPED structure + memset(&mReadOverlap, 0, sizeof(mReadOverlap)); + mReadOverlap.hEvent = mReadableEvent; + } + + // start the next overlapped read + if (needAnotherRead && !ReadFile(mSocketHandle, + mReadBuffer + mBytesInBuffer, + sizeof(mReadBuffer) - mBytesInBuffer, + NULL, &mReadOverlap)) + { + DWORD err = GetLastError(); + if (err == ERROR_IO_PENDING) + { + // Don't reset yet, there might be data + // in the buffer waiting to be read, + // will check below. + // ResetEvent(mReadableEvent); + } + else if (err == ERROR_HANDLE_EOF) + { + mReadClosed = true; + } + else if (err == ERROR_BROKEN_PIPE) + { + BOX_ERROR("Control client disconnected"); + mReadClosed = true; + } + else + { + BOX_ERROR("Failed to start overlapped read: " + << GetErrorMessage(err)); + Close(); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } + } + else + { + if (!ReadFile( + mSocketHandle, // pipe handle + pBuffer, // buffer to receive reply + NBytes, // size of buffer + &NumBytesRead, // number of bytes read + NULL)) // not overlapped + { + DWORD err = GetLastError(); + + Close(); + + // ERROR_NO_DATA is a strange name for + // "The pipe is being closed". No exception wanted. + + if (err == ERROR_NO_DATA || + err == ERROR_PIPE_NOT_CONNECTED) + { + NumBytesRead = 0; + } + else + { + BOX_ERROR("Failed to read from control socket: " + << GetErrorMessage(err)); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } + + // Closed for reading at EOF? + if (NumBytesRead == 0) + { + mReadClosed = true; + } + } + + return NumBytesRead; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::Write(void *pBuffer, int NBytes) +// Purpose: Writes data, blocking until it's all done. +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void WinNamedPipeStream::Write(const void *pBuffer, int NBytes) +{ + if (mSocketHandle == INVALID_HANDLE_VALUE || !mIsConnected) + { + THROW_EXCEPTION(ServerException, BadSocketHandle) + } + + // Buffer in byte sized type. + ASSERT(sizeof(char) == 1); + const char *pByteBuffer = (char *)pBuffer; + + int NumBytesWrittenTotal = 0; + + while (NumBytesWrittenTotal < NBytes) + { + DWORD NumBytesWrittenThisTime = 0; + + bool Success = WriteFile( + mSocketHandle, // pipe handle + pByteBuffer + NumBytesWrittenTotal, // message + NBytes - NumBytesWrittenTotal, // message length + &NumBytesWrittenThisTime, // bytes written this time + NULL); // not overlapped + + if (!Success) + { + // ERROR_NO_DATA is a strange name for + // "The pipe is being closed". + + DWORD err = GetLastError(); + + if (err != ERROR_NO_DATA) + { + BOX_ERROR("Failed to write to control " + "socket: " << GetErrorMessage(err)); + } + + Close(); + + THROW_EXCEPTION(ConnectionException, + Conn_SocketWriteError) + } + + NumBytesWrittenTotal += NumBytesWrittenThisTime; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::Close() +// Purpose: Closes connection to remote socket +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void WinNamedPipeStream::Close() +{ + if (mSocketHandle == INVALID_HANDLE_VALUE && mIsConnected) + { + BOX_ERROR("Named pipe: inconsistent connected state"); + mIsConnected = false; + } + + if (mSocketHandle == INVALID_HANDLE_VALUE) + { + THROW_EXCEPTION(ServerException, BadSocketHandle) + } + + if (mIsServer) + { + if (!CancelIo(mSocketHandle)) + { + BOX_ERROR("Failed to cancel outstanding I/O: " << + GetErrorMessage(GetLastError())); + } + + if (mReadableEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to destroy Readable event: " + "invalid handle"); + } + else if (!CloseHandle(mReadableEvent)) + { + BOX_ERROR("Failed to destroy Readable event: " << + GetErrorMessage(GetLastError())); + } + + mReadableEvent = INVALID_HANDLE_VALUE; + + if (!FlushFileBuffers(mSocketHandle)) + { + BOX_ERROR("Failed to FlushFileBuffers: " << + GetErrorMessage(GetLastError())); + } + + if (!DisconnectNamedPipe(mSocketHandle)) + { + DWORD err = GetLastError(); + if (err != ERROR_PIPE_NOT_CONNECTED) + { + BOX_ERROR("Failed to DisconnectNamedPipe: " << + GetErrorMessage(err)); + } + } + + mIsServer = false; + } + + bool result = CloseHandle(mSocketHandle); + + mSocketHandle = INVALID_HANDLE_VALUE; + mIsConnected = false; + mReadClosed = true; + mWriteClosed = true; + + if (!result) + { + BOX_ERROR("Failed to CloseHandle: " << + GetErrorMessage(GetLastError())); + THROW_EXCEPTION(ServerException, SocketCloseError) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::StreamDataLeft() +// Purpose: Still capable of reading data? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool WinNamedPipeStream::StreamDataLeft() +{ + return !mReadClosed; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::StreamClosed() +// Purpose: Connection been closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool WinNamedPipeStream::StreamClosed() +{ + return mWriteClosed; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::WriteAllBuffered() +// Purpose: Ensures that any data which has been buffered is written to the stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void WinNamedPipeStream::WriteAllBuffered() +{ + if (mSocketHandle == INVALID_HANDLE_VALUE || !mIsConnected) + { + THROW_EXCEPTION(ServerException, BadSocketHandle) + } + + if (!FlushFileBuffers(mSocketHandle)) + { + BOX_ERROR("Failed to FlushFileBuffers: " << + GetErrorMessage(GetLastError())); + } +} + + +#endif // WIN32 diff --git a/lib/server/WinNamedPipeStream.h b/lib/server/WinNamedPipeStream.h new file mode 100644 index 00000000..386ff7e3 --- /dev/null +++ b/lib/server/WinNamedPipeStream.h @@ -0,0 +1,67 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: WinNamedPipeStream.h +// Purpose: I/O stream interface for Win32 named pipes +// Created: 2005/12/07 +// +// -------------------------------------------------------------------------- + +#if ! defined WINNAMEDPIPESTREAM__H && defined WIN32 +#define WINNAMEDPIPESTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: WinNamedPipeStream +// Purpose: I/O stream interface for Win32 named pipes +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +class WinNamedPipeStream : public IOStream +{ +public: + WinNamedPipeStream(); + WinNamedPipeStream(HANDLE hNamedPipe); + ~WinNamedPipeStream(); + + // server side - create the named pipe and listen for connections + // use WinNamedPipeListener to do this instead. + + // client side - connect to a waiting server + void Connect(const std::string& rName); + + // both sides + virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes); + virtual void WriteAllBuffered(); + virtual void Close(); + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +protected: + void MarkAsReadClosed() {mReadClosed = true;} + void MarkAsWriteClosed() {mWriteClosed = true;} + +private: + WinNamedPipeStream(const WinNamedPipeStream &rToCopy) + { /* do not call */ } + + HANDLE mSocketHandle; + HANDLE mReadableEvent; + OVERLAPPED mReadOverlap; + uint8_t mReadBuffer[4096]; + size_t mBytesInBuffer; + bool mReadClosed; + bool mWriteClosed; + bool mIsServer; + bool mIsConnected; + +public: + static std::string sPipeNamePrefix; +}; + +#endif // WINNAMEDPIPESTREAM__H diff --git a/lib/server/makeprotocol.pl.in b/lib/server/makeprotocol.pl.in new file mode 100755 index 00000000..91ba55b0 --- /dev/null +++ b/lib/server/makeprotocol.pl.in @@ -0,0 +1,1093 @@ +#!@PERL@ +use strict; + +use lib "../../infrastructure"; +use BoxPlatform; + +# Make protocol C++ classes from a protocol description file + +# built in type info (values are is basic type, C++ typename) +# may get stuff added to it later if protocol uses extra types +my %translate_type_info = +( + 'int64' => [1, 'int64_t'], + 'int32' => [1, 'int32_t'], + 'int16' => [1, 'int16_t'], + 'int8' => [1, 'int8_t'], + 'bool' => [1, 'bool'], + 'string' => [0, 'std::string'] +); + +# built in instructions for logging various types +# may be added to +my %log_display_types = +( + 'int64' => ['0x%llx', 'VAR'], + 'int32' => ['0x%x', 'VAR'], + 'int16' => ['0x%x', 'VAR'], + 'int8' => ['0x%x', 'VAR'], + 'bool' => ['%s', '((VAR)?"true":"false")'], + 'string' => ['%s', 'VAR.c_str()'] +); + + + +my ($type, $file) = @ARGV; + +if($type ne 'Server' && $type ne 'Client') +{ + die "Neither Server or Client is specified on command line\n"; +} + +open IN, $file or die "Can't open input file $file\n"; + +print "Making $type protocol classes from $file...\n"; + +my @extra_header_files; + +my $implement_syslog = 0; +my $implement_filelog = 0; + +# read attributes +my %attr; +while(<IN>) +{ + # get and clean line + my $l = $_; $l =~ s/#.*\Z//; $l =~ s/\A\s+//; $l =~ s/\s+\Z//; next unless $l =~ m/\S/; + + last if $l eq 'BEGIN_OBJECTS'; + + my ($k,$v) = split /\s+/,$l,2; + + if($k eq 'ClientType') + { + add_type($v) if $type eq 'Client'; + } + elsif($k eq 'ServerType') + { + add_type($v) if $type eq 'Server'; + } + elsif($k eq 'ImplementLog') + { + my ($log_if_type,$log_type) = split /\s+/,$v; + if($type eq $log_if_type) + { + if($log_type eq 'syslog') + { + $implement_syslog = 1; + } + elsif($log_type eq 'file') + { + $implement_filelog = 1; + } + else + { + printf("ERROR: Unknown log type for implementation: $log_type\n"); + exit(1); + } + } + } + elsif($k eq 'LogTypeToText') + { + my ($log_if_type,$type_name,$printf_format,$arg_template) = split /\s+/,$v; + if($type eq $log_if_type) + { + $log_display_types{$type_name} = [$printf_format,$arg_template] + } + } + else + { + $attr{$k} = $v; + } +} + +sub add_type +{ + my ($protocol_name, $cpp_name, $header_file) = split /\s+/,$_[0]; + + $translate_type_info{$protocol_name} = [0, $cpp_name]; + push @extra_header_files, $header_file; +} + +# check attributes +for(qw/Name ServerContextClass IdentString/) +{ + if(!exists $attr{$_}) + { + die "Attribute $_ is required, but not specified\n"; + } +} + +my $protocol_name = $attr{'Name'}; +my ($context_class, $context_class_inc) = split /\s+/,$attr{'ServerContextClass'}; +my $ident_string = $attr{'IdentString'}; + +my $current_cmd = ''; +my %cmd_contents; +my %cmd_attributes; +my %cmd_constants; +my %cmd_id; +my @cmd_list; + +# read in the command definitions +while(<IN>) +{ + # get and clean line + my $l = $_; $l =~ s/#.*\Z//; $l =~ s/\s+\Z//; next unless $l =~ m/\S/; + + # definitions or new command thing? + if($l =~ m/\A\s+/) + { + die "No command defined yet" if $current_cmd eq ''; + + # definition of component + $l =~ s/\A\s+//; + + my ($type,$name,$value) = split /\s+/,$l; + if($type eq 'CONSTANT') + { + push @{$cmd_constants{$current_cmd}},"$name = $value" + } + else + { + push @{$cmd_contents{$current_cmd}},$type,$name; + } + } + else + { + # new command + my ($name,$id,@attributes) = split /\s+/,$l; + $cmd_attributes{$name} = [@attributes]; + $cmd_id{$name} = int($id); + $current_cmd = $name; + push @cmd_list,$name; + } +} + +close IN; + + + +# open files +my $h_filename = 'autogen_'.$protocol_name.'Protocol'.$type.'.h'; +open CPP,'>autogen_'.$protocol_name.'Protocol'.$type.'.cpp'; +open H,">$h_filename"; + +print CPP <<__E; + +// Auto-generated file -- do not edit + +#include "Box.h" + +#include <sstream> + +#include "$h_filename" +#include "IOStream.h" + +__E + +if($implement_syslog) +{ + print H <<EOF; +#ifndef WIN32 +#include <syslog.h> +#endif +EOF +} + + +my $guardname = uc 'AUTOGEN_'.$protocol_name.'Protocol'.$type.'_H'; +print H <<__E; + +// Auto-generated file -- do not edit + +#ifndef $guardname +#define $guardname + +#include "Protocol.h" +#include "ProtocolObject.h" +#include "ServerException.h" + +class IOStream; + +__E + +if($implement_filelog) +{ + print H qq~#include <stdio.h>\n~; +} + +# extra headers +for(@extra_header_files) +{ + print H qq~#include "$_"\n~ +} +print H "\n"; + +if($type eq 'Server') +{ + # need utils file for the server + print H '#include "Utils.h"',"\n\n" +} + + +my $derive_objects_from = 'ProtocolObject'; +my $objects_extra_h = ''; +my $objects_extra_cpp = ''; +if($type eq 'Server') +{ + # define the context + print H "class $context_class;\n\n"; + print CPP "#include \"$context_class_inc\"\n\n"; + + # change class we derive the objects from + $derive_objects_from = $protocol_name.'ProtocolObject'; + + $objects_extra_h = <<__E; + virtual std::auto_ptr<ProtocolObject> DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext); +__E + $objects_extra_cpp = <<__E; +std::auto_ptr<ProtocolObject> ${derive_objects_from}::DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext) +{ + THROW_EXCEPTION(ConnectionException, Conn_Protocol_TriedToExecuteReplyCommand) +} +__E +} + +print CPP qq~#include "MemLeakFindOn.h"\n~; + +if($type eq 'Client' && ($implement_syslog || $implement_filelog)) +{ + # change class we derive the objects from + $derive_objects_from = $protocol_name.'ProtocolObjectCl'; +} +if($implement_syslog) +{ + $objects_extra_h .= <<__E; + virtual void LogSysLog(const char *Action) const = 0; +__E +} +if($implement_filelog) +{ + $objects_extra_h .= <<__E; + virtual void LogFile(const char *Action, FILE *file) const = 0; +__E +} + +if($derive_objects_from ne 'ProtocolObject') +{ + # output a definition for the protocol object derived class + print H <<__E; +class ${protocol_name}ProtocolServer; + +class $derive_objects_from : public ProtocolObject +{ +public: + $derive_objects_from(); + virtual ~$derive_objects_from(); + $derive_objects_from(const $derive_objects_from &rToCopy); + +$objects_extra_h +}; +__E + + # and some cpp definitions + print CPP <<__E; +${derive_objects_from}::${derive_objects_from}() +{ +} +${derive_objects_from}::~${derive_objects_from}() +{ +} +${derive_objects_from}::${derive_objects_from}(const $derive_objects_from &rToCopy) +{ +} +$objects_extra_cpp +__E +} + + + +my $classname_base = $protocol_name.'Protocol'.$type; + +# output the classes +for my $cmd (@cmd_list) +{ + print H <<__E; +class $classname_base$cmd : public $derive_objects_from +{ +public: + $classname_base$cmd(); + $classname_base$cmd(const $classname_base$cmd &rToCopy); + ~$classname_base$cmd(); + int GetType() const; + enum + { + TypeID = $cmd_id{$cmd} + }; +__E + # constants + if(exists $cmd_constants{$cmd}) + { + print H "\tenum\n\t{\n\t\t"; + print H join(",\n\t\t",@{$cmd_constants{$cmd}}); + print H "\n\t};\n"; + } + # flags + if(obj_is_type($cmd,'EndsConversation')) + { + print H "\tbool IsConversationEnd() const;\n"; + } + if(obj_is_type($cmd,'IsError')) + { + print H "\tbool IsError(int &rTypeOut, int &rSubTypeOut) const;\n"; + print H "\tstd::string GetMessage() const;\n"; + } + if($type eq 'Server' && obj_is_type($cmd, 'Command')) + { + print H "\tstd::auto_ptr<ProtocolObject> DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext); // IMPLEMENT THIS\n" + } + + # want to be able to read from streams? + my $read_from_streams = (obj_is_type($cmd,'Command') && $type eq 'Server') || (obj_is_type($cmd,'Reply') && $type eq 'Client'); + my $write_to_streams = (obj_is_type($cmd,'Command') && $type eq 'Client') || (obj_is_type($cmd,'Reply') && $type eq 'Server'); + + if($read_from_streams) + { + print H "\tvoid SetPropertiesFromStreamData(Protocol &rProtocol);\n"; + + # write Get functions + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + print H "\t".translate_type_to_arg_type($ty)." Get$nm() {return m$nm;}\n"; + } + } + my $param_con_args = ''; + if($write_to_streams) + { + # extra constructor? + if($#{$cmd_contents{$cmd}} >= 0) + { + my @a; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + push @a,translate_type_to_arg_type($ty)." $nm"; + } + $param_con_args = join(', ',@a); + print H "\t$classname_base$cmd(".$param_con_args.");\n"; + } + print H "\tvoid WritePropertiesToStreamData(Protocol &rProtocol) const;\n"; + # set functions + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + print H "\tvoid Set$nm(".translate_type_to_arg_type($ty)." $nm) {m$nm = $nm;}\n"; + } + } + + if($implement_syslog) + { + print H "\tvirtual void LogSysLog(const char *Action) const;\n"; + } + if($implement_filelog) + { + print H "\tvirtual void LogFile(const char *Action, FILE *file) const;\n"; + } + + + # write member variables and setup for cpp file + my @def_constructor_list; + my @copy_constructor_list; + my @param_constructor_list; + + print H "private:\n"; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + print H "\t".translate_type_to_member_type($ty)." m$nm;\n"; + + my ($basic,$typename) = translate_type($ty); + if($basic) + { + push @def_constructor_list, "m$nm(0)"; + } + push @copy_constructor_list, "m$nm(rToCopy.m$nm)"; + push @param_constructor_list, "m$nm($nm)"; + } + + # finish off + print H "};\n\n"; + + # now the cpp file... + my $def_con_vars = join(",\n\t ",@def_constructor_list); + $def_con_vars = "\n\t: ".$def_con_vars if $def_con_vars ne ''; + my $copy_con_vars = join(",\n\t ",@copy_constructor_list); + $copy_con_vars = "\n\t: ".$copy_con_vars if $copy_con_vars ne ''; + my $param_con_vars = join(",\n\t ",@param_constructor_list); + $param_con_vars = "\n\t: ".$param_con_vars if $param_con_vars ne ''; + + my $class = "$classname_base$cmd".'::'; + print CPP <<__E; +$class$classname_base$cmd()$def_con_vars +{ +} +$class$classname_base$cmd(const $classname_base$cmd &rToCopy)$copy_con_vars +{ +} +$class~$classname_base$cmd() +{ +} +int ${class}GetType() const +{ + return $cmd_id{$cmd}; +} +__E + if($read_from_streams) + { + print CPP "void ${class}SetPropertiesFromStreamData(Protocol &rProtocol)\n{\n"; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + if($ty =~ m/\Avector/) + { + print CPP "\trProtocol.ReadVector(m$nm);\n"; + } + else + { + print CPP "\trProtocol.Read(m$nm);\n"; + } + } + print CPP "}\n"; + } + if($write_to_streams) + { + # implement extra constructor? + if($param_con_vars ne '') + { + print CPP "$class$classname_base$cmd($param_con_args)$param_con_vars\n{\n}\n"; + } + print CPP "void ${class}WritePropertiesToStreamData(Protocol &rProtocol) const\n{\n"; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + if($ty =~ m/\Avector/) + { + print CPP "\trProtocol.WriteVector(m$nm);\n"; + } + else + { + print CPP "\trProtocol.Write(m$nm);\n"; + } + } + print CPP "}\n"; + } + if(obj_is_type($cmd,'EndsConversation')) + { + print CPP "bool ${class}IsConversationEnd() const\n{\n\treturn true;\n}\n"; + } + if(obj_is_type($cmd,'IsError')) + { + # get parameters + my ($mem_type,$mem_subtype) = split /,/,obj_get_type_params($cmd,'IsError'); + print CPP <<__E; +bool ${class}IsError(int &rTypeOut, int &rSubTypeOut) const +{ + rTypeOut = m$mem_type; + rSubTypeOut = m$mem_subtype; + return true; +} +std::string ${class}GetMessage() const +{ + switch(m$mem_subtype) + { +__E + foreach my $const (@{$cmd_constants{$cmd}}) + { + next unless $const =~ /^Err_(.*)/; + my $shortname = $1; + $const =~ s/ = .*//; + print CPP <<__E; + case $const: return "$shortname"; +__E + } + print CPP <<__E; + default: + std::ostringstream out; + out << "Unknown subtype " << m$mem_subtype; + return out.str(); + } +} +__E + } + + if($implement_syslog) + { + my ($log) = make_log_strings_framework($cmd); + print CPP <<__E; +void ${class}LogSysLog(const char *Action) const +{ + BOX_TRACE($log); +} +__E + } + if($implement_filelog) + { + my ($log) = make_log_strings_framework($cmd); + print CPP <<__E; +void ${class}LogFile(const char *Action, FILE *File) const +{ + std::ostringstream oss; + oss << $log; + ::fprintf(File, "%s\\n", oss.str().c_str()); + ::fflush(File); +} +__E + } +} + +# finally, the protocol object itself +print H <<__E; +class $classname_base : public Protocol +{ +public: + $classname_base(IOStream &rStream); + virtual ~$classname_base(); + + std::auto_ptr<$derive_objects_from> Receive(); + void Send(const ${derive_objects_from} &rObject); +__E +if($implement_syslog) +{ + print H "\tvoid SetLogToSysLog(bool Log = false) {mLogToSysLog = Log;}\n"; +} +if($implement_filelog) +{ + print H "\tvoid SetLogToFile(FILE *File = 0) {mLogToFile = File;}\n"; +} +if($type eq 'Server') +{ + # need to put in the conversation function + print H "\tvoid DoServer($context_class &rContext);\n\n"; + # and the send vector thing + print H "\tvoid SendStreamAfterCommand(IOStream *pStream);\n\n"; +} +if($type eq 'Client') +{ + # add plain object taking query functions + my $with_params; + for my $cmd (@cmd_list) + { + if(obj_is_type($cmd,'Command')) + { + my $has_stream = obj_is_type($cmd,'StreamWithCommand'); + my $argextra = $has_stream?', IOStream &rStream':''; + my $queryextra = $has_stream?', rStream':''; + my $reply = obj_get_type_params($cmd,'Command'); + print H "\tstd::auto_ptr<$classname_base$reply> Query(const $classname_base$cmd &rQuery$argextra);\n"; + my @a; + my @na; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + push @a,translate_type_to_arg_type($ty)." $nm"; + push @na,"$nm"; + } + my $ar = join(', ',@a); + my $nar = join(', ',@na); + $nar = "($nar)" if $nar ne ''; + + $with_params .= "\tinline std::auto_ptr<$classname_base$reply> Query$cmd($ar$argextra)\n\t{\n"; + $with_params .= "\t\t$classname_base$cmd send$nar;\n"; + $with_params .= "\t\treturn Query(send$queryextra);\n"; + $with_params .= "\t}\n"; + } + } + # quick hack to correct bad argument lists for commands with zero paramters but with streams + $with_params =~ s/\(, /(/g; + print H "\n",$with_params,"\n"; +} +print H <<__E; +private: + $classname_base(const $classname_base &rToCopy); +__E +if($type eq 'Server') +{ + # need to put the streams to send vector + print H "\tstd::vector<IOStream*> mStreamsToSend;\n\tvoid DeleteStreamsToSend();\n"; +} + +if($implement_filelog || $implement_syslog) +{ + print H <<__E; + virtual void InformStreamReceiving(u_int32_t Size); + virtual void InformStreamSending(u_int32_t Size); +__E +} + +if($implement_syslog) +{ + print H "private:\n\tbool mLogToSysLog;\n"; +} +if($implement_filelog) +{ + print H "private:\n\tFILE *mLogToFile;\n"; +} +print H <<__E; + +protected: + virtual std::auto_ptr<ProtocolObject> MakeProtocolObject(int ObjType); + virtual const char *GetIdentString(); +}; + +__E + +my $constructor_extra = ''; +$constructor_extra .= ', mLogToSysLog(false)' if $implement_syslog; +$constructor_extra .= ', mLogToFile(0)' if $implement_filelog; + +my $destructor_extra = ($type eq 'Server')?"\n\tDeleteStreamsToSend();":''; + +my $prefix = $classname_base.'::'; +print CPP <<__E; +$prefix$classname_base(IOStream &rStream) + : Protocol(rStream)$constructor_extra +{ +} +$prefix~$classname_base() +{$destructor_extra +} +const char *${prefix}GetIdentString() +{ + return "$ident_string"; +} +std::auto_ptr<ProtocolObject> ${prefix}MakeProtocolObject(int ObjType) +{ + switch(ObjType) + { +__E + +# do objects within this +for my $cmd (@cmd_list) +{ + print CPP <<__E; + case $cmd_id{$cmd}: + return std::auto_ptr<ProtocolObject>(new $classname_base$cmd); + break; +__E +} + +print CPP <<__E; + default: + THROW_EXCEPTION(ConnectionException, Conn_Protocol_UnknownCommandRecieved) + } +} +__E +# write receive and send functions +print CPP <<__E; +std::auto_ptr<$derive_objects_from> ${prefix}Receive() +{ + std::auto_ptr<${derive_objects_from}> preply((${derive_objects_from}*)(Protocol::Receive().release())); + +__E + if($implement_syslog) + { + print CPP <<__E; + if(mLogToSysLog) + { + preply->LogSysLog("Receive"); + } +__E + } + if($implement_filelog) + { + print CPP <<__E; + if(mLogToFile != 0) + { + preply->LogFile("Receive", mLogToFile); + } +__E + } +print CPP <<__E; + + return preply; +} + +void ${prefix}Send(const ${derive_objects_from} &rObject) +{ +__E + if($implement_syslog) + { + print CPP <<__E; + if(mLogToSysLog) + { + rObject.LogSysLog("Send"); + } +__E + } + if($implement_filelog) + { + print CPP <<__E; + if(mLogToFile != 0) + { + rObject.LogFile("Send", mLogToFile); + } +__E + } + +print CPP <<__E; + Protocol::Send(rObject); +} + +__E +# write server function? +if($type eq 'Server') +{ + print CPP <<__E; +void ${prefix}DoServer($context_class &rContext) +{ + // Handshake with client + Handshake(); + + // Command processing loop + bool inProgress = true; + while(inProgress) + { + // Get an object from the conversation + std::auto_ptr<${derive_objects_from}> pobj(Receive()); + + // Run the command + std::auto_ptr<${derive_objects_from}> preply((${derive_objects_from}*)(pobj->DoCommand(*this, rContext).release())); + + // Send the reply + Send(*(preply.get())); + + // Send any streams + for(unsigned int s = 0; s < mStreamsToSend.size(); s++) + { + // Send the streams + SendStream(*mStreamsToSend[s]); + } + // Delete these streams + DeleteStreamsToSend(); + + // Does this end the conversation? + if(pobj->IsConversationEnd()) + { + inProgress = false; + } + } +} + +void ${prefix}SendStreamAfterCommand(IOStream *pStream) +{ + ASSERT(pStream != NULL); + mStreamsToSend.push_back(pStream); +} + +void ${prefix}DeleteStreamsToSend() +{ + for(std::vector<IOStream*>::iterator i(mStreamsToSend.begin()); i != mStreamsToSend.end(); ++i) + { + delete (*i); + } + mStreamsToSend.clear(); +} + +__E +} + +# write logging functions? +if($implement_filelog || $implement_syslog) +{ + my ($fR,$fS); + + if($implement_syslog) + { + $fR .= <<__E; + if(mLogToSysLog) + { + if(Size==Protocol::ProtocolStream_SizeUncertain) + { + BOX_TRACE("Receiving stream, size uncertain"); + } + else + { + BOX_TRACE("Receiving stream, size " << Size); + } + } +__E + + $fS .= <<__E; + if(mLogToSysLog) + { + if(Size==Protocol::ProtocolStream_SizeUncertain) + { + BOX_TRACE("Sending stream, size uncertain"); + } + else + { + BOX_TRACE("Sending stream, size " << Size); + } + } +__E + } + + if($implement_filelog) + { + $fR .= <<__E; + if(mLogToFile) + { + ::fprintf(mLogToFile, + (Size==Protocol::ProtocolStream_SizeUncertain) + ?"Receiving stream, size uncertain\\n" + :"Receiving stream, size %d\\n", Size); + ::fflush(mLogToFile); + } +__E + $fS .= <<__E; + if(mLogToFile) + { + ::fprintf(mLogToFile, + (Size==Protocol::ProtocolStream_SizeUncertain) + ?"Sending stream, size uncertain\\n" + :"Sending stream, size %d\\n", Size); + ::fflush(mLogToFile); + } +__E + } + + print CPP <<__E; + +void ${prefix}InformStreamReceiving(u_int32_t Size) +{ +$fR} + +void ${prefix}InformStreamSending(u_int32_t Size) +{ +$fS} + +__E +} + + +# write client Query functions? +if($type eq 'Client') +{ + for my $cmd (@cmd_list) + { + if(obj_is_type($cmd,'Command')) + { + my $reply = obj_get_type_params($cmd,'Command'); + my $reply_id = $cmd_id{$reply}; + my $has_stream = obj_is_type($cmd,'StreamWithCommand'); + my $argextra = $has_stream?', IOStream &rStream':''; + my $send_stream_extra = ''; + if($has_stream) + { + $send_stream_extra = <<__E; + + // Send stream after the command + SendStream(rStream); +__E + } + print CPP <<__E; +std::auto_ptr<$classname_base$reply> ${classname_base}::Query(const $classname_base$cmd &rQuery$argextra) +{ + // Send query + Send(rQuery); + $send_stream_extra + // Wait for the reply + std::auto_ptr<${derive_objects_from}> preply(Receive().release()); + + if(preply->GetType() == $reply_id) + { + // Correct response + return std::auto_ptr<$classname_base$reply>(($classname_base$reply*)preply.release()); + } + else + { + // Set protocol error + int type, subType; + if(preply->IsError(type, subType)) + { + SetError(type, subType); + BOX_WARNING("$cmd command failed: received error " << + ((${classname_base}Error&)*preply).GetMessage()); + } + else + { + SetError(Protocol::UnknownError, Protocol::UnknownError); + BOX_WARNING("$cmd command failed: received " + "unexpected response type " << + preply->GetType()); + } + + // Throw an exception + THROW_EXCEPTION(ConnectionException, Conn_Protocol_UnexpectedReply) + } +} +__E + } + } +} + + + +print H <<__E; +#endif // $guardname + +__E + +# close files +close H; +close CPP; + + +sub obj_is_type +{ + my ($c,$ty) = @_; + for(@{$cmd_attributes{$c}}) + { + return 1 if $_ =~ m/\A$ty/; + } + + return 0; +} + +sub obj_get_type_params +{ + my ($c,$ty) = @_; + for(@{$cmd_attributes{$c}}) + { + return $1 if $_ =~ m/\A$ty\((.+?)\)\Z/; + } + die "Can't find attribute $ty\n" +} + +# returns (is basic type, typename) +sub translate_type +{ + my $ty = $_[0]; + + if($ty =~ m/\Avector\<(.+?)\>\Z/) + { + my $v_type = $1; + my (undef,$v_ty) = translate_type($v_type); + return (0, 'std::vector<'.$v_ty.'>') + } + else + { + if(!exists $translate_type_info{$ty}) + { + die "Don't know about type name $ty\n"; + } + return @{$translate_type_info{$ty}} + } +} + +sub translate_type_to_arg_type +{ + my ($basic,$typename) = translate_type(@_); + return $basic?$typename:'const '.$typename.'&' +} + +sub translate_type_to_member_type +{ + my ($basic,$typename) = translate_type(@_); + return $typename +} + +sub make_log_strings +{ + my ($cmd) = @_; + + my @str; + my @arg; + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + if(exists $log_display_types{$ty}) + { + # need to translate it + my ($format,$arg) = @{$log_display_types{$ty}}; + $arg =~ s/VAR/m$nm/g; + + if ($format eq "0x%llx" and $target_windows) + { + $format = "0x%I64x"; + $arg = "(uint64_t)$arg"; + } + + push @str,$format; + push @arg,$arg; + } + else + { + # is opaque + push @str,'OPAQUE'; + } + } + return ($cmd.'('.join(',',@str).')', join(',','',@arg)); +} + +sub make_log_strings_framework +{ + my ($cmd) = @_; + + my @args; + + for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) + { + my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); + + if(exists $log_display_types{$ty}) + { + # need to translate it + my ($format,$arg) = @{$log_display_types{$ty}}; + $arg =~ s/VAR/m$nm/g; + + if ($format eq '\\"%s\\"') + { + $arg = "\"\\\"\" << $arg << \"\\\"\""; + } + elsif ($format =~ m'x$') + { + # my $width = 0; + # $ty =~ /^int(\d+)$/ and $width = $1 / 4; + $arg = "($arg == 0 ? \"0x\" : \"\") " . + "<< std::hex " . + "<< std::showbase " . + # "<< std::setw($width) " . + # "<< std::setfill('0') " . + # "<< std::internal " . + "<< $arg " . + "<< std::dec"; + } + + push @args, $arg; + } + else + { + # is opaque + push @args, '"OPAQUE"'; + } + } + + my $log_cmd = "Action << \" $cmd(\" "; + foreach my $arg (@args) + { + $arg = "<< $arg "; + } + $log_cmd .= join('<< "," ',@args); + $log_cmd .= '<< ")"'; + return $log_cmd; +} + + diff --git a/lib/win32/MSG00001.bin b/lib/win32/MSG00001.bin Binary files differnew file mode 100755 index 00000000..b4377b85 --- /dev/null +++ b/lib/win32/MSG00001.bin diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp new file mode 100644 index 00000000..db9974d2 --- /dev/null +++ b/lib/win32/emu.cpp @@ -0,0 +1,1843 @@ +// Box Backup Win32 native port by Nick Knight + +// Need at least 0x0500 to use GetFileSizeEx on Cygwin/MinGW +#define WINVER 0x0500 + +#include "emu.h" + +#ifdef WIN32 + +#include <assert.h> +#include <fcntl.h> +#include <process.h> +#include <windows.h> + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <string> +#include <list> +#include <sstream> + +// message resource definitions for syslog() +#include "messages.h" + +DWORD winerrno; +struct passwd gTempPasswd; + +bool EnableBackupRights() +{ + HANDLE hToken; + TOKEN_PRIVILEGES token_priv; + + //open current process to adjust privileges + if(!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, + &hToken)) + { + ::syslog(LOG_ERR, "Failed to open process token: %s", + GetErrorMessage(GetLastError()).c_str()); + return false; + } + + //let's build the token privilege struct - + //first, look up the LUID for the backup privilege + + if (!LookupPrivilegeValue( + NULL, //this system + SE_BACKUP_NAME, //the name of the privilege + &( token_priv.Privileges[0].Luid ))) //result + { + ::syslog(LOG_ERR, "Failed to lookup backup privilege: %s", + GetErrorMessage(GetLastError()).c_str()); + CloseHandle(hToken); + return false; + } + + token_priv.PrivilegeCount = 1; + token_priv.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + + // now set the privilege + // because we're going exit right after dumping the streams, there isn't + // any need to save current state + + if (!AdjustTokenPrivileges( + hToken, //our process token + false, //we're not disabling everything + &token_priv, //address of structure + sizeof(token_priv), //size of structure + NULL, NULL)) //don't save current state + { + //this function is a little tricky - if we were adjusting + //more than one privilege, it could return success but not + //adjust them all - in the general case, you need to trap this + ::syslog(LOG_ERR, "Failed to enable backup privilege: %s", + GetErrorMessage(GetLastError()).c_str()); + CloseHandle(hToken); + return false; + + } + + CloseHandle(hToken); + return true; +} + +// forward declaration +char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage); + +// -------------------------------------------------------------------------- +// +// Function +// Name: GetDefaultConfigFilePath(std::string name) +// Purpose: Calculates the default configuration file name, +// by using the directory location of the currently +// executing program, and appending the provided name. +// In case of fire, returns an empty string. +// Created: 26th May 2007 +// +// -------------------------------------------------------------------------- +std::string GetDefaultConfigFilePath(const std::string& rName) +{ + WCHAR exePathWide[MAX_PATH]; + GetModuleFileNameW(NULL, exePathWide, MAX_PATH-1); + + char* exePathUtf8 = ConvertFromWideString(exePathWide, CP_UTF8); + if (exePathUtf8 == NULL) + { + return ""; + } + + std::string configfile = exePathUtf8; + delete [] exePathUtf8; + + // make the default config file name, + // based on the program path + configfile = configfile.substr(0, + configfile.rfind('\\')); + configfile += "\\"; + configfile += rName; + + return configfile; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ConvertToWideString +// Purpose: Converts a string from specified codepage to +// a wide string (WCHAR*). Returns a buffer which +// MUST be freed by the caller with delete[]. +// In case of fire, logs the error and returns NULL. +// Created: 4th February 2006 +// +// -------------------------------------------------------------------------- +WCHAR* ConvertToWideString(const char* pString, unsigned int codepage, + bool logErrors) +{ + int len = MultiByteToWideChar + ( + codepage, // source code page + 0, // character-type options + pString, // string to map + -1, // number of bytes in string - auto detect + NULL, // wide-character buffer + 0 // size of buffer - work out + // how much space we need + ); + + if (len == 0) + { + winerrno = GetLastError(); + if (logErrors) + { + ::syslog(LOG_WARNING, + "Failed to convert string to wide string: " + "%s", GetErrorMessage(winerrno).c_str()); + } + errno = EINVAL; + return NULL; + } + + WCHAR* buffer = new WCHAR[len]; + + if (buffer == NULL) + { + if (logErrors) + { + ::syslog(LOG_WARNING, + "Failed to convert string to wide string: " + "out of memory"); + } + winerrno = ERROR_OUTOFMEMORY; + errno = ENOMEM; + return NULL; + } + + len = MultiByteToWideChar + ( + codepage, // source code page + 0, // character-type options + pString, // string to map + -1, // number of bytes in string - auto detect + buffer, // wide-character buffer + len // size of buffer + ); + + if (len == 0) + { + winerrno = GetLastError(); + if (logErrors) + { + ::syslog(LOG_WARNING, + "Failed to convert string to wide string: " + "%s", GetErrorMessage(winerrno).c_str()); + } + errno = EACCES; + delete [] buffer; + return NULL; + } + + return buffer; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ConvertUtf8ToWideString +// Purpose: Converts a string from UTF-8 to a wide string. +// Returns a buffer which MUST be freed by the caller +// with delete[]. +// In case of fire, logs the error and returns NULL. +// Created: 4th February 2006 +// +// -------------------------------------------------------------------------- +WCHAR* ConvertUtf8ToWideString(const char* pString) +{ + return ConvertToWideString(pString, CP_UTF8, true); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ConvertFromWideString +// Purpose: Converts a wide string to a narrow string in the +// specified code page. Returns a buffer which MUST +// be freed by the caller with delete[]. +// In case of fire, logs the error and returns NULL. +// Created: 4th February 2006 +// +// -------------------------------------------------------------------------- +char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) +{ + int len = WideCharToMultiByte + ( + codepage, // destination code page + 0, // character-type options + pString, // string to map + -1, // number of bytes in string - auto detect + NULL, // output buffer + 0, // size of buffer - work out + // how much space we need + NULL, // replace unknown chars with system default + NULL // don't tell us when that happened + ); + + if (len == 0) + { + ::syslog(LOG_WARNING, + "Failed to convert wide string to narrow: " + "error %d", GetLastError()); + errno = EINVAL; + return NULL; + } + + char* buffer = new char[len]; + + if (buffer == NULL) + { + ::syslog(LOG_WARNING, + "Failed to convert wide string to narrow: " + "out of memory"); + errno = ENOMEM; + return NULL; + } + + len = WideCharToMultiByte + ( + codepage, // source code page + 0, // character-type options + pString, // string to map + -1, // number of bytes in string - auto detect + buffer, // output buffer + len, // size of buffer + NULL, // replace unknown chars with system default + NULL // don't tell us when that happened + ); + + if (len == 0) + { + ::syslog(LOG_WARNING, + "Failed to convert wide string to narrow: " + "error %i", GetLastError()); + errno = EACCES; + delete [] buffer; + return NULL; + } + + return buffer; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ConvertEncoding(const std::string&, int, +// std::string&, int) +// Purpose: Converts a string from one code page to another. +// On success, replaces contents of rDest and returns +// true. In case of fire, logs the error and returns +// false. +// Created: 15th October 2006 +// +// -------------------------------------------------------------------------- +bool ConvertEncoding(const std::string& rSource, int sourceCodePage, + std::string& rDest, int destCodePage) +{ + WCHAR* pWide = ConvertToWideString(rSource.c_str(), sourceCodePage, + true); + if (pWide == NULL) + { + ::syslog(LOG_ERR, "Failed to convert string '%s' from " + "current code page %d to wide string: %s", + rSource.c_str(), sourceCodePage, + GetErrorMessage(GetLastError()).c_str()); + return false; + } + + char* pConsole = ConvertFromWideString(pWide, destCodePage); + delete [] pWide; + + if (!pConsole) + { + // Error should have been logged by ConvertFromWideString + return false; + } + + rDest = pConsole; + delete [] pConsole; + + return true; +} + +bool ConvertToUtf8(const std::string& rSource, std::string& rDest, + int sourceCodePage) +{ + return ConvertEncoding(rSource, sourceCodePage, rDest, CP_UTF8); +} + +bool ConvertFromUtf8(const std::string& rSource, std::string& rDest, + int destCodePage) +{ + return ConvertEncoding(rSource, CP_UTF8, rDest, destCodePage); +} + +bool ConvertConsoleToUtf8(const std::string& rSource, std::string& rDest) +{ + return ConvertToUtf8(rSource, rDest, GetConsoleCP()); +} + +bool ConvertUtf8ToConsole(const std::string& rSource, std::string& rDest) +{ + return ConvertFromUtf8(rSource, rDest, GetConsoleOutputCP()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ConvertPathToAbsoluteUnicode +// Purpose: Converts relative paths to absolute (with unicode marker) +// Created: 4th February 2006 +// +// -------------------------------------------------------------------------- +std::string ConvertPathToAbsoluteUnicode(const char *pFileName) +{ + std::string filename; + for (int i = 0; pFileName[i] != 0; i++) + { + if (pFileName[i] == '/') + { + filename += '\\'; + } + else + { + filename += pFileName[i]; + } + } + + std::string tmpStr("\\\\?\\"); + + // Is the path relative or absolute? + // Absolute paths on Windows are always a drive letter + // followed by ':' + + char wd[PATH_MAX]; + if (::getcwd(wd, PATH_MAX) == 0) + { + ::syslog(LOG_WARNING, + "Failed to open '%s': path too long", + pFileName); + errno = ENAMETOOLONG; + winerrno = ERROR_INVALID_NAME; + tmpStr = ""; + return tmpStr; + } + + if (filename.length() > 2 && filename[0] == '\\' && + filename[1] == '\\') + { + tmpStr += "UNC\\"; + filename.replace(0, 2, ""); + // \\?\UNC\<server>\<share> + // see http://msdn2.microsoft.com/en-us/library/aa365247.aspx + } + else if (filename.length() >= 1 && filename[0] == '\\') + { + // root directory of current drive. + tmpStr = wd; + tmpStr.resize(2); // drive letter and colon + } + else if (filename.length() >= 2 && filename[1] != ':') + { + // Must be relative. We need to get the + // current directory to make it absolute. + tmpStr += wd; + if (tmpStr[tmpStr.length()] != '\\') + { + tmpStr += '\\'; + } + } + + tmpStr += filename; + + // We are using direct filename access, which does not support .., + // so we need to implement it ourselves. + + for (std::string::size_type i = 1; i < tmpStr.size() - 3; i++) + { + if (tmpStr.substr(i, 3) == "\\..") + { + std::string::size_type lastSlash = + tmpStr.rfind('\\', i - 1); + + if (lastSlash == std::string::npos) + { + // no previous directory, ignore it, + // CreateFile will fail with error 123 + } + else + { + tmpStr.replace(lastSlash, i + 3 - lastSlash, + ""); + } + + i = lastSlash; + } + } + + return tmpStr; +} + +std::string GetErrorMessage(DWORD errorCode) +{ + char* pMsgBuf = NULL; + + DWORD chars = FormatMessage + ( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + errorCode, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (char *)(&pMsgBuf), + 0, NULL + ); + + if (chars == 0 || pMsgBuf == NULL) + { + return std::string("failed to get error message"); + } + + // remove embedded newline + pMsgBuf[chars - 1] = 0; + pMsgBuf[chars - 2] = 0; + + std::ostringstream line; + line << pMsgBuf << " (" << errorCode << ")"; + LocalFree(pMsgBuf); + + return line.str(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: openfile +// Purpose: replacement for any open calls - handles unicode +// filenames - supplied in utf8 +// Created: 25th October 2004 +// +// -------------------------------------------------------------------------- +HANDLE openfile(const char *pFileName, int flags, int mode) +{ + winerrno = ERROR_INVALID_FUNCTION; + + std::string AbsPathWithUnicode = + ConvertPathToAbsoluteUnicode(pFileName); + + if (AbsPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return INVALID_HANDLE_VALUE; + } + + WCHAR* pBuffer = ConvertUtf8ToWideString(AbsPathWithUnicode.c_str()); + // We are responsible for freeing pBuffer + + if (pBuffer == NULL) + { + // error already logged by ConvertUtf8ToWideString() + return INVALID_HANDLE_VALUE; + } + + // flags could be O_WRONLY | O_CREAT | O_RDONLY + DWORD createDisposition = OPEN_EXISTING; + DWORD shareMode = FILE_SHARE_READ | FILE_SHARE_WRITE + | FILE_SHARE_DELETE; + DWORD accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY + | FILE_READ_EA; + + if (flags & O_WRONLY) + { + accessRights = FILE_WRITE_DATA; + } + else if (flags & O_RDWR) + { + accessRights |= FILE_WRITE_ATTRIBUTES + | FILE_WRITE_DATA | FILE_WRITE_EA; + } + + if (flags & O_CREAT) + { + createDisposition = OPEN_ALWAYS; + } + + if (flags & O_TRUNC) + { + createDisposition = CREATE_ALWAYS; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) + { + createDisposition = CREATE_NEW; + } + + if (flags & O_LOCK) + { + shareMode = 0; + } + + DWORD winFlags = FILE_FLAG_BACKUP_SEMANTICS; + if (flags & O_TEMPORARY) + { + winFlags |= FILE_FLAG_DELETE_ON_CLOSE; + } + + HANDLE hdir = CreateFileW(pBuffer, + accessRights, + shareMode, + NULL, + createDisposition, + winFlags, + NULL); + + delete [] pBuffer; + + if (hdir == INVALID_HANDLE_VALUE) + { + winerrno = GetLastError(); + switch(winerrno) + { + case ERROR_SHARING_VIOLATION: + errno = EBUSY; + break; + + default: + errno = EINVAL; + } + + ::syslog(LOG_WARNING, "Failed to open file '%s': " + "%s", pFileName, + GetErrorMessage(GetLastError()).c_str()); + + return INVALID_HANDLE_VALUE; + } + + if (flags & O_APPEND) + { + if (SetFilePointer(hdir, 0, NULL, FILE_END) == + INVALID_SET_FILE_POINTER) + { + winerrno = GetLastError(); + errno = EINVAL; + CloseHandle(hdir); + return INVALID_HANDLE_VALUE; + } + } + + winerrno = NO_ERROR; + return hdir; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: emu_fstat +// Purpose: replacement for fstat. Supply a windows handle. +// Returns a struct emu_stat to have room for 64-bit +// file identifier in st_ino (mingw allows only 16!) +// Created: 25th October 2004 +// +// -------------------------------------------------------------------------- +int emu_fstat(HANDLE hdir, struct emu_stat * st) +{ + if (hdir == INVALID_HANDLE_VALUE) + { + ::syslog(LOG_ERR, "Error: invalid file handle in emu_fstat()"); + errno = EBADF; + return -1; + } + + BY_HANDLE_FILE_INFORMATION fi; + if (!GetFileInformationByHandle(hdir, &fi)) + { + ::syslog(LOG_WARNING, "Failed to read file information: " + "%s", GetErrorMessage(GetLastError()).c_str()); + errno = EACCES; + return -1; + } + + if (INVALID_FILE_ATTRIBUTES == fi.dwFileAttributes) + { + ::syslog(LOG_WARNING, "Failed to get file attributes: " + "%s", GetErrorMessage(GetLastError()).c_str()); + errno = EACCES; + return -1; + } + + memset(st, 0, sizeof(*st)); + + // This is how we get our INODE (equivalent) information + ULARGE_INTEGER conv; + conv.HighPart = fi.nFileIndexHigh; + conv.LowPart = fi.nFileIndexLow; + st->st_ino = conv.QuadPart; + + // get the time information + st->st_ctime = ConvertFileTimeToTime_t(&fi.ftCreationTime); + st->st_atime = ConvertFileTimeToTime_t(&fi.ftLastAccessTime); + st->st_mtime = ConvertFileTimeToTime_t(&fi.ftLastWriteTime); + + if (fi.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { + st->st_size = 0; + } + else + { + conv.HighPart = fi.nFileSizeHigh; + conv.LowPart = fi.nFileSizeLow; + st->st_size = (_off_t)conv.QuadPart; + } + + // at the mo + st->st_uid = 0; + st->st_gid = 0; + st->st_nlink = 1; + + // the mode of the file + // mode zero will make it impossible to restore on Unix + // (no access to anybody, including the owner). + // we'll fake a sensible mode: + // all objects get user read (0400) + // if it's a directory it gets user execute (0100) + // if it's not read-only it gets user write (0200) + st->st_mode = S_IREAD; + + if (fi.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { + st->st_mode |= S_IFDIR | S_IEXEC; + } + else + { + st->st_mode |= S_IFREG; + } + + if (!(fi.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) + { + st->st_mode |= S_IWRITE; + } + + // st_dev is normally zero, regardless of the drive letter, + // since backup locations can't normally span drives. However, + // a reparse point does allow all kinds of weird stuff to happen. + // We set st_dev to 1 for a reparse point, so that Box will detect + // a change of device number (from 0) and refuse to recurse down + // the reparse point (which could lead to havoc). + + if (fi.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + { + st->st_dev = 1; + } + else + { + st->st_dev = 0; + } + + return 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: OpenFileByNameUtf8 +// Purpose: Converts filename to Unicode and returns +// a handle to it. In case of error, sets errno, +// logs the error and returns NULL. +// Created: 10th December 2004 +// +// -------------------------------------------------------------------------- +HANDLE OpenFileByNameUtf8(const char* pFileName, DWORD flags) +{ + std::string AbsPathWithUnicode = + ConvertPathToAbsoluteUnicode(pFileName); + + if (AbsPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return NULL; + } + + WCHAR* pBuffer = ConvertUtf8ToWideString(AbsPathWithUnicode.c_str()); + // We are responsible for freeing pBuffer + + if (pBuffer == NULL) + { + // error already logged by ConvertUtf8ToWideString() + return NULL; + } + + HANDLE handle = CreateFileW(pBuffer, + flags, + FILE_SHARE_READ | FILE_SHARE_DELETE | FILE_SHARE_WRITE, + NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + NULL); + + if (handle == INVALID_HANDLE_VALUE) + { + // if our open fails we should always be able to + // open in this mode - to get the inode information + // at least one process must have the file open - + // in this case someone else does. + handle = CreateFileW(pBuffer, + READ_CONTROL, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + NULL); + } + + delete [] pBuffer; + + if (handle == INVALID_HANDLE_VALUE) + { + DWORD err = GetLastError(); + + if (err == ERROR_FILE_NOT_FOUND || + err == ERROR_PATH_NOT_FOUND) + { + errno = ENOENT; + } + else + { + ::syslog(LOG_WARNING, "Failed to open '%s': " + "%s", pFileName, + GetErrorMessage(err).c_str()); + errno = EACCES; + } + + return NULL; + } + + return handle; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: emu_stat +// Purpose: replacement for the lstat and stat functions. +// Works with unicode filenames supplied in utf8. +// Returns a struct emu_stat to have room for 64-bit +// file identifier in st_ino (mingw allows only 16!) +// Created: 25th October 2004 +// +// -------------------------------------------------------------------------- +int emu_stat(const char * pName, struct emu_stat * st) +{ + HANDLE handle = OpenFileByNameUtf8(pName, + FILE_READ_ATTRIBUTES | FILE_READ_EA); + + if (handle == NULL) + { + // errno already set and error logged by OpenFileByNameUtf8() + return -1; + } + + int retVal = emu_fstat(handle, st); + if (retVal != 0) + { + // error logged, but without filename + ::syslog(LOG_WARNING, "Failed to get file information " + "for '%s'", pName); + } + + // close the handle + CloseHandle(handle); + + return retVal; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: statfs +// Purpose: returns the mount point of where a file is located - +// in this case the volume serial number +// Created: 25th October 2004 +// +// -------------------------------------------------------------------------- +int statfs(const char * pName, struct statfs * s) +{ + HANDLE handle = OpenFileByNameUtf8(pName, + FILE_READ_ATTRIBUTES | FILE_READ_EA); + + if (handle == NULL) + { + // errno already set and error logged by OpenFileByNameUtf8() + return -1; + } + + BY_HANDLE_FILE_INFORMATION fi; + if (!GetFileInformationByHandle(handle, &fi)) + { + ::syslog(LOG_WARNING, "Failed to get file information " + "for '%s': %s", pName, + GetErrorMessage(GetLastError()).c_str()); + CloseHandle(handle); + errno = EACCES; + return -1; + } + + // convert volume serial number to a string + _ui64toa(fi.dwVolumeSerialNumber, s->f_mntonname + 1, 16); + + // pseudo unix mount point + s->f_mntonname[0] = '\\'; + + CloseHandle(handle); // close the handle + + return 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: emu_utimes +// Purpose: replacement for the POSIX utimes() function, +// works with unicode filenames supplied in utf8 format, +// sets creation time instead of last access time. +// Created: 25th July 2006 +// +// -------------------------------------------------------------------------- +int emu_utimes(const char * pName, const struct timeval times[]) +{ + FILETIME creationTime; + if (!ConvertTime_tToFileTime(times[0].tv_sec, &creationTime)) + { + errno = EINVAL; + return -1; + } + + FILETIME modificationTime; + if (!ConvertTime_tToFileTime(times[1].tv_sec, &modificationTime)) + { + errno = EINVAL; + return -1; + } + + HANDLE handle = OpenFileByNameUtf8(pName, FILE_WRITE_ATTRIBUTES); + + if (handle == NULL) + { + // errno already set and error logged by OpenFileByNameUtf8() + return -1; + } + + if (!SetFileTime(handle, &creationTime, NULL, &modificationTime)) + { + ::syslog(LOG_ERR, "Failed to set times on '%s': %s", pName, + GetErrorMessage(GetLastError()).c_str()); + CloseHandle(handle); + return 1; + } + + CloseHandle(handle); + return 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: emu_chmod +// Purpose: replacement for the POSIX chmod function, +// works with unicode filenames supplied in utf8 format +// Created: 26th July 2006 +// +// -------------------------------------------------------------------------- +int emu_chmod(const char * pName, mode_t mode) +{ + std::string AbsPathWithUnicode = + ConvertPathToAbsoluteUnicode(pName); + + if (AbsPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return -1; + } + + WCHAR* pBuffer = ConvertUtf8ToWideString(AbsPathWithUnicode.c_str()); + // We are responsible for freeing pBuffer + + if (pBuffer == NULL) + { + // error already logged by ConvertUtf8ToWideString() + free(pBuffer); + return -1; + } + + DWORD attribs = GetFileAttributesW(pBuffer); + if (attribs == INVALID_FILE_ATTRIBUTES) + { + ::syslog(LOG_ERR, "Failed to get file attributes of '%s': %s", + pName, GetErrorMessage(GetLastError()).c_str()); + errno = EACCES; + free(pBuffer); + return -1; + } + + if (mode & S_IWRITE) + { + attribs &= ~FILE_ATTRIBUTE_READONLY; + } + else + { + attribs |= FILE_ATTRIBUTE_READONLY; + } + + if (!SetFileAttributesW(pBuffer, attribs)) + { + ::syslog(LOG_ERR, "Failed to set file attributes of '%s': %s", + pName, GetErrorMessage(GetLastError()).c_str()); + errno = EACCES; + free(pBuffer); + return -1; + } + + delete [] pBuffer; + return 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: opendir +// Purpose: replacement for unix function, uses win32 findfirst routines +// Created: 25th October 2004 +// +// -------------------------------------------------------------------------- +DIR *opendir(const char *name) +{ + if (!name || !name[0]) + { + errno = EINVAL; + return NULL; + } + + std::string dirName(name); + + //append a '\' win32 findfirst is sensitive to this + if ( dirName[dirName.size()] != '\\' || dirName[dirName.size()] != '/' ) + { + dirName += '\\'; + } + + // what is the search string? - everything + dirName += '*'; + + DIR *pDir = new DIR; + if (pDir == NULL) + { + errno = ENOMEM; + return NULL; + } + + pDir->name = ConvertUtf8ToWideString(dirName.c_str()); + // We are responsible for freeing dir->name with delete[] + + if (pDir->name == NULL) + { + delete pDir; + return NULL; + } + + pDir->fd = _wfindfirst((const wchar_t*)pDir->name, &(pDir->info)); + + if (pDir->fd == -1) + { + delete [] pDir->name; + delete pDir; + return NULL; + } + + pDir->result.d_name = 0; + return pDir; +} + +// this kinda makes it not thread friendly! +// but I don't think it needs to be. +char tempbuff[MAX_PATH]; + +// -------------------------------------------------------------------------- +// +// Function +// Name: readdir +// Purpose: as function above +// Created: 25th October 2004 +// +// -------------------------------------------------------------------------- +struct dirent *readdir(DIR *dp) +{ + try + { + struct dirent *den = NULL; + + if (dp && dp->fd != -1) + { + if (!dp->result.d_name || + _wfindnext(dp->fd, &dp->info) != -1) + { + den = &dp->result; + std::wstring input(dp->info.name); + memset(tempbuff, 0, sizeof(tempbuff)); + WideCharToMultiByte(CP_UTF8, 0, dp->info.name, + -1, &tempbuff[0], sizeof (tempbuff), + NULL, NULL); + //den->d_name = (char *)dp->info.name; + den->d_name = &tempbuff[0]; + if (dp->info.attrib & FILE_ATTRIBUTE_DIRECTORY) + { + den->d_type = S_IFDIR; + } + else + { + den->d_type = S_IFREG; + } + } + } + else + { + errno = EBADF; + } + return den; + } + catch (...) + { + printf("Caught readdir"); + } + return NULL; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: closedir +// Purpose: as function above +// Created: 25th October 2004 +// +// -------------------------------------------------------------------------- +int closedir(DIR *dp) +{ + try + { + int finres = -1; + if (dp) + { + if(dp->fd != -1) + { + finres = _findclose(dp->fd); + } + + delete [] dp->name; + delete dp; + } + + if (finres == -1) // errors go to EBADF + { + errno = EBADF; + } + + return finres; + } + catch (...) + { + printf("Caught closedir"); + } + return -1; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: poll +// Purpose: a weak implimentation (just enough for box) +// of the unix poll for winsock2 +// Created: 25th October 2004 +// +// -------------------------------------------------------------------------- +int poll (struct pollfd *ufds, unsigned long nfds, int timeout) +{ + try + { + fd_set readfd; + fd_set writefd; + + FD_ZERO(&readfd); + FD_ZERO(&writefd); + + // struct pollfd *ufdsTmp = ufds; + + timeval timOut; + timeval *tmpptr; + + if (timeout == INFTIM) + tmpptr = NULL; + else + tmpptr = &timOut; + + timOut.tv_sec = timeout / 1000; + timOut.tv_usec = timeout * 1000; + + for (unsigned long i = 0; i < nfds; i++) + { + struct pollfd* ufd = &(ufds[i]); + + if (ufd->events & POLLIN) + { + FD_SET(ufd->fd, &readfd); + } + + if (ufd->events & POLLOUT) + { + FD_SET(ufd->fd, &writefd); + } + + if (ufd->events & ~(POLLIN | POLLOUT)) + { + printf("Unsupported poll bits %d", + ufd->events); + return -1; + } + } + + int nready = select(0, &readfd, &writefd, 0, tmpptr); + + if (nready == SOCKET_ERROR) + { + // int errval = WSAGetLastError(); + + struct pollfd* pufd = ufds; + for (unsigned long i = 0; i < nfds; i++) + { + pufd->revents = POLLERR; + pufd++; + } + return (-1); + } + else if (nready > 0) + { + for (unsigned long i = 0; i < nfds; i++) + { + struct pollfd *ufd = &(ufds[i]); + + if (FD_ISSET(ufd->fd, &readfd)) + { + ufd->revents |= POLLIN; + } + + if (FD_ISSET(ufd->fd, &writefd)) + { + ufd->revents |= POLLOUT; + } + } + } + + return nready; + } + catch (...) + { + printf("Caught poll"); + } + + return -1; +} + +// copied from MSDN: http://msdn.microsoft.com/library/default.asp?url=/library/en-us/eventlog/base/adding_a_source_to_the_registry.asp + +BOOL AddEventSource +( + LPTSTR pszSrcName, // event source name + DWORD dwNum // number of categories +) +{ + // Work out the executable file name, to register ourselves + // as the event source + + WCHAR cmd[MAX_PATH]; + DWORD len = GetModuleFileNameW(NULL, cmd, MAX_PATH); + + if (len == 0) + { + ::syslog(LOG_ERR, "Failed to get the program file name: %s", + GetErrorMessage(GetLastError()).c_str()); + return FALSE; + } + + // Create the event source as a subkey of the log. + + std::string regkey("SYSTEM\\CurrentControlSet\\Services\\EventLog\\" + "Application\\"); + regkey += pszSrcName; + + HKEY hk; + DWORD dwDisp; + + if (RegCreateKeyEx(HKEY_LOCAL_MACHINE, regkey.c_str(), + 0, NULL, REG_OPTION_NON_VOLATILE, + KEY_WRITE, NULL, &hk, &dwDisp)) + { + ::syslog(LOG_ERR, "Failed to create the registry key: %s", + GetErrorMessage(GetLastError()).c_str()); + return FALSE; + } + + // Set the name of the message file. + + if (RegSetValueExW(hk, // subkey handle + L"EventMessageFile", // value name + 0, // must be zero + REG_EXPAND_SZ, // value type + (LPBYTE)cmd, // pointer to value data + len*sizeof(WCHAR))) // data size + { + ::syslog(LOG_ERR, "Failed to set the event message file: %s", + GetErrorMessage(GetLastError()).c_str()); + RegCloseKey(hk); + return FALSE; + } + + // Set the supported event types. + + DWORD dwData = EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | + EVENTLOG_INFORMATION_TYPE; + + if (RegSetValueEx(hk, // subkey handle + "TypesSupported", // value name + 0, // must be zero + REG_DWORD, // value type + (LPBYTE) &dwData, // pointer to value data + sizeof(DWORD))) // length of value data + { + ::syslog(LOG_ERR, "Failed to set the supported types: %s", + GetErrorMessage(GetLastError()).c_str()); + RegCloseKey(hk); + return FALSE; + } + + // Set the category message file and number of categories. + + if (RegSetValueExW(hk, // subkey handle + L"CategoryMessageFile", // value name + 0, // must be zero + REG_EXPAND_SZ, // value type + (LPBYTE)cmd, // pointer to value data + len*sizeof(WCHAR))) // data size + { + ::syslog(LOG_ERR, "Failed to set the category message file: " + "%s", GetErrorMessage(GetLastError()).c_str()); + RegCloseKey(hk); + return FALSE; + } + + if (RegSetValueEx(hk, // subkey handle + "CategoryCount", // value name + 0, // must be zero + REG_DWORD, // value type + (LPBYTE) &dwNum, // pointer to value data + sizeof(DWORD))) // length of value data + { + ::syslog(LOG_ERR, "Failed to set the category count: %s", + GetErrorMessage(GetLastError()).c_str()); + RegCloseKey(hk); + return FALSE; + } + + RegCloseKey(hk); + return TRUE; +} + +static HANDLE gSyslogH = 0; +static bool sHaveWarnedEventLogFull = false; + +void openlog(const char * daemonName, int, int) +{ + std::string nameStr = "Box Backup ("; + nameStr += daemonName; + nameStr += ")"; + + // register a default event source, so that we can + // log errors with the process of adding or registering our own. + gSyslogH = RegisterEventSource( + NULL, // uses local computer + nameStr.c_str()); // source name + if (gSyslogH == NULL) + { + } + + char* name = strdup(nameStr.c_str()); + BOOL success = AddEventSource(name, 0); + free(name); + + if (!success) + { + ::syslog(LOG_ERR, "Failed to add our own event source"); + return; + } + + HANDLE newSyslogH = RegisterEventSource(NULL, nameStr.c_str()); + if (newSyslogH == NULL) + { + ::syslog(LOG_ERR, "Failed to register our own event source: " + "%s", GetErrorMessage(GetLastError()).c_str()); + return; + } + + DeregisterEventSource(gSyslogH); + gSyslogH = newSyslogH; +} + +void closelog(void) +{ + DeregisterEventSource(gSyslogH); +} + +void syslog(int loglevel, const char *frmt, ...) +{ + WORD errinfo; + char buffer[4096]; + std::string sixfour(frmt); + + switch (loglevel) + { + case LOG_INFO: + errinfo = EVENTLOG_INFORMATION_TYPE; + break; + case LOG_ERR: + errinfo = EVENTLOG_ERROR_TYPE; + break; + case LOG_WARNING: + errinfo = EVENTLOG_WARNING_TYPE; + break; + default: + errinfo = EVENTLOG_WARNING_TYPE; + break; + } + + // taken from MSDN + int sixfourpos; + while ( (sixfourpos = (int)sixfour.find("%ll")) != -1 ) + { + // maintain portability - change the 64 bit formater... + std::string temp = sixfour.substr(0,sixfourpos); + temp += "%I64"; + temp += sixfour.substr(sixfourpos+3, sixfour.length()); + sixfour = temp; + } + + // printf("parsed string is:%s\r\n", sixfour.c_str()); + + va_list args; + va_start(args, frmt); + + int len = vsnprintf(buffer, sizeof(buffer)-1, sixfour.c_str(), args); + assert(len >= 0); + if (len < 0) + { + printf("%s\r\n", buffer); + fflush(stdout); + return; + } + + assert((size_t)len < sizeof(buffer)); + buffer[sizeof(buffer)-1] = 0; + + va_end(args); + + if (gSyslogH == 0) + { + printf("%s\r\n", buffer); + fflush(stdout); + return; + } + + WCHAR* pWide = ConvertToWideString(buffer, CP_UTF8, false); + // must delete[] pWide + + DWORD result; + + if (pWide == NULL) + { + std::string buffer2 = buffer; + buffer2 += " (failed to convert string encoding)"; + LPCSTR strings[] = { buffer2.c_str(), NULL }; + + result = ReportEventA(gSyslogH, // event log handle + errinfo, // event type + 0, // category zero + MSG_ERR, // event identifier - + // we will call them all the same + NULL, // no user security identifier + 1, // one substitution string + 0, // no data + strings, // pointer to string array + NULL); // pointer to data + } + else + { + LPCWSTR strings[] = { pWide, NULL }; + result = ReportEventW(gSyslogH, // event log handle + errinfo, // event type + 0, // category zero + MSG_ERR, // event identifier - + // we will call them all the same + NULL, // no user security identifier + 1, // one substitution string + 0, // no data + strings, // pointer to string array + NULL); // pointer to data + delete [] pWide; + } + + if (result == 0) + { + DWORD err = GetLastError(); + if (err == ERROR_LOG_FILE_FULL) + { + if (!sHaveWarnedEventLogFull) + { + printf("Unable to send message to Event Log " + "(Event Log is full):\r\n"); + fflush(stdout); + sHaveWarnedEventLogFull = TRUE; + } + } + else + { + printf("Unable to send message to Event Log: %s:\r\n", + GetErrorMessage(err).c_str()); + fflush(stdout); + } + } + else + { + sHaveWarnedEventLogFull = false; + } +} + +int emu_chdir(const char* pDirName) +{ + /* + std::string AbsPathWithUnicode = + ConvertPathToAbsoluteUnicode(pDirName); + + if (AbsPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return -1; + } + + WCHAR* pBuffer = ConvertUtf8ToWideString(AbsPathWithUnicode.c_str()); + */ + + WCHAR* pBuffer = ConvertUtf8ToWideString(pDirName); + if (!pBuffer) return -1; + + int result = SetCurrentDirectoryW(pBuffer); + delete [] pBuffer; + + if (result != 0) return 0; + + errno = EACCES; + fprintf(stderr, "Failed to change directory to '%s': %s\n", + pDirName, GetErrorMessage(GetLastError()).c_str()); + return -1; +} + +char* emu_getcwd(char* pBuffer, int BufSize) +{ + DWORD len = GetCurrentDirectoryW(0, NULL); + if (len == 0) + { + errno = EINVAL; + return NULL; + } + + if ((int)len > BufSize) + { + errno = ENAMETOOLONG; + return NULL; + } + + WCHAR* pWide = new WCHAR [len]; + if (!pWide) + { + errno = ENOMEM; + return NULL; + } + + DWORD result = GetCurrentDirectoryW(len, pWide); + if (result <= 0 || result >= len) + { + errno = EACCES; + delete [] pWide; + return NULL; + } + + char* pUtf8 = ConvertFromWideString(pWide, CP_UTF8); + delete [] pWide; + + if (!pUtf8) + { + return NULL; + } + + strncpy(pBuffer, pUtf8, BufSize - 1); + pBuffer[BufSize - 1] = 0; + delete [] pUtf8; + + return pBuffer; +} + +int emu_mkdir(const char* pPathName) +{ + std::string AbsPathWithUnicode = + ConvertPathToAbsoluteUnicode(pPathName); + + if (AbsPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return -1; + } + + WCHAR* pBuffer = ConvertUtf8ToWideString(AbsPathWithUnicode.c_str()); + if (!pBuffer) + { + return -1; + } + + BOOL result = CreateDirectoryW(pBuffer, NULL); + delete [] pBuffer; + + if (!result) + { + errno = EACCES; + return -1; + } + + return 0; +} + +int emu_unlink(const char* pFileName) +{ + std::string AbsPathWithUnicode = + ConvertPathToAbsoluteUnicode(pFileName); + + if (AbsPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return -1; + } + + WCHAR* pBuffer = ConvertUtf8ToWideString(AbsPathWithUnicode.c_str()); + if (!pBuffer) + { + return -1; + } + + BOOL result = DeleteFileW(pBuffer); + DWORD err = GetLastError(); + delete [] pBuffer; + + if (!result) + { + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + { + errno = ENOENT; + } + else if (err == ERROR_SHARING_VIOLATION) + { + errno = EBUSY; + } + else if (err == ERROR_ACCESS_DENIED) + { + errno = EACCES; + } + else + { + ::syslog(LOG_WARNING, "Failed to delete file " + "'%s': %s", pFileName, + GetErrorMessage(err).c_str()); + errno = ENOSYS; + } + return -1; + } + + return 0; +} + +int emu_rename(const char* pOldFileName, const char* pNewFileName) +{ + std::string OldPathWithUnicode = + ConvertPathToAbsoluteUnicode(pOldFileName); + + if (OldPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return -1; + } + + WCHAR* pOldBuffer = ConvertUtf8ToWideString(OldPathWithUnicode.c_str()); + if (!pOldBuffer) + { + return -1; + } + + std::string NewPathWithUnicode = + ConvertPathToAbsoluteUnicode(pNewFileName); + + if (NewPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + delete [] pOldBuffer; + return -1; + } + + WCHAR* pNewBuffer = ConvertUtf8ToWideString(NewPathWithUnicode.c_str()); + if (!pNewBuffer) + { + delete [] pOldBuffer; + return -1; + } + + BOOL result = MoveFileW(pOldBuffer, pNewBuffer); + DWORD err = GetLastError(); + delete [] pOldBuffer; + delete [] pNewBuffer; + + if (!result) + { + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + { + errno = ENOENT; + } + else if (err == ERROR_SHARING_VIOLATION) + { + errno = EBUSY; + } + else if (err == ERROR_ACCESS_DENIED) + { + errno = EACCES; + } + else + { + ::syslog(LOG_WARNING, "Failed to rename file " + "'%s' to '%s': %s", pOldFileName, pNewFileName, + GetErrorMessage(err).c_str()); + errno = ENOSYS; + } + return -1; + } + + return 0; +} + +int console_read(char* pBuffer, size_t BufferSize) +{ + HANDLE hConsole = GetStdHandle(STD_INPUT_HANDLE); + + if (hConsole == INVALID_HANDLE_VALUE) + { + ::fprintf(stderr, "Failed to get a handle on standard input: " + "%s", GetErrorMessage(GetLastError()).c_str()); + return -1; + } + + size_t WideSize = BufferSize / 5; + WCHAR* pWideBuffer = new WCHAR [WideSize + 1]; + + if (!pWideBuffer) + { + ::perror("Failed to allocate wide character buffer"); + return -1; + } + + DWORD numCharsRead = 0; + + if (!ReadConsoleW( + hConsole, + pWideBuffer, + WideSize, // will not be null terminated by ReadConsole + &numCharsRead, + NULL // reserved + )) + { + ::fprintf(stderr, "Failed to read from console: %s\n", + GetErrorMessage(GetLastError()).c_str()); + return -1; + } + + pWideBuffer[numCharsRead] = 0; + + char* pUtf8 = ConvertFromWideString(pWideBuffer, GetConsoleCP()); + delete [] pWideBuffer; + + strncpy(pBuffer, pUtf8, BufferSize); + delete [] pUtf8; + + return strlen(pBuffer); +} + +int readv (int filedes, const struct iovec *vector, size_t count) +{ + int bytes = 0; + + for (size_t i = 0; i < count; i++) + { + int result = read(filedes, vector[i].iov_base, + vector[i].iov_len); + if (result < 0) + { + return result; + } + bytes += result; + } + + return bytes; +} + +int writev(int filedes, const struct iovec *vector, size_t count) +{ + int bytes = 0; + + for (size_t i = 0; i < count; i++) + { + int result = write(filedes, vector[i].iov_base, + vector[i].iov_len); + if (result < 0) + { + return result; + } + bytes += result; + } + + return bytes; +} + +// need this for conversions +time_t ConvertFileTimeToTime_t(FILETIME *fileTime) +{ + SYSTEMTIME stUTC; + struct tm timeinfo; + + // Convert the last-write time to local time. + FileTimeToSystemTime(fileTime, &stUTC); + + memset(&timeinfo, 0, sizeof(timeinfo)); + timeinfo.tm_sec = stUTC.wSecond; + timeinfo.tm_min = stUTC.wMinute; + timeinfo.tm_hour = stUTC.wHour; + timeinfo.tm_mday = stUTC.wDay; + timeinfo.tm_wday = stUTC.wDayOfWeek; + timeinfo.tm_mon = stUTC.wMonth - 1; + // timeinfo.tm_yday = ...; + timeinfo.tm_year = stUTC.wYear - 1900; + + time_t retVal = mktime(&timeinfo) - _timezone; + return retVal; +} + +bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo) +{ + time_t adjusted = from + _timezone; + struct tm *time_breakdown = gmtime(&adjusted); + if (time_breakdown == NULL) + { + ::syslog(LOG_ERR, "Error: failed to convert time format: " + "%d is not a valid time\n", adjusted); + return false; + } + + SYSTEMTIME stUTC; + stUTC.wSecond = time_breakdown->tm_sec; + stUTC.wMinute = time_breakdown->tm_min; + stUTC.wHour = time_breakdown->tm_hour; + stUTC.wDay = time_breakdown->tm_mday; + stUTC.wDayOfWeek = time_breakdown->tm_wday; + stUTC.wMonth = time_breakdown->tm_mon + 1; + stUTC.wYear = time_breakdown->tm_year + 1900; + stUTC.wMilliseconds = 0; + + // Convert the last-write time to local time. + if (!SystemTimeToFileTime(&stUTC, pTo)) + { + syslog(LOG_ERR, "Failed to convert between time formats: %s", + GetErrorMessage(GetLastError()).c_str()); + return false; + } + + return true; +} + +#endif // WIN32 + diff --git a/lib/win32/emu.h b/lib/win32/emu.h new file mode 100644 index 00000000..f3389590 --- /dev/null +++ b/lib/win32/emu.h @@ -0,0 +1,424 @@ +// emulates unix syscalls to win32 functions + +#ifdef WIN32 + #define EMU_STRUCT_STAT struct emu_stat + #define EMU_STAT emu_stat + #define EMU_FSTAT emu_fstat + #define EMU_LSTAT emu_stat +#else + #define EMU_STRUCT_STAT struct stat + #define EMU_STAT ::stat + #define EMU_FSTAT ::fstat + #define EMU_LSTAT ::lstat +#endif + +#if ! defined EMU_INCLUDE && defined WIN32 +#define EMU_INCLUDE + +// basic types, may be required by other headers since we +// don't include sys/types.h + +#ifdef __MINGW32__ + #include <stdint.h> +#else // MSVC + typedef unsigned __int64 u_int64_t; + typedef unsigned __int64 uint64_t; + typedef __int64 int64_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int32 u_int32_t; + typedef __int32 int32_t; + typedef unsigned __int16 uint16_t; + typedef __int16 int16_t; + typedef unsigned __int8 uint8_t; + typedef __int8 int8_t; +#endif + +// emulated types, present on MinGW but not MSVC or vice versa + +#ifdef __MINGW32__ + typedef uint32_t u_int32_t; +#else + typedef unsigned int mode_t; + typedef unsigned int pid_t; +#endif + +// set up to include the necessary parts of Windows headers + +#define WIN32_LEAN_AND_MEAN + +#ifndef __MSVCRT_VERSION__ +#define __MSVCRT_VERSION__ 0x0601 +#endif + +// Windows headers + +#include <winsock2.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <direct.h> +#include <errno.h> +#include <io.h> +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <stdarg.h> +#include <time.h> + +#include <string> + +// emulated functions + +#define gmtime_r( _clock, _result ) \ + ( *(_result) = *gmtime( (_clock) ), \ + (_result) ) + +#define ITIMER_REAL 0 + +#ifdef _MSC_VER +// Microsoft decided to deprecate the standard POSIX functions. Great! +#define open(file,flags,mode) _open(file,flags,mode) +#define close(fd) _close(fd) +#define dup(fd) _dup(fd) +#define read(fd,buf,count) _read(fd,buf,count) +#define write(fd,buf,count) _write(fd,buf,count) +#define lseek(fd,off,whence) _lseek(fd,off,whence) +#define fileno(struct_file) _fileno(struct_file) +#endif + +struct passwd { + char *pw_name; + char *pw_passwd; + int pw_uid; + int pw_gid; + time_t pw_change; + char *pw_class; + char *pw_gecos; + char *pw_dir; + char *pw_shell; + time_t pw_expire; +}; + +extern passwd gTempPasswd; +inline struct passwd * getpwnam(const char * name) +{ + //for the mo pretend to be root + gTempPasswd.pw_uid = 0; + gTempPasswd.pw_gid = 0; + + return &gTempPasswd; +} + +#define S_IRWXG 1 +#define S_IRWXO 2 +#define S_ISUID 4 +#define S_ISGID 8 +#define S_ISVTX 16 + +#ifndef __MINGW32__ + //not sure if these are correct + //S_IWRITE - writing permitted + //_S_IREAD - reading permitted + //_S_IREAD | _S_IWRITE - + #define S_IRUSR S_IWRITE + #define S_IWUSR S_IREAD + #define S_IRWXU (S_IREAD|S_IWRITE|S_IEXEC) + + #define S_ISREG(x) (S_IFREG & x) + #define S_ISDIR(x) (S_IFDIR & x) +#endif + +inline int chown(const char * Filename, u_int32_t uid, u_int32_t gid) +{ + //important - this needs implementing + //If a large restore is required then + //it needs to restore files AND permissions + //reference AdjustTokenPrivileges + //GetAccountSid + //InitializeSecurityDescriptor + //SetSecurityDescriptorOwner + //The next function looks like the guy to use... + //SetFileSecurity + + //indicate success + return 0; +} + +// Windows and Unix owners and groups are pretty fundamentally different. +// Ben prefers that we kludge here rather than litter the code with #ifdefs. +// Pretend to be root, and pretend that set...() operations succeed. +inline int setegid(int) +{ + return true; +} +inline int seteuid(int) +{ + return true; +} +inline int setgid(int) +{ + return true; +} +inline int setuid(int) +{ + return true; +} +inline int getgid(void) +{ + return 0; +} +inline int getuid(void) +{ + return 0; +} +inline int geteuid(void) +{ + return 0; +} + +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +// MinGW provides a getopt implementation +#ifndef __MINGW32__ +#include "getopt.h" +#endif // !__MINGW32__ + +#define timespec timeval + +//win32 deals in usec not nsec - so need to ensure this follows through +#define tv_nsec tv_usec + +#ifndef __MINGW32__ + typedef int socklen_t; +#endif + +#define S_IRGRP S_IWRITE +#define S_IWGRP S_IREAD +#define S_IROTH S_IWRITE | S_IREAD +#define S_IWOTH S_IREAD | S_IREAD + +//again need to verify these +#define S_IFLNK 1 +#define S_IFSOCK 0 + +#define S_ISLNK(x) ( false ) + +#define vsnprintf _vsnprintf + +#ifndef __MINGW32__ +inline int strcasecmp(const char *s1, const char *s2) +{ + return _stricmp(s1,s2); +} +#endif + +#ifdef _DIRENT_H_ +#error You must not include the MinGW dirent.h! +#endif + +struct dirent +{ + char *d_name; + unsigned long d_type; +}; + +struct DIR +{ + intptr_t fd; // filedescriptor + // struct _finddata_t info; + struct _wfinddata_t info; + // struct _finddata_t info; + struct dirent result; // d_name (first time null) + wchar_t *name; // null-terminated byte string +}; + +DIR *opendir(const char *name); +struct dirent *readdir(DIR *dp); +int closedir(DIR *dp); + +// local constant to open file exclusively without shared access +#define O_LOCK 0x10000 + +extern DWORD winerrno; /* used to report errors from openfile() */ +HANDLE openfile(const char *filename, int flags, int mode); +inline int closefile(HANDLE handle) +{ + if (CloseHandle(handle) != TRUE) + { + errno = EINVAL; + return -1; + } + return 0; +} + +#define LOG_DEBUG LOG_INFO +#define LOG_INFO 6 +#define LOG_NOTICE LOG_INFO +#define LOG_WARNING 4 +#define LOG_ERR 3 +#define LOG_CRIT LOG_ERR +#define LOG_PID 0 +#define LOG_LOCAL5 0 +#define LOG_LOCAL6 0 + +void openlog (const char * daemonName, int, int); +void closelog(void); +void syslog (int loglevel, const char *fmt, ...); + +#define LOG_LOCAL0 0 +#define LOG_LOCAL1 0 +#define LOG_LOCAL2 0 +#define LOG_LOCAL3 0 +#define LOG_LOCAL4 0 +#define LOG_LOCAL5 0 +#define LOG_LOCAL6 0 +#define LOG_DAEMON 0 + +#ifndef __MINGW32__ +#define strtoll _strtoi64 +#endif + +inline unsigned int sleep(unsigned int secs) +{ + Sleep(secs*1000); + return(ERROR_SUCCESS); +} + +#define INFTIM -1 +#define POLLIN 0x1 +#define POLLERR 0x8 +#define POLLOUT 0x4 + +#define SHUT_RDWR SD_BOTH +#define SHUT_RD SD_RECEIVE +#define SHUT_WR SD_SEND + +struct pollfd +{ + SOCKET fd; + short int events; + short int revents; +}; + +inline int ioctl(SOCKET sock, int flag, int * something) +{ + //indicate success + return 0; +} + +extern "C" inline int getpid() +{ + return (int)GetCurrentProcessId(); +} + +inline int waitpid(pid_t pid, int *status, int) +{ + return 0; +} + +//this shouldn't be needed. +struct statfs +{ + TCHAR f_mntonname[MAX_PATH]; +}; + +struct emu_stat { + int st_dev; + uint64_t st_ino; + DWORD st_mode; + short st_nlink; + short st_uid; + short st_gid; + //_dev_t st_rdev; + uint64_t st_size; + time_t st_atime; + time_t st_mtime; + time_t st_ctime; +}; + +// need this for conversions +time_t ConvertFileTimeToTime_t(FILETIME *fileTime); +bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo); + +int emu_chdir (const char* pDirName); +int emu_mkdir (const char* pPathName); +int emu_unlink (const char* pFileName); +int emu_fstat (HANDLE file, struct emu_stat* st); +int emu_stat (const char* pName, struct emu_stat* st); +int emu_utimes (const char* pName, const struct timeval[]); +int emu_chmod (const char* pName, mode_t mode); +char* emu_getcwd (char* pBuffer, int BufSize); +int emu_rename (const char* pOldName, const char* pNewName); + +#define chdir(directory) emu_chdir (directory) +#define mkdir(path, mode) emu_mkdir (path) +#define unlink(file) emu_unlink (file) +#define utimes(buffer, times) emu_utimes (buffer, times) +#define chmod(file, mode) emu_chmod (file, mode) +#define getcwd(buffer, size) emu_getcwd (buffer, size) +#define rename(oldname, newname) emu_rename (oldname, newname) + +// Not safe to replace stat/fstat/lstat on mingw at least, as struct stat +// has a 16-bit st_ino and we need a 64-bit one. +// +// #define stat(filename, struct) emu_stat (filename, struct) +// #define lstat(filename, struct) emu_stat (filename, struct) +// #define fstat(handle, struct) emu_fstat (handle, struct) +// +// But lstat doesn't exist on Windows, so we have to provide something: + +#define lstat(filename, struct) stat(filename, struct) + +int statfs(const char * name, struct statfs * s); + +int poll(struct pollfd *ufds, unsigned long nfds, int timeout); + +struct iovec { + void *iov_base; /* Starting address */ + size_t iov_len; /* Number of bytes */ +}; + +int readv (int filedes, const struct iovec *vector, size_t count); +int writev(int filedes, const struct iovec *vector, size_t count); + +// The following functions are not emulations, but utilities for other +// parts of the code where Windows API is used or windows-specific stuff +// is needed, like codepage conversion. + +bool EnableBackupRights( void ); + +bool ConvertEncoding (const std::string& rSource, int sourceCodePage, + std::string& rDest, int destCodePage); +bool ConvertToUtf8 (const std::string& rSource, std::string& rDest, + int sourceCodePage); +bool ConvertFromUtf8 (const std::string& rSource, std::string& rDest, + int destCodePage); +bool ConvertUtf8ToConsole(const std::string& rSource, std::string& rDest); +bool ConvertConsoleToUtf8(const std::string& rSource, std::string& rDest); + +// Utility function which returns a default config file name, +// based on the path of the current executable. +std::string GetDefaultConfigFilePath(const std::string& rName); + +// GetErrorMessage() returns a system error message, like strerror() +// but for Windows error codes. +std::string GetErrorMessage(DWORD errorCode); + +// console_read() is a replacement for _cgetws which requires a +// relatively recent C runtime lib +int console_read(char* pBuffer, size_t BufferSize); + +#ifdef _MSC_VER + /* disable certain compiler warnings to be able to actually see the show-stopper ones */ + #pragma warning(disable:4101) // unreferenced local variable + #pragma warning(disable:4244) // conversion, possible loss of data + #pragma warning(disable:4267) // conversion, possible loss of data + #pragma warning(disable:4311) // pointer truncation + #pragma warning(disable:4700) // uninitialized local variable used (hmmmmm...) + #pragma warning(disable:4805) // unsafe mix of type and type 'bool' in operation + #pragma warning(disable:4800) // forcing value to bool 'true' or 'false' (performance warning) + #pragma warning(disable:4996) // POSIX name for this item is deprecated +#endif // _MSC_VER + +#endif // !EMU_INCLUDE && WIN32 diff --git a/lib/win32/getopt.h b/lib/win32/getopt.h new file mode 100755 index 00000000..7c290343 --- /dev/null +++ b/lib/win32/getopt.h @@ -0,0 +1,98 @@ +/* $OpenBSD: getopt.h,v 1.1 2002/12/03 20:24:29 millert Exp $ */
+/* $NetBSD: getopt.h,v 1.4 2000/07/07 10:43:54 ad Exp $ */
+
+/*-
+ * Copyright (c) 2000 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Dieter Baron and Thomas Klausner.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by the NetBSD
+ * Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _GETOPT_H_
+#define _GETOPT_H_
+
+// copied from: http://www.la.utexas.edu/lab/software/devtool/gnu/libtool/C_header_files.html
+
+/* __BEGIN_DECLS should be used at the beginning of your declarations,
+ so that C++ compilers don't mangle their names. Use __END_DECLS at
+ the end of C declarations. */
+#undef __BEGIN_DECLS
+#undef __END_DECLS
+#ifdef __cplusplus
+# define __BEGIN_DECLS extern "C" {
+# define __END_DECLS }
+#else
+# define __BEGIN_DECLS /* empty */
+# define __END_DECLS /* empty */
+#endif
+
+/*
+ * GNU-like getopt_long() and 4.4BSD getsubopt()/optreset extensions
+ */
+#define no_argument 0
+#define required_argument 1
+#define optional_argument 2
+
+struct option {
+ /* name of long option */
+ const char *name;
+ /*
+ * one of no_argument, required_argument, and optional_argument:
+ * whether option takes an argument
+ */
+ int has_arg;
+ /* if not NULL, set *flag to val when option found */
+ int *flag;
+ /* if flag not NULL, value to set *flag to; else return value */
+ int val;
+};
+
+__BEGIN_DECLS
+int getopt_long(int, char * const *, const char *,
+ const struct option *, int *);
+int getopt_long_only(int, char * const *, const char *,
+ const struct option *, int *);
+#ifndef _GETOPT_DEFINED_
+#define _GETOPT_DEFINED_
+int getopt(int, char * const *, const char *);
+int getsubopt(char **, char * const *, char **);
+
+extern char *optarg; /* getopt(3) external variables */
+extern int opterr;
+extern int optind;
+extern int optopt;
+extern int optreset;
+extern char *suboptarg; /* getsubopt(3) external variable */
+#endif
+__END_DECLS
+
+#endif /* !_GETOPT_H_ */
diff --git a/lib/win32/getopt_long.cpp b/lib/win32/getopt_long.cpp new file mode 100755 index 00000000..5d910e1b --- /dev/null +++ b/lib/win32/getopt_long.cpp @@ -0,0 +1,550 @@ +/* $OpenBSD: getopt_long.c,v 1.20 2005/10/25 15:49:37 jmc Exp $ */
+/* $NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $ */
+// Adapted for Box Backup by Chris Wilson <chris+boxbackup@qwirx.com>
+
+/*
+ * Copyright (c) 2002 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Sponsored in part by the Defense Advanced Research Projects
+ * Agency (DARPA) and Air Force Research Laboratory, Air Force
+ * Materiel Command, USAF, under agreement number F39502-99-1-0512.
+ */
+/*-
+ * Copyright (c) 2000 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Dieter Baron and Thomas Klausner.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by the NetBSD
+ * Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// #include "Box.h"
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "getopt.h"
+
+#if defined _MSC_VER || defined __MINGW32__
+#define REPLACE_GETOPT /* use this getopt as the system getopt(3) */
+
+#ifdef REPLACE_GETOPT
+int opterr = 1; /* if error message should be printed */
+int optind = 1; /* index into parent argv vector */
+int optopt = '?'; /* character checked for validity */
+int optreset; /* reset getopt */
+char *optarg; /* argument associated with option */
+#endif
+
+#define PRINT_ERROR ((opterr) && (*options != ':'))
+
+#define FLAG_PERMUTE 0x01 /* permute non-options to the end of argv */
+#define FLAG_ALLARGS 0x02 /* treat non-options as args to option "-1" */
+#define FLAG_LONGONLY 0x04 /* operate as getopt_long_only */
+
+/* return values */
+#define BADCH (int)'?'
+#define BADARG ((*options == ':') ? (int)':' : (int)'?')
+#define INORDER (int)1
+
+#define EMSG ""
+
+static void warnx(const char* fmt, ...)
+{
+ va_list ap;
+ va_start(ap, fmt);
+ vfprintf(stderr, fmt, ap);
+ va_end(ap);
+ fprintf(stderr, "\n");
+}
+
+static int getopt_internal(int, char * const *, const char *,
+ const struct option *, int *, int);
+static int parse_long_options(char * const *, const char *,
+ const struct option *, int *, int);
+static int gcd(int, int);
+static void permute_args(int, int, int, char * const *);
+
+static char *place = EMSG; /* option letter processing */
+
+/* XXX: set optreset to 1 rather than these two */
+static int nonopt_start = -1; /* first non option argument (for permute) */
+static int nonopt_end = -1; /* first option after non options (for permute) */
+
+/* Error messages */
+static const char recargchar[] = "option requires an argument -- %c";
+static const char recargstring[] = "option requires an argument -- %s";
+static const char ambig[] = "ambiguous option -- %.*s";
+static const char noarg[] = "option doesn't take an argument -- %.*s";
+static const char illoptchar[] = "unknown option -- %c";
+static const char illoptstring[] = "unknown option -- %s";
+
+/*
+ * Compute the greatest common divisor of a and b.
+ */
+static int
+gcd(int a, int b)
+{
+ int c;
+
+ c = a % b;
+ while (c != 0) {
+ a = b;
+ b = c;
+ c = a % b;
+ }
+
+ return (b);
+}
+
+/*
+ * Exchange the block from nonopt_start to nonopt_end with the block
+ * from nonopt_end to opt_end (keeping the same order of arguments
+ * in each block).
+ */
+static void
+permute_args(int panonopt_start, int panonopt_end, int opt_end,
+ char * const *nargv)
+{
+ int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos;
+ char *swap;
+
+ /*
+ * compute lengths of blocks and number and size of cycles
+ */
+ nnonopts = panonopt_end - panonopt_start;
+ nopts = opt_end - panonopt_end;
+ ncycle = gcd(nnonopts, nopts);
+ cyclelen = (opt_end - panonopt_start) / ncycle;
+
+ for (i = 0; i < ncycle; i++) {
+ cstart = panonopt_end+i;
+ pos = cstart;
+ for (j = 0; j < cyclelen; j++) {
+ if (pos >= panonopt_end)
+ pos -= nnonopts;
+ else
+ pos += nopts;
+ swap = nargv[pos];
+ /* LINTED const cast */
+ ((char **) nargv)[pos] = nargv[cstart];
+ /* LINTED const cast */
+ ((char **)nargv)[cstart] = swap;
+ }
+ }
+}
+
+/*
+ * parse_long_options --
+ * Parse long options in argc/argv argument vector.
+ * Returns -1 if short_too is set and the option does not match long_options.
+ */
+static int
+parse_long_options(char * const *nargv, const char *options,
+ const struct option *long_options, int *idx, int short_too)
+{
+ char *current_argv, *has_equal;
+ size_t current_argv_len;
+ int i, match;
+
+ current_argv = place;
+ match = -1;
+
+ optind++;
+
+ if ((has_equal = strchr(current_argv, '=')) != NULL) {
+ /* argument found (--option=arg) */
+ current_argv_len = has_equal - current_argv;
+ has_equal++;
+ } else
+ current_argv_len = strlen(current_argv);
+
+ for (i = 0; long_options[i].name; i++) {
+ /* find matching long option */
+ if (strncmp(current_argv, long_options[i].name,
+ current_argv_len))
+ continue;
+
+ if (strlen(long_options[i].name) == current_argv_len) {
+ /* exact match */
+ match = i;
+ break;
+ }
+ /*
+ * If this is a known short option, don't allow
+ * a partial match of a single character.
+ */
+ if (short_too && current_argv_len == 1)
+ continue;
+
+ if (match == -1) /* partial match */
+ match = i;
+ else {
+ /* ambiguous abbreviation */
+ if (PRINT_ERROR)
+ warnx(ambig, (int)current_argv_len,
+ current_argv);
+ optopt = 0;
+ return (BADCH);
+ }
+ }
+ if (match != -1) { /* option found */
+ if (long_options[match].has_arg == no_argument
+ && has_equal) {
+ if (PRINT_ERROR)
+ warnx(noarg, (int)current_argv_len,
+ current_argv);
+ /*
+ * XXX: GNU sets optopt to val regardless of flag
+ */
+ if (long_options[match].flag == NULL)
+ optopt = long_options[match].val;
+ else
+ optopt = 0;
+ return (BADARG);
+ }
+ if (long_options[match].has_arg == required_argument ||
+ long_options[match].has_arg == optional_argument) {
+ if (has_equal)
+ optarg = has_equal;
+ else if (long_options[match].has_arg ==
+ required_argument) {
+ /*
+ * optional argument doesn't use next nargv
+ */
+ optarg = nargv[optind++];
+ }
+ }
+ if ((long_options[match].has_arg == required_argument)
+ && (optarg == NULL)) {
+ /*
+ * Missing argument; leading ':' indicates no error
+ * should be generated.
+ */
+ if (PRINT_ERROR)
+ warnx(recargstring,
+ current_argv);
+ /*
+ * XXX: GNU sets optopt to val regardless of flag
+ */
+ if (long_options[match].flag == NULL)
+ optopt = long_options[match].val;
+ else
+ optopt = 0;
+ --optind;
+ return (BADARG);
+ }
+ } else { /* unknown option */
+ if (short_too) {
+ --optind;
+ return (-1);
+ }
+ if (PRINT_ERROR)
+ warnx(illoptstring, current_argv);
+ optopt = 0;
+ return (BADCH);
+ }
+ if (idx)
+ *idx = match;
+ if (long_options[match].flag) {
+ *long_options[match].flag = long_options[match].val;
+ return (0);
+ } else
+ return (long_options[match].val);
+}
+
+/*
+ * getopt_internal --
+ * Parse argc/argv argument vector. Called by user level routines.
+ */
+static int
+getopt_internal(int nargc, char * const *nargv, const char *options,
+ const struct option *long_options, int *idx, int flags)
+{
+ const char * oli; /* option letter list index */
+ int optchar, short_too;
+ static int posixly_correct = -1;
+
+ if (options == NULL)
+ return (-1);
+
+ /*
+ * Disable GNU extensions if POSIXLY_CORRECT is set or options
+ * string begins with a '+'.
+ */
+ if (posixly_correct == -1)
+ posixly_correct = (getenv("POSIXLY_CORRECT") != NULL);
+ if (posixly_correct || *options == '+')
+ flags &= ~FLAG_PERMUTE;
+ else if (*options == '-')
+ flags |= FLAG_ALLARGS;
+ if (*options == '+' || *options == '-')
+ options++;
+
+ /*
+ * XXX Some GNU programs (like cvs) set optind to 0 instead of
+ * XXX using optreset. Work around this braindamage.
+ */
+ if (optind == 0)
+ optind = optreset = 1;
+
+ optarg = NULL;
+ if (optreset)
+ nonopt_start = nonopt_end = -1;
+start:
+ if (optreset || !*place) { /* update scanning pointer */
+ optreset = 0;
+ if (optind >= nargc) { /* end of argument vector */
+ place = EMSG;
+ if (nonopt_end != -1) {
+ /* do permutation, if we have to */
+ permute_args(nonopt_start, nonopt_end,
+ optind, nargv);
+ optind -= nonopt_end - nonopt_start;
+ }
+ else if (nonopt_start != -1) {
+ /*
+ * If we skipped non-options, set optind
+ * to the first of them.
+ */
+ optind = nonopt_start;
+ }
+ nonopt_start = nonopt_end = -1;
+ return (-1);
+ }
+ if (*(place = nargv[optind]) != '-' ||
+ (place[1] == '\0' && strchr(options, '-') == NULL)) {
+ place = EMSG; /* found non-option */
+ if (flags & FLAG_ALLARGS) {
+ /*
+ * GNU extension:
+ * return non-option as argument to option 1
+ */
+ optarg = nargv[optind++];
+ return (INORDER);
+ }
+ if (!(flags & FLAG_PERMUTE)) {
+ /*
+ * If no permutation wanted, stop parsing
+ * at first non-option.
+ */
+ return (-1);
+ }
+ /* do permutation */
+ if (nonopt_start == -1)
+ nonopt_start = optind;
+ else if (nonopt_end != -1) {
+ permute_args(nonopt_start, nonopt_end,
+ optind, nargv);
+ nonopt_start = optind -
+ (nonopt_end - nonopt_start);
+ nonopt_end = -1;
+ }
+ optind++;
+ /* process next argument */
+ goto start;
+ }
+ if (nonopt_start != -1 && nonopt_end == -1)
+ nonopt_end = optind;
+
+ /*
+ * If we have "-" do nothing, if "--" we are done.
+ */
+ if (place[1] != '\0' && *++place == '-' && place[1] == '\0') {
+ optind++;
+ place = EMSG;
+ /*
+ * We found an option (--), so if we skipped
+ * non-options, we have to permute.
+ */
+ if (nonopt_end != -1) {
+ permute_args(nonopt_start, nonopt_end,
+ optind, nargv);
+ optind -= nonopt_end - nonopt_start;
+ }
+ nonopt_start = nonopt_end = -1;
+ return (-1);
+ }
+ }
+
+ /*
+ * Check long options if:
+ * 1) we were passed some
+ * 2) the arg is not just "-"
+ * 3) either the arg starts with -- we are getopt_long_only()
+ */
+ if (long_options != NULL && place != nargv[optind] &&
+ (*place == '-' || (flags & FLAG_LONGONLY))) {
+ short_too = 0;
+ if (*place == '-')
+ place++; /* --foo long option */
+ else if (*place != ':' && strchr(options, *place) != NULL)
+ short_too = 1; /* could be short option too */
+
+ optchar = parse_long_options(nargv, options, long_options,
+ idx, short_too);
+ if (optchar != -1) {
+ place = EMSG;
+ return (optchar);
+ }
+ }
+
+ if ((optchar = (int)*place++) == (int)':' ||
+ optchar == (int)'-' && *place != '\0' ||
+ (oli = strchr(options, optchar)) == NULL) {
+ /*
+ * If the user specified "-" and '-' isn't listed in
+ * options, return -1 (non-option) as per POSIX.
+ * Otherwise, it is an unknown option character (or ':').
+ */
+ if (optchar == (int)'-' && *place == '\0')
+ return (-1);
+ if (!*place)
+ ++optind;
+ if (PRINT_ERROR)
+ warnx(illoptchar, optchar);
+ optopt = optchar;
+ return (BADCH);
+ }
+ if (long_options != NULL && optchar == 'W' && oli[1] == ';') {
+ /* -W long-option */
+ if (*place) /* no space */
+ /* NOTHING */;
+ else if (++optind >= nargc) { /* no arg */
+ place = EMSG;
+ if (PRINT_ERROR)
+ warnx(recargchar, optchar);
+ optopt = optchar;
+ return (BADARG);
+ } else /* white space */
+ place = nargv[optind];
+ optchar = parse_long_options(nargv, options, long_options,
+ idx, 0);
+ place = EMSG;
+ return (optchar);
+ }
+ if (*++oli != ':') { /* doesn't take argument */
+ if (!*place)
+ ++optind;
+ } else { /* takes (optional) argument */
+ optarg = NULL;
+ if (*place) /* no white space */
+ optarg = place;
+ /* XXX: disable test for :: if PC? (GNU doesn't) */
+ else if (oli[1] != ':') { /* arg not optional */
+ if (++optind >= nargc) { /* no arg */
+ place = EMSG;
+ if (PRINT_ERROR)
+ warnx(recargchar, optchar);
+ optopt = optchar;
+ return (BADARG);
+ } else
+ optarg = nargv[optind];
+ } else if (!(flags & FLAG_PERMUTE)) {
+ /*
+ * If permutation is disabled, we can accept an
+ * optional arg separated by whitespace so long
+ * as it does not start with a dash (-).
+ */
+ if (optind + 1 < nargc && *nargv[optind + 1] != '-')
+ optarg = nargv[++optind];
+ }
+ place = EMSG;
+ ++optind;
+ }
+ /* dump back option letter */
+ return (optchar);
+}
+
+#ifdef REPLACE_GETOPT
+/*
+ * getopt --
+ * Parse argc/argv argument vector.
+ *
+ * [eventually this will replace the BSD getopt]
+ */
+int
+getopt(int nargc, char * const *nargv, const char *options)
+{
+
+ /*
+ * We don't pass FLAG_PERMUTE to getopt_internal() since
+ * the BSD getopt(3) (unlike GNU) has never done this.
+ *
+ * Furthermore, since many privileged programs call getopt()
+ * before dropping privileges it makes sense to keep things
+ * as simple (and bug-free) as possible.
+ */
+ return (getopt_internal(nargc, nargv, options, NULL, NULL, 0));
+}
+#endif /* REPLACE_GETOPT */
+
+/*
+ * getopt_long --
+ * Parse argc/argv argument vector.
+ */
+int
+getopt_long(int nargc, char * const *nargv, const char *options,
+ const struct option *long_options, int *idx)
+{
+
+ return (getopt_internal(nargc, nargv, options, long_options, idx,
+ FLAG_PERMUTE));
+}
+
+/*
+ * getopt_long_only --
+ * Parse argc/argv argument vector.
+ */
+int
+getopt_long_only(int nargc, char * const *nargv, const char *options,
+ const struct option *long_options, int *idx)
+{
+
+ return (getopt_internal(nargc, nargv, options, long_options, idx,
+ FLAG_PERMUTE|FLAG_LONGONLY));
+}
+
+#endif // defined _MSC_VER || defined __MINGW32__
diff --git a/lib/win32/messages.h b/lib/win32/messages.h new file mode 100755 index 00000000..6959591b --- /dev/null +++ b/lib/win32/messages.h @@ -0,0 +1,57 @@ + // Message source file, to be compiled to a resource file with
+ // Microsoft Message Compiler (MC), to an object file with a Resource
+ // Compiler, and linked into the application.
+
+ // The main reason for this file is to work around Windows' stupid
+ // messages in the Event Log, which say:
+
+ // The description for Event ID ( 4 ) in Source ( Box Backup (bbackupd) )
+ // cannot be found. The local computer may not have the necessary
+ // registry information or message DLL files to display messages from a
+ // remote computer. The following information is part of the event:
+ // Message definitions follow
+//
+// Values are 32 bit values layed out as follows:
+//
+// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
+// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
+// +---+-+-+-----------------------+-------------------------------+
+// |Sev|C|R| Facility | Code |
+// +---+-+-+-----------------------+-------------------------------+
+//
+// where
+//
+// Sev - is the severity code
+//
+// 00 - Success
+// 01 - Informational
+// 10 - Warning
+// 11 - Error
+//
+// C - is the Customer code flag
+//
+// R - is a reserved bit
+//
+// Facility - is the facility code
+//
+// Code - is the facility's status code
+//
+//
+// Define the facility codes
+//
+
+
+//
+// Define the severity codes
+//
+
+
+//
+// MessageId: MSG_ERR
+//
+// MessageText:
+//
+// %1
+//
+#define MSG_ERR ((DWORD)0x40000001L)
+
diff --git a/lib/win32/messages.mc b/lib/win32/messages.mc new file mode 100644 index 00000000..75f17b0f --- /dev/null +++ b/lib/win32/messages.mc @@ -0,0 +1,22 @@ +; // Message source file, to be compiled to a resource file with +; // Microsoft Message Compiler (MC), to an object file with a Resource +; // Compiler, and linked into the application. +; +; // The main reason for this file is to work around Windows' stupid +; // messages in the Event Log, which say: +; +; // The description for Event ID ( 4 ) in Source ( Box Backup (bbackupd) ) +; // cannot be found. The local computer may not have the necessary +; // registry information or message DLL files to display messages from a +; // remote computer. The following information is part of the event: + +MessageIdTypedef = DWORD + +; // Message definitions follow + +MessageId = 0x1 +Severity = Informational +SymbolicName = MSG_ERR +Language = English +%1 +. diff --git a/lib/win32/messages.rc b/lib/win32/messages.rc new file mode 100755 index 00000000..116522b7 --- /dev/null +++ b/lib/win32/messages.rc @@ -0,0 +1,2 @@ +LANGUAGE 0x9,0x1
+1 11 MSG00001.bin
|