diff options
Diffstat (limited to 'lib')
109 files changed, 0 insertions, 12384 deletions
diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp deleted file mode 100644 index b25ed9c7..00000000 --- a/lib/backupclient/BackupClientFileAttributes.cpp +++ /dev/null @@ -1,1186 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientFileAttributes.cpp -// Purpose: Storage of file attributes -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - -#include <sys/types.h> -#include <sys/stat.h> -#include <errno.h> -#include <limits.h> - -#include <algorithm> -#include <cstring> -#include <new> -#include <vector> - -#ifdef HAVE_SYS_XATTR_H -#include <cerrno> -#include <sys/xattr.h> -#endif - -#include <cstring> - -#include "BackupClientFileAttributes.h" -#include "CommonException.h" -#include "FileModificationTime.h" -#include "BoxTimeToUnix.h" -#include "BackupStoreException.h" -#include "CipherContext.h" -#include "CipherBlowfish.h" -#include "MD5Digest.h" - -#include "MemLeakFindOn.h" - -// set packing to one byte -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "BeginStructPackForWire.h" -#else -BEGIN_STRUCTURE_PACKING_FOR_WIRE -#endif - -#define ATTRIBUTETYPE_GENERIC_UNIX 1 - -#define ATTRIBUTE_ENCODING_BLOWFISH 2 - -typedef struct -{ - int32_t AttributeType; - u_int32_t UID; - u_int32_t GID; - u_int64_t ModificationTime; - u_int64_t AttrModificationTime; - u_int32_t UserDefinedFlags; - u_int32_t FileGenerationNumber; - u_int16_t Mode; - // Symbolic link filename may follow - // Extended attribute (xattr) information may follow, format is: - // u_int32_t Size of extended attribute block (excluding this word) - // For each of NumberOfAttributes (sorted by AttributeName): - // u_int16_t AttributeNameLength - // char AttributeName[AttributeNameLength] - // u_int32_t AttributeValueLength - // unsigned char AttributeValue[AttributeValueLength] - // AttributeName is 0 terminated, AttributeValue is not (and may be binary data) -} attr_StreamFormat; - -// This has wire packing so it's compatible across platforms -// Use wider than necessary sizes, just to be careful. -typedef struct -{ - int32_t uid, gid, mode; - #ifdef WIN32 - int64_t fileCreationTime; - #endif -} attributeHashData; - -// Use default packing -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "EndStructPackForWire.h" -#else -END_STRUCTURE_PACKING_FOR_WIRE -#endif - - -#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH 256 - -// Hide private static variables from the rest of the world -// -- don't put them as static class variables to avoid openssl/evp.h being -// included all over the project. -namespace -{ - CipherContext sBlowfishEncrypt; - CipherContext sBlowfishDecrypt; - uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH]; - int sAttributeHashSecretLength = 0; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::BackupClientFileAttributes() -// Purpose: Default constructor -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -BackupClientFileAttributes::BackupClientFileAttributes() - : mpClearAttributes(0) -{ - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &) -// Purpose: Copy constructor -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy) - : StreamableMemBlock(rToCopy), // base class does the hard work - mpClearAttributes(0) -{ -} -BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy) - : StreamableMemBlock(rToCopy), // base class does the hard work - mpClearAttributes(0) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::~BackupClientFileAttributes() -// Purpose: Destructor -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -BackupClientFileAttributes::~BackupClientFileAttributes() -{ - if(mpClearAttributes) - { - delete mpClearAttributes; - mpClearAttributes = 0; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes &operator=(const BackupClientFileAttributes &) -// Purpose: Assignment operator -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr) -{ - StreamableMemBlock::Set(rAttr); - RemoveClear(); // make sure no decrypted version held - return *this; -} -// Assume users play nice -BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr) -{ - StreamableMemBlock::Set(rAttr); - RemoveClear(); // make sure no decrypted version held - return *this; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::operator==(const BackupClientFileAttributes &) -// Purpose: Comparison operator -// Created: 2003/10/09 -// -// -------------------------------------------------------------------------- -bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const -{ - EnsureClearAvailable(); - rAttr.EnsureClearAvailable(); - - return mpClearAttributes->operator==(*rAttr.mpClearAttributes); -} -// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data. -/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const -{ - StreamableMemBlock *pDecoded = 0; - - try - { - EnsureClearAvailable(); - StreamableMemBlock *pDecoded = MakeClear(rAttr); - - // Compare using clear version - bool compared = mpClearAttributes->operator==(rAttr); - - // Delete temporary - delete pDecoded; - - return compared; - } - catch(...) - { - delete pDecoded; - throw; - } -}*/ - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool) -// Purpose: Compare, optionally ignoring the attribute -// modification time and/or modification time, and some -// data which is irrelevant in practise (eg file -// generation number) -// Created: 10/12/03 -// -// -------------------------------------------------------------------------- -bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, - bool IgnoreAttrModTime, bool IgnoreModTime) const -{ - EnsureClearAvailable(); - rAttr.EnsureClearAvailable(); - - // Check sizes are the same, as a first check - if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize()) - { - BOX_TRACE("Attribute Compare: Attributes objects are " - "different sizes, cannot compare them: local " << - mpClearAttributes->GetSize() << " bytes, remote " << - rAttr.mpClearAttributes->GetSize() << " bytes"); - return false; - } - - // Then check the elements of the two things - // Bytes are checked in network order, but this doesn't matter as we're only checking for equality. - attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); - attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer(); - - #define COMPARE(attribute, message) \ - if (a1->attribute != a2->attribute) \ - { \ - BOX_TRACE("Attribute Compare: " << message << " differ: " \ - "local " << ntoh(a1->attribute) << ", " \ - "remote " << ntoh(a2->attribute)); \ - return false; \ - } - COMPARE(AttributeType, "Attribute types"); - COMPARE(UID, "UIDs"); - COMPARE(GID, "GIDs"); - COMPARE(UserDefinedFlags, "User-defined flags"); - COMPARE(Mode, "Modes"); - - if(!IgnoreModTime) - { - uint64_t t1 = box_ntoh64(a1->ModificationTime); - uint64_t t2 = box_ntoh64(a2->ModificationTime); - time_t s1 = BoxTimeToSeconds(t1); - time_t s2 = BoxTimeToSeconds(t2); - if(s1 != s2) - { - BOX_TRACE("Attribute Compare: File modification " - "times differ: local " << - FormatTime(t1, true) << " (" << s1 << "), " - "remote " << - FormatTime(t2, true) << " (" << s2 << ")"); - return false; - } - } - - if(!IgnoreAttrModTime) - { - uint64_t t1 = box_ntoh64(a1->AttrModificationTime); - uint64_t t2 = box_ntoh64(a2->AttrModificationTime); - time_t s1 = BoxTimeToSeconds(t1); - time_t s2 = BoxTimeToSeconds(t2); - if(s1 != s2) - { - BOX_TRACE("Attribute Compare: Attribute modification " - "times differ: local " << - FormatTime(t1, true) << " (" << s1 << "), " - "remote " << - FormatTime(t2, true) << " (" << s2 << ")"); - return false; - } - } - - // Check symlink string? - unsigned int size = mpClearAttributes->GetSize(); - if(size > sizeof(attr_StreamFormat)) - { - // Symlink strings don't match. This also compares xattrs - int datalen = size - sizeof(attr_StreamFormat); - - if(::memcmp(a1 + 1, a2 + 1, datalen) != 0) - { - std::string s1((char *)(a1 + 1), datalen); - std::string s2((char *)(a2 + 1), datalen); - BOX_TRACE("Attribute Compare: Symbolic link target " - "or extended attributes differ: " - "local " << PrintEscapedBinaryData(s1) << ", " - "remote " << PrintEscapedBinaryData(s2)); - return false; - } - } - - // Passes all test, must be OK - return true; -} - - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::ReadAttributes( -// const char *Filename, bool ZeroModificationTimes, -// box_time_t *pModTime, box_time_t *pAttrModTime, -// int64_t *pFileSize, InodeRefType *pInodeNumber, -// bool *pHasMultipleLinks) -// Purpose: Read the attributes of the file, and store them -// ready for streaming. Optionally retrieve the -// modification time and attribute modification time. -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::ReadAttributes(const char *Filename, - bool ZeroModificationTimes, box_time_t *pModTime, - box_time_t *pAttrModTime, int64_t *pFileSize, - InodeRefType *pInodeNumber, bool *pHasMultipleLinks) -{ - StreamableMemBlock *pnewAttr = 0; - try - { - EMU_STRUCT_STAT st; - if(EMU_LSTAT(Filename, &st) != 0) - { - BOX_LOG_SYS_ERROR("Failed to stat file: '" << - Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError) - } - - // Modification times etc - if(pModTime) {*pModTime = FileModificationTime(st);} - if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);} - if(pFileSize) {*pFileSize = st.st_size;} - if(pInodeNumber) {*pInodeNumber = st.st_ino;} - if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);} - - pnewAttr = new StreamableMemBlock; - - FillAttributes(*pnewAttr, Filename, st, ZeroModificationTimes); - -#ifndef WIN32 - // Is it a link? - if((st.st_mode & S_IFMT) == S_IFLNK) - { - FillAttributesLink(*pnewAttr, Filename, st); - } -#endif - - FillExtendedAttr(*pnewAttr, Filename); - -#ifdef WIN32 - //this is to catch those problems with invalid time stamps stored... - //need to find out the reason why - but also a catch as well. - - attr_StreamFormat *pattr = - (attr_StreamFormat*)pnewAttr->GetBuffer(); - ASSERT(pattr != 0); - - // __time64_t winTime = BoxTimeToSeconds( - // pnewAttr->ModificationTime); - - u_int64_t modTime = box_ntoh64(pattr->ModificationTime); - box_time_t modSecs = BoxTimeToSeconds(modTime); - __time64_t winTime = modSecs; - - // _MAX__TIME64_T doesn't seem to be defined, but the code below - // will throw an assertion failure if we exceed it :-) - // Microsoft says dates up to the year 3000 are valid, which - // is a bit more than 15 * 2^32. Even that doesn't seem - // to be true (still aborts), but it can at least hold 2^32. - if (winTime >= 0x100000000LL || _gmtime64(&winTime) == 0) - { - BOX_ERROR("Invalid Modification Time caught for " - "file: '" << Filename << "'"); - pattr->ModificationTime = 0; - } - - modTime = box_ntoh64(pattr->AttrModificationTime); - modSecs = BoxTimeToSeconds(modTime); - winTime = modSecs; - - if (winTime > 0x100000000LL || _gmtime64(&winTime) == 0) - { - BOX_ERROR("Invalid Attribute Modification Time " - "caught for file: '" << Filename << "'"); - pattr->AttrModificationTime = 0; - } -#endif - - // Attributes ready. Encrypt into this block - EncryptAttr(*pnewAttr); - - // Store the new attributes - RemoveClear(); - mpClearAttributes = pnewAttr; - pnewAttr = 0; - } - catch(...) - { - // clean up - delete pnewAttr; - pnewAttr = 0; - throw; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::ReadAttributesLink() -// Purpose: Private function, handles standard attributes for all objects -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, EMU_STRUCT_STAT &st, bool ZeroModificationTimes) -{ - outputBlock.ResizeBlock(sizeof(attr_StreamFormat)); - attr_StreamFormat *pattr = (attr_StreamFormat*)outputBlock.GetBuffer(); - ASSERT(pattr != 0); - - // Fill in the entries - pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX); - pattr->UID = htonl(st.st_uid); - pattr->GID = htonl(st.st_gid); - if(ZeroModificationTimes) - { - pattr->ModificationTime = 0; - pattr->AttrModificationTime = 0; - } - else - { - pattr->ModificationTime = box_hton64(FileModificationTime(st)); - pattr->AttrModificationTime = box_hton64(FileAttrModificationTime(st)); - } - pattr->Mode = htons(st.st_mode); - -#ifndef HAVE_STRUCT_STAT_ST_FLAGS - pattr->UserDefinedFlags = 0; - pattr->FileGenerationNumber = 0; -#else - pattr->UserDefinedFlags = htonl(st.st_flags); - pattr->FileGenerationNumber = htonl(st.st_gen); -#endif -} -#ifndef WIN32 -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::ReadAttributesLink() -// Purpose: Private function, handles the case where a symbolic link is needed -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st) -{ - // Make sure we're only called for symbolic links - ASSERT((st.st_mode & S_IFMT) == S_IFLNK); - - // Get the filename the link is linked to - char linkedTo[PATH_MAX+4]; - int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX); - if(linkedToSize == -1) - { - BOX_LOG_SYS_ERROR("Failed to readlink '" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError); - } - - int oldSize = outputBlock.GetSize(); - outputBlock.ResizeBlock(oldSize+linkedToSize+1); - char* buffer = static_cast<char*>(outputBlock.GetBuffer()); - - // Add the path name for the symbolic link, and add 0 termination - std::memcpy(buffer+oldSize, linkedTo, linkedToSize); - buffer[oldSize+linkedToSize] = '\0'; -} -#endif - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::ReadExtendedAttr(const char *, unsigned char**) -// Purpose: Private function, read the extended attributes of the file into the block -// Created: 2005/06/12 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename) -{ -#ifdef HAVE_SYS_XATTR_H - int listBufferSize = 10000; - char* list = new char[listBufferSize]; - - try - { - // This returns an unordered list of attribute names, each 0 terminated, - // concatenated together - int listSize = ::llistxattr(Filename, list, listBufferSize); - - if(listSize>listBufferSize) - { - delete[] list, list = NULL; - list = new char[listSize]; - listSize = ::llistxattr(Filename, list, listSize); - } - - if(listSize>0) - { - // Extract list of attribute names so we can sort them - std::vector<std::string> attrKeys; - for(int i = 0; i<listSize; ++i) - { - std::string attrKey(list+i); - i += attrKey.size(); - attrKeys.push_back(attrKey); - } - sort(attrKeys.begin(), attrKeys.end()); - - // Make initial space in block - int xattrSize = outputBlock.GetSize(); - int xattrBufferSize = (xattrSize+listSize)>500 ? (xattrSize+listSize)*2 : 1000; - outputBlock.ResizeBlock(xattrBufferSize); - unsigned char* buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); - - // Leave space for attr block size later - int xattrBlockSizeOffset = xattrSize; - xattrSize += sizeof(u_int32_t); - - // Loop for each attribute - for(std::vector<std::string>::const_iterator attrKeyI = attrKeys.begin(); attrKeyI!=attrKeys.end(); ++attrKeyI) - { - std::string attrKey(*attrKeyI); - - if(xattrSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t)>static_cast<unsigned int>(xattrBufferSize)) - { - xattrBufferSize = (xattrBufferSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t))*2; - outputBlock.ResizeBlock(xattrBufferSize); - buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); - } - - // Store length and text for attibute name - u_int16_t keyLength = htons(attrKey.size()+1); - std::memcpy(buffer+xattrSize, &keyLength, sizeof(u_int16_t)); - xattrSize += sizeof(u_int16_t); - std::memcpy(buffer+xattrSize, attrKey.c_str(), attrKey.size()+1); - xattrSize += attrKey.size()+1; - - // Leave space for value size - int valueSizeOffset = xattrSize; - xattrSize += sizeof(u_int32_t); - - // Find size of attribute (must call with buffer and length 0 on some platforms, - // as -1 is returned if the data doesn't fit.) - int valueSize = ::lgetxattr(Filename, attrKey.c_str(), 0, 0); - if(valueSize<0) - { - BOX_LOG_SYS_ERROR("Failed to get " - "extended attributes size " - "for '" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError); - } - - // Resize block, if needed - if(xattrSize+valueSize>xattrBufferSize) - { - xattrBufferSize = (xattrBufferSize+valueSize)*2; - outputBlock.ResizeBlock(xattrBufferSize); - buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); - } - - // This gets the attribute value (may be text or binary), no termination - valueSize = ::lgetxattr(Filename, attrKey.c_str(), buffer+xattrSize, xattrBufferSize-xattrSize); - if(valueSize<0) - { - BOX_LOG_SYS_ERROR("Failed to get " - "extended attributes for " - "'" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError); - } - xattrSize += valueSize; - - // Fill in value size - u_int32_t valueLength = htonl(valueSize); - std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(u_int32_t)); - } - - // Fill in attribute block size - u_int32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(u_int32_t)); - std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(u_int32_t)); - - outputBlock.ResizeBlock(xattrSize); - } - else if(listSize<0) - { - if(errno == EOPNOTSUPP || errno == EACCES) - { - // fail silently - } - else if(errno == ERANGE) - { - BOX_ERROR("Failed to list extended " - "attributes of '" << Filename << "': " - "buffer too small, not backed up"); - } - else - { - BOX_LOG_SYS_ERROR("Failed to list extended " - "attributes of '" << Filename << "', " - "not backed up"); - THROW_EXCEPTION(CommonException, OSFileError); - } - } - } - catch(...) - { - delete[] list; - throw; - } - delete[] list; -#endif -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::GetModificationTimes() -// Purpose: Returns the modification time embedded in the -// attributes. -// Created: 2010/02/24 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::GetModificationTimes( - box_time_t *pModificationTime, - box_time_t *pAttrModificationTime) const -{ - // Got something loaded - if(GetSize() <= 0) - { - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - // Make sure there are clear attributes to use - EnsureClearAvailable(); - ASSERT(mpClearAttributes != 0); - - // Check if the decrypted attributes are small enough, and the type of attributes stored - if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) - { - THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); - } - int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); - ASSERT(type != 0); - if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) - { - // Don't know what to do with these - THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); - } - - // Check there is enough space for an attributes block - if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) - { - // Too small - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - // Get pointer to structure - attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); - - if(pModificationTime) - { - *pModificationTime = box_ntoh64(pattr->ModificationTime); - } - - if(pAttrModificationTime) - { - *pAttrModificationTime = box_ntoh64(pattr->AttrModificationTime); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::WriteAttributes(const char *) -// Purpose: Apply the stored attributes to the file -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::WriteAttributes(const char *Filename, - bool MakeUserWritable) const -{ - // Got something loaded - if(GetSize() <= 0) - { - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - // Make sure there are clear attributes to use - EnsureClearAvailable(); - ASSERT(mpClearAttributes != 0); - - // Check if the decrypted attributes are small enough, and the type of attributes stored - if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) - { - THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); - } - int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); - ASSERT(type != 0); - if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) - { - // Don't know what to do with these - THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); - } - - // Check there is enough space for an attributes block - if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) - { - // Too small - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - // Get pointer to structure - attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); - int xattrOffset = sizeof(attr_StreamFormat); - - // is it a symlink? - int16_t mode = ntohs(pattr->Mode); - if((mode & S_IFMT) == S_IFLNK) - { - // Check things are sensible - if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1) - { - // Too small - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - -#ifdef WIN32 - BOX_WARNING("Cannot create symbolic links on Windows: '" << - Filename << "'"); -#else - // Make a symlink, first deleting anything in the way - ::unlink(Filename); - if(::symlink((char*)(pattr + 1), Filename) != 0) - { - BOX_LOG_SYS_ERROR("Failed to symlink '" << Filename << - "' to '" << (char*)(pattr + 1) << "'"); - THROW_EXCEPTION(CommonException, OSFileError) - } -#endif - - xattrOffset += std::strlen(reinterpret_cast<char*>(pattr+1))+1; - } - - // If working as root, set user IDs - if(::geteuid() == 0) - { - #ifndef HAVE_LCHOWN - // only if not a link, can't set their owner on this platform - if((mode & S_IFMT) != S_IFLNK) - { - // Not a link, use normal chown - if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) - { - BOX_LOG_SYS_ERROR("Failed to change " - "owner of file " - "'" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError) - } - } - #else - // use the version which sets things on symlinks - if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) - { - BOX_LOG_SYS_ERROR("Failed to change owner of " - "symbolic link '" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError) - } - #endif - } - - if(static_cast<int>(xattrOffset+sizeof(u_int32_t))<=mpClearAttributes->GetSize()) - { - WriteExtendedAttr(Filename, xattrOffset); - } - - // Stop now if symlink, because otherwise it'll just be applied to the target - if((mode & S_IFMT) == S_IFLNK) - { - return; - } - - // Set modification time? - box_time_t modtime = box_ntoh64(pattr->ModificationTime); - if(modtime != 0) - { - // Work out times as timevals - struct timeval times[2]; - - #ifdef WIN32 - BoxTimeToTimeval(box_ntoh64(pattr->ModificationTime), - times[1]); - BoxTimeToTimeval(box_ntoh64(pattr->AttrModificationTime), - times[0]); - // Because stat() returns the creation time in the ctime - // field under Windows, and this gets saved in the - // AttrModificationTime field of the serialised attributes, - // we subvert the first parameter of emu_utimes() to allow - // it to be reset to the right value on the restored file. - #else - BoxTimeToTimeval(modtime, times[1]); - // Copy access time as well, why not, got to set it to something - times[0] = times[1]; - // Attr modification time will be changed anyway, - // nothing that can be done about it - #endif - - // Try to apply - if(::utimes(Filename, times) != 0) - { - BOX_LOG_SYS_WARNING("Failed to change times of " - "file '" << Filename << "' to ctime=" << - BOX_FORMAT_TIMESPEC(times[0]) << ", mtime=" << - BOX_FORMAT_TIMESPEC(times[1])); - } - } - - if (MakeUserWritable) - { - mode |= S_IRWXU; - } - - // Apply everything else... (allowable mode flags only) - // Mode must be done last (think setuid) - if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID - | S_ISGID | S_ISVTX)) != 0) - { - BOX_LOG_SYS_ERROR("Failed to change permissions of file " - "'" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError) - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::IsSymLink() -// Purpose: Do these attributes represent a symbolic link? -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -bool BackupClientFileAttributes::IsSymLink() const -{ - EnsureClearAvailable(); - - // Got the right kind of thing? - if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) - { - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - // Get the type of attributes stored - int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); - ASSERT(type != 0); - if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat)) - { - // Check link - attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); - return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK; - } - - return false; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::RemoveClear() -// Purpose: Private. Deletes any clear version of the attributes that may be held -// Created: 3/12/03 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::RemoveClear() const -{ - if(mpClearAttributes) - { - delete mpClearAttributes; - } - mpClearAttributes = 0; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::EnsureClearAvailable() -// Purpose: Private. Makes sure the clear version is available -// Created: 3/12/03 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::EnsureClearAvailable() const -{ - if(mpClearAttributes == 0) - { - mpClearAttributes = MakeClear(*this); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) -// Purpose: Private function, apply the stored extended attributes to the file -// Created: 2005/06/13 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xattrOffset) const -{ -#ifdef HAVE_SYS_XATTR_H - const char* buffer = static_cast<char*>(mpClearAttributes->GetBuffer()); - - u_int32_t xattrBlockLength = 0; - std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(u_int32_t)); - int xattrBlockSize = ntohl(xattrBlockLength); - xattrOffset += sizeof(u_int32_t); - - int xattrEnd = xattrOffset+xattrBlockSize; - if(xattrEnd>mpClearAttributes->GetSize()) - { - // Too small - THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); - } - - while(xattrOffset<xattrEnd) - { - u_int16_t keyLength = 0; - std::memcpy(&keyLength, buffer+xattrOffset, sizeof(u_int16_t)); - int keySize = ntohs(keyLength); - xattrOffset += sizeof(u_int16_t); - - const char* key = buffer+xattrOffset; - xattrOffset += keySize; - - u_int32_t valueLength = 0; - std::memcpy(&valueLength, buffer+xattrOffset, sizeof(u_int32_t)); - int valueSize = ntohl(valueLength); - xattrOffset += sizeof(u_int32_t); - - // FIXME: Warn on EOPNOTSUPP - if(::lsetxattr(Filename, key, buffer+xattrOffset, valueSize, 0)!=0 && errno!=EOPNOTSUPP) - { - BOX_LOG_SYS_ERROR("Failed to set extended attributes " - "on file '" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileError); - } - - xattrOffset += valueSize; - } - - ASSERT(xattrOffset==xattrEnd); -#endif -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::MakeClear(const StreamableMemBlock &) -// Purpose: Static. Decrypts stored attributes. -// Created: 3/12/03 -// -// -------------------------------------------------------------------------- -StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted) -{ - // New block - StreamableMemBlock *pdecrypted = 0; - - try - { - // Check the block is big enough for IV and header - int ivSize = sBlowfishEncrypt.GetIVLength(); - if(rEncrypted.GetSize() <= (ivSize + 1)) - { - THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes); - } - - // How much space is needed for the output? - int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize); - - // Allocate it - pdecrypted = new StreamableMemBlock(maxDecryptedSize); - - // ptr to block - uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer(); - - // Check that the header has right type - if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH) - { - THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding); - } - - // Set IV - sBlowfishDecrypt.SetIV(encBlock + 1); - - // Decrypt - int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1)); - - // Resize block to fit - pdecrypted->ResizeBlock(decryptedSize); - } - catch(...) - { - delete pdecrypted; - pdecrypted = 0; - } - - return pdecrypted; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::SetBlowfishKey(const void *, int) -// Purpose: Static. Sets the key to use for encryption and decryption. -// Created: 3/12/03 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength) -{ - // IVs set later - sBlowfishEncrypt.Reset(); - sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); - sBlowfishDecrypt.Reset(); - sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &) -// Purpose: Private. Encrypt the given attributes into this block. -// Created: 3/12/03 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt) -{ - // Free any existing block - FreeBlock(); - - // Work out the maximum amount of space we need - int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize()); - // And the size of the IV - int ivSize = sBlowfishEncrypt.GetIVLength(); - - // Allocate this space - AllocateBlock(maxEncryptedSize + ivSize + 1); - - // Store the encoding byte - uint8_t *block = (uint8_t*)GetBuffer(); - block[0] = ATTRIBUTE_ENCODING_BLOWFISH; - - // Generate and store an IV for this attribute block - int ivSize2 = 0; - const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2); - ASSERT(ivSize == ivSize2); - - // Copy into the encrypted block - ::memcpy(block + 1, iv, ivSize); - - // Do the transform - int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize()); - - // Resize this block - ResizeBlock(encrytedSize + ivSize + 1); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::SetAttributeHashSecret(const void *, int) -// Purpose: Set the secret for the filename attribute hash -// Created: 25/4/04 -// -// -------------------------------------------------------------------------- -void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength) -{ - if(SecretLength > (int)sizeof(sAttributeHashSecret)) - { - SecretLength = sizeof(sAttributeHashSecret); - } - if(SecretLength < 0) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Copy - ::memcpy(sAttributeHashSecret, pSecret, SecretLength); - sAttributeHashSecretLength = SecretLength; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientFileAttributes::GenerateAttributeHash( -// struct stat &, const std::string &, -// const std::string &) -// Purpose: Generate a 64 bit hash from the attributes, used to -// detect changes. Include filename in the hash, so -// that it changes from one file to another, so don't -// reveal identical attributes. -// Created: 25/4/04 -// -// -------------------------------------------------------------------------- -uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, - const std::string &filename, const std::string &leafname) -{ - if(sAttributeHashSecretLength == 0) - { - THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet) - } - - // Assemble stuff we're interested in - attributeHashData hashData; - memset(&hashData, 0, sizeof(hashData)); - // Use network byte order and large sizes to be cross platform - hashData.uid = htonl(st.st_uid); - hashData.gid = htonl(st.st_gid); - hashData.mode = htonl(st.st_mode); - - #ifdef WIN32 - // On Windows, the "file attribute modification time" is the - // file creation time, and we want to back this up, restore - // it and compare it. - // - // On other platforms, it's not very important and can't - // reliably be set to anything other than the current time. - hashData.fileCreationTime = box_hton64(st.st_ctime); - #endif - - StreamableMemBlock xattr; - FillExtendedAttr(xattr, filename.c_str()); - - // Create a MD5 hash of the data, filename, and secret - MD5Digest digest; - digest.Add(&hashData, sizeof(hashData)); - digest.Add(xattr.GetBuffer(), xattr.GetSize()); - digest.Add(leafname.c_str(), leafname.size()); - digest.Add(sAttributeHashSecret, sAttributeHashSecretLength); - digest.Finish(); - - // Return the first 64 bits of the hash - uint64_t result = *((uint64_t *)(digest.DigestAsData())); - return result; -} diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h deleted file mode 100644 index f9a0d883..00000000 --- a/lib/backupclient/BackupClientFileAttributes.h +++ /dev/null @@ -1,78 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientFileAttributes.h -// Purpose: Storage of file attributes -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCLIENTFILEATTRIBUTES__H -#define BACKUPCLIENTFILEATTRIBUTES__H - -#include <string> - -#include "StreamableMemBlock.h" -#include "BoxTime.h" - -EMU_STRUCT_STAT; // declaration - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupClientFileAttributes -// Purpose: Storage, streaming and application of file attributes -// Created: 2003/10/07 -// -// -------------------------------------------------------------------------- -class BackupClientFileAttributes : public StreamableMemBlock -{ -public: - BackupClientFileAttributes(); - BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy); - BackupClientFileAttributes(const StreamableMemBlock &rToCopy); - ~BackupClientFileAttributes(); - BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr); - BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr); - bool operator==(const BackupClientFileAttributes &rAttr) const; -// bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous? - - bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const; - - // Prevent access to base class members accidently - void Set(); - - void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false, - box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0, - InodeRefType *pInodeNumber = 0, bool *pHasMultipleLinks = 0); - void WriteAttributes(const char *Filename, - bool MakeUserWritable = false) const; - void GetModificationTimes(box_time_t *pModificationTime, - box_time_t *pAttrModificationTime) const; - - bool IsSymLink() const; - - static void SetBlowfishKey(const void *pKey, int KeyLength); - static void SetAttributeHashSecret(const void *pSecret, int SecretLength); - - static uint64_t GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname); - static void FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename); - -private: - static void FillAttributes(StreamableMemBlock &outputBlock, - const char *Filename, EMU_STRUCT_STAT &st, - bool ZeroModificationTimes); - static void FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st); - void WriteExtendedAttr(const char *Filename, int xattrOffset) const; - - void RemoveClear() const; - void EnsureClearAvailable() const; - static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted); - void EncryptAttr(const StreamableMemBlock &rToEncrypt); - -private: - mutable StreamableMemBlock *mpClearAttributes; -}; - -#endif // BACKUPCLIENTFILEATTRIBUTES__H - diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp index f6b42af9..db72c4bd 100644 --- a/lib/backupclient/BackupClientRestore.cpp +++ b/lib/backupclient/BackupClientRestore.cpp @@ -22,11 +22,7 @@ #include <errno.h> #include "BackupClientRestore.h" -<<<<<<< HEAD -#include "autogen_BackupProtocolClient.h" -======= #include "autogen_BackupProtocol.h" ->>>>>>> 0.12 #include "CommonException.h" #include "BackupClientFileAttributes.h" #include "IOStream.h" @@ -447,13 +443,8 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, // list of files which is appropriate to the restore type rConnection.QueryListDirectory( DirectoryID, -<<<<<<< HEAD - Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING), - BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)), -======= Params.RestoreDeleted?(BackupProtocolListDirectory::Flags_Deleted):(BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING), BackupProtocolListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolListDirectory::Flags_Deleted)), ->>>>>>> 0.12 true /* want attributes */); // Retrieve the directory from the stream following @@ -578,11 +569,8 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, if (Params.ContinueAfterErrors) { Params.ContinuedAfterError = true; -<<<<<<< HEAD -======= // ensure that protocol remains usable objectStream->Flush(); ->>>>>>> 0.12 } else { @@ -853,13 +841,8 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, // // -------------------------------------------------------------------------- int BackupClientRestore(BackupProtocolClient &rConnection, -<<<<<<< HEAD - int64_t DirectoryID, const char *RemoteDirectoryName, - const char *LocalDirectoryName, bool PrintDots, bool RestoreDeleted, -======= int64_t DirectoryID, const std::string& RemoteDirectoryName, const std::string& LocalDirectoryName, bool PrintDots, bool RestoreDeleted, ->>>>>>> 0.12 bool UndeleteAfterRestoreDeleted, bool Resume, bool ContinueAfterErrors) { diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h index 79765f63..77f09c2e 100644 --- a/lib/backupclient/BackupClientRestore.h +++ b/lib/backupclient/BackupClientRestore.h @@ -24,15 +24,6 @@ enum int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, -<<<<<<< HEAD - const char *RemoteDirectoryName, - const char *LocalDirectoryName, - bool PrintDots = false, - bool RestoreDeleted = false, - bool UndeleteAfterRestoreDeleted = false, - bool Resume = false, - bool ContinueAfterErrors = false); -======= const std::string& RemoteDirectoryName, const std::string& LocalDirectoryName, bool PrintDots, @@ -40,7 +31,6 @@ int BackupClientRestore(BackupProtocolClient &rConnection, bool UndeleteAfterRestoreDeleted, bool Resume, bool ContinueAfterErrors); ->>>>>>> 0.12 #endif // BACKUPSCLIENTRESTORE__H diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index 3104ae8e..a3b95335 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -114,25 +114,19 @@ static const ConfigurationVerifyKey verifyrootkeys[] = ConfigurationVerifyKey("NotifyAlways", ConfigTest_IsBool, false), // option to disable the suppression of duplicate notifications -<<<<<<< HEAD -======= ConfigurationVerifyKey("MaxUploadRate", ConfigTest_IsInt), // optional maximum speed of uploads in kbytes per second ConfigurationVerifyKey("TcpNice", ConfigTest_IsBool, false), // optional enable of tcp nice/background mode ->>>>>>> 0.12 ConfigurationVerifyKey("CertificateFile", ConfigTest_Exists), ConfigurationVerifyKey("PrivateKeyFile", ConfigTest_Exists), ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_Exists), ConfigurationVerifyKey("KeysFile", ConfigTest_Exists), ConfigurationVerifyKey("DataDirectory", ConfigTest_Exists | ConfigTest_LastEntry), -<<<<<<< HEAD -======= ->>>>>>> 0.12 }; const ConfigurationVerify BackupDaemonConfigVerify = diff --git a/lib/backupclient/BackupStoreConstants.h b/lib/backupclient/BackupStoreConstants.h deleted file mode 100644 index 2c33fd8f..00000000 --- a/lib/backupclient/BackupStoreConstants.h +++ /dev/null @@ -1,44 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreContants.h -// Purpose: constants for the backup system -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTORECONSTANTS__H -#define BACKUPSTORECONSTANTS__H - -#define BACKUPSTORE_ROOT_DIRECTORY_ID 1 - -#define BACKUP_STORE_SERVER_VERSION 1 - -// Minimum size for a chunk to be compressed -#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE 256 - -// min and max sizes for blocks -#define BACKUP_FILE_MIN_BLOCK_SIZE 4096 -#define BACKUP_FILE_MAX_BLOCK_SIZE (512*1024) - -// Increase the block size if there are more than this number of blocks -#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 4096 - -// Avoid creating blocks smaller than this -#define BACKUP_FILE_AVOID_BLOCKS_LESS_THAN 128 - -// Maximum number of sizes to do an rsync-like scan for -#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES 64 - -// When doing rsync scans, do not scan for blocks smaller than -#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE 128 - -// A limit to stop diffing running out of control: If more than this -// times the number of blocks in the original index are found, stop -// looking. This stops really bad cases of diffing files containing -// all the same byte using huge amounts of memory and processor time. -// This is a multiple of the number of blocks in the diff from file. -#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE 4096 - -#endif // BACKUPSTORECONSTANTS__H - diff --git a/lib/backupclient/BackupStoreDirectory.cpp b/lib/backupclient/BackupStoreDirectory.cpp deleted file mode 100644 index 0d06da34..00000000 --- a/lib/backupclient/BackupStoreDirectory.cpp +++ /dev/null @@ -1,568 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreDirectory.h -// Purpose: Representation of a backup directory -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <sys/types.h> - -#include "BackupStoreDirectory.h" -#include "IOStream.h" -#include "BackupStoreException.h" -#include "BackupStoreObjectMagic.h" - -#include "MemLeakFindOn.h" - -// set packing to one byte -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "BeginStructPackForWire.h" -#else -BEGIN_STRUCTURE_PACKING_FOR_WIRE -#endif - -typedef struct -{ - int32_t mMagicValue; // also the version number - int32_t mNumEntries; - int64_t mObjectID; // this object ID - int64_t mContainerID; // ID of container - uint64_t mAttributesModTime; - int32_t mOptionsPresent; // bit mask of optional sections / features present - // Then a StreamableMemBlock for attributes -} dir_StreamFormat; - -typedef enum -{ - Option_DependencyInfoPresent = 1 -} dir_StreamFormatOptions; - -typedef struct -{ - uint64_t mModificationTime; - int64_t mObjectID; - int64_t mSizeInBlocks; - uint64_t mAttributesHash; - int16_t mFlags; // order smaller items after bigger ones (for alignment) - // Then a BackupStoreFilename - // Then a StreamableMemBlock for attributes -} en_StreamFormat; - -typedef struct -{ - int64_t mDependsNewer; - int64_t mDependsOlder; -} en_StreamFormatDepends; - -// Use default packing -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "EndStructPackForWire.h" -#else -END_STRUCTURE_PACKING_FOR_WIRE -#endif - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::BackupStoreDirectory() -// Purpose: Constructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::BackupStoreDirectory() - : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0) -{ - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); -} - - -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t) -// Purpose: Constructor giving object and container IDs -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID) - : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::~BackupStoreDirectory() -// Purpose: Destructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::~BackupStoreDirectory() -{ - for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) - { - delete (*i); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::ReadFromStream(IOStream &, int) -// Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads. -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) -{ - // Get the header - dir_StreamFormat hdr; - if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Check magic value... - if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue)) - { - THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat) - } - - // Get data - mObjectID = box_ntoh64(hdr.mObjectID); - mContainerID = box_ntoh64(hdr.mContainerID); - mAttributesModTime = box_ntoh64(hdr.mAttributesModTime); - - // Options - int32_t options = ntohl(hdr.mOptionsPresent); - - // Get attributes - mAttributes.ReadFromStream(rStream, Timeout); - - // Decode count - int count = ntohl(hdr.mNumEntries); - - // Clear existing list - for(std::vector<Entry*>::iterator i = mEntries.begin(); - i != mEntries.end(); i++) - { - delete (*i); - } - mEntries.clear(); - - // Read them in! - for(int c = 0; c < count; ++c) - { - Entry *pen = new Entry; - try - { - // Read from stream - pen->ReadFromStream(rStream, Timeout); - - // Add to list - mEntries.push_back(pen); - } - catch(...) - { - delete pen; - throw; - } - } - - // Read in dependency info? - if(options & Option_DependencyInfoPresent) - { - // Read in extra dependency data - for(int c = 0; c < count; ++c) - { - mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout); - } - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool) -// Purpose: Writes a selection of entries to a stream -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const -{ - // Get count of entries - int32_t count = mEntries.size(); - if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING) - { - // Need to count the entries - count = 0; - Iterator i(*this); - while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0) - { - count++; - } - } - - // Check that sensible IDs have been set - ASSERT(mObjectID != 0); - ASSERT(mContainerID != 0); - - // Need dependency info? - bool dependencyInfoRequired = false; - if(StreamDependencyInfo) - { - Iterator i(*this); - Entry *pen = 0; - while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) - { - if(pen->HasDependencies()) - { - dependencyInfoRequired = true; - } - } - } - - // Options - int32_t options = 0; - if(dependencyInfoRequired) options |= Option_DependencyInfoPresent; - - // Build header - dir_StreamFormat hdr; - hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE); - hdr.mNumEntries = htonl(count); - hdr.mObjectID = box_hton64(mObjectID); - hdr.mContainerID = box_hton64(mContainerID); - hdr.mAttributesModTime = box_hton64(mAttributesModTime); - hdr.mOptionsPresent = htonl(options); - - // Write header - rStream.Write(&hdr, sizeof(hdr)); - - // Write the attributes? - if(StreamAttributes) - { - mAttributes.WriteToStream(rStream); - } - else - { - // Write a blank header instead - StreamableMemBlock::WriteEmptyBlockToStream(rStream); - } - - // Then write all the entries - Iterator i(*this); - Entry *pen = 0; - while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) - { - pen->WriteToStream(rStream); - } - - // Write dependency info? - if(dependencyInfoRequired) - { - Iterator i(*this); - Entry *pen = 0; - while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) - { - pen->WriteToStreamDependencyInfo(rStream); - } - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::AddEntry(const Entry &) -// Purpose: Adds entry to directory (no checking) -// Created: 2003/08/27 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy) -{ - Entry *pnew = new Entry(rEntryToCopy); - try - { - mEntries.push_back(pnew); - } - catch(...) - { - delete pnew; - throw; - } - - return pnew; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t) -// Purpose: Adds entry to directory (no checking) -// Created: 2003/08/27 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime) -{ - Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime); - try - { - mEntries.push_back(pnew); - } - catch(...) - { - delete pnew; - throw; - } - - return pnew; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::DeleteEntry(int64_t) -// Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient) -// Created: 2003/08/27 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) -{ - for(std::vector<Entry*>::iterator i(mEntries.begin()); - i != mEntries.end(); ++i) - { - if((*i)->mObjectID == ObjectID) - { - // Delete - delete (*i); - // Remove from list - mEntries.erase(i); - // Done - return; - } - } - - // Not found - THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::FindEntryByID(int64_t) -// Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist. -// Created: 12/11/03 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const -{ - for(std::vector<Entry*>::const_iterator i(mEntries.begin()); - i != mEntries.end(); ++i) - { - if((*i)->mObjectID == ObjectID) - { - // Found - return (*i); - } - } - - // Not found - return 0; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::Entry() -// Purpose: Constructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry::Entry() - : mModificationTime(0), - mObjectID(0), - mSizeInBlocks(0), - mFlags(0), - mAttributesHash(0), - mMinMarkNumber(0), - mMarkNumber(0), - mDependsNewer(0), - mDependsOlder(0) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::~Entry() -// Purpose: Destructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry::~Entry() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::Entry(const Entry &) -// Purpose: Copy constructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) - : mName(rToCopy.mName), - mModificationTime(rToCopy.mModificationTime), - mObjectID(rToCopy.mObjectID), - mSizeInBlocks(rToCopy.mSizeInBlocks), - mFlags(rToCopy.mFlags), - mAttributesHash(rToCopy.mAttributesHash), - mAttributes(rToCopy.mAttributes), - mMinMarkNumber(rToCopy.mMinMarkNumber), - mMarkNumber(rToCopy.mMarkNumber), - mDependsNewer(rToCopy.mDependsNewer), - mDependsOlder(rToCopy.mDependsOlder) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t) -// Purpose: Constructor from values -// Created: 2003/08/27 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) - : mName(rName), - mModificationTime(ModificationTime), - mObjectID(ObjectID), - mSizeInBlocks(SizeInBlocks), - mFlags(Flags), - mAttributesHash(AttributesHash), - mMinMarkNumber(0), - mMarkNumber(0), - mDependsNewer(0), - mDependsOlder(0) -{ -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::TryReading(IOStream &, int) -// Purpose: Read an entry from a stream -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) -{ - // Grab the raw bytes from the stream which compose the header - en_StreamFormat entry; - if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Do reading first before modifying the variables, to be more exception safe - - // Get the filename - BackupStoreFilename name; - name.ReadFromStream(rStream, Timeout); - - // Get the attributes - mAttributes.ReadFromStream(rStream, Timeout); - - // Store the rest of the bits - mModificationTime = box_ntoh64(entry.mModificationTime); - mObjectID = box_ntoh64(entry.mObjectID); - mSizeInBlocks = box_ntoh64(entry.mSizeInBlocks); - mAttributesHash = box_ntoh64(entry.mAttributesHash); - mFlags = ntohs(entry.mFlags); - mName = name; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::WriteToStream(IOStream &) -// Purpose: Writes the entry to a stream -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const -{ - // Build a structure - en_StreamFormat entry; - entry.mModificationTime = box_hton64(mModificationTime); - entry.mObjectID = box_hton64(mObjectID); - entry.mSizeInBlocks = box_hton64(mSizeInBlocks); - entry.mAttributesHash = box_hton64(mAttributesHash); - entry.mFlags = htons(mFlags); - - // Write it - rStream.Write(&entry, sizeof(entry)); - - // Write the filename - mName.WriteToStream(rStream); - - // Write any attributes - mAttributes.WriteToStream(rStream); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int) -// Purpose: Read the optional dependency info from a stream -// Created: 13/7/04 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout) -{ - // Grab the raw bytes from the stream which compose the header - en_StreamFormatDepends depends; - if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Store the data - mDependsNewer = box_ntoh64(depends.mDependsNewer); - mDependsOlder = box_ntoh64(depends.mDependsOlder); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &) -// Purpose: Write the optional dependency info to a stream -// Created: 13/7/04 -// -// -------------------------------------------------------------------------- -void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const -{ - // Build structure - en_StreamFormatDepends depends; - depends.mDependsNewer = box_hton64(mDependsNewer); - depends.mDependsOlder = box_hton64(mDependsOlder); - // Write - rStream.Write(&depends, sizeof(depends)); -} - - - diff --git a/lib/backupclient/BackupStoreDirectory.h b/lib/backupclient/BackupStoreDirectory.h deleted file mode 100644 index 958eee81..00000000 --- a/lib/backupclient/BackupStoreDirectory.h +++ /dev/null @@ -1,268 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreDirectory.h -// Purpose: Representation of a backup directory -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREDIRECTORY__H -#define BACKUPSTOREDIRECTORY__H - -#include <string> -#include <vector> - -#include "BackupStoreFilenameClear.h" -#include "StreamableMemBlock.h" -#include "BoxTime.h" - -class IOStream; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupStoreDirectory -// Purpose: In memory representation of a directory -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -class BackupStoreDirectory -{ -public: - BackupStoreDirectory(); - BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID); -private: - // Copying not allowed - BackupStoreDirectory(const BackupStoreDirectory &rToCopy); -public: - ~BackupStoreDirectory(); - - class Entry - { - public: - friend class BackupStoreDirectory; - - Entry(); - ~Entry(); - Entry(const Entry &rToCopy); - Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash); - - void ReadFromStream(IOStream &rStream, int Timeout); - void WriteToStream(IOStream &rStream) const; - - const BackupStoreFilename &GetName() const {return mName;} - box_time_t GetModificationTime() const {return mModificationTime;} - int64_t GetObjectID() const {return mObjectID;} - int64_t GetSizeInBlocks() const {return mSizeInBlocks;} - int16_t GetFlags() const {return mFlags;} - void AddFlags(int16_t Flags) {mFlags |= Flags;} - void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;} - - // Some things can be changed - void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;} - void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;} - - // Attributes - bool HasAttributes() const {return !mAttributes.IsEmpty();} - void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;} - const StreamableMemBlock &GetAttributes() const {return mAttributes;} - uint64_t GetAttributesHash() const {return mAttributesHash;} - - // Marks - // The lowest mark number a version of a file of this name has ever had - uint32_t GetMinMarkNumber() const {return mMinMarkNumber;} - // The mark number on this file - uint32_t GetMarkNumber() const {return mMarkNumber;} - - // Make sure these flags are synced with those in backupprocotol.txt - // ListDirectory command - enum - { - Flags_INCLUDE_EVERYTHING = -1, - Flags_EXCLUDE_NOTHING = 0, - Flags_EXCLUDE_EVERYTHING = 31, // make sure this is kept as sum of ones below! - Flags_File = 1, - Flags_Dir = 2, - Flags_Deleted = 4, - Flags_OldVersion = 8, - Flags_RemoveASAP = 16 // if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion - }; - // characters for textual listing of files -- see bbackupquery/BackupQueries - #define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR" - - bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) - { - return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet)) - && ((mFlags & FlagsNotToBeSet) == 0); - }; - - // Get dependency info - // new version this depends on - int64_t GetDependsNewer() const {return mDependsNewer;} - void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;} - // older version which depends on this - int64_t GetDependsOlder() const {return mDependsOlder;} - void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;} - - // Dependency info saving - bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;} - void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout); - void WriteToStreamDependencyInfo(IOStream &rStream) const; - - private: - BackupStoreFilename mName; - box_time_t mModificationTime; - int64_t mObjectID; - int64_t mSizeInBlocks; - int16_t mFlags; - uint64_t mAttributesHash; - StreamableMemBlock mAttributes; - uint32_t mMinMarkNumber; - uint32_t mMarkNumber; - - uint64_t mDependsNewer; // new version this depends on - uint64_t mDependsOlder; // older version which depends on this - }; - - void ReadFromStream(IOStream &rStream, int Timeout); - void WriteToStream(IOStream &rStream, - int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, - int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING, - bool StreamAttributes = true, bool StreamDependencyInfo = true) const; - - Entry *AddEntry(const Entry &rEntryToCopy); - Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime); - void DeleteEntry(int64_t ObjectID); - Entry *FindEntryByID(int64_t ObjectID) const; - - int64_t GetObjectID() const {return mObjectID;} - int64_t GetContainerID() const {return mContainerID;} - - // Need to be able to update the container ID when moving objects - void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;} - - // Purely for use of server -- not serialised into streams - int64_t GetRevisionID() const {return mRevisionID;} - void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;} - - unsigned int GetNumberOfEntries() const {return mEntries.size();} - - // User info -- not serialised into streams - int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;} - void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;} - - // Attributes - bool HasAttributes() const {return !mAttributes.IsEmpty();} - void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;} - const StreamableMemBlock &GetAttributes() const {return mAttributes;} - box_time_t GetAttributesModTime() const {return mAttributesModTime;} - - class Iterator - { - public: - Iterator(const BackupStoreDirectory &rDir) - : mrDir(rDir), i(rDir.mEntries.begin()) - { - } - - BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) - { - // Skip over things which don't match the required flags - while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) - { - ++i; - } - // Not the last one? - if(i == mrDir.mEntries.end()) - { - return 0; - } - // Return entry, and increment - return (*(i++)); - } - - // WARNING: This function is really very inefficient. - // Only use when you want to look up ONE filename, not in a loop looking up lots. - // In a looping situation, cache the decrypted filenames in another memory structure. - BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) - { - // Skip over things which don't match the required flags or filename - while( (i != mrDir.mEntries.end()) - && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) - || (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) ) - { - ++i; - } - // Not the last one? - if(i == mrDir.mEntries.end()) - { - return 0; - } - // Return entry, and increment - return (*(i++)); - } - - private: - const BackupStoreDirectory &mrDir; - std::vector<Entry*>::const_iterator i; - }; - - friend class Iterator; - - class ReverseIterator - { - public: - ReverseIterator(const BackupStoreDirectory &rDir) - : mrDir(rDir), i(rDir.mEntries.rbegin()) - { - } - - BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) - { - // Skip over things which don't match the required flags - while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) - { - ++i; - } - // Not the last one? - if(i == mrDir.mEntries.rend()) - { - return 0; - } - // Return entry, and increment - return (*(i++)); - } - - private: - const BackupStoreDirectory &mrDir; - std::vector<Entry*>::const_reverse_iterator i; - }; - - friend class ReverseIterator; - - // For recovery of the store - // Implemented in BackupStoreCheck2.cpp - bool CheckAndFix(); - void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags); - bool NameInUse(const BackupStoreFilename &rName); - // Don't use these functions in normal code! - - // For testing - void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;} - - // Debug and diagonistics - void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere - -private: - int64_t mRevisionID; - int64_t mObjectID; - int64_t mContainerID; - std::vector<Entry*> mEntries; - box_time_t mAttributesModTime; - StreamableMemBlock mAttributes; - int64_t mUserInfo1; -}; - -#endif // BACKUPSTOREDIRECTORY__H - diff --git a/lib/backupclient/BackupStoreException.h b/lib/backupclient/BackupStoreException.h deleted file mode 100644 index 981dfa60..00000000 --- a/lib/backupclient/BackupStoreException.h +++ /dev/null @@ -1,17 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreException.h -// Purpose: Exception -// Created: 2003/07/08 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREEXCEPTION__H -#define BACKUPSTOREEXCEPTION__H - -// Compatibility -#include "autogen_BackupStoreException.h" - -#endif // BACKUPSTOREEXCEPTION__H - diff --git a/lib/backupclient/BackupStoreException.txt b/lib/backupclient/BackupStoreException.txt deleted file mode 100644 index 528a8c94..00000000 --- a/lib/backupclient/BackupStoreException.txt +++ /dev/null @@ -1,71 +0,0 @@ -EXCEPTION BackupStore 4 - -Internal 0 -BadAccountDatabaseFile 1 -AccountDatabaseNoSuchEntry 2 -InvalidBackupStoreFilename 3 -UnknownFilenameEncoding 4 -CouldntReadEntireStructureFromStream 5 -BadDirectoryFormat 6 -CouldNotFindEntryInDirectory 7 -OutputFileAlreadyExists 8 -OSFileError 9 -StreamDoesntHaveRequiredFeatures 10 -BadBackupStoreFile 11 -CouldNotLoadStoreInfo 12 -BadStoreInfoOnLoad 13 -StoreInfoIsReadOnly 14 -StoreInfoDirNotInList 15 -StoreInfoBlockDeltaMakesValueNegative 16 -DirectoryHasBeenDeleted 17 -StoreInfoNotInitialised 18 -StoreInfoAlreadyLoaded 19 -StoreInfoNotLoaded 20 -ReadFileFromStreamTimedOut 21 -FileWrongSizeAfterBeingStored 22 -AddedFileDoesNotVerify 23 -StoreInfoForWrongAccount 24 -ContextIsReadOnly 25 -AttributesNotLoaded 26 -AttributesNotUnderstood 27 -WrongServerVersion 28 # client side -ClientMarkerNotAsExpected 29 Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account. -NameAlreadyExistsInDirectory 30 -BerkelyDBFailure 31 # client side -InodeMapIsReadOnly 32 # client side -InodeMapNotOpen 33 # client side -FilenameEncryptionKeyNotKnown 34 -FilenameEncryptionNoKeyForSpecifiedMethod 35 -FilenameEncryptionNotSetup 36 -CouldntLoadClientKeyMaterial 37 -BadEncryptedAttributes 38 -EncryptedAttributesHaveUnknownEncoding 39 -OutputSizeTooSmallForChunk 40 -BadEncodedChunk 41 -NotEnoughSpaceToDecodeChunk 42 -ChunkHasUnknownEncoding 43 -ChunkContainsBadCompressedData 44 -CantWriteToEncodedFileStream 45 -Temp_FileEncodeStreamDidntReadBuffer 46 -CantWriteToDecodedFileStream 47 -WhenDecodingExpectedToReadButCouldnt 48 -BackupStoreFileFailedIntegrityCheck 49 -ThereIsNoDataInASymLink 50 -IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements 51 -BlockEntryEncodingDidntGiveExpectedLength 52 -CouldNotFindUnusedIDDuringAllocation 53 -AddedFileExceedsStorageLimit 54 -CannotDiffAnIncompleteStoreFile 55 -CannotDecodeDiffedFilesWithoutCombining 56 -FailedToReadBlockOnCombine 57 -OnCombineFromFileIsIncomplete 58 -BadNotifySysadminEventCode 59 -InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60 -CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server? -AttributeHashSecretNotSet 62 -AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES -SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection. -IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related -DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory -PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. -UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp deleted file mode 100644 index 17e145a3..00000000 --- a/lib/backupclient/BackupStoreFile.cpp +++ /dev/null @@ -1,1556 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFile.cpp -// Purpose: Utils for manipulating files -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - -#include <sys/stat.h> -#include <string.h> -#include <new> -#include <string.h> - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - #include <stdio.h> -#endif - -#include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreFileCryptVar.h" -#include "BackupStoreFilename.h" -#include "BackupStoreException.h" -#include "IOStream.h" -#include "Guards.h" -#include "FileModificationTime.h" -#include "FileStream.h" -#include "BackupClientFileAttributes.h" -#include "BackupStoreObjectMagic.h" -#include "Compress.h" -#include "CipherContext.h" -#include "CipherBlowfish.h" -#include "CipherAES.h" -#include "BackupStoreConstants.h" -#include "CollectInBufferStream.h" -#include "RollingChecksum.h" -#include "MD5Digest.h" -#include "ReadGatherStream.h" -#include "Random.h" -#include "BackupStoreFileEncodeStream.h" -#include "Logging.h" - -#include "MemLeakFindOn.h" - -using namespace BackupStoreFileCryptVar; - -// How big a buffer to use for copying files -#define COPY_BUFFER_SIZE (8*1024) - -// Statistics -BackupStoreFileStats BackupStoreFile::msStats = {0,0,0}; - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - bool sWarnedAboutBackwardsCompatiblity = false; -#endif - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &) -// Purpose: Encode a file into something for storing on file server. -// Requires a real filename so full info can be stored. -// -// Returns a stream. Most of the work is done by the stream -// when data is actually requested -- the file will be held -// open until the stream is deleted or the file finished. -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(const char *Filename, - int64_t ContainerID, const BackupStoreFilename &rStoreFilename, - int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) -{ - // Create the stream - std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); - - // Do the initial setup - ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, - 0 /* no recipe, just encode */, - ContainerID, rStoreFilename, pModificationTime, pLogger, - pRunStatusProvider); - - // Return the stream for the caller - return stream; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &) -// Purpose: Verify that an encoded file meets the format requirements. -// Doesn't verify that the data is intact and can be decoded. -// Optionally returns the ID of the file which it is diffed from, -// and the (original) container ID. -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut) -{ - // Get the size of the file - int64_t fileSize = rFile.BytesLeftToRead(); - if(fileSize == IOStream::SizeOfStreamUnknown) - { - THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) - } - - // Get the header... - file_StreamFormat hdr; - if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) - { - // Couldn't read header - return false; - } - - // Check magic number - if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif - ) - { - return false; - } - - // Get a filename, see if it loads OK - try - { - BackupStoreFilename fn; - fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); - } - catch(...) - { - // an error occured while reading it, so that's not good - return false; - } - - // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not - try - { - int32_t size_s; - if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) - { - THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) - } - int size = ntohl(size_s); - // Skip forward the size - rFile.Seek(size, IOStream::SeekType_Relative); - } - catch(...) - { - // an error occured while reading it, so that's not good - return false; - } - - // Get current position in file -- the end of the header - int64_t headerEnd = rFile.GetPosition(); - - // Get number of blocks - int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - - // Calculate where the block index will be, check it's reasonable - int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - if(blockIndexLoc < headerEnd) - { - // Not enough space left for the block index, let alone the blocks themselves - return false; - } - - // Load the block index header - rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute); - file_BlockIndexHeader blkhdr; - if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */)) - { - // Couldn't read block index header -- assume bad file - return false; - } - - // Check header - if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 -#endif - ) - || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != numBlocks) - { - // Bad header -- either magic value or number of blocks is wrong - return false; - } - - // Flag for recording whether a block is referenced from another file - bool blockFromOtherFileReferenced = false; - - // Read the index, checking that the length values all make sense - int64_t currentBlockStart = headerEnd; - for(int64_t b = 0; b < numBlocks; ++b) - { - // Read block entry - file_BlockIndexEntry blk; - if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */)) - { - // Couldn't read block index entry -- assume bad file - return false; - } - - // Check size and location - int64_t blkSize = box_ntoh64(blk.mEncodedSize); - if(blkSize <= 0) - { - // Mark that this file references another file - blockFromOtherFileReferenced = true; - } - else - { - // This block is actually in this file - if((currentBlockStart + blkSize) > blockIndexLoc) - { - // Encoded size makes the block run over the index - return false; - } - - // Move the current block start ot the end of this block - currentBlockStart += blkSize; - } - } - - // Check that there's no empty space - if(currentBlockStart != blockIndexLoc) - { - return false; - } - - // Check that if another block is references, then the ID is there, and if one isn't there is no ID. - int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); - if((otherID != 0 && blockFromOtherFileReferenced == false) - || (otherID == 0 && blockFromOtherFileReferenced == true)) - { - // Doesn't look good! - return false; - } - - // Does the caller want the other ID? - if(pDiffFromObjectIDOut) - { - *pDiffFromObjectIDOut = otherID; - } - - // Does the caller want the container ID? - if(pContainerIDOut) - { - *pContainerIDOut = box_ntoh64(hdr.mContainerID); - } - - // Passes all tests - return true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodeFile(IOStream &, const char *) -// Purpose: Decode a file. Will set file attributes. File must not exist. -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) -{ - // Does file exist? - EMU_STRUCT_STAT st; - if(EMU_STAT(DecodedFilename, &st) == 0) - { - THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) - } - - // Try, delete output file if error - try - { - // Make a stream for outputting this file - FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL); - - // Get the decoding stream - std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr)); - - // Is it a symlink? - if(!stream->IsSymLink()) - { - // Copy it out to the file - stream->CopyStreamTo(out); - } - - out.Close(); - - // The stream might have uncertain size, in which case - // we need to drain it to get the - // Protocol::ProtocolStreamHeader_EndOfStream byte - // out of our connection stream. - char buffer[1]; - int drained = rEncodedFile.Read(buffer, 1); - - // The Read will return 0 if we are actually at the end - // of the stream, but some tests decode files directly, - // in which case we are actually positioned at the start - // of the block index. I hope that reading an extra byte - // doesn't hurt! - // ASSERT(drained == 0); - - // Write the attributes - try - { - stream->GetAttributes().WriteAttributes(DecodedFilename); - } - catch (std::exception& e) - { - BOX_WARNING("Failed to restore attributes on " << - DecodedFilename << ": " << e.what()); - } - } - catch(...) - { - ::unlink(DecodedFilename); - throw; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *) -// Purpose: Return a stream which will decode the encrypted file data on the fly. -// Accepts streams in block index first, or main header first, order. In the latter case, -// the stream must be Seek()able. -// -// Before you use the returned stream, call IsSymLink() -- symlink streams won't allow -// you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation. -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) -{ - // Create stream - std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout)); - - // Get it ready - stream->Setup(pAlterativeAttr); - - // Return to caller - return stream; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int) -// Purpose: Constructor -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout) - : mrEncodedFile(rEncodedFile), - mTimeout(Timeout), - mNumBlocks(0), - mpBlockIndex(0), - mpEncodedData(0), - mpClearData(0), - mClearDataSize(0), - mCurrentBlock(-1), - mCurrentBlockClearSize(0), - mPositionInCurrentBlock(0), - mEntryIVBase(42) // different to default value in the encoded stream! -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - , mIsOldVersion(false) -#endif -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::~DecodedStream() -// Purpose: Desctructor -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -BackupStoreFile::DecodedStream::~DecodedStream() -{ - // Free any allocated memory - if(mpBlockIndex) - { - ::free(mpBlockIndex); - } - if(mpEncodedData) - { - BackupStoreFile::CodingChunkFree(mpEncodedData); - } - if(mpClearData) - { - ::free(mpClearData); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *) -// Purpose: Get the stream ready to decode -- reads in headers -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr) -{ - // Get the size of the file - int64_t fileSize = mrEncodedFile.BytesLeftToRead(); - - // Get the magic number to work out which order the stream is in - int32_t magic; - if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read magic value - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - - bool inFileOrder = true; - switch(ntohl(magic)) - { -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - case OBJECTMAGIC_FILE_MAGIC_VALUE_V0: - mIsOldVersion = true; - // control flows on -#endif - case OBJECTMAGIC_FILE_MAGIC_VALUE_V1: - inFileOrder = true; - break; - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0: - mIsOldVersion = true; - // control flows on -#endif - case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1: - inFileOrder = false; - break; - - default: - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // If not in file order, then the index list must be read now - if(!inFileOrder) - { - ReadBlockIndex(true /* have already read and verified the magic number */); - } - - // Get header - file_StreamFormat hdr; - if(inFileOrder) - { - // Read the header, without the magic number - if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic), - 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - // Put in magic number - hdr.mMagicValue = magic; - } - else - { - // Not in file order, so need to read the full header - if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - } - - // Check magic number - if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif - ) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Get the filename - mFilename.ReadFromStream(mrEncodedFile, mTimeout); - - // Get the attributes (either from stream, or supplied attributes) - if(pAlterativeAttr != 0) - { - // Read dummy attributes - BackupClientFileAttributes attr; - attr.ReadFromStream(mrEncodedFile, mTimeout); - - // Set to supplied attributes - mAttributes = *pAlterativeAttr; - } - else - { - // Read the attributes from the stream - mAttributes.ReadFromStream(mrEncodedFile, mTimeout); - } - - // If it is in file order, go and read the file attributes - // Requires that the stream can seek - if(inFileOrder) - { - // Make sure the file size is known - if(fileSize == IOStream::SizeOfStreamUnknown) - { - THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) - } - - // Store current location (beginning of encoded blocks) - int64_t endOfHeaderPos = mrEncodedFile.GetPosition(); - - // Work out where the index is - int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - - // Seek to that position - mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute); - - // Read the block index - ReadBlockIndex(false /* magic number still to be read */); - - // Seek back to the end of header position, ready for reading the chunks - mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute); - } - - // Check view of blocks from block header and file header match - if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks)) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Need to allocate some memory for the two blocks for reading encoded data, and clear data - if(mNumBlocks > 0) - { - // Find the maximum encoded data size - int32_t maxEncodedDataSize = 0; - const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; - ASSERT(entry != 0); - for(int64_t e = 0; e < mNumBlocks; e++) - { - // Get the clear and encoded size - int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize); - ASSERT(encodedSize > 0); - - // Larger? - if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize; - } - - // Allocate those blocks! - mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32); - - // Allocate the block for the clear data, using the hint from the header. - // If this is wrong, things will exception neatly later on, so it can't be used - // to do anything more than cause an error on downloading. - mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32; - mpClearData = (uint8_t*)::malloc(mClearDataSize); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool) -// Purpose: Read the block index from the stream, and store in internal buffer (minus header) -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) -{ - // Header - file_BlockIndexHeader blkhdr; - - // Read it in -- way depends on how whether the magic number has already been read - if(MagicAlreadyRead) - { - // Read the header, without the magic number - if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue), - 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - } - else - { - // Magic not already read, so need to read the full header - if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - - // Check magic value - if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 -#endif - ) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - } - - // Get the number of blocks out of the header - mNumBlocks = box_ntoh64(blkhdr.mNumBlocks); - - // Read the IV base - mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase); - - // Load the block entries in? - if(mNumBlocks > 0) - { - // How big is the index? - int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks; - - // Allocate some memory - mpBlockIndex = ::malloc(indexSize); - if(mpBlockIndex == 0) - { - throw std::bad_alloc(); - } - - // Read it in - if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::Read(void *, int, int) -// Purpose: As interface. Reads decrpyted data. -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) -{ - // Symlinks don't have data. So can't read it. Not even zero bytes. - if(IsSymLink()) - { - // Don't allow reading in this case - THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink); - } - - // Already finished? - if(mCurrentBlock >= mNumBlocks) - { - // At end of stream, nothing to do - return 0; - } - - int bytesToRead = NBytes; - uint8_t *output = (uint8_t*)pBuffer; - - while(bytesToRead > 0 && mCurrentBlock < mNumBlocks) - { - // Anything left in the current block? - if(mPositionInCurrentBlock < mCurrentBlockClearSize) - { - // Copy data out of this buffer - int s = mCurrentBlockClearSize - mPositionInCurrentBlock; - if(s > bytesToRead) s = bytesToRead; // limit to requested data - - // Copy - ::memcpy(output, mpClearData + mPositionInCurrentBlock, s); - - // Update positions - output += s; - mPositionInCurrentBlock += s; - bytesToRead -= s; - } - - // Need to get some more data? - if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize) - { - // Number of next block - ++mCurrentBlock; - if(mCurrentBlock >= mNumBlocks) - { - // Stop now! - break; - } - - // Get the size from the block index - const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; - int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize); - if(encodedSize <= 0) - { - // The caller is attempting to decode a file which is the direct result of a diff - // operation, and so does not contain all the data. - // It needs to be combined with the previous version first. - THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining) - } - - // Load in next block - if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) - } - - // Decode the data - mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize); - - // Calculate IV for this entry - uint64_t iv = mEntryIVBase; - iv += mCurrentBlock; - // Convert to network byte order before encrypting with it, so that restores work on - // platforms with different endiannesses. - iv = box_hton64(iv); - sBlowfishDecryptBlockEntry.SetIV(&iv); - - // Decrypt the encrypted section - file_BlockIndexEntryEnc entryEnc; - int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), - entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); - if(sectionSize != sizeof(entryEnc)) - { - THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) - } - - // Make sure this is the right size - if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) - { -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - if(!mIsOldVersion) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the - // IV for the encrypted section. Try again, with the thing the other way round - iv = box_swap64(iv); - sBlowfishDecryptBlockEntry.SetIV(&iv); - int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), - entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc)); - if(sectionSize != sizeof(entryEnc)) - { - THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) - } - if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize)) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - else - { - // Warn and log this issue - if(!sWarnedAboutBackwardsCompatiblity) - { - BOX_WARNING("WARNING: Decoded one or more files using backwards compatibility mode for block index."); - sWarnedAboutBackwardsCompatiblity = true; - } - } -#else - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) -#endif - } - - // Check the digest - MD5Digest md5; - md5.Add(mpClearData, mCurrentBlockClearSize); - md5.Finish(); - if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum)) - { - THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck) - } - - // Set vars to say what's happening - mPositionInCurrentBlock = 0; - } - } - - ASSERT(bytesToRead >= 0); - ASSERT(bytesToRead <= NBytes); - - return NBytes - bytesToRead; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::IsSymLink() -// Purpose: Is the unencoded file actually a symlink? -// Created: 10/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFile::DecodedStream::IsSymLink() -{ - // First, check in with the attributes - if(!mAttributes.IsSymLink()) - { - return false; - } - - // So the attributes think it is a symlink. - // Consistency check... - if(mNumBlocks != 0) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - return true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::Write(const void *, int) -// Purpose: As interface. Throws exception, as you can't write to this stream. -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes) -{ - THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream) -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::StreamDataLeft() -// Purpose: As interface. Any data left? -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFile::DecodedStream::StreamDataLeft() -{ - return mCurrentBlock < mNumBlocks; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodedStream::StreamClosed() -// Purpose: As interface. Always returns true, no writing allowed. -// Created: 9/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFile::DecodedStream::StreamClosed() -{ - // Can't write to this stream! - return true; -} - - - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::SetBlowfishKey(const void *, int) -// Purpose: Static. Sets the key to use for encryption and decryption. -// Created: 7/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength) -{ - // IVs set later - sBlowfishEncrypt.Reset(); - sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); - sBlowfishDecrypt.Reset(); - sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); - - sBlowfishEncryptBlockEntry.Reset(); - sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); - sBlowfishEncryptBlockEntry.UsePadding(false); - sBlowfishDecryptBlockEntry.Reset(); - sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength)); - sBlowfishDecryptBlockEntry.UsePadding(false); -} - - -#ifndef HAVE_OLD_SSL -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::SetAESKey(const void *, int) -// Purpose: Sets the AES key to use for file data encryption. Will select AES as -// the cipher to use when encrypting. -// Created: 27/4/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength) -{ - // Setup context - sAESEncrypt.Reset(); - sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); - sAESDecrypt.Reset(); - sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); - - // Set encryption to use this key, instead of the "default" blowfish key - spEncrypt = &sAESEncrypt; - sEncryptCipherType = HEADER_AES_ENCODING; -} -#endif - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::MaxBlockSizeForChunkSize(int) -// Purpose: The maximum output size of a block, given the chunk size -// Created: 7/12/03 -// -// -------------------------------------------------------------------------- -int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize) -{ - // Calculate... the maximum size of output by first the largest it could be after compression, - // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck - // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing - // an problem where a rather non-compressable file didn't fit in a block buffer.) - return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1 - + sBlowfishEncrypt.GetIVLength() + 128; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &) -// Purpose: Encodes a chunk (encryption, possible compressed beforehand) -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput) -{ - ASSERT(spEncrypt != 0); - - // Check there's some space in the output block - if(rOutput.mBufferSize < 256) - { - rOutput.Reallocate(256); - } - - // Check alignment of the block - ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); - - // Want to compress it? - bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE); - - // Build header - uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT; - if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED; - - // Store header - rOutput.mpBuffer[0] = header; - int outOffset = 1; - - // Setup cipher, and store the IV - int ivLen = 0; - const void *iv = spEncrypt->SetRandomIV(ivLen); - ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen); - outOffset += ivLen; - - // Start encryption process - spEncrypt->Begin(); - - #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \ - { \ - if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \ - { \ - rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \ - } \ - } - - // Encode the chunk - if(compressChunk) - { - // buffer to compress into - uint8_t buffer[2048]; - - // Set compressor with all the chunk as an input - Compress<true> compress; - compress.Input(Chunk, ChunkSize); - compress.FinishInput(); - - // Get and encrypt output - while(!compress.OutputHasFinished()) - { - int s = compress.Output(buffer, sizeof(buffer)); - if(s > 0) - { - ENCODECHUNK_CHECK_SPACE(s) - outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); - } - else - { - // Should never happen, as we put all the input in in one go. - // So if this happens, it means there's a logical problem somewhere - THROW_EXCEPTION(BackupStoreException, Internal) - } - } - ENCODECHUNK_CHECK_SPACE(16) - outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); - } - else - { - // Straight encryption - ENCODECHUNK_CHECK_SPACE(ChunkSize) - outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize); - ENCODECHUNK_CHECK_SPACE(16) - outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); - } - - ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check - - return outOffset; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int) -// Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find -// the extra output buffer size needed before calling. -// See notes in EncodeChunk() for notes re alignment of the -// encoded data. -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize) -{ - // Check alignment of the encoded block - ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); - - // First check - if(EncodedSize < 1) - { - THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) - } - - const uint8_t *input = (uint8_t*)Encoded; - - // Get header, make checks, etc - uint8_t header = input[0]; - bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED; - uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT); - if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING) - { - THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding) - } - -#ifndef HAVE_OLD_SSL - // Choose cipher - CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt); -#else - // AES not supported with this version of OpenSSL - if(encodingType == HEADER_AES_ENCODING) - { - THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL) - } - CipherContext &cipher(sBlowfishDecrypt); -#endif - - // Check enough space for header, an IV and one byte of input - int ivLen = cipher.GetIVLength(); - if(EncodedSize < (1 + ivLen + 1)) - { - THROW_EXCEPTION(BackupStoreException, BadEncodedChunk) - } - - // Set IV in decrypt context, and start - cipher.SetIV(input + 1); - cipher.Begin(); - - // Setup vars for code - int inOffset = 1 + ivLen; - uint8_t *output = (uint8_t*)Output; - int outOffset = 0; - - // Do action - if(chunkCompressed) - { - // Do things in chunks - uint8_t buffer[2048]; - int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer)); - - // Decompressor - Compress<false> decompress; - - while(inOffset < EncodedSize) - { - // Decrypt a block - int bl = inputBlockLen; - if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long - int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl); - inOffset += bl; - - // Decompress the decrypted data - if(s > 0) - { - decompress.Input(buffer, s); - int os = 0; - do - { - os = decompress.Output(output + outOffset, OutputSize - outOffset); - outOffset += os; - } while(os > 0); - - // Check that there's space left in the output buffer -- there always should be - if(outOffset >= OutputSize) - { - THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) - } - } - } - - // Get any compressed data remaining in the cipher context and compression - int s = cipher.Final(buffer, sizeof(buffer)); - decompress.Input(buffer, s); - decompress.FinishInput(); - while(!decompress.OutputHasFinished()) - { - int os = decompress.Output(output + outOffset, OutputSize - outOffset); - outOffset += os; - - // Check that there's space left in the output buffer -- there always should be - if(outOffset >= OutputSize) - { - THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk) - } - } - } - else - { - // Easy decryption - outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset); - outOffset += cipher.Final(output + outOffset, OutputSize - outOffset); - } - - return outOffset; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool) -// Purpose: Returns a stream which gives a Stream order version of the encoded file. -// If TakeOwnership == true, then the input stream will be deleted when the -// returned stream is deleted. -// The input stream must be seekable. -// Created: 10/12/03 -// -// -------------------------------------------------------------------------- -std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership) -{ - ASSERT(pStream != 0); - - // Get the size of the file - int64_t fileSize = pStream->BytesLeftToRead(); - if(fileSize == IOStream::SizeOfStreamUnknown) - { - THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) - } - - // Read the header - int bytesRead = 0; - file_StreamFormat hdr; - bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead); - - // Seek backwards to put the file pointer back where it was before we started this - pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative); - - // Check we got a block - if(!readBlock) - { - // Couldn't read header -- assume file bad - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Check magic number - if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif - ) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Get number of blocks - int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - - // Calculate where the block index will be, check it's reasonable - int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - int64_t blockIndexLoc = fileSize - blockIndexSize; - if(blockIndexLoc < 0) - { - // Doesn't look good! - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Build a reordered stream - std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership)); - - // Set it up... - ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get())); - int component = rreordered.AddComponent(pStream); - // Send out the block index - rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc); - // And then the rest of the file - rreordered.AddBlock(component, blockIndexLoc, true, 0); - - return reordered; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::ResetStats() -// Purpose: Reset the gathered statistics -// Created: 20/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::ResetStats() -{ - msStats.mBytesInEncodedFiles = 0; - msStats.mBytesAlreadyOnServer = 0; - msStats.mTotalFileStreamSize = 0; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &) -// Purpose: Compares the contents of a file against the checksums contained in the -// block index. Returns true if the checksums match, meaning the file is -// extremely likely to match the original. Will always consume the entire index. -// Created: 21/1/04 -// -// -------------------------------------------------------------------------- -bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout) -{ - // is it a symlink? - bool sourceIsSymlink = false; - { - EMU_STRUCT_STAT st; - if(EMU_LSTAT(Filename, &st) == -1) - { - THROW_EXCEPTION(CommonException, OSFileError) - } - if((st.st_mode & S_IFMT) == S_IFLNK) - { - sourceIsSymlink = true; - } - } - - // Open file, if it's not a symlink - std::auto_ptr<FileStream> in; - if(!sourceIsSymlink) - { - in.reset(new FileStream(Filename)); - } - - // Read header - file_BlockIndexHeader hdr; - if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Check magic - if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0) -#endif - ) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0); -#endif - - // Get basic information - int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase); - - //TODO: Verify that these sizes look reasonable - - // setup - void *data = 0; - int32_t dataSize = -1; - bool matches = true; - int64_t totalSizeInBlockIndex = 0; - - try - { - for(int64_t b = 0; b < numBlocks; ++b) - { - // Read an entry from the stream - file_BlockIndexEntry entry; - if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) - { - // Couldn't read entry - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Calculate IV for this entry - uint64_t iv = entryIVBase; - iv += b; - iv = box_hton64(iv); -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - if(isOldVersion) - { - // Reverse the IV for compatibility - iv = box_swap64(iv); - } -#endif - sBlowfishDecryptBlockEntry.SetIV(&iv); - - // Decrypt the encrypted section - file_BlockIndexEntryEnc entryEnc; - int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), - entry.mEnEnc, sizeof(entry.mEnEnc)); - if(sectionSize != sizeof(entryEnc)) - { - THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) - } - - // Size of block - int32_t blockClearSize = ntohl(entryEnc.mSize); - if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024)) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - totalSizeInBlockIndex += blockClearSize; - - // Make sure there's enough memory allocated to load the block in - if(dataSize < blockClearSize) - { - // Too small, free the block if it's already allocated - if(data != 0) - { - ::free(data); - data = 0; - } - // Allocate a block - data = ::malloc(blockClearSize + 128); - if(data == 0) - { - throw std::bad_alloc(); - } - dataSize = blockClearSize + 128; - } - - // Load in the block from the file, if it's not a symlink - if(!sourceIsSymlink) - { - if(in->Read(data, blockClearSize) != blockClearSize) - { - // Not enough data left in the file, can't possibly match - matches = false; - } - else - { - // Check the checksum - MD5Digest md5; - md5.Add(data, blockClearSize); - md5.Finish(); - if(!md5.DigestMatches(entryEnc.mStrongChecksum)) - { - // Checksum didn't match - matches = false; - } - } - } - - // Keep on going regardless, to make sure the entire block index stream is read - // -- must always be consistent about what happens with the stream. - } - } - catch(...) - { - // clean up in case of errors - if(data != 0) - { - ::free(data); - data = 0; - } - throw; - } - - // free block - if(data != 0) - { - ::free(data); - data = 0; - } - - // Check for data left over if it's not a symlink - if(!sourceIsSymlink) - { - // Anything left to read in the file? - if(in->BytesLeftToRead() != 0) - { - // File has extra data at the end - matches = false; - } - } - - // Symlinks must have zero size on server - if(sourceIsSymlink) - { - matches = (totalSizeInBlockIndex == 0); - } - - return matches; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodingBuffer::EncodingBuffer() -// Purpose: Constructor -// Created: 25/11/04 -// -// -------------------------------------------------------------------------- -BackupStoreFile::EncodingBuffer::EncodingBuffer() - : mpBuffer(0), - mBufferSize(0) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer() -// Purpose: Destructor -// Created: 25/11/04 -// -// -------------------------------------------------------------------------- -BackupStoreFile::EncodingBuffer::~EncodingBuffer() -{ - if(mpBuffer != 0) - { - BackupStoreFile::CodingChunkFree(mpBuffer); - mpBuffer = 0; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodingBuffer::Allocate(int) -// Purpose: Do initial allocation of block -// Created: 25/11/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::EncodingBuffer::Allocate(int Size) -{ - ASSERT(mpBuffer == 0); - uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size); - if(buffer == 0) - { - throw std::bad_alloc(); - } - mpBuffer = buffer; - mBufferSize = Size; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodingBuffer::Reallocate(int) -// Purpose: Reallocate the block. Try not to call this, it has to copy -// the entire contents as the block can't be reallocated straight. -// Created: 25/11/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) -{ - BOX_TRACE("Reallocating EncodingBuffer from " << mBufferSize << - " to " << NewSize); - ASSERT(mpBuffer != 0); - uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize); - if(buffer == 0) - { - throw std::bad_alloc(); - } - // Copy data - ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize); - - // Free old - BackupStoreFile::CodingChunkFree(mpBuffer); - - // Store new buffer - mpBuffer = buffer; - mBufferSize = NewSize; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: DiffTimer::DiffTimer(); -// Purpose: Constructor -// Created: 2005/02/01 -// -// -------------------------------------------------------------------------- -DiffTimer::DiffTimer() -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: DiffTimer::DiffTimer(); -// Purpose: Destructor -// Created: 2005/02/01 -// -// -------------------------------------------------------------------------- -DiffTimer::~DiffTimer() -{ -} diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h deleted file mode 100644 index f4c60919..00000000 --- a/lib/backupclient/BackupStoreFile.h +++ /dev/null @@ -1,228 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFile.h -// Purpose: Utils for manipulating files -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILE__H -#define BACKUPSTOREFILE__H - -#include <cstdlib> -#include <memory> -#include <cstdlib> - -#include "BackupClientFileAttributes.h" -#include "BackupStoreFilename.h" -#include "IOStream.h" -#include "ReadLoggingStream.h" -#include "RunStatusProvider.h" - -typedef struct -{ - int64_t mBytesInEncodedFiles; - int64_t mBytesAlreadyOnServer; - int64_t mTotalFileStreamSize; -} BackupStoreFileStats; - -// Uncomment to disable backwards compatibility -//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - - -// Output buffer to EncodeChunk and input data to DecodeChunk must -// have specific alignment, see function comments. -#define BACKUPSTOREFILE_CODING_BLOCKSIZE 16 -#define BACKUPSTOREFILE_CODING_OFFSET 15 - -// Have some memory allocation commands, note closing "Off" at end of file. -#include "MemLeakFindOn.h" - -// -------------------------------------------------------------------------- -// -// Class -// Name: DiffTimer -// Purpose: Interface for classes that can keep track of diffing time, -// and send SSL keepalive messages -// Created: 2006/01/19 -// -// -------------------------------------------------------------------------- -class DiffTimer -{ -public: - DiffTimer(); - virtual ~DiffTimer(); -public: - virtual void DoKeepAlive() = 0; - virtual int GetMaximumDiffingTime() = 0; - virtual bool IsManaged() = 0; -}; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupStoreFile -// Purpose: Class to hold together utils for manipulating files. -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -class BackupStoreFile -{ -public: - class DecodedStream : public IOStream - { - friend class BackupStoreFile; - private: - DecodedStream(IOStream &rEncodedFile, int Timeout); - DecodedStream(const DecodedStream &); // not allowed - DecodedStream &operator=(const DecodedStream &); // not allowed - public: - ~DecodedStream(); - - // Stream functions - virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); - virtual bool StreamDataLeft(); - virtual bool StreamClosed(); - - // Accessor functions - const BackupClientFileAttributes &GetAttributes() {return mAttributes;} - const BackupStoreFilename &GetFilename() {return mFilename;} - int64_t GetNumBlocks() {return mNumBlocks;} // primarily for tests - - bool IsSymLink(); - - private: - void Setup(const BackupClientFileAttributes *pAlterativeAttr); - void ReadBlockIndex(bool MagicAlreadyRead); - - private: - IOStream &mrEncodedFile; - int mTimeout; - BackupClientFileAttributes mAttributes; - BackupStoreFilename mFilename; - int64_t mNumBlocks; - void *mpBlockIndex; - uint8_t *mpEncodedData; - uint8_t *mpClearData; - int mClearDataSize; - int mCurrentBlock; - int mCurrentBlockClearSize; - int mPositionInCurrentBlock; - uint64_t mEntryIVBase; -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - bool mIsOldVersion; -#endif - }; - - - // Main interface - static std::auto_ptr<IOStream> EncodeFile(const char *Filename, - int64_t ContainerID, const BackupStoreFilename &rStoreFilename, - int64_t *pModificationTime = 0, - ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL); - static std::auto_ptr<IOStream> EncodeFileDiff - ( - const char *Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, - int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, - int Timeout, - DiffTimer *pDiffTimer, - int64_t *pModificationTime = 0, - bool *pIsCompletelyDifferent = 0 - ); - static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); - static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); - static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); - static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0); - static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); - static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0); - static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout); - static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false); - - // Stream manipulation - static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership); - static void MoveStreamPositionToBlockIndex(IOStream &rStream); - - // Crypto setup - static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength); -#ifndef HAVE_OLD_SSL - static void SetAESKey(const void *pKey, int KeyLength); -#endif - - // Allocation of properly aligning chunks for decoding and encoding chunks - inline static void *CodingChunkAlloc(int Size) - { - uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3)); - if(a == 0) return 0; - // Align to main block size - ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size - uint8_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE - - (uint8_t)(((unsigned long)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE); - uint8_t *b = (a + adjustment); - // Store adjustment - *b = adjustment; - // Return offset - return b + BACKUPSTOREFILE_CODING_OFFSET; - } - inline static void CodingChunkFree(void *Block) - { - // Check alignment is as expected - ASSERT(sizeof(unsigned long) >= sizeof(void*)); // make sure casting the right pointer size - ASSERT((uint8_t)(((unsigned long)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); - uint8_t *a = (uint8_t*)Block; - a -= BACKUPSTOREFILE_CODING_OFFSET; - // Adjust downwards... - a -= *a; - free(a); - } - - static void DiffTimerExpired(); - - // Building blocks - class EncodingBuffer - { - public: - EncodingBuffer(); - ~EncodingBuffer(); - private: - // No copying - EncodingBuffer(const EncodingBuffer &); - EncodingBuffer &operator=(const EncodingBuffer &); - public: - void Allocate(int Size); - void Reallocate(int NewSize); - - uint8_t *mpBuffer; - int mBufferSize; - }; - static int MaxBlockSizeForChunkSize(int ChunkSize); - static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput); - - // Caller should know how big the output size is, but also allocate a bit more memory to cover various - // overheads allowed for in checks - static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize) - { - // Plenty big enough - return KnownChunkSize + 256; - } - static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize); - - // Statisitics, not designed to be completely reliable - static void ResetStats(); - static BackupStoreFileStats msStats; - - // For debug -#ifndef BOX_RELEASE_BUILD - static bool TraceDetailsOfDiffProcess; -#endif - - // For decoding encoded files - static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile); -}; - -#include "MemLeakFindOff.h" - -#endif // BACKUPSTOREFILE__H diff --git a/lib/backupclient/BackupStoreFileCmbDiff.cpp b/lib/backupclient/BackupStoreFileCmbDiff.cpp deleted file mode 100644 index 1a88fa3f..00000000 --- a/lib/backupclient/BackupStoreFileCmbDiff.cpp +++ /dev/null @@ -1,326 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileCmbDiff.cpp -// Purpose: Combine two diffs together -// Created: 12/7/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <new> -#include <stdlib.h> - -#include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreObjectMagic.h" -#include "BackupStoreException.h" -#include "BackupStoreConstants.h" -#include "BackupStoreFilename.h" - -#include "MemLeakFindOn.h" - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::CombineDiffs(IOStream &, IOStream &, IOStream &rOut) -// Purpose: Given two diffs, combine them into a single diff, to produce a diff -// which, combined with the original file, creates the result of applying -// rDiff, then rDiff2. Two opens of rDiff2 are required -// Created: 12/7/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut) -{ - // Skip header of first diff, record where the data starts, and skip to the index - int64_t diff1DataStarts = 0; - { - // Read the header for the From file - file_StreamFormat diff1Hdr; - if(!rDiff1.ReadFullBuffer(&diff1Hdr, sizeof(diff1Hdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - if(ntohl(diff1Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - // Skip over the filename and attributes of the From file - // BLOCK - { - BackupStoreFilename filename2; - filename2.ReadFromStream(rDiff1, IOStream::TimeOutInfinite); - int32_t size_s; - if(!rDiff1.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) - { - THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) - } - int size = ntohl(size_s); - // Skip forward the size - rDiff1.Seek(size, IOStream::SeekType_Relative); - } - // Record position - diff1DataStarts = rDiff1.GetPosition(); - // Skip to index - rDiff1.Seek(0 - (((box_ntoh64(diff1Hdr.mNumBlocks)) * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); - } - - // Read the index of the first diff - // Header first - file_BlockIndexHeader diff1IdxHdr; - if(!rDiff1.ReadFullBuffer(&diff1IdxHdr, sizeof(diff1IdxHdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - if(ntohl(diff1IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - int64_t diff1NumBlocks = box_ntoh64(diff1IdxHdr.mNumBlocks); - // Allocate some memory - int64_t *diff1BlockStartPositions = (int64_t*)::malloc((diff1NumBlocks + 1) * sizeof(int64_t)); - if(diff1BlockStartPositions == 0) - { - throw std::bad_alloc(); - } - - // Buffer data - void *buffer = 0; - int bufferSize = 0; - - try - { - // Then the entries: - // For each entry, want to know if it's in the file, and if so, how big it is. - // We'll store this as an array of file positions in the file, with an additioal - // entry on the end so that we can work out the length of the last block. - // If an entry isn't in the file, then store 0 - (position in other file). - int64_t diff1Position = diff1DataStarts; - for(int64_t b = 0; b < diff1NumBlocks; ++b) - { - file_BlockIndexEntry e; - if(!rDiff1.ReadFullBuffer(&e, sizeof(e), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Where's the block? - int64_t blockEn = box_ntoh64(e.mEncodedSize); - if(blockEn <= 0) - { - // Just store the negated block number - diff1BlockStartPositions[b] = blockEn; - } - else - { - // Block is present in this file - diff1BlockStartPositions[b] = diff1Position; - diff1Position += blockEn; - } - } - - // Finish off the list, so the last entry can have it's size calcuated. - diff1BlockStartPositions[diff1NumBlocks] = diff1Position; - - // Now read the second diff's header, copying it to the out file - file_StreamFormat diff2Hdr; - if(!rDiff2.ReadFullBuffer(&diff2Hdr, sizeof(diff2Hdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - if(ntohl(diff2Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - // Copy - rOut.Write(&diff2Hdr, sizeof(diff2Hdr)); - // Copy over filename and attributes - // BLOCK - { - BackupStoreFilename filename; - filename.ReadFromStream(rDiff2, IOStream::TimeOutInfinite); - filename.WriteToStream(rOut); - StreamableMemBlock attr; - attr.ReadFromStream(rDiff2, IOStream::TimeOutInfinite); - attr.WriteToStream(rOut); - } - - // Get to the index of rDiff2b, and read the header - MoveStreamPositionToBlockIndex(rDiff2b); - file_BlockIndexHeader diff2IdxHdr; - if(!rDiff2b.ReadFullBuffer(&diff2IdxHdr, sizeof(diff2IdxHdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - if(ntohl(diff2IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - int64_t diff2NumBlocks = box_ntoh64(diff2IdxHdr.mNumBlocks); - int64_t diff2IndexEntriesStart = rDiff2b.GetPosition(); - - // Then read all the entries - int64_t diff2FilePosition = rDiff2.GetPosition(); - for(int64_t b = 0; b < diff2NumBlocks; ++b) - { - file_BlockIndexEntry e; - if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // What do to next about copying data - bool copyBlock = false; - int copySize = 0; - int64_t copyFrom = 0; - bool fromFileDiff1 = false; - - // Where's the block? - int64_t blockEn = box_ntoh64(e.mEncodedSize); - if(blockEn > 0) - { - // Block is present in this file -- copy to out - copyBlock = true; - copyFrom = diff2FilePosition; - copySize = (int)blockEn; - - // Move pointer onwards - diff2FilePosition += blockEn; - } - else - { - // Block isn't present here -- is it present in the old one? - int64_t blockIndex = 0 - blockEn; - if(blockIndex < 0 || blockIndex > diff1NumBlocks) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - if(diff1BlockStartPositions[blockIndex] > 0) - { - // Block is in the old diff file, copy it across - copyBlock = true; - copyFrom = diff1BlockStartPositions[blockIndex]; - int nb = blockIndex + 1; - while(diff1BlockStartPositions[nb] <= 0) - { - // This is safe, because the last entry will terminate it properly! - ++nb; - ASSERT(nb <= diff1NumBlocks); - } - copySize = diff1BlockStartPositions[nb] - copyFrom; - fromFileDiff1 = true; - } - } - //TRACE4("%d %d %lld %d\n", copyBlock, copySize, copyFrom, fromFileDiff1); - - // Copy data to the output file? - if(copyBlock) - { - // Allocate enough space - if(bufferSize < copySize || buffer == 0) - { - // Free old block - if(buffer != 0) - { - ::free(buffer); - buffer = 0; - bufferSize = 0; - } - // Allocate new block - buffer = ::malloc(copySize); - if(buffer == 0) - { - throw std::bad_alloc(); - } - bufferSize = copySize; - } - ASSERT(bufferSize >= copySize); - - // Load in the data - if(fromFileDiff1) - { - rDiff1.Seek(copyFrom, IOStream::SeekType_Absolute); - if(!rDiff1.ReadFullBuffer(buffer, copySize, 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - } - else - { - rDiff2.Seek(copyFrom, IOStream::SeekType_Absolute); - if(!rDiff2.ReadFullBuffer(buffer, copySize, 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - } - // Write out data - rOut.Write(buffer, copySize); - } - } - - // Write the modified header - diff2IdxHdr.mOtherFileID = diff1IdxHdr.mOtherFileID; - rOut.Write(&diff2IdxHdr, sizeof(diff2IdxHdr)); - - // Then we'll write out the index, reading the data again - rDiff2b.Seek(diff2IndexEntriesStart, IOStream::SeekType_Absolute); - for(int64_t b = 0; b < diff2NumBlocks; ++b) - { - file_BlockIndexEntry e; - if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Where's the block? - int64_t blockEn = box_ntoh64(e.mEncodedSize); - - // If it's not in this file, it needs modification... - if(blockEn <= 0) - { - int64_t blockIndex = 0 - blockEn; - // In another file. Need to translate this against the other diff - if(diff1BlockStartPositions[blockIndex] > 0) - { - // Block is in the first diff file, stick in size - int nb = blockIndex + 1; - while(diff1BlockStartPositions[nb] <= 0) - { - // This is safe, because the last entry will terminate it properly! - ++nb; - ASSERT(nb <= diff1NumBlocks); - } - int64_t size = diff1BlockStartPositions[nb] - diff1BlockStartPositions[blockIndex]; - e.mEncodedSize = box_hton64(size); - } - else - { - // Block in the original file, use translated value - e.mEncodedSize = box_hton64(diff1BlockStartPositions[blockIndex]); - } - } - - // Write entry - rOut.Write(&e, sizeof(e)); - } - } - catch(...) - { - // clean up - ::free(diff1BlockStartPositions); - if(buffer != 0) - { - ::free(buffer); - } - throw; - } - - // Clean up allocated memory - ::free(diff1BlockStartPositions); - if(buffer != 0) - { - ::free(buffer); - } -} - diff --git a/lib/backupclient/BackupStoreFileCmbIdx.cpp b/lib/backupclient/BackupStoreFileCmbIdx.cpp deleted file mode 100644 index c8bcc3b9..00000000 --- a/lib/backupclient/BackupStoreFileCmbIdx.cpp +++ /dev/null @@ -1,324 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileCmbIdx.cpp -// Purpose: Combine indicies of a delta file and the file it's a diff from. -// Created: 8/7/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <new> -#include <string.h> - -#include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreObjectMagic.h" -#include "BackupStoreException.h" -#include "BackupStoreConstants.h" -#include "BackupStoreFilename.h" - -#include "MemLeakFindOn.h" - -// Hide from outside world -namespace -{ - -class BSFCombinedIndexStream : public IOStream -{ -public: - BSFCombinedIndexStream(IOStream *pDiff); - ~BSFCombinedIndexStream(); - - virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); - virtual bool StreamDataLeft(); - virtual bool StreamClosed(); - virtual void Initialise(IOStream &rFrom); - -private: - IOStream *mpDiff; - bool mIsInitialised; - bool mHeaderWritten; - file_BlockIndexHeader mHeader; - int64_t mNumEntriesToGo; - int64_t mNumEntriesInFromFile; - int64_t *mFromBlockSizes; // NOTE: Entries in network byte order -}; - -}; - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::CombineFileIndices(IOStream &, IOStream &, bool) -// Purpose: Given a diff file and the file it's a diff from, return a stream from which -// can be read the index of the combined file, without actually combining them. -// The stream of the diff must have a lifetime greater than or equal to the -// lifetime of the returned stream object. The full "from" file stream -// only needs to exist during the actual function call. -// If you pass in dodgy files which aren't related, then you will either -// get an error or bad results. So don't do that. -// If DiffIsIndexOnly is true, then rDiff is assumed to be a stream positioned -// at the beginning of the block index. Similarly for FromIsIndexOnly. -// WARNING: Reads of the returned streams with buffer sizes less than 64 bytes -// will not return any data. -// Created: 8/7/04 -// -// -------------------------------------------------------------------------- -std::auto_ptr<IOStream> BackupStoreFile::CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly, bool FromIsIndexOnly) -{ - // Reposition file pointers? - if(!DiffIsIndexOnly) - { - MoveStreamPositionToBlockIndex(rDiff); - } - if(!FromIsIndexOnly) - { - MoveStreamPositionToBlockIndex(rFrom); - } - - // Create object - std::auto_ptr<IOStream> stream(new BSFCombinedIndexStream(&rDiff)); - - // Initialise it - ((BSFCombinedIndexStream *)stream.get())->Initialise(rFrom); - - // And return the stream - return stream; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BSFCombinedIndexStream::BSFCombinedIndexStream() -// Purpose: Private class. Constructor. -// Created: 8/7/04 -// -// -------------------------------------------------------------------------- -BSFCombinedIndexStream::BSFCombinedIndexStream(IOStream *pDiff) - : mpDiff(pDiff), - mIsInitialised(false), - mHeaderWritten(false), - mNumEntriesToGo(0), - mNumEntriesInFromFile(0), - mFromBlockSizes(0) -{ - ASSERT(mpDiff != 0); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BSFCombinedIndexStream::~BSFCombinedIndexStream() -// Purpose: Private class. Destructor. -// Created: 8/7/04 -// -// -------------------------------------------------------------------------- -BSFCombinedIndexStream::~BSFCombinedIndexStream() -{ - if(mFromBlockSizes != 0) - { - ::free(mFromBlockSizes); - mFromBlockSizes = 0; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BSFCombinedIndexStream::Initialise(IOStream &) -// Purpose: Private class. Initalise from the streams (diff passed in constructor). -// Both streams must have file pointer positioned at the block index. -// Created: 8/7/04 -// -// -------------------------------------------------------------------------- -void BSFCombinedIndexStream::Initialise(IOStream &rFrom) -{ - // Paranoia is good. - if(mIsInitialised) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Look at the diff file: Read in the header - if(!mpDiff->ReadFullBuffer(&mHeader, sizeof(mHeader), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - if(ntohl(mHeader.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Read relevant data. - mNumEntriesToGo = box_ntoh64(mHeader.mNumBlocks); - - // Adjust a bit to reflect the fact it's no longer a diff - mHeader.mOtherFileID = box_hton64(0); - - // Now look at the from file: Read header - file_BlockIndexHeader fromHdr; - if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Then... allocate memory for the list of sizes - mNumEntriesInFromFile = box_ntoh64(fromHdr.mNumBlocks); - mFromBlockSizes = (int64_t*)::malloc(mNumEntriesInFromFile * sizeof(int64_t)); - if(mFromBlockSizes == 0) - { - throw std::bad_alloc(); - } - - // And read them all in! - for(int64_t b = 0; b < mNumEntriesInFromFile; ++b) - { - file_BlockIndexEntry e; - if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Check that the from file isn't a delta in itself - if(box_ntoh64(e.mEncodedSize) <= 0) - { - THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) - } - - // Store size (in network byte order) - mFromBlockSizes[b] = e.mEncodedSize; - } - - // Flag as initialised - mIsInitialised = true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BSFCombinedIndexStream::Read(void *, int, int) -// Purpose: Private class. As interface. -// Created: 8/7/04 -// -// -------------------------------------------------------------------------- -int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout) -{ - // Paranoia is good. - if(!mIsInitialised || mFromBlockSizes == 0 || mpDiff == 0) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - int written = 0; - - // Header output yet? - if(!mHeaderWritten) - { - // Enough space? - if(NBytes < (int)sizeof(mHeader)) return 0; - - // Copy in - ::memcpy(pBuffer, &mHeader, sizeof(mHeader)); - NBytes -= sizeof(mHeader); - written += sizeof(mHeader); - - // Flag it's done - mHeaderWritten = true; - } - - // How many entries can be written? - int entriesToWrite = NBytes / sizeof(file_BlockIndexEntry); - if(entriesToWrite > mNumEntriesToGo) - { - entriesToWrite = mNumEntriesToGo; - } - - // Setup ready to go - file_BlockIndexEntry *poutput = (file_BlockIndexEntry*)(((uint8_t*)pBuffer) + written); - - // Write entries - for(int b = 0; b < entriesToWrite; ++b) - { - if(!mpDiff->ReadFullBuffer(&(poutput[b]), sizeof(file_BlockIndexEntry), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Does this need adjusting? - int s = box_ntoh64(poutput[b].mEncodedSize); - if(s <= 0) - { - // A reference to a block in the from file - int block = 0 - s; - ASSERT(block >= 0); - if(block >= mNumEntriesInFromFile) - { - // That's not good, the block doesn't exist - THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) - } - - // Adjust the entry in the buffer - poutput[b].mEncodedSize = mFromBlockSizes[block]; // stored in network byte order, no translation necessary - } - } - - // Update written count - written += entriesToWrite * sizeof(file_BlockIndexEntry); - mNumEntriesToGo -= entriesToWrite; - - return written; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BSFCombinedIndexStream::Write(const void *, int) -// Purpose: Private class. As interface. -// Created: 8/7/04 -// -// -------------------------------------------------------------------------- -void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes) -{ - THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BSFCombinedIndexStream::StreamDataLeft() -// Purpose: Private class. As interface -// Created: 8/7/04 -// -// -------------------------------------------------------------------------- -bool BSFCombinedIndexStream::StreamDataLeft() -{ - return (!mHeaderWritten) || (mNumEntriesToGo > 0); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BSFCombinedIndexStream::StreamClosed() -// Purpose: Private class. As interface. -// Created: 8/7/04 -// -// -------------------------------------------------------------------------- -bool BSFCombinedIndexStream::StreamClosed() -{ - return true; // doesn't do writing -} - diff --git a/lib/backupclient/BackupStoreFileCombine.cpp b/lib/backupclient/BackupStoreFileCombine.cpp deleted file mode 100644 index baa331f0..00000000 --- a/lib/backupclient/BackupStoreFileCombine.cpp +++ /dev/null @@ -1,410 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileCombine.cpp -// Purpose: File combining for BackupStoreFile -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <new> - -#include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreObjectMagic.h" -#include "BackupStoreException.h" -#include "BackupStoreConstants.h" -#include "BackupStoreFilename.h" -#include "FileStream.h" - -#include "MemLeakFindOn.h" - -typedef struct -{ - int64_t mFilePosition; -} FromIndexEntry; - -static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries); -static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut); -static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut); - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::CombineFile(IOStream &, IOStream &, IOStream &) -// Purpose: Where rDiff is a store file which is incomplete as a result of a -// diffing operation, rFrom is the file it is diffed from, and -// rOut is the stream in which to place the result, the old file -// and new file are combined into a file containing all the data. -// rDiff2 is the same file as rDiff, opened again to get two -// independent streams to the same file. -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut) -{ - // Read and copy the header. - file_StreamFormat hdr; - if(!rDiff.ReadFullBuffer(&hdr, sizeof(hdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - // Copy - rOut.Write(&hdr, sizeof(hdr)); - // Copy over filename and attributes - // BLOCK - { - BackupStoreFilename filename; - filename.ReadFromStream(rDiff, IOStream::TimeOutInfinite); - filename.WriteToStream(rOut); - StreamableMemBlock attr; - attr.ReadFromStream(rDiff, IOStream::TimeOutInfinite); - attr.WriteToStream(rOut); - } - - // Read the header for the From file - file_StreamFormat fromHdr; - if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - // Skip over the filename and attributes of the From file - // BLOCK - { - BackupStoreFilename filename2; - filename2.ReadFromStream(rFrom, IOStream::TimeOutInfinite); - int32_t size_s; - if(!rFrom.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) - { - THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) - } - int size = ntohl(size_s); - // Skip forward the size - rFrom.Seek(size, IOStream::SeekType_Relative); - } - - // Allocate memory for the block index of the From file - int64_t fromNumBlocks = box_ntoh64(fromHdr.mNumBlocks); - // NOTE: An extra entry is required so that the length of the last block can be calculated - FromIndexEntry *pFromIndex = (FromIndexEntry*)::malloc((fromNumBlocks+1) * sizeof(FromIndexEntry)); - if(pFromIndex == 0) - { - throw std::bad_alloc(); - } - - try - { - // Load the index from the From file, calculating the offsets in the - // file as we go along, and enforce that everything should be present. - LoadFromIndex(rFrom, pFromIndex, fromNumBlocks); - - // Read in the block index of the Diff file in small chunks, and output data - // for each block, either from this file, or the other file. - int64_t diffNumBlocks = box_ntoh64(hdr.mNumBlocks); - CopyData(rDiff /* positioned at start of data */, rDiff2, diffNumBlocks, rFrom, pFromIndex, fromNumBlocks, rOut); - - // Read in the block index again, and output the new block index, simply - // filling in the sizes of blocks from the old file. - WriteNewIndex(rDiff, diffNumBlocks, pFromIndex, fromNumBlocks, rOut); - - // Free buffers - ::free(pFromIndex); - pFromIndex = 0; - } - catch(...) - { - // Clean up - if(pFromIndex != 0) - { - ::free(pFromIndex); - pFromIndex = 0; - } - throw; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static LoadFromIndex(IOStream &, FromIndexEntry *, int64_t) -// Purpose: Static. Load the index from the From file -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries) -{ - ASSERT(pIndex != 0); - ASSERT(NumEntries >= 0); - - // Get the starting point in the file - int64_t filePos = rFrom.GetPosition(); - - // Jump to the end of the file to read the index - rFrom.Seek(0 - ((NumEntries * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); - - // Read block index header - file_BlockIndexHeader blkhdr; - if(!rFrom.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 - || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != NumEntries) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // And then the block entries - for(int64_t b = 0; b < NumEntries; ++b) - { - // Read - file_BlockIndexEntry en; - if(!rFrom.ReadFullBuffer(&en, sizeof(en), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - - // Add to list - pIndex[b].mFilePosition = filePos; - - // Encoded size? - int64_t encodedSize = box_ntoh64(en.mEncodedSize); - // Check that the block is actually there - if(encodedSize <= 0) - { - THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete) - } - - // Move file pointer on - filePos += encodedSize; - } - - // Store the position in the very last entry, so the size of the last entry can be calculated - pIndex[NumEntries].mFilePosition = filePos; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static CopyData(IOStream &, IOStream &, int64_t, IOStream &, FromIndexEntry *, int64_t, IOStream &) -// Purpose: Static. Copy data from the Diff and From file to the out file. -// rDiffData is at beginning of data. -// rDiffIndex at any position. -// rFrom is at any position. -// rOut is after the header, ready for data -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, - IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut) -{ - // Jump to the end of the diff file to read the index - rDiffIndex.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); - - // Read block index header - file_BlockIndexHeader diffBlkhdr; - if(!rDiffIndex.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 - || (int64_t)box_ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Record where the From file is - int64_t fromPos = rFrom.GetPosition(); - - // Buffer data - void *buffer = 0; - int bufferSize = 0; - - try - { - // Read the blocks in! - for(int64_t b = 0; b < DiffNumBlocks; ++b) - { - // Read - file_BlockIndexEntry en; - if(!rDiffIndex.ReadFullBuffer(&en, sizeof(en), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - - // What's the size value stored in the entry - int64_t encodedSize = box_ntoh64(en.mEncodedSize); - - // How much data will be read? - int32_t blockSize = 0; - if(encodedSize > 0) - { - // The block is actually in the diff file - blockSize = encodedSize; - } - else - { - // It's in the from file. First, check to see if it's valid - int64_t blockIdx = (0 - encodedSize); - if(blockIdx > FromNumBlocks) - { - // References a block which doesn't actually exist - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - // Calculate size. This operation is safe because of the extra entry at the end - blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition; - } - ASSERT(blockSize > 0); - - // Make sure there's memory available to copy this - if(bufferSize < blockSize || buffer == 0) - { - // Free old block - if(buffer != 0) - { - ::free(buffer); - buffer = 0; - bufferSize = 0; - } - // Allocate new block - buffer = ::malloc(blockSize); - if(buffer == 0) - { - throw std::bad_alloc(); - } - bufferSize = blockSize; - } - ASSERT(bufferSize >= blockSize); - - // Load in data from one of the files - if(encodedSize > 0) - { - // Load from diff file - if(!rDiffData.ReadFullBuffer(buffer, blockSize, 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - } - else - { - // Locate and read the data from the from file - int64_t blockIdx = (0 - encodedSize); - // Seek if necessary - if(fromPos != pFromIndex[blockIdx].mFilePosition) - { - rFrom.Seek(pFromIndex[blockIdx].mFilePosition, IOStream::SeekType_Absolute); - fromPos = pFromIndex[blockIdx].mFilePosition; - } - // Read - if(!rFrom.ReadFullBuffer(buffer, blockSize, 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - - // Update fromPos to current position - fromPos += blockSize; - } - - // Write data to out file - rOut.Write(buffer, blockSize); - } - - // Free buffer, if allocated - if(buffer != 0) - { - ::free(buffer); - buffer = 0; - } - } - catch(...) - { - if(buffer != 0) - { - ::free(buffer); - buffer = 0; - } - throw; - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static WriteNewIndex(IOStream &, int64_t, FromIndexEntry *, int64_t, IOStream &) -// Purpose: Write the index to the out file, just copying from the diff file and -// adjusting the entries. -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut) -{ - // Jump to the end of the diff file to read the index - rDiff.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End); - - // Read block index header - file_BlockIndexHeader diffBlkhdr; - if(!rDiff.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 - || (int64_t)box_ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Write it out with a blanked out other file ID - diffBlkhdr.mOtherFileID = box_hton64(0); - rOut.Write(&diffBlkhdr, sizeof(diffBlkhdr)); - - // Rewrite the index - for(int64_t b = 0; b < DiffNumBlocks; ++b) - { - file_BlockIndexEntry en; - if(!rDiff.ReadFullBuffer(&en, sizeof(en), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - - // What's the size value stored in the entry - int64_t encodedSize = box_ntoh64(en.mEncodedSize); - - // Need to adjust it? - if(encodedSize <= 0) - { - // This actually refers to a block in the from file. So rewrite this. - int64_t blockIdx = (0 - encodedSize); - if(blockIdx > FromNumBlocks) - { - // References a block which doesn't actually exist - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - // Calculate size. This operation is safe because of the extra entry at the end - int32_t blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition; - // Then replace entry - en.mEncodedSize = box_hton64(((uint64_t)blockSize)); - } - - // Write entry - rOut.Write(&en, sizeof(en)); - } -} - - - - - diff --git a/lib/backupclient/BackupStoreFileCryptVar.cpp b/lib/backupclient/BackupStoreFileCryptVar.cpp deleted file mode 100644 index e826de4e..00000000 --- a/lib/backupclient/BackupStoreFileCryptVar.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileCryptVar.cpp -// Purpose: Cryptographic keys for backup store files -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include "BackupStoreFileCryptVar.h" -#include "BackupStoreFileWire.h" - -#include "MemLeakFindOn.h" - -CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt; -CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt; - -#ifndef HAVE_OLD_SSL - CipherContext BackupStoreFileCryptVar::sAESEncrypt; - CipherContext BackupStoreFileCryptVar::sAESDecrypt; -#endif - -// Default to blowfish -CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt; -uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING; - -CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry; -CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry; - diff --git a/lib/backupclient/BackupStoreFileCryptVar.h b/lib/backupclient/BackupStoreFileCryptVar.h deleted file mode 100644 index 566813c8..00000000 --- a/lib/backupclient/BackupStoreFileCryptVar.h +++ /dev/null @@ -1,39 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileCryptVar.h -// Purpose: Cryptographic keys for backup store files -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILECRYPTVAR__H -#define BACKUPSTOREFILECRYPTVAR__H - -#include "CipherContext.h" - -// Hide private static variables from the rest of the world by putting them -// as static variables in a namespace. -// -- don't put them as static class variables to avoid openssl/evp.h being -// included all over the project. -namespace BackupStoreFileCryptVar -{ - // Keys for the main file data - extern CipherContext sBlowfishEncrypt; - extern CipherContext sBlowfishDecrypt; - // Use AES when available -#ifndef HAVE_OLD_SSL - extern CipherContext sAESEncrypt; - extern CipherContext sAESDecrypt; -#endif - // How encoding will be done - extern CipherContext *spEncrypt; - extern uint8_t sEncryptCipherType; - - // Keys for the block indicies - extern CipherContext sBlowfishEncryptBlockEntry; - extern CipherContext sBlowfishDecryptBlockEntry; -} - -#endif // BACKUPSTOREFILECRYPTVAR__H - diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp deleted file mode 100644 index 5705c3aa..00000000 --- a/lib/backupclient/BackupStoreFileDiff.cpp +++ /dev/null @@ -1,1046 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileDiff.cpp -// Purpose: Functions relating to diffing BackupStoreFiles -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <string.h> - -#include <new> -#include <map> - -#ifdef HAVE_TIME_H - #include <time.h> -#elif HAVE_SYS_TIME_H - #include <sys/time.h> -#endif - -#include "BackupStoreConstants.h" -#include "BackupStoreException.h" -#include "BackupStoreFile.h" -#include "BackupStoreFileCryptVar.h" -#include "BackupStoreFileEncodeStream.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreObjectMagic.h" -#include "CommonException.h" -#include "FileStream.h" -#include "MD5Digest.h" -#include "RollingChecksum.h" -#include "Timer.h" - -#include "MemLeakFindOn.h" - -#include <cstring> - -using namespace BackupStoreFileCryptVar; -using namespace BackupStoreFileCreation; - -// By default, don't trace out details of the diff as we go along -- would fill up logs significantly. -// But it's useful for the test. -#ifndef BOX_RELEASE_BUILD - bool BackupStoreFile::TraceDetailsOfDiffProcess = false; -#endif - -static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis); -static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]); -static void SearchForMatchingBlocks(IOStream &rFile, - std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, - int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], - DiffTimer *pDiffTimer); -static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable); -static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum &fastSum, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber, -BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks); -static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile); - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &) -// Purpose: Move the file pointer in this stream to just before the block index. -// Assumes that the stream is at the beginning, seekable, and -// reading from the stream is OK. -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream) -{ - // Size of file - int64_t fileSize = rStream.BytesLeftToRead(); - - // Get header - file_StreamFormat hdr; - - // Read the header - if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Check magic number - if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif - ) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Work out where the index is - int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - int64_t blockHeaderPosFromEnd = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - - // Sanity check - if(blockHeaderPosFromEnd > static_cast<int64_t>(fileSize - sizeof(file_StreamFormat))) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Seek to that position - rStream.Seek(0 - blockHeaderPosFromEnd, IOStream::SeekType_End); - - // Done. Stream now in right position (as long as the file is formatted correctly) -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *) -// Purpose: Similar to EncodeFile, but takes the object ID of the file it's -// diffing from, and the index of the blocks in a stream. It'll then -// calculate which blocks can be reused from that old file. -// The timeout is the timeout value for reading the diff block index. -// If pIsCompletelyDifferent != 0, it will be set to true if the -// the two files are completely different (do not share any block), false otherwise. -// -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- -std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff -( - const char *Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, - IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, - int64_t *pModificationTime, bool *pIsCompletelyDifferent) -{ - // Is it a symlink? - { - EMU_STRUCT_STAT st; - if(EMU_LSTAT(Filename, &st) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError) - } - if((st.st_mode & S_IFLNK) == S_IFLNK) - { - // Don't do diffs for symlinks - if(pIsCompletelyDifferent != 0) - { - *pIsCompletelyDifferent = true; - } - return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); - } - } - - // Load in the blocks - BlocksAvailableEntry *pindex = 0; - int64_t blocksInIndex = 0; - bool canDiffFromThis = false; - LoadIndex(rDiffFromBlockIndex, DiffFromObjectID, &pindex, blocksInIndex, Timeout, canDiffFromThis); - // BOX_TRACE("Diff: Blocks in index: " << blocksInIndex); - - if(!canDiffFromThis) - { - // Don't do diffing... - if(pIsCompletelyDifferent != 0) - { - *pIsCompletelyDifferent = true; - } - return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); - } - - // Pointer to recipe we're going to create - BackupStoreFileEncodeStream::Recipe *precipe = 0; - - try - { - // Find which sizes should be scanned - int32_t sizesToScan[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]; - FindMostUsedSizes(pindex, blocksInIndex, sizesToScan); - - // Flag for reporting to the user - bool completelyDifferent; - - // BLOCK - { - // Search the file to find matching blocks - std::map<int64_t, int64_t> foundBlocks; // map of offset in file to index in block index - int64_t sizeOfInputFile = 0; - // BLOCK - { - FileStream file(Filename); - // Get size of file - sizeOfInputFile = file.BytesLeftToRead(); - // Find all those lovely matching blocks - SearchForMatchingBlocks(file, foundBlocks, pindex, - blocksInIndex, sizesToScan, pDiffTimer); - - // Is it completely different? - completelyDifferent = (foundBlocks.size() == 0); - } - - // Create a recipe -- if the two files are completely different, don't put the from file ID in the recipe. - precipe = new BackupStoreFileEncodeStream::Recipe(pindex, blocksInIndex, completelyDifferent?(0):(DiffFromObjectID)); - BlocksAvailableEntry *pindexKeptRef = pindex; // we need this later, but must set pindex == 0 now, because of exceptions - pindex = 0; // Recipe now has ownership - - // Fill it in - GenerateRecipe(*precipe, pindexKeptRef, blocksInIndex, foundBlocks, sizeOfInputFile); - } - // foundBlocks no longer required - - // Create the stream - std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); - - // Do the initial setup - ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime); - precipe = 0; // Stream has taken ownership of this - - // Tell user about completely different status? - if(pIsCompletelyDifferent != 0) - { - *pIsCompletelyDifferent = completelyDifferent; - } - - // Return the stream for the caller - return stream; - } - catch(...) - { - // cleanup - if(pindex != 0) - { - ::free(pindex); - pindex = 0; - } - if(precipe != 0) - { - delete precipe; - precipe = 0; - } - throw; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: static LoadIndex(IOStream &, int64_t, BlocksAvailableEntry **, int64_t, bool &) -// Purpose: Read in an index, and decrypt, and store in the in memory block format. -// rCanDiffFromThis is set to false if the version of the from file is too old. -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- -static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis) -{ - // Reset - rNumBlocksOut = 0; - rCanDiffFromThis = false; - - // Read header - file_BlockIndexHeader hdr; - if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) - { - // Couldn't read header - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - // Check against backwards comptaibility stuff - if(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)) - { - // Won't diff against old version - - // Absorb rest of stream - char buffer[2048]; - while(rBlockIndex.StreamDataLeft()) - { - rBlockIndex.Read(buffer, sizeof(buffer), 1000 /* 1 sec timeout */); - } - - // Tell caller - rCanDiffFromThis = false; - return; - } -#endif - - // Check magic - if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Check that we're not trying to diff against a file which references blocks from another file - if(((int64_t)box_ntoh64(hdr.mOtherFileID)) != 0) - { - THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile) - } - - // Mark as an acceptable diff. - rCanDiffFromThis = true; - - // Get basic information - int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase); - - //TODO: Verify that these sizes look reasonable - - // Allocate space for the index - BlocksAvailableEntry *pindex = (BlocksAvailableEntry*)::malloc(sizeof(BlocksAvailableEntry) * numBlocks); - if(pindex == 0) - { - throw std::bad_alloc(); - } - - try - { - for(int64_t b = 0; b < numBlocks; ++b) - { - // Read an entry from the stream - file_BlockIndexEntry entry; - if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) - { - // Couldn't read entry - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Calculate IV for this entry - uint64_t iv = entryIVBase; - iv += b; - // Network byte order - iv = box_hton64(iv); - sBlowfishDecryptBlockEntry.SetIV(&iv); - - // Decrypt the encrypted section - file_BlockIndexEntryEnc entryEnc; - int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), - entry.mEnEnc, sizeof(entry.mEnEnc)); - if(sectionSize != sizeof(entryEnc)) - { - THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) - } - - // Check that we're not trying to diff against a file which references blocks from another file - if(((int64_t)box_ntoh64(entry.mEncodedSize)) <= 0) - { - THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile) - } - - // Store all the required information - pindex[b].mpNextInHashList = 0; // hash list not set up yet - pindex[b].mSize = ntohl(entryEnc.mSize); - pindex[b].mWeakChecksum = ntohl(entryEnc.mWeakChecksum); - ::memcpy(pindex[b].mStrongChecksum, entryEnc.mStrongChecksum, sizeof(pindex[b].mStrongChecksum)); - } - - // Store index pointer for called - ASSERT(ppIndex != 0); - *ppIndex = pindex; - - // Store number of blocks for caller - rNumBlocksOut = numBlocks; - - } - catch(...) - { - // clean up and send the exception along its way - ::free(pindex); - throw; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static FindMostUsedSizes(BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) -// Purpose: Finds the most commonly used block sizes in the index -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- -static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) -{ - // Array for lengths - int64_t sizeCounts[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]; - - // Set arrays to lots of zeros (= unused entries) - for(int l = 0; l < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++l) - { - Sizes[l] = 0; - sizeCounts[l] = 0; - } - - // Array for collecting sizes - std::map<int32_t, int64_t> foundSizes; - - // Run through blocks and make a count of the entries - for(int64_t b = 0; b < NumBlocks; ++b) - { - // Only if the block size is bigger than the minimum size we'll scan for - if(pIndex[b].mSize > BACKUP_FILE_DIFF_MIN_BLOCK_SIZE) - { - // Find entry? - std::map<int32_t, int64_t>::const_iterator f(foundSizes.find(pIndex[b].mSize)); - if(f != foundSizes.end()) - { - // Increment existing entry - foundSizes[pIndex[b].mSize] = foundSizes[pIndex[b].mSize] + 1; - } - else - { - // New entry - foundSizes[pIndex[b].mSize] = 1; - } - } - } - - // Make the block sizes - for(std::map<int32_t, int64_t>::const_iterator i(foundSizes.begin()); i != foundSizes.end(); ++i) - { - // Find the position of the size in the array - for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t) - { - // Instead of sorting on the raw count of blocks, - // take the file area covered by this block size. - if(i->second * i->first > sizeCounts[t] * Sizes[t]) - { - // Then this size belong before this entry -- shuffle them up - for(int s = (BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1); s >= t; --s) - { - Sizes[s] = Sizes[s-1]; - sizeCounts[s] = sizeCounts[s-1]; - } - - // Insert this size - Sizes[t] = i->first; - sizeCounts[t] = i->second; - - // Shouldn't do any more searching - break; - } - } - } - - // trace the size table in debug builds -#ifndef BOX_RELEASE_BUILD - if(BackupStoreFile::TraceDetailsOfDiffProcess) - { - for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t) - { - BOX_TRACE("Diff block size " << t << ": " << - Sizes[t] << " (count = " << - sizeCounts[t] << ")"); - } - } -#endif -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) -// Purpose: Find the matching blocks within the file. -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- -static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, - BlocksAvailableEntry *pIndex, int64_t NumBlocks, - int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer) -{ - Timer maximumDiffingTime(0, "MaximumDiffingTime"); - - if(pDiffTimer && pDiffTimer->IsManaged()) - { - maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime(), - "MaximumDiffingTime"); - } - - std::map<int64_t, int32_t> goodnessOfFit; - - // Allocate the hash lookup table - BlocksAvailableEntry **phashTable = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024)); - - // Choose a size for the buffer, just a little bit more than the maximum block size - int32_t bufSize = Sizes[0]; - for(int z = 1; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z) - { - if(Sizes[z] > bufSize) bufSize = Sizes[z]; - } - bufSize += 4; - ASSERT(bufSize > Sizes[0]); - ASSERT(bufSize > 0); - if(bufSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024)) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // TODO: Because we read in the file a scanned block size at a time, - // it is likely to be inefficient. Probably will be much better to - // calculate checksums for all block sizes in a single pass. - - // Allocate the buffers. - uint8_t *pbuffer0 = (uint8_t *)::malloc(bufSize); - uint8_t *pbuffer1 = (uint8_t *)::malloc(bufSize); - try - { - // Check buffer allocation - if(pbuffer0 == 0 || pbuffer1 == 0 || phashTable == 0) - { - // If a buffer got allocated, it will be cleaned up in the catch block - throw std::bad_alloc(); - } - - // Flag to abort the run, if too many blocks are found -- avoid using - // huge amounts of processor time when files contain many similar blocks. - bool abortSearch = false; - - // Search for each block size in turn - // NOTE: Do the smallest size first, so that the scheme for adding - // entries in the found list works as expected and replaces smallers block - // with larger blocks when it finds matches at the same offset in the file. - for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s) - { - ASSERT(Sizes[s] <= bufSize); - BOX_TRACE("Diff pass " << s << ", for block size " << - Sizes[s]); - - // Check we haven't finished - if(Sizes[s] == 0) - { - // empty entry, try next size - continue; - } - - // Set up the hash table entries - SetupHashTable(pIndex, NumBlocks, Sizes[s], phashTable); - - // Shift file position to beginning - rFile.Seek(0, IOStream::SeekType_Absolute); - - // Read first block - if(rFile.Read(pbuffer0, Sizes[s]) != Sizes[s]) - { - // Size of file too short to match -- do next size - continue; - } - - // Setup block pointers - uint8_t *beginnings = pbuffer0; - uint8_t *endings = pbuffer1; - int offset = 0; - - // Calculate the first checksum, ready for rolling - RollingChecksum rolling(beginnings, Sizes[s]); - - // Then roll, until the file is exhausted - int64_t fileBlockNumber = 0; - int64_t fileOffset = 0; - int rollOverInitialBytes = 0; - while(true) - { - if(maximumDiffingTime.HasExpired()) - { - ASSERT(pDiffTimer != NULL); - BOX_INFO("MaximumDiffingTime reached - " - "suspending file diff"); - abortSearch = true; - break; - } - - if(pDiffTimer) - { - pDiffTimer->DoKeepAlive(); - } - - // Load in another block of data, and record how big it is - int bytesInEndings = rFile.Read(endings, Sizes[s]); - int tmp; - - // Skip any bytes from a previous matched block - if(rollOverInitialBytes > 0 && offset < bytesInEndings) - { - int spaceLeft = bytesInEndings - offset; - int thisRoll = (rollOverInitialBytes > spaceLeft) ? spaceLeft : rollOverInitialBytes; - - rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll); - - offset += thisRoll; - fileOffset += thisRoll; - rollOverInitialBytes -= thisRoll; - - if(rollOverInitialBytes) - { - goto refresh; - } - } - - if(goodnessOfFit.count(fileOffset)) - { - tmp = goodnessOfFit[fileOffset]; - } - else - { - tmp = 0; - } - - if(tmp >= Sizes[s]) - { - // Skip over bigger ready-matched blocks completely - rollOverInitialBytes = tmp; - int spaceLeft = bytesInEndings - offset; - int thisRoll = (rollOverInitialBytes > spaceLeft) ? spaceLeft : rollOverInitialBytes; - - rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll); - - offset += thisRoll; - fileOffset += thisRoll; - rollOverInitialBytes -= thisRoll; - - if(rollOverInitialBytes) - { - goto refresh; - } - } - - while(offset < bytesInEndings) - { - // Is current checksum in hash list? - uint16_t hash = rolling.GetComponentForHashing(); - if(phashTable[hash] != 0 && (goodnessOfFit.count(fileOffset) == 0 || goodnessOfFit[fileOffset] < Sizes[s])) - { - if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks)) - { - BOX_TRACE("Found block match for " << hash << " of " << Sizes[s] << " bytes at offset " << fileOffset); - goodnessOfFit[fileOffset] = Sizes[s]; - - // Block matched, roll the checksum forward to the next block without doing - // any more comparisons, because these are pointless (as any more matches will be ignored when - // the recipe is generated) and just take up valuable processor time. Edge cases are - // especially nasty, using huge amounts of time and memory. - int skip = Sizes[s]; - if(offset < bytesInEndings && skip > 0) - { - int spaceLeft = bytesInEndings - offset; - int thisRoll = (skip > spaceLeft) ? spaceLeft : skip; - - rolling.RollForwardSeveral(beginnings+offset, endings+offset, Sizes[s], thisRoll); - - offset += thisRoll; - fileOffset += thisRoll; - skip -= thisRoll; - } - // Not all the bytes necessary will have been skipped, so get them - // skipped after the next block is loaded. - rollOverInitialBytes = skip; - - // End this loop, so the final byte isn't used again - break; - } - else - { - BOX_TRACE("False alarm match for " << hash << " of " << Sizes[s] << " bytes at offset " << fileOffset); - } - - int64_t NumBlocksFound = static_cast<int64_t>( - rFoundBlocks.size()); - int64_t MaxBlocksFound = NumBlocks * - BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE; - - if(NumBlocksFound > MaxBlocksFound) - { - abortSearch = true; - break; - } - } - - // Roll checksum forward - rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]); - - // Increment offsets - ++offset; - ++fileOffset; - } - - if(abortSearch) break; - - refresh: - // Finished? - if(bytesInEndings != Sizes[s]) - { - // No more data in file -- check the final block - // (Do a copy and paste of 5 lines of code instead of introducing a comparison for - // each byte of the file) - uint16_t hash = rolling.GetComponentForHashing(); - if(phashTable[hash] != 0 && (goodnessOfFit.count(fileOffset) == 0 || goodnessOfFit[fileOffset] < Sizes[s])) - { - if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks)) - { - goodnessOfFit[fileOffset] = Sizes[s]; - } - } - - // finish - break; - } - - // Switch buffers, reset offset - beginnings = endings; - endings = (beginnings == pbuffer0)?(pbuffer1):(pbuffer0); // ie the other buffer - offset = 0; - - // And count the blocks which have been done - ++fileBlockNumber; - } - - if(abortSearch) break; - } - - // Free buffers and hash table - ::free(pbuffer1); - pbuffer1 = 0; - ::free(pbuffer0); - pbuffer0 = 0; - ::free(phashTable); - phashTable = 0; - } - catch(...) - { - // Cleanup and throw - if(pbuffer1 != 0) ::free(pbuffer1); - if(pbuffer0 != 0) ::free(pbuffer0); - if(phashTable != 0) ::free(phashTable); - throw; - } - -#ifndef BOX_RELEASE_BUILD - if(BackupStoreFile::TraceDetailsOfDiffProcess) - { - // Trace out the found blocks in debug mode - BOX_TRACE("Diff: list of found blocks"); - BOX_TRACE("======== ======== ======== ========"); - BOX_TRACE(" Offset BlkIdx Size Movement"); - for(std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); i != rFoundBlocks.end(); ++i) - { - int64_t orgLoc = 0; - for(int64_t b = 0; b < i->second; ++b) - { - orgLoc += pIndex[b].mSize; - } - BOX_TRACE(std::setw(8) << i->first << " " << - std::setw(8) << i->second << " " << - std::setw(8) << pIndex[i->second].mSize << - " " << - std::setw(8) << (i->first - orgLoc)); - } - BOX_TRACE("======== ======== ======== ========"); - } -#endif -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static SetupHashTable(BlocksAvailableEntry *, int64_t, in32_t, BlocksAvailableEntry **) -// Purpose: Set up the hash table ready for a scan -// Created: 14/1/04 -// -// -------------------------------------------------------------------------- -static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable) -{ - // Set all entries in the hash table to zero - ::memset(pHashTable, 0, (sizeof(BlocksAvailableEntry *) * (64*1024))); - - // Scan through the blocks, building the hash table - for(int64_t b = 0; b < NumBlocks; ++b) - { - // Only look at the required block size - if(pIndex[b].mSize == BlockSize) - { - // Get the value under which to hash this entry - uint16_t hash = RollingChecksum::ExtractHashingComponent(pIndex[b].mWeakChecksum); - - // Already present in table? - if(pHashTable[hash] != 0) - { - //BOX_TRACE("Another hash entry for " << hash << " found"); - // Yes -- need to set the pointer in this entry to the current entry to build the linked list - pIndex[b].mpNextInHashList = pHashTable[hash]; - } - - // Put a pointer to this entry in the hash table - pHashTable[hash] = pIndex + b; - } - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static bool SecondStageMatch(xxx) -// Purpose: When a match in the hash table is found, scan for second stage match using strong checksum. -// Created: 14/1/04 -// -// -------------------------------------------------------------------------- -static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum &fastSum, uint8_t *pBeginnings, uint8_t *pEndings, - int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks) -{ - // Check parameters - ASSERT(pBeginnings != 0); - ASSERT(pEndings != 0); - ASSERT(Offset >= 0); - ASSERT(BlockSize > 0); - ASSERT(pFirstInHashList != 0); - ASSERT(pIndex != 0); - -#ifndef BOX_RELEASE_BUILD - uint16_t DEBUG_Hash = fastSum.GetComponentForHashing(); -#endif - uint32_t Checksum = fastSum.GetChecksum(); - - // Before we go to the expense of the MD5, make sure it's a darn good match on the checksum we already know. - BlocksAvailableEntry *scan = pFirstInHashList; - bool found=false; - while(scan != 0) - { - if(scan->mWeakChecksum == Checksum) - { - found = true; - break; - } - scan = scan->mpNextInHashList; - } - if(!found) - { - return false; - } - - // Calculate the strong MD5 digest for this block - MD5Digest strong; - // Add the data from the beginnings - strong.Add(pBeginnings + Offset, BlockSize - Offset); - // Add any data from the endings - if(Offset > 0) - { - strong.Add(pEndings, Offset); - } - strong.Finish(); - - // Then go through the entries in the hash list, comparing with the strong digest calculated - scan = pFirstInHashList; - //BOX_TRACE("second stage match"); - while(scan != 0) - { - //BOX_TRACE("scan size " << scan->mSize << - // ", block size " << BlockSize << - // ", hash " << Hash); - ASSERT(scan->mSize == BlockSize); - ASSERT(RollingChecksum::ExtractHashingComponent(scan->mWeakChecksum) == DEBUG_Hash); - - // Compare? - if(strong.DigestMatches(scan->mStrongChecksum)) - { - //BOX_TRACE("Match!\n"); - // Found! Add to list of found blocks... - int64_t fileOffset = (FileBlockNumber * BlockSize) + Offset; - int64_t blockIndex = (scan - pIndex); // pointer arthmitic is frowned upon. But most efficient way of doing it here -- alternative is to use more memory - - // We do NOT search for smallest blocks first, as this code originally assumed. - // To prevent this from potentially overwriting a better match, the caller must determine - // the relative "goodness" of any existing match and this one, and avoid the call if it - // could be detrimental. - rFoundBlocks[fileOffset] = blockIndex; - - // No point in searching further, report success - return true; - } - - // Next - scan = scan->mpNextInHashList; - } - - // Not matched - return false; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static GenerateRecipe(BackupStoreFileEncodeStream::Recipe &, BlocksAvailableEntry *, int64_t, std::map<int64_t, int64_t> &) -// Purpose: Fills in the recipe from the found block list -// Created: 15/1/04 -// -// -------------------------------------------------------------------------- -static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, - int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile) -{ - // NOTE: This function could be a lot more sophisiticated. For example, if - // a small block overlaps a big block like this - // **** - // ******************************* - // then the small block will be used, not the big one. But it'd be better to - // just ignore the small block and keep the big one. However, some stats should - // be gathered about real world files before writing complex code which might - // go wrong. - - // Initialise a blank instruction - BackupStoreFileEncodeStream::RecipeInstruction instruction; - #define RESET_INSTRUCTION \ - instruction.mSpaceBefore = 0; \ - instruction.mBlocks = 0; \ - instruction.mpStartBlock = 0; - RESET_INSTRUCTION - - // First, a special case for when there are no found blocks - if(rFoundBlocks.size() == 0) - { - // No blocks, just a load of space - instruction.mSpaceBefore = SizeOfInputFile; - rRecipe.push_back(instruction); - - #ifndef BOX_RELEASE_BUILD - if(BackupStoreFile::TraceDetailsOfDiffProcess) - { - BOX_TRACE("Diff: Default recipe generated, " << - SizeOfInputFile << " bytes of file"); - } - #endif - - // Don't do anything - return; - } - - // Current location - int64_t loc = 0; - - // Then iterate through the list, generating the recipe - std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); - ASSERT(i != rFoundBlocks.end()); // check logic - - // Counting for debug tracing -#ifndef BOX_RELEASE_BUILD - int64_t debug_NewBytesFound = 0; - int64_t debug_OldBlocksUsed = 0; -#endif - - for(; i != rFoundBlocks.end(); ++i) - { - // Remember... map is (position in file) -> (index of block in pIndex) - - if(i->first < loc) - { - // This block overlaps the last one - continue; - } - else if(i->first > loc) - { - // There's a gap between the end of the last thing and this block. - // If there's an instruction waiting, push it onto the list - if(instruction.mSpaceBefore != 0 || instruction.mpStartBlock != 0) - { - rRecipe.push_back(instruction); - } - // Start a new instruction, with the gap ready - RESET_INSTRUCTION - instruction.mSpaceBefore = i->first - loc; - // Move location forward to match - loc += instruction.mSpaceBefore; -#ifndef BOX_RELEASE_BUILD - debug_NewBytesFound += instruction.mSpaceBefore; -#endif - } - - // First, does the current instruction need pushing back, because this block is not - // sequential to the last one? - if(instruction.mpStartBlock != 0 && (pIndex + i->second) != (instruction.mpStartBlock + instruction.mBlocks)) - { - rRecipe.push_back(instruction); - RESET_INSTRUCTION - } - - // Add in this block - if(instruction.mpStartBlock == 0) - { - // This block starts a new instruction - instruction.mpStartBlock = pIndex + i->second; - instruction.mBlocks = 1; - } - else - { - // It continues the previous section of blocks - instruction.mBlocks += 1; - } - -#ifndef BOX_RELEASE_BUILD - debug_OldBlocksUsed++; -#endif - - // Move location forward - loc += pIndex[i->second].mSize; - } - - // Push the last instruction generated - rRecipe.push_back(instruction); - - // Is there any space left at the end which needs sending? - if(loc != SizeOfInputFile) - { - RESET_INSTRUCTION - instruction.mSpaceBefore = SizeOfInputFile - loc; -#ifndef BOX_RELEASE_BUILD - debug_NewBytesFound += instruction.mSpaceBefore; -#endif - rRecipe.push_back(instruction); - } - - // dump out the recipe -#ifndef BOX_RELEASE_BUILD - BOX_TRACE("Diff: " << - debug_NewBytesFound << " new bytes found, " << - debug_OldBlocksUsed << " old blocks used"); - if(BackupStoreFile::TraceDetailsOfDiffProcess) - { - BOX_TRACE("Diff: Recipe generated (size " << rRecipe.size()); - BOX_TRACE("======== ========= ========"); - BOX_TRACE("Space b4 FirstBlk NumBlks"); - { - for(unsigned int e = 0; e < rRecipe.size(); ++e) - { - char b[64]; -#ifdef WIN32 - sprintf(b, "%8I64d", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); -#else - sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); -#endif - BOX_TRACE(std::setw(8) << - rRecipe[e].mSpaceBefore << - " " << - ((rRecipe[e].mpStartBlock == 0)?" -":b) << - " " << std::setw(8) << - rRecipe[e].mBlocks); - } - } - BOX_TRACE("======== ========= ========"); - } -#endif -} diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp deleted file mode 100644 index 54c2463d..00000000 --- a/lib/backupclient/BackupStoreFileEncodeStream.cpp +++ /dev/null @@ -1,715 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileEncodeStream.cpp -// Purpose: Implement stream-based file encoding for the backup store -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <string.h> - -#include "BackupClientFileAttributes.h" -#include "BackupStoreConstants.h" -#include "BackupStoreException.h" -#include "BackupStoreFile.h" -#include "BackupStoreFileCryptVar.h" -#include "BackupStoreFileEncodeStream.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreObjectMagic.h" -#include "BoxTime.h" -#include "FileStream.h" -#include "Random.h" -#include "RollingChecksum.h" - -#include "MemLeakFindOn.h" - -#include <cstring> - -using namespace BackupStoreFileCryptVar; - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::BackupStoreFileEncodeStream -// Purpose: Constructor (opens file) -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() - : mpRecipe(0), - mpFile(0), - mpLogging(0), - mpRunStatusProvider(NULL), - mStatus(Status_Header), - mSendData(true), - mTotalBlocks(0), - mAbsoluteBlockNumber(-1), - mInstructionNumber(-1), - mNumBlocks(0), - mCurrentBlock(-1), - mCurrentBlockEncodedSize(0), - mPositionInCurrentBlock(0), - mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), - mLastBlockSize(0), - mpRawBuffer(0), - mAllocatedBufferSize(0), - mEntryIVBase(0) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() -// Purpose: Destructor -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() -{ - // Free buffers - if(mpRawBuffer) - { - ::free(mpRawBuffer); - mpRawBuffer = 0; - } - - // Close the file, which we might have open - if(mpFile) - { - delete mpFile; - mpFile = 0; - } - - // Clear up logging stream - if(mpLogging) - { - delete mpLogging; - mpLogging = 0; - } - - // Free the recipe - if(mpRecipe != 0) - { - delete mpRecipe; - mpRecipe = 0; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *) -// Purpose: Reads file information, and builds file header reading for sending. -// Takes ownership of the Recipe. -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::Setup(const char *Filename, - BackupStoreFileEncodeStream::Recipe *pRecipe, - int64_t ContainerID, const BackupStoreFilename &rStoreFilename, - int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) -{ - // Pointer to a blank recipe which we might create - BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; - - try - { - // Get file attributes - box_time_t modTime = 0; - int64_t fileSize = 0; - BackupClientFileAttributes attr; - attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime, - 0 /* not interested in attr mod time */, &fileSize); - - // Might need to create a blank recipe... - if(pRecipe == 0) - { - pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); - - BackupStoreFileEncodeStream::RecipeInstruction instruction; - instruction.mSpaceBefore = fileSize; // whole file - instruction.mBlocks = 0; // no blocks - instruction.mpStartBlock = 0; // no block - pblankRecipe->push_back(instruction); - - pRecipe = pblankRecipe; - } - - // Tell caller? - if(pModificationTime != 0) - { - *pModificationTime = modTime; - } - - // Go through each instruction in the recipe and work out how many blocks - // it will add, and the max clear size of these blocks - int maxBlockClearSize = 0; - for(uint64_t inst = 0; inst < pRecipe->size(); ++inst) - { - if((*pRecipe)[inst].mSpaceBefore > 0) - { - // Calculate the number of blocks the space before requires - int64_t numBlocks; - int32_t blockSize, lastBlockSize; - CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize); - // Add to accumlated total - mTotalBlocks += numBlocks; - // Update maximum clear size - if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize; - if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize; - } - - // Add number of blocks copied from the previous file - mTotalBlocks += (*pRecipe)[inst].mBlocks; - - // Check for bad things - if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0)) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Run through blocks to get the max clear size - for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b) - { - if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize; - } - } - - // Send data? (symlinks don't have any data in them) - mSendData = !attr.IsSymLink(); - - // If not data is being sent, then the max clear block size is zero - if(!mSendData) - { - maxBlockClearSize = 0; - } - - // Header - file_StreamFormat hdr; - hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1); - hdr.mNumBlocks = (mSendData)?(box_hton64(mTotalBlocks)):(0); - hdr.mContainerID = box_hton64(ContainerID); - hdr.mModificationTime = box_hton64(modTime); - // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size - hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128); - hdr.mOptions = 0; // no options defined yet - - // Write header to stream - mData.Write(&hdr, sizeof(hdr)); - - // Write filename to stream - rStoreFilename.WriteToStream(mData); - - // Write attributes to stream - attr.WriteToStream(mData); - - // Allocate some buffers for writing data - if(mSendData) - { - // Open the file - mpFile = new FileStream(Filename); - - if (pLogger) - { - // Create logging stream - mpLogging = new ReadLoggingStream(*mpFile, - *pLogger); - } - else - { - // re-use FileStream instead - mpLogging = mpFile; - mpFile = NULL; - } - - // Work out the largest possible block required for the encoded data - mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); - - // Then allocate two blocks of this size - mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize); - if(mpRawBuffer == 0) - { - throw std::bad_alloc(); - } -#ifndef BOX_RELEASE_BUILD - // In debug builds, make sure that the reallocation code is exercised. - mEncodedBuffer.Allocate(mAllocatedBufferSize / 4); -#else - mEncodedBuffer.Allocate(mAllocatedBufferSize); -#endif - } - else - { - // Write an empty block index for the symlink - file_BlockIndexHeader blkhdr; - blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); - blkhdr.mOtherFileID = box_hton64(0); // not other file ID - blkhdr.mEntryIVBase = box_hton64(0); - blkhdr.mNumBlocks = box_hton64(0); - mData.Write(&blkhdr, sizeof(blkhdr)); - } - - // Ready for reading - mData.SetForReading(); - - // Update stats - BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize; - - // Finally, store the pointer to the recipe, when we know exceptions won't occur - mpRecipe = pRecipe; - } - catch(...) - { - // Clean up any blank recipe - if(pblankRecipe != 0) - { - delete pblankRecipe; - pblankRecipe = 0; - } - throw; - } - - mpRunStatusProvider = pRunStatusProvider; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &) -// Purpose: Calculates the sizes of blocks in a section of the file -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut) -{ - // How many blocks, and how big? - rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2; - do - { - rBlockSizeOut *= 2; - - rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut; - - } while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); - - // Last block size - rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut); - - // Avoid small blocks? - if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN - && rNumBlocksOut > 1) - { - // Add the small bit of data to the last block - --rNumBlocksOut; - rLastBlockSizeOut += rBlockSizeOut; - } - - // checks! - ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize); - //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut); -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::Read(void *, int, int) -// Purpose: As interface -- generates encoded file data on the fly from the raw file -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) -{ - // Check there's something to do. - if(mStatus == Status_Finished) - { - return 0; - } - - if(mpRunStatusProvider && mpRunStatusProvider->StopRun()) - { - THROW_EXCEPTION(BackupStoreException, SignalReceived); - } - - int bytesToRead = NBytes; - uint8_t *buffer = (uint8_t*)pBuffer; - - while(bytesToRead > 0 && mStatus != Status_Finished) - { - if(mStatus == Status_Header || mStatus == Status_BlockListing) - { - // Header or block listing phase -- send from the buffered stream - - // Send bytes from the data buffer - int b = mData.Read(buffer, bytesToRead, Timeout); - bytesToRead -= b; - buffer += b; - - // Check to see if all the data has been used from this stream - if(!mData.StreamDataLeft()) - { - // Yes, move on to next phase (or finish, if there's no file data) - if(!mSendData) - { - mStatus = Status_Finished; - } - else - { - // Reset the buffer so it can be used for the next phase - mData.Reset(); - - // Get buffer ready for index? - if(mStatus == Status_Header) - { - // Just finished doing the stream header, create the block index header - file_BlockIndexHeader blkhdr; - blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1); - ASSERT(mpRecipe != 0); - blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID()); - blkhdr.mNumBlocks = box_hton64(mTotalBlocks); - - // Generate the IV base - Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase)); - blkhdr.mEntryIVBase = box_hton64(mEntryIVBase); - - mData.Write(&blkhdr, sizeof(blkhdr)); - } - - ++mStatus; - } - } - } - else if(mStatus == Status_Blocks) - { - // Block sending phase - - if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize) - { - // Next block! - ++mCurrentBlock; - ++mAbsoluteBlockNumber; - if(mCurrentBlock >= mNumBlocks) - { - // Output extra blocks for this instruction and move forward in file - if(mInstructionNumber >= 0) - { - SkipPreviousBlocksInInstruction(); - } - - // Is there another instruction to go? - ++mInstructionNumber; - - // Skip instructions which don't contain any data - while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size()) - && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0) - { - SkipPreviousBlocksInInstruction(); - ++mInstructionNumber; - } - - if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size())) - { - // End of blocks, go to next phase - ++mStatus; - - // Set the data to reading so the index can be written - mData.SetForReading(); - } - else - { - // Get ready for this instruction - SetForInstruction(); - } - } - - // Can't use 'else' here as SetForInstruction() will change this - if(mCurrentBlock < mNumBlocks) - { - EncodeCurrentBlock(); - } - } - - // Send data from the current block (if there's data to send) - if(mPositionInCurrentBlock < mCurrentBlockEncodedSize) - { - // How much data to put in the buffer? - int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock; - if(s > bytesToRead) s = bytesToRead; - - // Copy it in - ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s); - - // Update variables - bytesToRead -= s; - buffer += s; - mPositionInCurrentBlock += s; - } - } - else - { - // Should never get here, as it'd be an invalid status - ASSERT(false); - } - } - - // Add encoded size to stats - BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead); - - // Return size of data to caller - return NBytes - bytesToRead; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction() -// Purpose: Private. Stores the blocks of the old file referenced in the current -// instruction into the index and skips over the data in the file -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction() -{ - // Check something is necessary - if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0) - { - return; - } - - // Index of the first block in old file (being diffed from) - int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock); - - int64_t sizeToSkip = 0; - - for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b) - { - // Update stats - BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; - - // Store the entry - StoreBlockIndexEntry(0 - (firstIndex + b), - (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize, - (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum, - (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); - - // Increment the absolute block number -- kept encryption IV in sync - ++mAbsoluteBlockNumber; - - // Add the size of this block to the size to skip - sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; - } - - // Move forward in the stream - mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::SetForInstruction() -// Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::SetForInstruction() -{ - // Calculate block sizes - CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize); - - // Set variables - mCurrentBlock = 0; - mCurrentBlockEncodedSize = 0; - mPositionInCurrentBlock = 0; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::EncodeCurrentBlock() -// Purpose: Private. Encodes the current block, and writes the block data to the index -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::EncodeCurrentBlock() -{ - // How big is the block, raw? - int blockRawSize = mBlockSize; - if(mCurrentBlock == (mNumBlocks - 1)) - { - blockRawSize = mLastBlockSize; - } - ASSERT(blockRawSize < mAllocatedBufferSize); - - // Check file open - if(mpLogging == 0) - { - // File should be open, but isn't. So logical error. - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Read the data in - if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, - 0 /* not interested in size if failure */)) - { - // TODO: Do something more intelligent, and abort - // this upload because the file has changed. - THROW_EXCEPTION(BackupStoreException, - Temp_FileEncodeStreamDidntReadBuffer) - } - - // Encode it - mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, - blockRawSize, mEncodedBuffer); - - //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); - - // Create block listing data -- generate checksums - RollingChecksum weakChecksum(mpRawBuffer, blockRawSize); - MD5Digest strongChecksum; - strongChecksum.Add(mpRawBuffer, blockRawSize); - strongChecksum.Finish(); - - // Add entry to the index - StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, - weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); - - // Set vars to reading this block - mPositionInCurrentBlock = 0; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *) -// Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream. -// Created: 16/1/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum) -{ - // First, the encrypted section - file_BlockIndexEntryEnc entryEnc; - entryEnc.mSize = htonl(ClearSize); - entryEnc.mWeakChecksum = htonl(WeakChecksum); - ::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum)); - - // Then the clear section - file_BlockIndexEntry entry; - entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex)); - - // Then encrypt the encryted section - // Generate the IV from the block number - if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase)) - { - THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements) - } - uint64_t iv = mEntryIVBase; - iv += mAbsoluteBlockNumber; - // Convert to network byte order before encrypting with it, so that restores work on - // platforms with different endiannesses. - iv = box_hton64(iv); - sBlowfishEncryptBlockEntry.SetIV(&iv); - - // Encode the data - int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc)); - if(encodedSize != sizeof(entry.mEnEnc)) - { - THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength) - } - - // Save to data block for sending at the end of the stream - mData.Write(&entry, sizeof(entry)); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::Write(const void *, int) -// Purpose: As interface. Exceptions. -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes) -{ - THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream) -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::StreamDataLeft() -// Purpose: As interface -- end of stream reached? -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFileEncodeStream::StreamDataLeft() -{ - return (mStatus != Status_Finished); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::StreamClosed() -// Purpose: As interface -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFileEncodeStream::StreamClosed() -{ - return true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t) -// Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted -// Created: 15/1/04 -// -// -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, - int64_t NumBlocksInIndex, int64_t OtherFileID) - : mpBlockIndex(pBlockIndex), - mNumBlocksInIndex(NumBlocksInIndex), - mOtherFileID(OtherFileID) -{ - ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFileEncodeStream::Recipe::~Recipe() -// Purpose: Destructor -// Created: 15/1/04 -// -// -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::Recipe::~Recipe() -{ - // Free the block index, if there is one - if(mpBlockIndex != 0) - { - ::free(mpBlockIndex); - } -} - - - - diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h deleted file mode 100644 index c5fa780a..00000000 --- a/lib/backupclient/BackupStoreFileEncodeStream.h +++ /dev/null @@ -1,135 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileEncodeStream.h -// Purpose: Implement stream-based file encoding for the backup store -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILEENCODESTREAM__H -#define BACKUPSTOREFILEENCODESTREAM__H - -#include <vector> - -#include "IOStream.h" -#include "BackupStoreFilename.h" -#include "CollectInBufferStream.h" -#include "MD5Digest.h" -#include "BackupStoreFile.h" -#include "ReadLoggingStream.h" -#include "RunStatusProvider.h" - -namespace BackupStoreFileCreation -{ - // Diffing and creation of files share some implementation details. - typedef struct _BlocksAvailableEntry - { - struct _BlocksAvailableEntry *mpNextInHashList; - int32_t mSize; // size in clear - uint32_t mWeakChecksum; // weak, rolling checksum - uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum - } BlocksAvailableEntry; - -} - - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupStoreFileEncodeStream -// Purpose: Encode a file into a stream -// Created: 8/12/03 -// -// -------------------------------------------------------------------------- -class BackupStoreFileEncodeStream : public IOStream -{ -public: - BackupStoreFileEncodeStream(); - ~BackupStoreFileEncodeStream(); - - typedef struct - { - int64_t mSpaceBefore; // amount of bytes which aren't taken out of blocks which go - int32_t mBlocks; // number of block to reuse, starting at this one - BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock; // may be null - } RecipeInstruction; - - class Recipe : public std::vector<RecipeInstruction> - { - // NOTE: This class is rather tied in with the implementation of diffing. - public: - Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex, - int64_t OtherFileID = 0); - ~Recipe(); - - int64_t GetOtherFileID() {return mOtherFileID;} - int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock) - { - return pBlock - mpBlockIndex; - } - - private: - BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex; - int64_t mNumBlocksInIndex; - int64_t mOtherFileID; - }; - - void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, - int64_t *pModificationTime, - ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL); - - virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); - virtual bool StreamDataLeft(); - virtual bool StreamClosed(); - -private: - enum - { - Status_Header = 0, - Status_Blocks = 1, - Status_BlockListing = 2, - Status_Finished = 3 - }; - -private: - void EncodeCurrentBlock(); - void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); - void SkipPreviousBlocksInInstruction(); - void SetForInstruction(); - void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum); - -private: - Recipe *mpRecipe; - IOStream *mpFile; // source file - CollectInBufferStream mData; // buffer for header and index entries - IOStream *mpLogging; - RunStatusProvider* mpRunStatusProvider; - int mStatus; - bool mSendData; // true if there's file data to send (ie not a symlink) - int64_t mTotalBlocks; // Total number of blocks in the file - int64_t mAbsoluteBlockNumber; // The absolute block number currently being output - // Instruction number - int64_t mInstructionNumber; - // All the below are within the current instruction - int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases - int64_t mCurrentBlock; - int32_t mCurrentBlockEncodedSize; - int32_t mPositionInCurrentBlock; // for reading out - int32_t mBlockSize; // Basic block size of most of the blocks in the file - int32_t mLastBlockSize; // the size (unencoded) of the last block in the file - // Buffers - uint8_t *mpRawBuffer; // buffer for raw data - BackupStoreFile::EncodingBuffer mEncodedBuffer; - // buffer for encoded data - int32_t mAllocatedBufferSize; // size of above two allocated blocks - uint64_t mEntryIVBase; // base for block entry IV -}; - - - -#endif // BACKUPSTOREFILEENCODESTREAM__H - diff --git a/lib/backupclient/BackupStoreFileRevDiff.cpp b/lib/backupclient/BackupStoreFileRevDiff.cpp deleted file mode 100644 index 509eef61..00000000 --- a/lib/backupclient/BackupStoreFileRevDiff.cpp +++ /dev/null @@ -1,258 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileRevDiff.cpp -// Purpose: Reverse a patch, to build a new patch from new to old files -// Created: 12/7/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <new> -#include <stdlib.h> - -#include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreObjectMagic.h" -#include "BackupStoreException.h" -#include "BackupStoreConstants.h" -#include "BackupStoreFilename.h" - -#include "MemLeakFindOn.h" - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t) -// Purpose: Reverse a patch, to build a new patch from new to old files. Takes -// two independent copies to the From file, for efficiency. -// Created: 12/7/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent) -{ - // Read and copy the header from the from file to the out file -- beginnings of the patch - file_StreamFormat hdr; - if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - // Copy - rOut.Write(&hdr, sizeof(hdr)); - // Copy over filename and attributes - // BLOCK - { - BackupStoreFilename filename; - filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite); - filename.WriteToStream(rOut); - StreamableMemBlock attr; - attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite); - attr.WriteToStream(rOut); - } - - // Build an index of common blocks. - // For each block in the from file, we want to know it's index in the - // diff file. Allocate memory for this information. - int64_t fromNumBlocks = box_ntoh64(hdr.mNumBlocks); - int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t)); - if(pfromIndexInfo == 0) - { - throw std::bad_alloc(); - } - - // Buffer data - void *buffer = 0; - int bufferSize = 0; - - // flag - bool isCompletelyDifferent = true; - - try - { - // Initialise the index to be all 0, ie not filled in yet - for(int64_t i = 0; i < fromNumBlocks; ++i) - { - pfromIndexInfo[i] = 0; - } - - // Within the from file, skip to the index - MoveStreamPositionToBlockIndex(rDiff); - - // Read in header of index - file_BlockIndexHeader diffIdxHdr; - if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // And then read in each entry - int64_t diffNumBlocks = box_ntoh64(diffIdxHdr.mNumBlocks); - for(int64_t b = 0; b < diffNumBlocks; ++b) - { - file_BlockIndexEntry e; - if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Where's the block? - int64_t blockEn = box_ntoh64(e.mEncodedSize); - if(blockEn > 0) - { - // Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file - } - else - { - // Block is in the original file, store which block it is in this file - int64_t fromIndex = 0 - blockEn; - if(fromIndex < 0 || fromIndex >= fromNumBlocks) - { - THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles) - } - - // Store information about where it is in the new file - // NOTE: This is slight different to how it'll be stored in the final index. - pfromIndexInfo[fromIndex] = -1 - b; - } - } - - // Open the index for the second copy of the from file - MoveStreamPositionToBlockIndex(rFrom2); - - // Read in header of index - file_BlockIndexHeader fromIdxHdr; - if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 - || box_ntoh64(fromIdxHdr.mOtherFileID) != 0) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // So, we can now start building the data in the file - int64_t filePosition = rFrom.GetPosition(); - for(int64_t b = 0; b < fromNumBlocks; ++b) - { - // Read entry from from index - file_BlockIndexEntry e; - if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Get size - int64_t blockSize = box_hton64(e.mEncodedSize); - if(blockSize < 0) - { - THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) - } - - // Copy this block? - if(pfromIndexInfo[b] == 0) - { - // Copy it, first move to file location - rFrom.Seek(filePosition, IOStream::SeekType_Absolute); - - // Make sure there's memory available to copy this - if(bufferSize < blockSize || buffer == 0) - { - // Free old block - if(buffer != 0) - { - ::free(buffer); - buffer = 0; - bufferSize = 0; - } - // Allocate new block - buffer = ::malloc(blockSize); - if(buffer == 0) - { - throw std::bad_alloc(); - } - bufferSize = blockSize; - } - ASSERT(bufferSize >= blockSize); - - // Copy the block - if(!rFrom.ReadFullBuffer(buffer, blockSize, 0)) - { - THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine) - } - rOut.Write(buffer, blockSize); - - // Store the size - pfromIndexInfo[b] = blockSize; - } - else - { - // Block isn't needed, so it's not completely different - isCompletelyDifferent = false; - } - filePosition += blockSize; - } - - // Then write the index, modified header first - fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(box_hton64(ObjectIDOfFrom)); - rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr)); - - // Move to start of index entries - rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute); - - // Then copy modified entries - for(int64_t b = 0; b < fromNumBlocks; ++b) - { - // Read entry from from index - file_BlockIndexEntry e; - if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // Modify... - int64_t s = pfromIndexInfo[b]; - // Adjust to reflect real block index (remember 0 has a different meaning here) - if(s < 0) ++s; - // Insert - e.mEncodedSize = box_hton64(s); - // Write - rOut.Write(&e, sizeof(e)); - } - } - catch(...) - { - ::free(pfromIndexInfo); - if(buffer != 0) - { - ::free(buffer); - } - throw; - } - - // Free memory used (oh for finally {} blocks) - ::free(pfromIndexInfo); - if(buffer != 0) - { - ::free(buffer); - } - - // return completely different flag - if(pIsCompletelyDifferent != 0) - { - *pIsCompletelyDifferent = isCompletelyDifferent; - } -} - - - diff --git a/lib/backupclient/BackupStoreFileWire.h b/lib/backupclient/BackupStoreFileWire.h deleted file mode 100644 index 49e94aa5..00000000 --- a/lib/backupclient/BackupStoreFileWire.h +++ /dev/null @@ -1,74 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFileWire.h -// Purpose: On the wire / disc formats for backup store files -// Created: 12/1/04 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILEWIRE__H -#define BACKUPSTOREFILEWIRE__H - -#include "MD5Digest.h" - -// set packing to one byte -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "BeginStructPackForWire.h" -#else -BEGIN_STRUCTURE_PACKING_FOR_WIRE -#endif - -typedef struct -{ - int32_t mMagicValue; // also the version number - int64_t mNumBlocks; // number of blocks contained in the file - int64_t mContainerID; - int64_t mModificationTime; - int32_t mMaxBlockClearSize; // Maximum clear size that can be expected for a block - int32_t mOptions; // bitmask of options used - // Then a BackupStoreFilename - // Then a BackupClientFileAttributes -} file_StreamFormat; - -typedef struct -{ - int32_t mMagicValue; // different magic value - int64_t mOtherFileID; // the file ID of the 'other' file which may be referenced by the index - uint64_t mEntryIVBase; // base value for block IV - int64_t mNumBlocks; // repeat of value in file header -} file_BlockIndexHeader; - -typedef struct -{ - int32_t mSize; // size in clear - uint32_t mWeakChecksum; // weak, rolling checksum - uint8_t mStrongChecksum[MD5Digest::DigestLength]; // strong digest based checksum -} file_BlockIndexEntryEnc; - -typedef struct -{ - union - { - int64_t mEncodedSize; // size encoded, if > 0 - int64_t mOtherBlockIndex; // 0 - block number in other file, if <= 0 - }; - uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)]; // Encoded section -} file_BlockIndexEntry; - -// Use default packing -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "EndStructPackForWire.h" -#else -END_STRUCTURE_PACKING_FOR_WIRE -#endif - -// header for blocks of compressed data in files -#define HEADER_CHUNK_IS_COMPRESSED 1 // bit -#define HEADER_ENCODING_SHIFT 1 // shift value -#define HEADER_BLOWFISH_ENCODING 1 // value stored in bits 1 -- 7 -#define HEADER_AES_ENCODING 2 // value stored in bits 1 -- 7 - - -#endif // BACKUPSTOREFILEWIRE__H - diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp deleted file mode 100644 index 72cd1acd..00000000 --- a/lib/backupclient/BackupStoreFilename.cpp +++ /dev/null @@ -1,281 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFilename.cpp -// Purpose: Filename for the backup store -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#include "Box.h" -#include "BackupStoreFilename.h" -#include "Protocol.h" -#include "BackupStoreException.h" -#include "IOStream.h" -#include "Guards.h" - -#include "MemLeakFindOn.h" - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::BackupStoreFilename() -// Purpose: Default constructor -- creates an invalid filename -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilename::BackupStoreFilename() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &) -// Purpose: Copy constructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy) - : mEncryptedName(rToCopy.mEncryptedName) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::~BackupStoreFilename() -// Purpose: Destructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilename::~BackupStoreFilename() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::CheckValid(bool) -// Purpose: Checks the encoded filename for validity -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const -{ - bool ok = true; - - if(mEncryptedName.size() < 2) - { - // Isn't long enough to have a header - ok = false; - } - else - { - // Check size is consistent - unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(this->mEncryptedName); - if(dsize != mEncryptedName.size()) - { - ok = false; - } - - // And encoding is an accepted value - unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName); - if(encoding < Encoding_Min || encoding > Encoding_Max) - { - ok = false; - } - } - - // Exception? - if(!ok && ExceptionIfInvalid) - { - THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename) - } - - return ok; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::ReadFromProtocol(Protocol &) -// Purpose: Reads the filename from the protocol object -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol) -{ - // Read the header - char hdr[2]; - rProtocol.Read(hdr, 2); - - // How big is it? - int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); - - // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us - std::string data; - rProtocol.Read(data, dsize - 2); - - // assign to this string, storing the header and the extra data - mEncryptedName.assign(hdr, 2); - mEncryptedName.append(data.c_str(), data.size()); - - // Check it - CheckValid(); - - // Alert derived classes - EncodedFilenameChanged(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::WriteToProtocol(Protocol &) -// Purpose: Writes the filename to the protocol object -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const -{ - CheckValid(); - - rProtocol.Write(mEncryptedName.c_str(), mEncryptedName.size()); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::ReadFromStream(IOStream &) -// Purpose: Reads the filename from a stream -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout) -{ - // Read the header - char hdr[2]; - if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - - // How big is it? - unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr); - - // Assume most filenames are small - char buf[256]; - if(dsize < sizeof(buf)) - { - // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us - if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - // Copy in header - buf[0] = hdr[0]; buf[1] = hdr[1]; - - // assign to this string, storing the header and the extra data - mEncryptedName.assign(buf, dsize); - } - else - { - // Block of memory to hold it - MemoryBlockGuard<char*> dataB(dsize+2); - char *data = dataB; - - // Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us - if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout)) - { - THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - // Copy in header - data[0] = hdr[0]; data[1] = hdr[1]; - - // assign to this string, storing the header and the extra data - mEncryptedName.assign(data, dsize); - } - - // Check it - CheckValid(); - - // Alert derived classes - EncodedFilenameChanged(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::WriteToStream(IOStream &) -// Purpose: Writes the filename to a stream -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::WriteToStream(IOStream &rStream) const -{ - CheckValid(); - - rStream.Write(mEncryptedName.c_str(), mEncryptedName.size()); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::EncodedFilenameChanged() -// Purpose: The encoded filename stored has changed -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::EncodedFilenameChanged() -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::IsEncrypted() -// Purpose: Returns true if the filename is stored using an encrypting encoding -// Created: 1/12/03 -// -// -------------------------------------------------------------------------- -bool BackupStoreFilename::IsEncrypted() const -{ - return BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName) != - Encoding_Clear; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilename::SetAsClearFilename(const char *) -// Purpose: Sets this object to be a valid filename, but with a -// filename in the clear. Used on the server to create -// filenames when there's no way of encrypting it. -// Created: 22/4/04 -// -// -------------------------------------------------------------------------- -void BackupStoreFilename::SetAsClearFilename(const char *Clear) -{ - // Make std::string from the clear name - std::string toEncode(Clear); - - // Make an encoded string - char hdr[2]; - BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear); - std::string encoded(hdr, 2); - encoded += toEncode; - ASSERT(encoded.size() == toEncode.size() + 2); - - // Store the encoded string - mEncryptedName.assign(encoded); - - // Stuff which must be done - EncodedFilenameChanged(); - CheckValid(false); -} - - - diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h deleted file mode 100644 index 80db9516..00000000 --- a/lib/backupclient/BackupStoreFilename.h +++ /dev/null @@ -1,107 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFilename.h -// Purpose: Filename for the backup store -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILENAME__H -#define BACKUPSTOREFILENAME__H - -#include <string> - -class Protocol; -class IOStream; - -// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE -// don't define this -- the problem of memory usage still appears without this. -// It's just that this class really showed up the problem. Instead, malloc allocation -// is globally defined in BoxPlatform.h, for troublesome libraries. - -#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE - // Use a malloc_allocated string, because the STL default allocators really screw up with - // memory allocation, particularly with this class. - // Makes a few things a bit messy and inefficient with conversions. - // Given up using this, and use global malloc allocation instead, but thought it - // worth leaving this code in just in case it's useful for the future. - typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base; - // If this is changed, change GetClearFilename() back to returning a reference. -#else - typedef std::string BackupStoreFilename_base; -#endif - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupStoreFilename -// Purpose: Filename for the backup store -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -class BackupStoreFilename /* : public BackupStoreFilename_base */ -{ -private: - std::string mEncryptedName; - -public: - BackupStoreFilename(); - BackupStoreFilename(const BackupStoreFilename &rToCopy); - virtual ~BackupStoreFilename(); - - bool CheckValid(bool ExceptionIfInvalid = true) const; - - void ReadFromProtocol(Protocol &rProtocol); - void WriteToProtocol(Protocol &rProtocol) const; - - void ReadFromStream(IOStream &rStream, int Timeout); - void WriteToStream(IOStream &rStream) const; - - void SetAsClearFilename(const char *Clear); - - // Check that it's encrypted - bool IsEncrypted() const; - - // These enumerated types belong in the base class so - // the CheckValid() function can make sure that the encoding - // is a valid encoding - enum - { - Encoding_Min = 1, - Encoding_Clear = 1, - Encoding_Blowfish = 2, - Encoding_Max = 2 - }; - - const std::string& GetEncodedFilename() const - { - return mEncryptedName; - } - - bool operator==(const BackupStoreFilename& rOther) const - { - return mEncryptedName == rOther.mEncryptedName; - } - - bool operator!=(const BackupStoreFilename& rOther) const - { - return mEncryptedName != rOther.mEncryptedName; - } - -protected: - virtual void EncodedFilenameChanged(); - void SetEncodedFilename(const std::string &rEncoded) - { - mEncryptedName = rEncoded; - } -}; - -// On the wire utilities for class and derived class -#define BACKUPSTOREFILENAME_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2) -#define BACKUPSTOREFILENAME_GET_ENCODING(hdr) (((hdr)[0]) & 0x3) - -#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding) {uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;} - -#endif // BACKUPSTOREFILENAME__H - diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp deleted file mode 100644 index e529d8d3..00000000 --- a/lib/backupclient/BackupStoreFilenameClear.cpp +++ /dev/null @@ -1,335 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFilenameClear.cpp -// Purpose: BackupStoreFilenames in the clear -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#include "Box.h" -#include "BackupStoreFilenameClear.h" -#include "BackupStoreException.h" -#include "CipherContext.h" -#include "CipherBlowfish.h" -#include "Guards.h" -#include "Logging.h" - -#include "MemLeakFindOn.h" - -// Hide private variables from the rest of the world -namespace -{ - int sEncodeMethod = BackupStoreFilename::Encoding_Clear; - CipherContext sBlowfishEncrypt; - CipherContext sBlowfishDecrypt; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::BackupStoreFilenameClear() -// Purpose: Default constructor, creates an invalid filename -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilenameClear::BackupStoreFilenameClear() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &) -// Purpose: Creates a filename, encoding from the given string -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode) -{ - SetClearFilename(rToEncode); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &) -// Purpose: Copy constructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy) - : BackupStoreFilename(rToCopy), - mClearFilename(rToCopy.mClearFilename) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) -// Purpose: Copy from base class -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy) - : BackupStoreFilename(rToCopy) -{ - // Will get a clear filename when it's required -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::~BackupStoreFilenameClear() -// Purpose: Destructor -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -BackupStoreFilenameClear::~BackupStoreFilenameClear() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::GetClearFilename() -// Purpose: Get the unencoded filename -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE -const std::string BackupStoreFilenameClear::GetClearFilename() const -{ - MakeClearAvailable(); - // When modifying, remember to change back to reference return if at all possible - // -- returns an object rather than a reference to allow easy use with other code. - return std::string(mClearFilename.c_str(), mClearFilename.size()); -} -#else -const std::string &BackupStoreFilenameClear::GetClearFilename() const -{ - MakeClearAvailable(); - return mClearFilename; -} -#endif - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::SetClearFilename(const std::string &) -// Purpose: Encode and make available the clear filename -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode) -{ - // Only allow Blowfish encodings - if(sEncodeMethod != Encoding_Blowfish) - { - THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup) - } - - // Make an encoded string with blowfish encryption - EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish); - - // Store the clear filename - mClearFilename.assign(rToEncode.c_str(), rToEncode.size()); - - // Make sure we did the right thing - if(!CheckValid(false)) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::MakeClearAvailable() -// Purpose: Private. Make sure the clear filename is available -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::MakeClearAvailable() const -{ - if(!mClearFilename.empty()) - return; // nothing to do - - // Check valid - CheckValid(); - - // Decode the header - int size = BACKUPSTOREFILENAME_GET_SIZE(GetEncodedFilename()); - int encoding = BACKUPSTOREFILENAME_GET_ENCODING(GetEncodedFilename()); - - // Decode based on encoding given in the header - switch(encoding) - { - case Encoding_Clear: - BOX_TRACE("**** BackupStoreFilename encoded with " - "Clear encoding ****"); - mClearFilename.assign(GetEncodedFilename().c_str() + 2, - size - 2); - break; - - case Encoding_Blowfish: - DecryptEncoded(sBlowfishDecrypt); - break; - - default: - THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding) - break; - } -} - - -// Buffer for encoding and decoding -- do this all in one single buffer to -// avoid lots of string allocation, which stuffs up memory usage. -// These static memory vars are, of course, not thread safe, but we don't use threads. -static int sEncDecBufferSize = 0; -static MemoryBlockGuard<uint8_t *> *spEncDecBuffer = 0; - -static void EnsureEncDecBufferSize(int BufSize) -{ - if(spEncDecBuffer == 0) - { -#ifndef WIN32 - BOX_TRACE("Allocating filename encoding/decoding buffer " - "with size " << BufSize); -#endif - spEncDecBuffer = new MemoryBlockGuard<uint8_t *>(BufSize); - MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer); - MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer); - sEncDecBufferSize = BufSize; - } - else - { - if(sEncDecBufferSize < BufSize) - { - BOX_TRACE("Reallocating filename encoding/decoding " - "buffer from " << sEncDecBufferSize << - " to " << BufSize); - spEncDecBuffer->Resize(BufSize); - sEncDecBufferSize = BufSize; - MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer); - } - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int) -// Purpose: Private. Assigns the encoded filename string, encrypting. -// Created: 1/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding) -{ - // Work out max size - int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4; - - // Make sure encode/decode buffer has enough space - EnsureEncDecBufferSize(maxOutSize); - - // Pointer to buffer - uint8_t *buffer = *spEncDecBuffer; - - // Encode -- do entire block in one go - int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size()); - // and add in header size - encSize += 2; - - // Adjust header - BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding); - - // Store the encoded string - SetEncodedFilename(std::string((char*)buffer, encSize)); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::DecryptEncoded(CipherContext &) -// Purpose: Decrypt the encoded filename using the cipher context -// Created: 1/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const -{ - const std::string& rEncoded = GetEncodedFilename(); - - // Work out max size - int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rEncoded.size()) + 4; - - // Make sure encode/decode buffer has enough space - EnsureEncDecBufferSize(maxOutSize); - - // Pointer to buffer - uint8_t *buffer = *spEncDecBuffer; - - // Decrypt - const char *str = rEncoded.c_str() + 2; - int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, rEncoded.size() - 2); - - // Assign to this - mClearFilename.assign((char*)buffer, sizeOut); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::EncodedFilenameChanged() -// Purpose: The encoded filename stored has changed -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::EncodedFilenameChanged() -{ - BackupStoreFilename::EncodedFilenameChanged(); - - // Delete stored filename in clear - mClearFilename.erase(); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::SetBlowfishKey(const void *, int) -// Purpose: Set the key used for Blowfish encryption of filenames -// Created: 1/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength) -{ - // Initialisation vector not used. Can't use a different vector for each filename as - // that would stop comparisions on the server working. - sBlowfishEncrypt.Reset(); - sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); - ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength); - sBlowfishEncrypt.SetIV(pIV); - sBlowfishDecrypt.Reset(); - sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength)); - ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength); - sBlowfishDecrypt.SetIV(pIV); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFilenameClear::SetEncodingMethod(int) -// Purpose: Set the encoding method used for filenames -// Created: 1/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreFilenameClear::SetEncodingMethod(int Method) -{ - sEncodeMethod = Method; -} - - - diff --git a/lib/backupclient/BackupStoreFilenameClear.h b/lib/backupclient/BackupStoreFilenameClear.h deleted file mode 100644 index d4c45701..00000000 --- a/lib/backupclient/BackupStoreFilenameClear.h +++ /dev/null @@ -1,60 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreFilenameClear.h -// Purpose: BackupStoreFilenames in the clear -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREFILENAMECLEAR__H -#define BACKUPSTOREFILENAMECLEAR__H - -#include "BackupStoreFilename.h" - -class CipherContext; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupStoreFilenameClear -// Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version -// Created: 2003/08/26 -// -// -------------------------------------------------------------------------- -class BackupStoreFilenameClear : public BackupStoreFilename -{ -public: - BackupStoreFilenameClear(); - BackupStoreFilenameClear(const std::string &rToEncode); - BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy); - BackupStoreFilenameClear(const BackupStoreFilename &rToCopy); - virtual ~BackupStoreFilenameClear(); - - // Because we need to use a different allocator for this class to avoid - // nasty things happening, can't return this as a reference. Which is a - // pity. But probably not too bad. -#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE - const std::string GetClearFilename() const; -#else - const std::string &GetClearFilename() const; -#endif - void SetClearFilename(const std::string &rToEncode); - - // Setup for encryption of filenames - static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength); - static void SetEncodingMethod(int Method); - -protected: - void MakeClearAvailable() const; - virtual void EncodedFilenameChanged(); - void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding); - void DecryptEncoded(CipherContext &rCipherContext) const; - -private: - mutable BackupStoreFilename_base mClearFilename; -}; - -#endif // BACKUPSTOREFILENAMECLEAR__H - - diff --git a/lib/backupclient/BackupStoreObjectMagic.h b/lib/backupclient/BackupStoreObjectMagic.h deleted file mode 100644 index 7ee600a2..00000000 --- a/lib/backupclient/BackupStoreObjectMagic.h +++ /dev/null @@ -1,31 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupStoreObjectMagic.h -// Purpose: Magic values for the start of objects in the backup store -// Created: 19/11/03 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPSTOREOBJECTMAGIC__H -#define BACKUPSTOREOBJECTMAGIC__H - -// Each of these values is the first 4 bytes of the object file. -// Remember to swap from network to host byte order. - -// Magic value for file streams -#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1 0x66696C65 -// Do not use v0 in any new code! -#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0 0x46494C45 - -// Magic for the block index at the file stream -- used to -// ensure streams are reordered as expected -#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478 -// Do not use v0 in any new code! -#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B - -// Magic value for directory streams -#define OBJECTMAGIC_DIR_MAGIC_VALUE 0x4449525F - -#endif // BACKUPSTOREOBJECTMAGIC__H - diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra deleted file mode 100644 index df3319df..00000000 --- a/lib/backupclient/Makefile.extra +++ /dev/null @@ -1,16 +0,0 @@ - -MAKEPROTOCOL = ../../lib/server/makeprotocol.pl - -GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt - -# AUTOGEN SEEDING -autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt - $(_PERL) $(GEN_CMD_SRV) - - -MAKEEXCEPTION = ../../lib/common/makeexception.pl - -# AUTOGEN SEEDING -autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt - $(_PERL) $(MAKEEXCEPTION) BackupStoreException.txt - diff --git a/lib/backupclient/RunStatusProvider.h b/lib/backupclient/RunStatusProvider.h deleted file mode 100644 index 89f361ca..00000000 --- a/lib/backupclient/RunStatusProvider.h +++ /dev/null @@ -1,29 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: RunStatusProvider.h -// Purpose: Declares the RunStatusProvider interface. -// Created: 2008/08/14 -// -// -------------------------------------------------------------------------- - -#ifndef RUNSTATUSPROVIDER__H -#define RUNSTATUSPROVIDER__H - -// -------------------------------------------------------------------------- -// -// Class -// Name: RunStatusProvider -// Purpose: Provides a StopRun() method which returns true if -// the current backup should be halted. -// Created: 2005/11/15 -// -// -------------------------------------------------------------------------- -class RunStatusProvider -{ - public: - virtual ~RunStatusProvider() { } - virtual bool StopRun() = 0; -}; - -#endif // RUNSTATUSPROVIDER__H diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp index ec44de0e..201491a3 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.cpp +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -40,11 +40,7 @@ public: // Created: 2003/08/20 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *Filename) -======= BackupStoreAccountDatabase::BackupStoreAccountDatabase(const std::string& Filename) ->>>>>>> 0.12 : pImpl(new _BackupStoreAccountDatabase) { pImpl->mFilename = Filename; @@ -127,11 +123,7 @@ BackupStoreAccountDatabase::Entry::~Entry() // Created: 2003/08/21 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -std::auto_ptr<BackupStoreAccountDatabase> BackupStoreAccountDatabase::Read(const char *Filename) -======= std::auto_ptr<BackupStoreAccountDatabase> BackupStoreAccountDatabase::Read(const std::string& Filename) ->>>>>>> 0.12 { // Database object to use std::auto_ptr<BackupStoreAccountDatabase> db(new BackupStoreAccountDatabase(Filename)); diff --git a/lib/backupstore/BackupStoreAccountDatabase.h b/lib/backupstore/BackupStoreAccountDatabase.h index cb19b01b..f9665c7d 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.h +++ b/lib/backupstore/BackupStoreAccountDatabase.h @@ -31,19 +31,11 @@ public: friend class _BackupStoreAccountDatabase; // to stop compiler warnings ~BackupStoreAccountDatabase(); private: -<<<<<<< HEAD - BackupStoreAccountDatabase(const char *Filename); - BackupStoreAccountDatabase(const BackupStoreAccountDatabase &); -public: - - static std::auto_ptr<BackupStoreAccountDatabase> Read(const char *Filename); -======= BackupStoreAccountDatabase(const std::string& Filename); BackupStoreAccountDatabase(const BackupStoreAccountDatabase &); public: static std::auto_ptr<BackupStoreAccountDatabase> Read(const std::string& Filename); ->>>>>>> 0.12 void Write(); class Entry diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp index 8cb23a6a..18500fc1 100644 --- a/lib/backupstore/BackupStoreAccounts.cpp +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -11,16 +11,6 @@ #include <stdio.h> -<<<<<<< HEAD -#include "BoxPortsAndFiles.h" -#include "BackupStoreAccounts.h" -#include "BackupStoreAccountDatabase.h" -#include "BackupStoreRefCountDatabase.h" -#include "RaidFileWrite.h" -#include "BackupStoreInfo.h" -#include "BackupStoreDirectory.h" -#include "BackupStoreConstants.h" -======= #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreConstants.h" @@ -31,7 +21,6 @@ #include "BoxPortsAndFiles.h" #include "RaidFileWrite.h" #include "StoreStructure.h" ->>>>>>> 0.12 #include "UnixUser.h" #include "MemLeakFindOn.h" @@ -68,13 +57,8 @@ BackupStoreAccounts::~BackupStoreAccounts() // Function // Name: BackupStoreAccounts::Create(int32_t, int, int64_t, int64_t, const std::string &) // Purpose: Create a new account on the specified disc set. -<<<<<<< HEAD -// If rAsUsername is not empty, then the account information will be written under the -// username specified. -======= // If rAsUsername is not empty, then the account information will be written under the // username specified. ->>>>>>> 0.12 // Created: 2003/08/21 // // -------------------------------------------------------------------------- @@ -120,10 +104,7 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, dirName, DiscSet, false /* ReadWrite */)); info->ChangeBlocksUsed(rootDirSize); info->ChangeBlocksInDirectories(rootDirSize); -<<<<<<< HEAD -======= info->AdjustNumDirectories(1); ->>>>>>> 0.12 // Save it back info->Save(); @@ -189,9 +170,6 @@ bool BackupStoreAccounts::AccountExists(int32_t ID) return mrDatabase.EntryExists(ID); } -<<<<<<< HEAD - -======= void BackupStoreAccounts::LockAccount(int32_t ID, NamedLock& rNamedLock) { const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID)); @@ -223,4 +201,3 @@ void BackupStoreAccounts::LockAccount(int32_t ID, NamedLock& rNamedLock) "lock on account " << ID); } } ->>>>>>> 0.12 diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h index e0e420bb..3163f15c 100644 --- a/lib/backupstore/BackupStoreAccounts.h +++ b/lib/backupstore/BackupStoreAccounts.h @@ -13,10 +13,7 @@ #include <string> #include "BackupStoreAccountDatabase.h" -<<<<<<< HEAD -======= #include "NamedLock.h" ->>>>>>> 0.12 // -------------------------------------------------------------------------- // @@ -35,12 +32,8 @@ private: BackupStoreAccounts(const BackupStoreAccounts &rToCopy); public: -<<<<<<< HEAD - void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername); -======= void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername); ->>>>>>> 0.12 bool AccountExists(int32_t ID); void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const; @@ -49,10 +42,7 @@ public: { return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet()); } -<<<<<<< HEAD -======= void LockAccount(int32_t ID, NamedLock& rNamedLock); ->>>>>>> 0.12 private: static std::string MakeAccountRootDir(int32_t ID, int DiscSet); diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index 79a61a77..f2302337 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -11,19 +11,6 @@ #include <stdio.h> #include <string.h> -<<<<<<< HEAD -#include <unistd.h> - -#include "BackupStoreCheck.h" -#include "StoreStructure.h" -#include "RaidFileRead.h" -#include "RaidFileWrite.h" -#include "autogen_BackupStoreException.h" -#include "BackupStoreObjectMagic.h" -#include "BackupStoreFile.h" -#include "BackupStoreDirectory.h" -#include "BackupStoreConstants.h" -======= #ifdef HAVE_UNISTD_H # include <unistd.h> @@ -42,7 +29,6 @@ #include "RaidFileWrite.h" #include "StoreStructure.h" #include "Utils.h" ->>>>>>> 0.12 #include "MemLeakFindOn.h" @@ -68,11 +54,6 @@ BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNum mLostDirNameSerial(0), mLostAndFoundDirectoryID(0), mBlocksUsed(0), -<<<<<<< HEAD - mBlocksInOldFiles(0), - mBlocksInDeletedFiles(0), - mBlocksInDirectories(0) -======= mBlocksInCurrentFiles(0), mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), @@ -81,7 +62,6 @@ BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNum mNumOldFiles(0), mNumDeletedFiles(0), mNumDirectories(0) ->>>>>>> 0.12 { } @@ -105,51 +85,16 @@ BackupStoreCheck::~BackupStoreCheck() // // Function // Name: BackupStoreCheck::Check() -<<<<<<< HEAD -// Purpose: Perform the check on the given account -======= // Purpose: Perform the check on the given account. You need to // hold a lock on the account before calling this! ->>>>>>> 0.12 // Created: 21/4/04 // // -------------------------------------------------------------------------- void BackupStoreCheck::Check() { -<<<<<<< HEAD - // Lock the account - { - std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); - - bool gotLock = false; - int triesLeft = 8; - do - { - gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */); - - if(!gotLock) - { - --triesLeft; - ::sleep(1); - } - } while(!gotLock && triesLeft > 0); - - if(!gotLock) - { - // Couldn't lock the account -- just stop now - if(!mQuiet) - { - BOX_ERROR("Failed to lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server."); - } - THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount) - } - } -======= std::string writeLockFilename; StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); ASSERT(FileExists(writeLockFilename)); ->>>>>>> 0.12 if(!mQuiet && mFixErrors) { @@ -339,8 +284,6 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const // Read in all the directories, and recurse downwards { -<<<<<<< HEAD -======= // If any of the directories is missing, create it. RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); @@ -368,7 +311,6 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const } } ->>>>>>> 0.12 std::vector<std::string> dirs; RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName, RaidFileRead::DirReadType_DirsOnly, dirs); @@ -391,11 +333,7 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const } else { -<<<<<<< HEAD - BOX_WARNING("Spurious or invalid directory " << -======= BOX_ERROR("Spurious or invalid directory " << ->>>>>>> 0.12 rDirName << DIRECTORY_SEPARATOR << (*i) << " found, " << (mFixErrors?"deleting":"delete manually")); @@ -412,12 +350,8 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const // // Function // Name: BackupStoreCheck::CheckObjectsDir(int64_t) -<<<<<<< HEAD -// Purpose: Check all the files within this directory which has the given starting ID. -======= // Purpose: Check all the files within this directory which has // the given starting ID. ->>>>>>> 0.12 // Created: 22/4/04 // // -------------------------------------------------------------------------- @@ -465,15 +399,6 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) // Filename is valid, mark as existing idsPresent[n] = true; } -<<<<<<< HEAD - else - { - // info file in root dir is OK! - if(StartID != 0 || ::strcmp("info", (*i).c_str()) != 0) - { - fileOK = false; - } -======= // No other files should be present in subdirectories else if(StartID != 0) { @@ -487,17 +412,12 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) else { fileOK = false; ->>>>>>> 0.12 } if(!fileOK) { // Unexpected or bad file, delete it -<<<<<<< HEAD - BOX_WARNING("Spurious file " << dirName << -======= BOX_ERROR("Spurious file " << dirName << ->>>>>>> 0.12 DIRECTORY_SEPARATOR << (*i) << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; @@ -520,11 +440,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) if(!CheckAndAddObject(StartID | i, dirName + leaf)) { // File was bad, delete it -<<<<<<< HEAD - BOX_WARNING("Corrupted file " << dirName << -======= BOX_ERROR("Corrupted file " << dirName << ->>>>>>> 0.12 leaf << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; @@ -542,15 +458,6 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) // -------------------------------------------------------------------------- // // Function -<<<<<<< HEAD -// Name: BackupStoreCheck::CheckAndAddObject(int64_t, const std::string &) -// Purpose: Check a specific object and add it to the list if it's OK -- if -// there are any errors with the reading, return false and it'll be deleted. -// Created: 21/4/04 -// -// -------------------------------------------------------------------------- -bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename) -======= // Name: BackupStoreCheck::CheckAndAddObject(int64_t, // const std::string &) // Purpose: Check a specific object and add it to the list @@ -561,7 +468,6 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF // -------------------------------------------------------------------------- bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename) ->>>>>>> 0.12 { // Info on object... bool isFile = true; @@ -571,19 +477,12 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, try { // Open file -<<<<<<< HEAD - std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, rFilename)); - size = file->GetDiscUsageInBlocks(); - - // Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile -======= std::auto_ptr<RaidFileRead> file( RaidFileRead::Open(mDiscSetNumber, rFilename)); size = file->GetDiscUsageInBlocks(); // Read in first four bytes -- don't have to worry about // retrying if not all bytes read as is RaidFile ->>>>>>> 0.12 uint32_t signature; if(file->Read(&signature, sizeof(signature)) != sizeof(signature)) { @@ -614,16 +513,6 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, return false; break; } -<<<<<<< HEAD - - // Add to usage counts - mBlocksUsed += size; - if(!isFile) - { - mBlocksInDirectories += size; - } -======= ->>>>>>> 0.12 } catch(...) { @@ -636,12 +525,6 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, { return false; } -<<<<<<< HEAD - - // Add to list of IDs known about - AddID(ObjectID, containerID, size, isFile); - -======= // Add to list of IDs known about AddID(ObjectID, containerID, size, isFile); @@ -693,7 +576,6 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, } } ->>>>>>> 0.12 // Report success return true; } @@ -703,23 +585,15 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, // // Function // Name: BackupStoreCheck::CheckFile(int64_t, IOStream &) -<<<<<<< HEAD -// Purpose: Do check on file, return original container ID if OK, or -1 on error -======= // Purpose: Do check on file, return original container ID // if OK, or -1 on error ->>>>>>> 0.12 // Created: 22/4/04 // // -------------------------------------------------------------------------- int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) { -<<<<<<< HEAD - // Check that it's not the root directory ID. Having a file as the root directory would be bad. -======= // Check that it's not the root directory ID. Having a file as // the root directory would be bad. ->>>>>>> 0.12 if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID) { // Get that dodgy thing deleted! @@ -729,12 +603,8 @@ int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) // Check the format of the file, and obtain the container ID int64_t originalContainerID = -1; -<<<<<<< HEAD - if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */, -======= if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */, ->>>>>>> 0.12 &originalContainerID)) { // Didn't verify @@ -749,12 +619,8 @@ int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) // // Function // Name: BackupStoreCheck::CheckDirInitial(int64_t, IOStream &) -<<<<<<< HEAD -// Purpose: Do initial check on directory, return container ID if OK, or -1 on error -======= // Purpose: Do initial check on directory, return container ID // if OK, or -1 on error ->>>>>>> 0.12 // Created: 22/4/04 // // -------------------------------------------------------------------------- @@ -793,16 +659,12 @@ void BackupStoreCheck::CheckDirectories() // This phase will check all the files in the directories, make // a note of all directories which are missing, and do initial fixing. -<<<<<<< HEAD - // Scan all objects -======= // The root directory is not contained inside another directory, so // it has no directory entry to scan, but we have to count it // somewhere, so we'll count it here. mNumDirectories++; // Scan all objects. ->>>>>>> 0.12 for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) { IDBlock *pblock = i->second; @@ -823,24 +685,6 @@ void BackupStoreCheck::CheckDirectories() } // Flag for modifications -<<<<<<< HEAD - bool isModified = false; - - // Check for validity - if(dir.CheckAndFix()) - { - // Wasn't quite right, and has been modified - BOX_WARNING("Directory ID " << - BOX_FORMAT_OBJECTID(pblock->mID[e]) << - " has bad structure"); - ++mNumberErrorsFound; - isModified = true; - } - - // Go through, and check that everything in that directory exists and is valid - std::vector<int64_t> toDelete; - -======= bool isModified = CheckDirectory(dir); // Check the directory again, now that entries have been removed @@ -871,157 +715,10 @@ void BackupStoreCheck::CheckDirectories() } // Count valid entries ->>>>>>> 0.12 BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { -<<<<<<< HEAD - // Lookup the item - int32_t iIndex; - IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); - bool badEntry = false; - if(piBlock != 0) - { - // Found. Get flags - uint8_t iflags = GetFlags(piBlock, iIndex); - - // Is the type the same? - if(((iflags & Flags_IsDir) == Flags_IsDir) - != ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir)) - { - // Entry is of wrong type - BOX_WARNING("Directory ID " << - BOX_FORMAT_OBJECTID(pblock->mID[e]) << - " references object " << - BOX_FORMAT_OBJECTID(en->GetObjectID()) << - " which has a different type than expected."); - badEntry = true; - } - else - { - // Check that the entry is not already contained. - if(iflags & Flags_IsContained) - { - BOX_WARNING("Directory ID " << - BOX_FORMAT_OBJECTID(pblock->mID[e]) << - " references object " << - BOX_FORMAT_OBJECTID(en->GetObjectID()) << - " which is already contained."); - badEntry = true; - } - else - { - // Not already contained -- mark as contained - SetFlags(piBlock, iIndex, iflags | Flags_IsContained); - - // Check that the container ID of the object is correct - if(piBlock->mContainer[iIndex] != pblock->mID[e]) - { - // Needs fixing... - if(iflags & Flags_IsDir) - { - // Add to will fix later list - BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " has wrong container ID."); - mDirsWithWrongContainerID.push_back(en->GetObjectID()); - } - else - { - // This is OK for files, they might move - BOX_WARNING("File ID " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " has different container ID, probably moved"); - } - - // Fix entry for now - piBlock->mContainer[iIndex] = pblock->mID[e]; - } - } - } - - // Check the object size, if it's OK and a file - if(!badEntry && !((iflags & Flags_IsDir) == Flags_IsDir)) - { - if(en->GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[iIndex]) - { - // Correct - en->SetSizeInBlocks(piBlock->mObjectSizeInBlocks[iIndex]); - // Mark as changed - isModified = true; - // Tell user - BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " has wrong size for object " << BOX_FORMAT_OBJECTID(en->GetObjectID())); - } - } - } - else - { - // Item can't be found. Is it a directory? - if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) - { - // Store the directory for later attention - mDirsWhichContainLostDirs[en->GetObjectID()] = pblock->mID[e]; - } - else - { - // Just remove the entry - badEntry = true; - BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " references object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " which does not exist."); - } - } - - // Is this entry worth keeping? - if(badEntry) - { - toDelete.push_back(en->GetObjectID()); - } - else - { - // Add to sizes? - if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) - { - mBlocksInOldFiles += en->GetSizeInBlocks(); - } - if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) - { - mBlocksInDeletedFiles += en->GetSizeInBlocks(); - } - } - } - - if(toDelete.size() > 0) - { - // Delete entries from directory - for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) - { - dir.DeleteEntry(*d); - } - - // Mark as modified - isModified = true; - - // Check the directory again, now that entries have been removed - dir.CheckAndFix(); - - // Errors found - ++mNumberErrorsFound; - } - - if(isModified && mFixErrors) - { - BOX_WARNING("Fixing directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e])); - - // Save back to disc - RaidFileWrite fixed(mDiscSetNumber, filename); - fixed.Open(true /* allow overwriting */); - dir.WriteToStream(fixed); - // Commit it - fixed.Commit(true /* convert to raid representation now */); - } - } - } - } - -} - -======= int32_t iIndex; IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); @@ -1254,5 +951,4 @@ bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, return !badEntry; } ->>>>>>> 0.12 diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h index cddd8a4d..178a873a 100644 --- a/lib/backupstore/BackupStoreCheck.h +++ b/lib/backupstore/BackupStoreCheck.h @@ -16,11 +16,8 @@ #include <set> #include "NamedLock.h" -<<<<<<< HEAD -======= #include "BackupStoreDirectory.h" ->>>>>>> 0.12 class IOStream; class BackupStoreFilename; @@ -31,14 +28,8 @@ The following problems can be fixed: * Spurious files deleted * Corrupted files deleted * Root ID as file, deleted -<<<<<<< HEAD - * Dirs with wrong object id inside, deleted - * Direcetory entries pointing to non-existant files, deleted - * Doubly references files have second reference deleted -======= * Dirs with wrong object id in header, deleted * Doubly referenced files have second reference deleted ->>>>>>> 0.12 * Wrong directory container IDs fixed * Missing root recreated * Reattach files which exist, but aren't referenced @@ -51,13 +42,9 @@ The following problems can be fixed: * Inside directories, - only one object per name has old version clear - IDs aren't duplicated -<<<<<<< HEAD - * Bad store info files regenerated -======= - entries pointing to non-existant files are deleted - patches depending on non-existent objects are deleted * Bad store info and refcount files regenerated ->>>>>>> 0.12 * Bad sizes of files in directories fixed */ @@ -98,13 +85,10 @@ public: void Check(); bool ErrorsFound() {return mNumberErrorsFound > 0;} -<<<<<<< HEAD -======= inline int64_t GetNumErrorsFound() { return mNumberErrorsFound; } ->>>>>>> 0.12 private: enum @@ -143,12 +127,9 @@ private: int64_t CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName); void CheckObjectsDir(int64_t StartID); bool CheckAndAddObject(int64_t ObjectID, const std::string &rFilename); -<<<<<<< HEAD -======= bool CheckDirectory(BackupStoreDirectory& dir); bool CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified); ->>>>>>> 0.12 int64_t CheckFile(int64_t ObjectID, IOStream &rStream); int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); @@ -185,19 +166,12 @@ private: #else #define DUMP_OBJECT_INFO #endif -<<<<<<< HEAD - -======= ->>>>>>> 0.12 private: std::string mStoreRoot; int mDiscSetNumber; int32_t mAccountID; -<<<<<<< HEAD -======= std::string mAccountName; ->>>>>>> 0.12 bool mFixErrors; bool mQuiet; @@ -216,12 +190,8 @@ private: // List of stuff to fix std::vector<BackupStoreCheck_ID_t> mDirsWithWrongContainerID; // This is a map of lost dir ID -> existing dir ID -<<<<<<< HEAD - std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t> mDirsWhichContainLostDirs; -======= std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t> mDirsWhichContainLostDirs; ->>>>>>> 0.12 // Set of extra directories added std::set<BackupStoreCheck_ID_t> mDirsAdded; @@ -232,11 +202,6 @@ private: // Usage int64_t mBlocksUsed; -<<<<<<< HEAD - int64_t mBlocksInOldFiles; - int64_t mBlocksInDeletedFiles; - int64_t mBlocksInDirectories; -======= int64_t mBlocksInCurrentFiles; int64_t mBlocksInOldFiles; int64_t mBlocksInDeletedFiles; @@ -245,7 +210,6 @@ private: int64_t mNumOldFiles; int64_t mNumDeletedFiles; int64_t mNumDirectories; ->>>>>>> 0.12 }; #endif // BACKUPSTORECHECK__H diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp index 341ac524..90e21e7f 100644 --- a/lib/backupstore/BackupStoreCheck2.cpp +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -12,19 +12,6 @@ #include <stdio.h> #include <string.h> -<<<<<<< HEAD -#include "BackupStoreCheck.h" -#include "StoreStructure.h" -#include "RaidFileRead.h" -#include "RaidFileWrite.h" -#include "autogen_BackupStoreException.h" -#include "BackupStoreObjectMagic.h" -#include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" -#include "BackupStoreDirectory.h" -#include "BackupStoreConstants.h" -#include "BackupStoreInfo.h" -======= #include "autogen_BackupStoreException.h" #include "BackupStoreCheck.h" #include "BackupStoreConstants.h" @@ -37,7 +24,6 @@ #include "RaidFileRead.h" #include "RaidFileWrite.h" #include "StoreStructure.h" ->>>>>>> 0.12 #include "MemLeakFindOn.h" @@ -152,11 +138,7 @@ void BackupStoreCheck::CheckUnattachedObjects() if((flags & Flags_IsContained) == 0) { // Unattached object... -<<<<<<< HEAD - BOX_WARNING("Object " << -======= BOX_ERROR("Object " << ->>>>>>> 0.12 BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached."); ++mNumberErrorsFound; @@ -176,10 +158,7 @@ void BackupStoreCheck::CheckUnattachedObjects() int64_t diffFromObjectID = 0; std::string filename; StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */); -<<<<<<< HEAD -======= ->>>>>>> 0.12 // The easiest way to do this is to verify it again. Not such a bad penalty, because // this really shouldn't be done very often. { @@ -405,11 +384,7 @@ void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, } // Add a new entry in an appropriate place -<<<<<<< HEAD - mDirectory.AddUnattactedObject(objectStoreFilename, modTime, -======= mDirectory.AddUnattachedObject(objectStoreFilename, modTime, ->>>>>>> 0.12 ObjectID, sizeInBlocks, IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); } @@ -600,19 +575,6 @@ void BackupStoreCheck::FixDirsWithLostDirs() void BackupStoreCheck::WriteNewStoreInfo() { // Attempt to load the existing store info file -<<<<<<< HEAD - std::auto_ptr<BackupStoreInfo> poldInfo; - try - { - poldInfo.reset(BackupStoreInfo::Load(mAccountID, mStoreRoot, mDiscSetNumber, true /* read only */).release()); - } - catch(...) - { - BOX_WARNING("Load of existing store info failed, regenerating."); - ++mNumberErrorsFound; - } - -======= std::auto_ptr<BackupStoreInfo> pOldInfo; try { @@ -630,19 +592,14 @@ void BackupStoreCheck::WriteNewStoreInfo() "deleted files: " << mNumDeletedFiles << "), " "directories: " << mNumDirectories); ->>>>>>> 0.12 // Minimum soft and hard limits int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024; int64_t minHard = ((minSoft * 11) / 10) + 1024; // Need to do anything? -<<<<<<< HEAD - if(poldInfo.get() != 0 && mNumberErrorsFound == 0 && poldInfo->GetAccountID() == mAccountID) -======= if(pOldInfo.get() != 0 && mNumberErrorsFound == 0 && pOldInfo->GetAccountID() == mAccountID) ->>>>>>> 0.12 { // Leave the store info as it is, no need to alter it because nothing really changed, // and the only essential thing was that the account ID was correct, which is was. @@ -654,23 +611,6 @@ void BackupStoreCheck::WriteNewStoreInfo() // Work out the new limits int64_t softLimit = minSoft; int64_t hardLimit = minHard; -<<<<<<< HEAD - if(poldInfo.get() != 0 && poldInfo->GetBlocksSoftLimit() > minSoft) - { - softLimit = poldInfo->GetBlocksSoftLimit(); - } - else - { - BOX_WARNING("Soft limit for account changed to ensure housekeeping doesn't delete files on next run."); - } - if(poldInfo.get() != 0 && poldInfo->GetBlocksHardLimit() > minHard) - { - hardLimit = poldInfo->GetBlocksHardLimit(); - } - else - { - BOX_WARNING("Hard limit for account changed to ensure housekeeping doesn't delete files on next run."); -======= if(pOldInfo.get() != 0 && pOldInfo->GetBlocksSoftLimit() > minSoft) { softLimit = pOldInfo->GetBlocksSoftLimit(); @@ -688,7 +628,6 @@ void BackupStoreCheck::WriteNewStoreInfo() { BOX_WARNING("Hard limit for account changed to ensure " "housekeeping doesn't delete files on next run."); ->>>>>>> 0.12 } // Object ID @@ -699,10 +638,6 @@ void BackupStoreCheck::WriteNewStoreInfo() } // Build a new store info -<<<<<<< HEAD - std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::CreateForRegeneration( - mAccountID, -======= std::auto_ptr<MemBlockStream> extra_data; if(pOldInfo.get()) { @@ -715,22 +650,15 @@ void BackupStoreCheck::WriteNewStoreInfo() std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::CreateForRegeneration( mAccountID, mAccountName, ->>>>>>> 0.12 mStoreRoot, mDiscSetNumber, lastObjID, mBlocksUsed, -<<<<<<< HEAD -======= mBlocksInCurrentFiles, ->>>>>>> 0.12 mBlocksInOldFiles, mBlocksInDeletedFiles, mBlocksInDirectories, softLimit, -<<<<<<< HEAD - hardLimit)); -======= hardLimit, (pOldInfo.get() ? pOldInfo->IsAccountEnabled() : true), *extra_data)); @@ -743,7 +671,6 @@ void BackupStoreCheck::WriteNewStoreInfo() { mNumberErrorsFound += info->ReportChangesTo(*pOldInfo); } ->>>>>>> 0.12 // Save to disc? if(mFixErrors) @@ -770,16 +697,12 @@ bool BackupStoreDirectory::CheckAndFix() bool changed = false; // Check that if a file depends on a new version, that version is in this directory -<<<<<<< HEAD - { -======= bool restart; do { restart = false; ->>>>>>> 0.12 std::vector<Entry*>::iterator i(mEntries.begin()); for(; i != mEntries.end(); ++i) { @@ -790,11 +713,7 @@ bool BackupStoreDirectory::CheckAndFix() if(newerEn == 0) { // Depends on something, but it isn't there. -<<<<<<< HEAD - BOX_TRACE("Entry id " << FMT_i << -======= BOX_WARNING("Entry id " << FMT_i << ->>>>>>> 0.12 " removed because depends " "on newer version " << FMT_OID(dependsNewer) << @@ -804,20 +723,12 @@ bool BackupStoreDirectory::CheckAndFix() delete *i; mEntries.erase(i); -<<<<<<< HEAD - // Start again at the beginning of the vector, the iterator is now invalid - i = mEntries.begin(); - - // Mark as changed - changed = true; -======= // Mark as changed changed = true; // Start again at the beginning of the vector, the iterator is now invalid restart = true; break; ->>>>>>> 0.12 } else { @@ -839,10 +750,7 @@ bool BackupStoreDirectory::CheckAndFix() } } } -<<<<<<< HEAD -======= while(restart); ->>>>>>> 0.12 // Check that if a file has a dependency marked, it exists, and remove it if it doesn't { @@ -967,11 +875,7 @@ bool BackupStoreDirectory::CheckAndFix() // erase the thing from the list Entry *pentry = (*i); mEntries.erase(i); -<<<<<<< HEAD - -======= ->>>>>>> 0.12 // And delete the entry object delete pentry; @@ -989,20 +893,12 @@ bool BackupStoreDirectory::CheckAndFix() // -------------------------------------------------------------------------- // // Function -<<<<<<< HEAD -// Name: BackupStoreDirectory::AddUnattactedObject(...) -======= // Name: BackupStoreDirectory::AddUnattachedObject(...) ->>>>>>> 0.12 // Purpose: Adds an object which is currently unattached. Assume that CheckAndFix() will be called afterwards. // Created: 22/4/04 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -void BackupStoreDirectory::AddUnattactedObject(const BackupStoreFilename &rName, -======= void BackupStoreDirectory::AddUnattachedObject(const BackupStoreFilename &rName, ->>>>>>> 0.12 box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags) { Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp index c89b5082..ec606d52 100644 --- a/lib/backupstore/BackupStoreCheckData.cpp +++ b/lib/backupstore/BackupStoreCheckData.cpp @@ -65,23 +65,11 @@ void BackupStoreCheck::AddID(BackupStoreCheck_ID_t ID, if(mpInfoLastBlock == 0 || mInfoLastBlockEntries >= BACKUPSTORECHECK_BLOCK_SIZE) { // No. Allocate a new one -<<<<<<< HEAD - IDBlock *pblk = (IDBlock*)::malloc(sizeof(IDBlock)); -======= IDBlock *pblk = (IDBlock*)calloc(1, sizeof(IDBlock)); ->>>>>>> 0.12 if(pblk == 0) { throw std::bad_alloc(); } -<<<<<<< HEAD - // Zero all the flags entries - for(int z = 0; z < (BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry); ++z) - { - pblk->mFlags[z] = 0; - } -======= ->>>>>>> 0.12 // Store in map mInfo[ID] = pblk; // Allocated and stored OK, setup for use @@ -148,13 +136,8 @@ BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, pblock = ib->second; } -<<<<<<< HEAD - ASSERT(pblock != 0); - if(pblock == 0) return 0; -======= if(pblock == 0) return 0; ASSERT(pblock != 0); ->>>>>>> 0.12 // How many entries are there in the block int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp index c2344634..921adfa4 100644 --- a/lib/backupstore/BackupStoreConfigVerify.cpp +++ b/lib/backupstore/BackupStoreConfigVerify.cpp @@ -38,17 +38,7 @@ static const ConfigurationVerifyKey verifyrootkeys[] = ConfigTest_Exists | ConfigTest_IsInt), ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), // make value "yes" to enable in config file -<<<<<<< HEAD - - #ifdef WIN32 - ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry) - #else - ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry, - BOX_FILE_RAIDFILE_DEFAULT_CONFIG) - #endif -======= ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry) ->>>>>>> 0.12 }; const ConfigurationVerify BackupConfigFileVerify = diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp index c58ae99a..b6714709 100644 --- a/lib/backupstore/BackupStoreInfo.cpp +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -11,10 +11,7 @@ #include <algorithm> -<<<<<<< HEAD -======= #include "Archive.h" ->>>>>>> 0.12 #include "BackupStoreInfo.h" #include "BackupStoreException.h" #include "RaidFileWrite.h" @@ -22,57 +19,12 @@ #include "MemLeakFindOn.h" -<<<<<<< HEAD -// set packing to one byte -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "BeginStructPackForWire.h" -#else -BEGIN_STRUCTURE_PACKING_FOR_WIRE -#endif - -// ****************** -// make sure the defaults in CreateNew are modified! -// ****************** -typedef struct -{ - int32_t mMagicValue; // also the version number - int32_t mAccountID; - int64_t mClientStoreMarker; - int64_t mLastObjectIDUsed; - int64_t mBlocksUsed; - int64_t mBlocksInOldFiles; - int64_t mBlocksInDeletedFiles; - int64_t mBlocksInDirectories; - int64_t mBlocksSoftLimit; - int64_t mBlocksHardLimit; - uint32_t mCurrentMarkNumber; - uint32_t mOptionsPresent; // bit mask of optional elements present - int64_t mNumberDeletedDirectories; - // Then loads of int64_t IDs for the deleted directories -} info_StreamFormat; - -#define INFO_MAGIC_VALUE 0x34832476 - -// Use default packing -#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#include "EndStructPackForWire.h" -#else -END_STRUCTURE_PACKING_FOR_WIRE -#endif - -======= ->>>>>>> 0.12 #ifdef BOX_RELEASE_BUILD #define NUM_DELETED_DIRS_BLOCK 256 #else #define NUM_DELETED_DIRS_BLOCK 2 #endif -<<<<<<< HEAD -#define INFO_FILENAME "info" - -======= ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Function @@ -89,10 +41,6 @@ BackupStoreInfo::BackupStoreInfo() mClientStoreMarker(0), mLastObjectIDUsed(-1), mBlocksUsed(0), -<<<<<<< HEAD - mBlocksInOldFiles(0), - mBlocksInDeletedFiles(0) -======= mBlocksInCurrentFiles(0), mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), @@ -102,7 +50,6 @@ BackupStoreInfo::BackupStoreInfo() mNumDeletedFiles(0), mNumDirectories(0), mAccountEnabled(true) ->>>>>>> 0.12 { } @@ -128,41 +75,6 @@ BackupStoreInfo::~BackupStoreInfo() // -------------------------------------------------------------------------- void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit) { -<<<<<<< HEAD - // Initial header (is entire file) - info_StreamFormat hdr = { - htonl(INFO_MAGIC_VALUE), // mMagicValue - htonl(AccountID), // mAccountID - 0, // mClientStoreMarker - box_hton64(1), // mLastObjectIDUsed (which is the root directory) - 0, // mBlocksUsed - 0, // mBlocksInOldFiles - 0, // mBlocksInDeletedFiles - 0, // mBlocksInDirectories - box_hton64(BlockSoftLimit), // mBlocksSoftLimit - box_hton64(BlockHardLimit), // mBlocksHardLimit - 0, // mCurrentMarkNumber - 0, // mOptionsPresent - 0 // mNumberDeletedDirectories - }; - - // Generate the filename - ASSERT(rRootDir[rRootDir.size() - 1] == '/' || - rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); - std::string fn(rRootDir + INFO_FILENAME); - - // Open the file for writing - RaidFileWrite rf(DiscSet, fn); - rf.Open(false); // no overwriting, as this is a new file - - // Write header - rf.Write(&hdr, sizeof(hdr)); - - // Commit it to disc, converting it to RAID now - rf.Commit(true); - - // Done. -======= BackupStoreInfo info; info.mAccountID = AccountID; info.mDiscSet = DiscSet; @@ -178,40 +90,11 @@ void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, info.mExtraData.SetForReading(); // extra data is empty in this case info.Save(false); ->>>>>>> 0.12 } // -------------------------------------------------------------------------- // // Function -<<<<<<< HEAD -// Name: BackupStoreInfo::Load(int32_t, const std::string &, int, bool) -// Purpose: Loads the info from disc, given the root information. Can be marked as read only. -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID) -{ - // Generate the filename - std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME); - - // Open the file for reading (passing on optional request for revision ID) - std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID)); - - // Read in a header - info_StreamFormat hdr; - if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) - { - THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) - } - - // Check it - if(ntohl(hdr.mMagicValue) != INFO_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != AccountID) - { - THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) - } - -======= // Name: BackupStoreInfo::Load(int32_t, const std::string &, // int, bool) // Purpose: Loads the info from disc, given the root @@ -255,7 +138,6 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, fn, BackupStoreException, BadStoreInfoOnLoad); } ->>>>>>> 0.12 // Make new object std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); @@ -264,23 +146,6 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, info->mDiscSet = DiscSet; info->mFilename = fn; info->mReadOnly = ReadOnly; -<<<<<<< HEAD - - // Insert info from file - info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker); - info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed); - info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); - info->mBlocksInOldFiles = box_ntoh64(hdr.mBlocksInOldFiles); - info->mBlocksInDeletedFiles = box_ntoh64(hdr.mBlocksInDeletedFiles); - info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories); - info->mBlocksSoftLimit = box_ntoh64(hdr.mBlocksSoftLimit); - info->mBlocksHardLimit = box_ntoh64(hdr.mBlocksHardLimit); - - // Load up array of deleted objects - int64_t numDelObj = box_ntoh64(hdr.mNumberDeletedDirectories); - - // Then load them in -======= int64_t numDelObj = 0; if (v1) @@ -348,7 +213,6 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, } // Then load the list of deleted directories ->>>>>>> 0.12 if(numDelObj > 0) { int64_t objs[NUM_DELETED_DIRS_BLOCK]; @@ -380,8 +244,6 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) } -<<<<<<< HEAD -======= if(v2) { @@ -402,7 +264,6 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, rf->CopyStreamTo(info->mExtraData); } info->mExtraData.SetForReading(); ->>>>>>> 0.12 // return it to caller return info; @@ -417,14 +278,6 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, // Created: 23/4/04 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration(int32_t AccountID, const std::string &rRootDir, - int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles, - int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit) -{ - // Generate the filename - std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME); -======= std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( int32_t AccountID, const std::string& rAccountName, const std::string &rRootDir, int DiscSet, @@ -436,23 +289,12 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( { // Generate the filename std::string fn(rRootDir + INFO_FILENAME); ->>>>>>> 0.12 // Make new object std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); // Put in basic info info->mAccountID = AccountID; -<<<<<<< HEAD - info->mDiscSet = DiscSet; - info->mFilename = fn; - info->mReadOnly = false; - - // Insert info starting info - info->mClientStoreMarker = 0; - info->mLastObjectIDUsed = LastObjectID; - info->mBlocksUsed = BlocksUsed; -======= info->mAccountName = rAccountName; info->mDiscSet = DiscSet; info->mFilename = fn; @@ -463,21 +305,16 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( info->mLastObjectIDUsed = LastObjectID; info->mBlocksUsed = BlocksUsed; info->mBlocksInCurrentFiles = BlocksInCurrentFiles; ->>>>>>> 0.12 info->mBlocksInOldFiles = BlocksInOldFiles; info->mBlocksInDeletedFiles = BlocksInDeletedFiles; info->mBlocksInDirectories = BlocksInDirectories; info->mBlocksSoftLimit = BlockSoftLimit; info->mBlocksHardLimit = BlockHardLimit; -<<<<<<< HEAD - -======= info->mAccountEnabled = AccountEnabled; ExtraData.CopyStreamTo(info->mExtraData); info->mExtraData.SetForReading(); ->>>>>>> 0.12 // return it to caller return info; } @@ -486,20 +323,12 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( // -------------------------------------------------------------------------- // // Function -<<<<<<< HEAD -// Name: BackupStoreInfo::Save() -======= // Name: BackupStoreInfo::Save(bool allowOverwrite) ->>>>>>> 0.12 // Purpose: Save modified info back to disc // Created: 2003/08/28 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -void BackupStoreInfo::Save() -======= void BackupStoreInfo::Save(bool allowOverwrite) ->>>>>>> 0.12 { // Make sure we're initialised (although should never come to this) if(mFilename.empty() || mAccountID == -1 || mDiscSet == -1) @@ -515,29 +344,6 @@ void BackupStoreInfo::Save(bool allowOverwrite) // Then... open a write file RaidFileWrite rf(mDiscSet, mFilename); -<<<<<<< HEAD - rf.Open(true); // allow overwriting - - // Make header - info_StreamFormat hdr; - hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); - hdr.mAccountID = htonl(mAccountID); - hdr.mClientStoreMarker = box_hton64(mClientStoreMarker); - hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed); - hdr.mBlocksUsed = box_hton64(mBlocksUsed); - hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles); - hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles); - hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories); - hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit); - hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit); - hdr.mCurrentMarkNumber = 0; - hdr.mOptionsPresent = 0; - hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size()); - - // Write header - rf.Write(&hdr, sizeof(hdr)); - -======= rf.Open(allowOverwrite); // Make header @@ -564,7 +370,6 @@ void BackupStoreInfo::Save(bool allowOverwrite) int64_t numDelObj = mDeletedDirectories.size(); archive.Write(numDelObj); ->>>>>>> 0.12 // Write the deleted object list if(mDeletedDirectories.size() > 0) { @@ -592,15 +397,12 @@ void BackupStoreInfo::Save(bool allowOverwrite) tosave -= b; } } -<<<<<<< HEAD -======= archive.Write(mAccountEnabled); mExtraData.Seek(0, IOStream::SeekType_Absolute); mExtraData.CopyStreamTo(rf); mExtraData.Seek(0, IOStream::SeekType_Absolute); ->>>>>>> 0.12 // Commit it to disc, converting it to RAID now rf.Commit(true); @@ -609,9 +411,6 @@ void BackupStoreInfo::Save(bool allowOverwrite) mIsModified = false; } -<<<<<<< HEAD - -======= int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) { int numChanges = 0; @@ -661,7 +460,6 @@ int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) \ field += delta; \ mIsModified = true; ->>>>>>> 0.12 // -------------------------------------------------------------------------- // @@ -673,20 +471,6 @@ int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) // -------------------------------------------------------------------------- void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta) { -<<<<<<< HEAD - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) - } - if((mBlocksUsed + Delta) < 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) - } - - mBlocksUsed += Delta; - - mIsModified = true; -======= APPLY_DELTA(mBlocksUsed, Delta); } @@ -702,7 +486,6 @@ void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta) void BackupStoreInfo::ChangeBlocksInCurrentFiles(int64_t Delta) { APPLY_DELTA(mBlocksInCurrentFiles, Delta); ->>>>>>> 0.12 } // -------------------------------------------------------------------------- @@ -715,22 +498,7 @@ void BackupStoreInfo::ChangeBlocksInCurrentFiles(int64_t Delta) // -------------------------------------------------------------------------- void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta) { -<<<<<<< HEAD - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) - } - if((mBlocksInOldFiles + Delta) < 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) - } - - mBlocksInOldFiles += Delta; - - mIsModified = true; -======= APPLY_DELTA(mBlocksInOldFiles, Delta); ->>>>>>> 0.12 } // -------------------------------------------------------------------------- @@ -743,22 +511,7 @@ void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta) // -------------------------------------------------------------------------- void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta) { -<<<<<<< HEAD - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) - } - if((mBlocksInDeletedFiles + Delta) < 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) - } - - mBlocksInDeletedFiles += Delta; - - mIsModified = true; -======= APPLY_DELTA(mBlocksInDeletedFiles, Delta); ->>>>>>> 0.12 } // -------------------------------------------------------------------------- @@ -771,22 +524,6 @@ void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta) // -------------------------------------------------------------------------- void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta) { -<<<<<<< HEAD - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) - } - if((mBlocksInDirectories + Delta) < 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) - } - - mBlocksInDirectories += Delta; - - mIsModified = true; -} - -======= APPLY_DELTA(mBlocksInDirectories, Delta); } @@ -809,7 +546,6 @@ void BackupStoreInfo::AdjustNumDirectories(int64_t increase) { APPLY_DELTA(mNumDirectories, increase); } ->>>>>>> 0.12 // -------------------------------------------------------------------------- // @@ -953,8 +689,6 @@ void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker) } -<<<<<<< HEAD -======= // -------------------------------------------------------------------------- // // Function @@ -974,5 +708,4 @@ void BackupStoreInfo::SetAccountName(const std::string& rName) mIsModified = true; } ->>>>>>> 0.12 diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h index db2a62ef..752cc44a 100644 --- a/lib/backupstore/BackupStoreInfo.h +++ b/lib/backupstore/BackupStoreInfo.h @@ -14,10 +14,6 @@ #include <string> #include <vector> -<<<<<<< HEAD -class BackupStoreCheck; - -======= #include "CollectInBufferStream.h" class BackupStoreCheck; @@ -63,7 +59,6 @@ END_STRUCTURE_PACKING_FOR_WIRE #define INFO_FILENAME "info" ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Class @@ -94,33 +89,19 @@ public: bool IsModified() const {return mIsModified;} // Save modified infomation back to store -<<<<<<< HEAD - void Save(); -======= void Save(bool allowOverwrite = true); ->>>>>>> 0.12 // Data access functions int32_t GetAccountID() const {return mAccountID;} int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;} int64_t GetBlocksUsed() const {return mBlocksUsed;} -<<<<<<< HEAD -======= int64_t GetBlocksInCurrentFiles() const {return mBlocksInCurrentFiles;} ->>>>>>> 0.12 int64_t GetBlocksInOldFiles() const {return mBlocksInOldFiles;} int64_t GetBlocksInDeletedFiles() const {return mBlocksInDeletedFiles;} int64_t GetBlocksInDirectories() const {return mBlocksInDirectories;} const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;} int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;} int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;} -<<<<<<< HEAD - bool IsReadOnly() const {return mReadOnly;} - int GetDiscSetNumber() const {return mDiscSet;} - - // Data modification functions - void ChangeBlocksUsed(int64_t Delta); -======= int64_t GetNumFiles() const {return mNumFiles;} int64_t GetNumOldFiles() const {return mNumOldFiles;} int64_t GetNumDeletedFiles() const {return mNumDeletedFiles;} @@ -134,7 +115,6 @@ public: // Data modification functions void ChangeBlocksUsed(int64_t Delta); void ChangeBlocksInCurrentFiles(int64_t Delta); ->>>>>>> 0.12 void ChangeBlocksInOldFiles(int64_t Delta); void ChangeBlocksInDeletedFiles(int64_t Delta); void ChangeBlocksInDirectories(int64_t Delta); @@ -142,31 +122,15 @@ public: void AddDeletedDirectory(int64_t DirID); void RemovedDeletedDirectory(int64_t DirID); void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit); -<<<<<<< HEAD -======= void AdjustNumFiles(int64_t increase); void AdjustNumOldFiles(int64_t increase); void AdjustNumDeletedFiles(int64_t increase); void AdjustNumDirectories(int64_t increase); ->>>>>>> 0.12 // Object IDs int64_t AllocateObjectID(); // Client marker set and get -<<<<<<< HEAD - int64_t GetClientStoreMarker() {return mClientStoreMarker;} - void SetClientStoreMarker(int64_t ClientStoreMarker); - -private: - static std::auto_ptr<BackupStoreInfo> CreateForRegeneration(int32_t AccountID, const std::string &rRootDir, - int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles, - int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit); - -private: - // Location information - int32_t mAccountID; -======= int64_t GetClientStoreMarker() const {return mClientStoreMarker;} void SetClientStoreMarker(int64_t ClientStoreMarker); @@ -194,7 +158,6 @@ private: // they now define the sizes of fields on disk (via Archive). int32_t mAccountID; std::string mAccountName; ->>>>>>> 0.12 int mDiscSet; std::string mFilename; bool mReadOnly; @@ -206,21 +169,12 @@ private: // Account information int64_t mLastObjectIDUsed; int64_t mBlocksUsed; -<<<<<<< HEAD -======= int64_t mBlocksInCurrentFiles; ->>>>>>> 0.12 int64_t mBlocksInOldFiles; int64_t mBlocksInDeletedFiles; int64_t mBlocksInDirectories; int64_t mBlocksSoftLimit; int64_t mBlocksHardLimit; -<<<<<<< HEAD - std::vector<int64_t> mDeletedDirectories; -}; - - -======= int64_t mNumFiles; int64_t mNumOldFiles; int64_t mNumDeletedFiles; @@ -230,7 +184,6 @@ private: CollectInBufferStream mExtraData; }; ->>>>>>> 0.12 #endif // BACKUPSTOREINFO__H diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp index 642e260c..26f9acca 100644 --- a/lib/backupstore/BackupStoreRefCountDatabase.cpp +++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp @@ -61,11 +61,7 @@ std::string BackupStoreRefCountDatabase::GetFilename(const ASSERT(RootDir[RootDir.size() - 1] == '/' || RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); -<<<<<<< HEAD - std::string fn(RootDir + "refcount.db"); -======= std::string fn(RootDir + REFCOUNT_FILENAME ".db"); ->>>>>>> 0.12 RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet())); return RaidFileUtil::MakeWriteFileName(rdiscSet, fn); @@ -95,15 +91,9 @@ void BackupStoreRefCountDatabase::Create(const // Open the file for writing if (FileExists(Filename) && !AllowOverwrite) { -<<<<<<< HEAD - BOX_ERROR("Attempted to overwrite refcount database file: " << - Filename); - THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile); -======= THROW_FILE_ERROR("Failed to overwrite refcount database: " "not allowed here", Filename, RaidFileException, CannotOverwriteExistingFile); ->>>>>>> 0.12 } int flags = O_CREAT | O_BINARY | O_RDWR; @@ -144,26 +134,18 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( refcount_StreamFormat hdr; if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) { -<<<<<<< HEAD - THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) -======= THROW_FILE_ERROR("Failed to read refcount database: " "short read", filename, BackupStoreException, CouldNotLoadStoreInfo); ->>>>>>> 0.12 } // Check it if(ntohl(hdr.mMagicValue) != REFCOUNT_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID()) { -<<<<<<< HEAD - THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) -======= THROW_FILE_ERROR("Failed to read refcount database: " "bad magic number", filename, BackupStoreException, BadStoreInfoOnLoad); ->>>>>>> 0.12 } // Make new object @@ -180,92 +162,6 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( // -------------------------------------------------------------------------- // // Function -<<<<<<< HEAD -// Name: BackupStoreRefCountDatabase::Save() -// Purpose: Save modified info back to disc -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -/* -void BackupStoreRefCountDatabase::Save() -{ - // Make sure we're initialised (although should never come to this) - if(mFilename.empty() || mAccount.GetID() == 0) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Can we do this? - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) - } - - // Then... open a write file - RaidFileWrite rf(mAccount.GetDiscSet(), mFilename); - rf.Open(true); // allow overwriting - - // Make header - info_StreamFormat hdr; - hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); - hdr.mAccountID = htonl(mAccountID); - hdr.mClientStoreMarker = box_hton64(mClientStoreMarker); - hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed); - hdr.mBlocksUsed = box_hton64(mBlocksUsed); - hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles); - hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles); - hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories); - hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit); - hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit); - hdr.mCurrentMarkNumber = 0; - hdr.mOptionsPresent = 0; - hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size()); - - // Write header - rf.Write(&hdr, sizeof(hdr)); - - // Write the deleted object list - if(mDeletedDirectories.size() > 0) - { - int64_t objs[NUM_DELETED_DIRS_BLOCK]; - - int tosave = mDeletedDirectories.size(); - std::vector<int64_t>::iterator i(mDeletedDirectories.begin()); - while(tosave > 0) - { - // How many in this one? - int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); - - // Add them - for(int t = 0; t < b; ++t) - { - ASSERT(i != mDeletedDirectories.end()); - objs[t] = box_hton64((*i)); - i++; - } - - // Write - rf.Write(objs, b * sizeof(int64_t)); - - // Number saved - tosave -= b; - } - } - - // Commit it to disc, converting it to RAID now - rf.Commit(true); - - // Mark is as not modified - mIsModified = false; -} -*/ - - -// -------------------------------------------------------------------------- -// -// Function -======= ->>>>>>> 0.12 // Name: BackupStoreRefCountDatabase::GetRefCount(int64_t // ObjectID) // Purpose: Get the number of references to the specified object @@ -280,17 +176,10 @@ BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const if (GetSize() < offset + GetEntrySize()) { -<<<<<<< HEAD - BOX_ERROR("attempted read of unknown refcount for object " << - BOX_FORMAT_OBJECTID(ObjectID)); - THROW_EXCEPTION(BackupStoreException, - UnknownObjectRefCountRequested); -======= THROW_FILE_ERROR("Failed to read refcount database: " "attempted read of unknown refcount for object " << BOX_FORMAT_OBJECTID(ObjectID), mFilename, BackupStoreException, UnknownObjectRefCountRequested); ->>>>>>> 0.12 } mapDatabaseFile->Seek(offset, SEEK_SET); @@ -299,15 +188,9 @@ BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const if (mapDatabaseFile->Read(&refcount, sizeof(refcount)) != sizeof(refcount)) { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("short read on refcount database: " << - mFilename); - THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo); -======= THROW_FILE_ERROR("Failed to read refcount database: " "short read at offset " << offset, mFilename, BackupStoreException, CouldNotLoadStoreInfo); ->>>>>>> 0.12 } return ntohl(refcount); diff --git a/lib/common/Archive.h b/lib/common/Archive.h index 139cc5fd..6d5ce88b 100644 --- a/lib/common/Archive.h +++ b/lib/common/Archive.h @@ -45,10 +45,7 @@ public: { Write((int) Item); } -<<<<<<< HEAD -======= void WriteExact(uint32_t Item) { Write((int)Item); } ->>>>>>> 0.12 void Write(int Item) { int32_t privItem = htonl(Item); @@ -59,10 +56,7 @@ public: int64_t privItem = box_hton64(Item); mrStream.Write(&privItem, sizeof(privItem)); } -<<<<<<< HEAD -======= void WriteExact(uint64_t Item) { Write(Item); } ->>>>>>> 0.12 void Write(uint64_t Item) { uint64_t privItem = box_hton64(Item); @@ -87,11 +81,7 @@ public: int privItem; Read(privItem); -<<<<<<< HEAD - if (privItem) -======= if(privItem) ->>>>>>> 0.12 { rItemOut = true; } @@ -100,8 +90,6 @@ public: rItemOut = false; } } -<<<<<<< HEAD -======= void ReadIfPresent(bool &rItemOut, bool ValueIfNotPresent) { int privItem; @@ -109,7 +97,6 @@ public: rItemOut = privItem ? true : false; } void ReadExact(uint32_t &rItemOut) { Read((int&)rItemOut); } ->>>>>>> 0.12 void Read(int &rItemOut) { int32_t privItem; @@ -119,8 +106,6 @@ public: } rItemOut = ntohl(privItem); } -<<<<<<< HEAD -======= void ReadIfPresent(int &rItemOut, int ValueIfNotPresent) { int32_t privItem; @@ -140,7 +125,6 @@ public: THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) } } ->>>>>>> 0.12 void Read(int64_t &rItemOut) { int64_t privItem; @@ -150,10 +134,7 @@ public: } rItemOut = box_ntoh64(privItem); } -<<<<<<< HEAD -======= void ReadExact(uint64_t &rItemOut) { Read(rItemOut); } ->>>>>>> 0.12 void Read(uint64_t &rItemOut) { uint64_t privItem; diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h index ae3ff841..f0772c9c 100644 --- a/lib/common/BannerText.h +++ b/lib/common/BannerText.h @@ -10,11 +10,6 @@ #ifndef BANNERTEXT__H #define BANNERTEXT__H -<<<<<<< HEAD -#define BANNER_TEXT(UtilityName) \ - "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers and " \ - "contributors 2003-2010" -======= #ifdef NEED_BOX_VERSION_H # include "BoxVersion.h" #endif @@ -22,7 +17,6 @@ #define BANNER_TEXT(UtilityName) \ "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers and " \ "contributors 2003-2011" ->>>>>>> 0.12 #endif // BANNERTEXT__H diff --git a/lib/common/Box.h b/lib/common/Box.h index 91b4967b..316f4364 100644 --- a/lib/common/Box.h +++ b/lib/common/Box.h @@ -17,11 +17,8 @@ #include "BoxPlatform.h" -<<<<<<< HEAD -======= #include <memory> ->>>>>>> 0.12 // uncomment this line to enable full memory leak finding on all // malloc-ed blocks (at least, ones used by the STL) //#define MEMLEAKFINDER_FULL_MALLOC_MONITORING @@ -43,10 +40,6 @@ #include "Logging.h" #ifndef BOX_RELEASE_BUILD -<<<<<<< HEAD - -======= ->>>>>>> 0.12 extern bool AssertFailuresToSyslog; #define ASSERT_FAILS_TO_SYSLOG_ON {AssertFailuresToSyslog = true;} void BoxDebugAssertFailed(const char *cond, const char *file, int line); @@ -77,10 +70,6 @@ // Exception names #define EXCEPTION_CODENAMES_EXTENDED -<<<<<<< HEAD - -======= ->>>>>>> 0.12 #else #define ASSERT_FAILS_TO_SYSLOG_ON #define ASSERT(cond) @@ -91,11 +80,6 @@ // Box Backup builds release get extra information for exception logging #define EXCEPTION_CODENAMES_EXTENDED #define EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION -<<<<<<< HEAD - -#endif - -======= #endif #if defined DEBUG_LEAKS @@ -110,7 +94,6 @@ #endif #endif // DEBUG_LEAKS || BOX_RELEASE_BUILD ->>>>>>> 0.12 #ifdef BOX_MEMORY_LEAK_TESTING // Memory leak testing #include "MemLeakFinder.h" @@ -131,10 +114,6 @@ #define THROW_EXCEPTION(type, subtype) \ { \ -<<<<<<< HEAD - if(!HideExceptionMessageGuard::ExceptionsHidden()) \ - { \ -======= if((!HideExceptionMessageGuard::ExceptionsHidden() \ && !HideSpecificExceptionGuard::IsHidden( \ type::ExceptionType, type::subtype)) \ @@ -147,7 +126,6 @@ guard.reset(new Logging::Guard(Log::EVERYTHING)); \ } \ \ ->>>>>>> 0.12 OPTIONAL_DO_BACKTRACE \ BOX_WARNING("Exception thrown: " \ #type "(" #subtype ") " \ @@ -160,14 +138,6 @@ { \ std::ostringstream _box_throw_line; \ _box_throw_line << message; \ -<<<<<<< HEAD - if(!HideExceptionMessageGuard::ExceptionsHidden()) \ - { \ - OPTIONAL_DO_BACKTRACE \ - BOX_WARNING("Exception thrown: " \ - #type "(" #subtype ") (" << message << \ - ") at " __FILE__ "(" << __LINE__ << ")") \ -======= if((!HideExceptionMessageGuard::ExceptionsHidden() \ && !HideSpecificExceptionGuard::IsHidden( \ type::ExceptionType, type::subtype)) \ @@ -185,7 +155,6 @@ #type "(" #subtype ") (" << \ _box_throw_line.str() << \ ") at " __FILE__ ":" << __LINE__) \ ->>>>>>> 0.12 } \ throw type(type::subtype, _box_throw_line.str()); \ } diff --git a/lib/common/BoxConfig-MSVC.h b/lib/common/BoxConfig-MSVC.h index bfa0dcaf..eeb25d2e 100644 --- a/lib/common/BoxConfig-MSVC.h +++ b/lib/common/BoxConfig-MSVC.h @@ -76,12 +76,9 @@ /* Define to 1 if you have the <execinfo.h> header file. */ /* #undef HAVE_EXECINFO_H */ -<<<<<<< HEAD -======= /* Define to 1 if you have the <fcntl.h> header file. */ #define HAVE_FCNTL_H 1 ->>>>>>> 0.12 /* Define to 1 if you have the `flock' function. */ /* #undef HAVE_FLOCK */ @@ -188,11 +185,7 @@ /* Define to 1 if you have the `setproctitle' function. */ /* #undef HAVE_SETPROCTITLE */ -<<<<<<< HEAD - -======= #define HAVE_SETPROCTITLE 1 ->>>>>>> 0.12 /* Define to 1 if you have the `setxattr' function. */ /* #undef HAVE_SETXATTR */ diff --git a/lib/common/BoxException.h b/lib/common/BoxException.h index ad5aba4f..361f04e8 100644 --- a/lib/common/BoxException.h +++ b/lib/common/BoxException.h @@ -29,10 +29,7 @@ public: virtual unsigned int GetType() const throw() = 0; virtual unsigned int GetSubType() const throw() = 0; -<<<<<<< HEAD -======= virtual const std::string& GetMessage() const = 0; ->>>>>>> 0.12 private: }; diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h index 2c7ffcf6..53a967e8 100644 --- a/lib/common/BoxPlatform.h +++ b/lib/common/BoxPlatform.h @@ -23,11 +23,7 @@ #ifdef _MSC_VER #include "BoxConfig-MSVC.h" -<<<<<<< HEAD -#include "BoxVersion.h" -======= #define NEED_BOX_VERSION_H ->>>>>>> 0.12 #else #include "BoxConfig.h" #endif @@ -163,9 +159,6 @@ #define INFTIM -1 #endif -<<<<<<< HEAD -// for Unix compatibility with Windows :-) -======= // Define O_BINARY for Unix compatibility with Windows :-) // MSVC 2010 and newer MinGW define this in fcntl.h, which is probably // not included by this point, so include it now so that we can detect @@ -175,7 +168,6 @@ # include <fcntl.h> #endif ->>>>>>> 0.12 #ifndef O_BINARY #define O_BINARY 0 #endif diff --git a/lib/common/BoxPortsAndFiles.h.in b/lib/common/BoxPortsAndFiles.h.in index 55616da7..047a828f 100644 --- a/lib/common/BoxPortsAndFiles.h.in +++ b/lib/common/BoxPortsAndFiles.h.in @@ -32,14 +32,6 @@ #define BOX_GET_DEFAULT_BBSTORED_CONFIG_FILE \ GetDefaultConfigFilePath("bbstored.conf").c_str() #else -<<<<<<< HEAD -#define BOX_FILE_BBACKUPD_DEFAULT_CONFIG "@sysconfdir_expanded@/boxbackup/bbackupd.conf" -#define BOX_FILE_RAIDFILE_DEFAULT_CONFIG "@sysconfdir_expanded@/boxbackup/raidfile.conf" -#define BOX_FILE_BBSTORED_DEFAULT_CONFIG "@sysconfdir_expanded@/boxbackup/bbstored.conf" -#define BOX_FILE_BBACKUPD_OLD_CONFIG "@sysconfdir_expanded@/box/bbackupd.conf" -#define BOX_FILE_RAIDFILE_OLD_CONFIG "@sysconfdir_expanded@/box/raidfile.conf" -#define BOX_FILE_BBSTORED_OLD_CONFIG "@sysconfdir_expanded@/box/bbstored.conf" -======= #define BOX_FILE_BBACKUPD_OLD_CONFIG "@sysconfdir_expanded@/box/bbackupd.conf" #define BOX_FILE_RAIDFILE_OLD_CONFIG "@sysconfdir_expanded@/box/raidfile.conf" #define BOX_FILE_BBSTORED_OLD_CONFIG "@sysconfdir_expanded@/box/bbstored.conf" @@ -49,7 +41,6 @@ std::string("@sysconfdir_expanded@/boxbackup/raidfile.conf") #define BOX_GET_DEFAULT_BBSTORED_CONFIG_FILE \ std::string("@sysconfdir_expanded@/boxbackup/bbstored.conf") ->>>>>>> 0.12 #endif #endif // BOXPORTSANDFILES__H diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp index ead3410b..f62b1c35 100644 --- a/lib/common/BoxTime.cpp +++ b/lib/common/BoxTime.cpp @@ -94,8 +94,6 @@ std::string FormatTime(box_time_t time, bool includeDate, bool showMicros) return buf.str(); } -<<<<<<< HEAD -======= // -------------------------------------------------------------------------- // // Function @@ -145,4 +143,3 @@ void ShortSleep(box_time_t duration, bool logDuration) #endif } ->>>>>>> 0.12 diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h index d688ff10..3108d809 100644 --- a/lib/common/BoxTime.h +++ b/lib/common/BoxTime.h @@ -11,25 +11,16 @@ #define BOXTIME__H // Time is presented as an unsigned 64 bit integer, in microseconds -<<<<<<< HEAD -typedef uint64_t box_time_t; -======= typedef int64_t box_time_t; ->>>>>>> 0.12 #define NANO_SEC_IN_SEC (1000000000LL) #define NANO_SEC_IN_USEC (1000) #define NANO_SEC_IN_USEC_LL (1000LL) #define MICRO_SEC_IN_SEC (1000000) #define MICRO_SEC_IN_SEC_LL (1000000LL) -<<<<<<< HEAD -#define MILLI_SEC_IN_NANO_SEC (1000) -#define MILLI_SEC_IN_NANO_SEC_LL (1000LL) -======= #define MICRO_SEC_IN_MILLI_SEC (1000) #define MILLI_SEC_IN_SEC (1000) #define MILLI_SEC_IN_SEC_LL (1000LL) ->>>>>>> 0.12 box_time_t GetCurrentBoxTime(); @@ -37,24 +28,17 @@ inline box_time_t SecondsToBoxTime(time_t Seconds) { return ((box_time_t)Seconds * MICRO_SEC_IN_SEC_LL); } -<<<<<<< HEAD -======= inline uint64_t MilliSecondsToBoxTime(int64_t milliseconds) { return ((box_time_t)milliseconds * 1000); } ->>>>>>> 0.12 inline time_t BoxTimeToSeconds(box_time_t Time) { return Time / MICRO_SEC_IN_SEC_LL; } inline uint64_t BoxTimeToMilliSeconds(box_time_t Time) { -<<<<<<< HEAD - return Time / MILLI_SEC_IN_NANO_SEC_LL; -======= return Time / MILLI_SEC_IN_SEC_LL; ->>>>>>> 0.12 } inline uint64_t BoxTimeToMicroSeconds(box_time_t Time) { @@ -64,9 +48,6 @@ inline uint64_t BoxTimeToMicroSeconds(box_time_t Time) std::string FormatTime(box_time_t time, bool includeDate, bool showMicros = false); -<<<<<<< HEAD -======= void ShortSleep(box_time_t duration, bool logDuration); ->>>>>>> 0.12 #endif // BOXTIME__H diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt index 885a9197..05da2709 100644 --- a/lib/common/CommonException.txt +++ b/lib/common/CommonException.txt @@ -45,8 +45,6 @@ IOStreamGetLineNotEnoughDataToIgnore 37 Bad value passed to IOStreamGetLine::Ign TempDirPathTooLong 38 Your temporary directory path is too long. Check the TMP and TEMP environment variables. ArchiveBlockIncompleteRead 39 The Store Object Info File is too short or corrupted, and will be rewritten automatically when the next backup completes. AccessDenied 40 Access to the file or directory was denied. Please check the permissions. -<<<<<<< HEAD -======= DatabaseOpenFailed 41 Failed to open the database file DatabaseReadFailed 42 Failed to read a record from the database file DatabaseWriteFailed 43 Failed to write a record from the database file @@ -57,4 +55,3 @@ DatabaseRecordAlreadyExists 47 The database already contains a record with this DatabaseRecordBadSize 48 The database contains a record with an invalid size DatabaseIterateFailed 49 Failed to iterate over the database keys ReferenceNotFound 50 The database does not contain an expected reference ->>>>>>> 0.12 diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp index ecc4eb12..0b123675 100644 --- a/lib/common/DebugMemLeakFinder.cpp +++ b/lib/common/DebugMemLeakFinder.cpp @@ -7,18 +7,10 @@ // // -------------------------------------------------------------------------- -<<<<<<< HEAD - -#ifndef BOX_RELEASE_BUILD - -#include "Box.h" - -======= #include "Box.h" #ifdef BOX_MEMORY_LEAK_TESTING ->>>>>>> 0.12 #undef malloc #undef realloc #undef free @@ -27,13 +19,6 @@ #include <unistd.h> #endif -<<<<<<< HEAD -#include <map> -#include <stdio.h> -#include <string.h> -#include <set> -#include <cstdlib> // for std::atexit -======= #include <signal.h> #include <stdio.h> #include <string.h> @@ -41,7 +26,6 @@ #include <cstdlib> // for std::atexit #include <map> #include <set> ->>>>>>> 0.12 #include "MemLeakFinder.h" @@ -90,8 +74,6 @@ namespace size_t sNotLeaksPreNum = 0; } -<<<<<<< HEAD -======= void memleakfinder_report_on_signal(int unused) { // this is not safe! do not send SIGUSR1 to a process @@ -99,7 +81,6 @@ void memleakfinder_report_on_signal(int unused) memleakfinder_report_usage_summary(); } ->>>>>>> 0.12 void memleakfinder_init() { ASSERT(!memleakfinder_initialised); @@ -111,8 +92,6 @@ void memleakfinder_init() } memleakfinder_initialised = true; -<<<<<<< HEAD -======= #if defined WIN32 // no signals, no way to trigger event yet @@ -128,7 +107,6 @@ void memleakfinder_init() } ASSERT(oldact.sa_handler == 0); #endif // WIN32 ->>>>>>> 0.12 } MemLeakSuppressionGuard::MemLeakSuppressionGuard() @@ -186,8 +164,6 @@ void *memleakfinder_malloc(size_t size, const char *file, int line) return b; } -<<<<<<< HEAD -======= void *memleakfinder_calloc(size_t blocks, size_t size, const char *file, int line) { void *block = memleakfinder_malloc(blocks * size, file, line); @@ -198,7 +174,6 @@ void *memleakfinder_calloc(size_t blocks, size_t size, const char *file, int lin return block; } ->>>>>>> 0.12 void *memleakfinder_realloc(void *ptr, size_t size) { InternalAllocGuard guard; @@ -404,8 +379,6 @@ int memleakfinder_numleaks() return n; } -<<<<<<< HEAD -======= // Summarise all blocks allocated and still allocated, for memory usage // diagnostics. void memleakfinder_report_usage_summary() @@ -485,7 +458,6 @@ void memleakfinder_report_usage_summary() #endif // DEBUG_LEAKS } ->>>>>>> 0.12 void memleakfinder_reportleaks_file(FILE *file) { InternalAllocGuard guard; @@ -689,8 +661,4 @@ void operator delete(void *ptr) throw () internal_delete(ptr); } -<<<<<<< HEAD -#endif // BOX_RELEASE_BUILD -======= #endif // BOX_MEMORY_LEAK_TESTING ->>>>>>> 0.12 diff --git a/lib/common/ExcludeList.cpp b/lib/common/ExcludeList.cpp index c7c80ed2..213c4f8e 100644 --- a/lib/common/ExcludeList.cpp +++ b/lib/common/ExcludeList.cpp @@ -101,14 +101,6 @@ std::string ExcludeList::ReplaceSlashesRegex(const std::string& input) const output.replace(pos, 1, "\\" DIRECTORY_SEPARATOR); } -<<<<<<< HEAD - for (std::string::iterator i = output.begin(); i != output.end(); i++) - { - *i = tolower(*i); - } - -======= ->>>>>>> 0.12 return output; } #endif @@ -188,30 +180,19 @@ void ExcludeList::AddRegexEntries(const std::string &rEntries) try { std::string entry = *i; -<<<<<<< HEAD -======= int flags = REG_EXTENDED | REG_NOSUB; ->>>>>>> 0.12 // Convert any forward slashes in the string // to appropriately escaped backslashes #ifdef WIN32 entry = ReplaceSlashesRegex(entry); -<<<<<<< HEAD - #endif - - // Compile - int errcode = ::regcomp(pregex, entry.c_str(), - REG_EXTENDED | REG_NOSUB); -======= flags |= REG_ICASE; // Windows convention #endif // Compile int errcode = ::regcomp(pregex, entry.c_str(), flags); ->>>>>>> 0.12 if (errcode != 0) { @@ -254,10 +235,7 @@ bool ExcludeList::IsExcluded(const std::string &rTest) const std::string test = rTest; #ifdef WIN32 -<<<<<<< HEAD -======= // converts to lower case as well ->>>>>>> 0.12 test = ReplaceSlashesDefinite(test); #endif diff --git a/lib/common/FdGetLine.cpp b/lib/common/FdGetLine.cpp index 2d2f7c6a..30409d92 100644 --- a/lib/common/FdGetLine.cpp +++ b/lib/common/FdGetLine.cpp @@ -20,16 +20,6 @@ #include "MemLeakFindOn.h" -<<<<<<< HEAD -// utility whitespace function -inline bool iw(int c) -{ - return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded -} - - -======= ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Function @@ -39,16 +29,7 @@ inline bool iw(int c) // // -------------------------------------------------------------------------- FdGetLine::FdGetLine(int fd) -<<<<<<< HEAD - : mFileHandle(fd), - mLineNumber(0), - mBufferBegin(0), - mBytesInBuffer(0), - mPendingEOF(false), - mEOF(false) -======= : mFileHandle(fd) ->>>>>>> 0.12 { if(mFileHandle < 0) {THROW_EXCEPTION(CommonException, BadArguments)} //printf("FdGetLine buffer size = %d\n", sizeof(mBuffer)); @@ -81,128 +62,6 @@ FdGetLine::~FdGetLine() std::string FdGetLine::GetLine(bool Preprocess) { if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)} -<<<<<<< HEAD - - // EOF? - if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)} - - std::string r; - - bool foundLineEnd = false; - - while(!foundLineEnd && !mEOF) - { - // Use any bytes left in the buffer - while(mBufferBegin < mBytesInBuffer) - { - int c = mBuffer[mBufferBegin++]; - if(c == '\r') - { - // Ignore nasty Windows line ending extra chars - } - else if(c == '\n') - { - // Line end! - foundLineEnd = true; - break; - } - else - { - // Add to string - r += c; - } - - // Implicit line ending at EOF - if(mBufferBegin >= mBytesInBuffer && mPendingEOF) - { - foundLineEnd = true; - } - } - - // Check size - if(r.size() > FDGETLINE_MAX_LINE_SIZE) - { - THROW_EXCEPTION(CommonException, GetLineTooLarge) - } - - // Read more in? - if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF) - { -#ifdef WIN32 - int bytes; - - if (mFileHandle == _fileno(stdin)) - { - bytes = console_read(mBuffer, sizeof(mBuffer)); - } - else - { - bytes = ::read(mFileHandle, mBuffer, - sizeof(mBuffer)); - } -#else // !WIN32 - int bytes = ::read(mFileHandle, mBuffer, sizeof(mBuffer)); -#endif // WIN32 - - // Error? - if(bytes == -1) - { - THROW_EXCEPTION(CommonException, OSFileError) - } - - // Adjust buffer info - mBytesInBuffer = bytes; - mBufferBegin = 0; - - // EOF / closed? - if(bytes == 0) - { - mPendingEOF = true; - } - } - - // EOF? - if(mPendingEOF && mBufferBegin >= mBytesInBuffer) - { - // File is EOF, and now we've depleted the buffer completely, so tell caller as well. - mEOF = true; - } - } - - if(!Preprocess) - { - return r; - } - else - { - // Check for comment char, but char before must be whitespace - int end = 0; - int size = r.size(); - while(end < size) - { - if(r[end] == '#' && (end == 0 || (iw(r[end-1])))) - { - break; - } - end++; - } - - // Remove whitespace - int begin = 0; - while(begin < size && iw(r[begin])) - { - begin++; - } - if(!iw(r[end])) end--; - while(end > begin && iw(r[end])) - { - end--; - } - - // Return a sub string - return r.substr(begin, end - begin + 1); - } -======= std::string r; bool result = GetLineInternal(r, Preprocess); @@ -250,7 +109,6 @@ int FdGetLine::ReadMore(int Timeout) } return bytes; ->>>>>>> 0.12 } @@ -259,11 +117,7 @@ int FdGetLine::ReadMore(int Timeout) // Function // Name: FdGetLine::DetachFile() // Purpose: Detaches the file handle, setting the file pointer correctly. -<<<<<<< HEAD -// Probably not good for sockets... -======= // Probably not good for sockets... ->>>>>>> 0.12 // Created: 2003/07/24 // // -------------------------------------------------------------------------- @@ -286,7 +140,3 @@ void FdGetLine::DetachFile() mFileHandle = -1; } -<<<<<<< HEAD - -======= ->>>>>>> 0.12 diff --git a/lib/common/FdGetLine.h b/lib/common/FdGetLine.h index 6b609fe8..2b9c268f 100644 --- a/lib/common/FdGetLine.h +++ b/lib/common/FdGetLine.h @@ -12,23 +12,7 @@ #include <string> -<<<<<<< HEAD -#ifdef BOX_RELEASE_BUILD - #define FDGETLINE_BUFFER_SIZE 1024 -#elif defined WIN32 - // need enough space for at least one unicode character - // in UTF-8 when calling console_read() from bbackupquery - #define FDGETLINE_BUFFER_SIZE 5 -#else - #define FDGETLINE_BUFFER_SIZE 4 -#endif - -// Just a very large upper bound for line size to avoid -// people sending lots of data over sockets and causing memory problems. -#define FDGETLINE_MAX_LINE_SIZE (1024*256) -======= #include "GetLine.h" ->>>>>>> 0.12 // -------------------------------------------------------------------------- // @@ -38,41 +22,15 @@ // Created: 2003/07/24 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -class FdGetLine -{ -public: - FdGetLine(int fd); - ~FdGetLine(); -======= class FdGetLine : public GetLine { public: FdGetLine(int fd); virtual ~FdGetLine(); ->>>>>>> 0.12 private: FdGetLine(const FdGetLine &rToCopy); public: -<<<<<<< HEAD - std::string GetLine(bool Preprocess = false); - bool IsEOF() {return mEOF;} - int GetLineNumber() {return mLineNumber;} - - // Call to detach, setting file pointer correctly to last bit read. - // Only works for lseek-able file descriptors. - void DetachFile(); - -private: - char mBuffer[FDGETLINE_BUFFER_SIZE]; - int mFileHandle; - int mLineNumber; - int mBufferBegin; - int mBytesInBuffer; - bool mPendingEOF; - bool mEOF; -======= virtual std::string GetLine(bool Preprocess = false); // Call to detach, setting file pointer correctly to last bit read. // Only works for lseek-able file descriptors. @@ -86,7 +44,6 @@ protected: private: int mFileHandle; ->>>>>>> 0.12 }; #endif // FDGETLINE__H diff --git a/lib/common/FileModificationTime.cpp b/lib/common/FileModificationTime.cpp index bc35b7e6..06fc7887 100644 --- a/lib/common/FileModificationTime.cpp +++ b/lib/common/FileModificationTime.cpp @@ -16,11 +16,7 @@ #include "MemLeakFindOn.h" -<<<<<<< HEAD -box_time_t FileModificationTime(EMU_STRUCT_STAT &st) -======= box_time_t FileModificationTime(const EMU_STRUCT_STAT &st) ->>>>>>> 0.12 { #ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); @@ -32,11 +28,7 @@ box_time_t FileModificationTime(const EMU_STRUCT_STAT &st) return datamodified; } -<<<<<<< HEAD -box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st) -======= box_time_t FileAttrModificationTime(const EMU_STRUCT_STAT &st) ->>>>>>> 0.12 { box_time_t statusmodified = #ifdef HAVE_STRUCT_STAT_ST_MTIMESPEC @@ -55,11 +47,7 @@ box_time_t FileAttrModificationTime(const EMU_STRUCT_STAT &st) return statusmodified; } -<<<<<<< HEAD -box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st) -======= box_time_t FileModificationTimeMaxModAndAttr(const EMU_STRUCT_STAT &st) ->>>>>>> 0.12 { #ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); diff --git a/lib/common/FileModificationTime.h b/lib/common/FileModificationTime.h index ffff29c7..85424842 100644 --- a/lib/common/FileModificationTime.h +++ b/lib/common/FileModificationTime.h @@ -14,15 +14,9 @@ #include "BoxTime.h" -<<<<<<< HEAD -box_time_t FileModificationTime(EMU_STRUCT_STAT &st); -box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st); -box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st); -======= box_time_t FileModificationTime(const EMU_STRUCT_STAT &st); box_time_t FileAttrModificationTime(const EMU_STRUCT_STAT &st); box_time_t FileModificationTimeMaxModAndAttr(const EMU_STRUCT_STAT &st); ->>>>>>> 0.12 #endif // FILEMODIFICATIONTIME__H diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp index 68209b15..fc0319da 100644 --- a/lib/common/FileStream.cpp +++ b/lib/common/FileStream.cpp @@ -190,23 +190,6 @@ int FileStream::Read(void *pBuffer, int NBytes, int Timeout) } else { -<<<<<<< HEAD - BOX_LOG_WIN_ERROR("Failed to read from file: " << mFileName); - r = -1; - } -#else - int r = ::read(mOSFileHandle, pBuffer, NBytes); - if(r == -1) - { - BOX_LOG_SYS_ERROR("Failed to read from file: " << mFileName); - } -#endif - - if(r == -1) - { - THROW_EXCEPTION(CommonException, OSFileReadError) - } -======= THROW_WIN_FILE_ERROR("Failed to read from file", mFileName, CommonException, OSFileReadError); } @@ -223,7 +206,6 @@ int FileStream::Read(void *pBuffer, int NBytes, int Timeout) CommonException, OSFileReadError); } #endif ->>>>>>> 0.12 if(r == 0) { @@ -247,11 +229,7 @@ IOStream::pos_type FileStream::BytesLeftToRead() EMU_STRUCT_STAT st; if(EMU_FSTAT(mOSFileHandle, &st) != 0) { -<<<<<<< HEAD - THROW_EXCEPTION(CommonException, OSFileError) -======= BOX_LOG_SYS_ERROR(BOX_FILE_MESSAGE("Failed to stat file", mFileName)); ->>>>>>> 0.12 } return st.st_size - GetPosition(); @@ -285,24 +263,14 @@ void FileStream::Write(const void *pBuffer, int NBytes) if ((res == 0) || (numBytesWritten != (DWORD)NBytes)) { -<<<<<<< HEAD - // DWORD err = GetLastError(); - THROW_EXCEPTION(CommonException, OSFileWriteError) -======= THROW_WIN_FILE_ERROR("Failed to write to file", mFileName, CommonException, OSFileWriteError); ->>>>>>> 0.12 } #else if(::write(mOSFileHandle, pBuffer, NBytes) != NBytes) { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("Failed to write to file: " << mFileName); - THROW_EXCEPTION(CommonException, OSFileWriteError) -======= THROW_SYS_FILE_ERROR("Failed to write to file", mFileName, CommonException, OSFileWriteError); ->>>>>>> 0.12 } #endif } @@ -325,14 +293,6 @@ IOStream::pos_type FileStream::GetPosition() const #ifdef WIN32 LARGE_INTEGER conv; -<<<<<<< HEAD - - conv.HighPart = 0; - conv.LowPart = 0; - - conv.LowPart = SetFilePointer(this->mOSFileHandle, 0, &conv.HighPart, FILE_CURRENT); - -======= conv.HighPart = 0; conv.LowPart = SetFilePointer(this->mOSFileHandle, 0, &conv.HighPart, FILE_CURRENT); @@ -342,18 +302,13 @@ IOStream::pos_type FileStream::GetPosition() const CommonException, OSFileError); } ->>>>>>> 0.12 return (IOStream::pos_type)conv.QuadPart; #else // ! WIN32 off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR); if(p == -1) { -<<<<<<< HEAD - THROW_EXCEPTION(CommonException, OSFileError) -======= THROW_SYS_FILE_ERROR("Failed to seek in file", mFileName, CommonException, OSFileError); ->>>>>>> 0.12 } return (IOStream::pos_type)p; @@ -378,31 +333,19 @@ void FileStream::Seek(IOStream::pos_type Offset, int SeekType) #ifdef WIN32 LARGE_INTEGER conv; -<<<<<<< HEAD - -======= ->>>>>>> 0.12 conv.QuadPart = Offset; DWORD retVal = SetFilePointer(this->mOSFileHandle, conv.LowPart, &conv.HighPart, ConvertSeekTypeToOSWhence(SeekType)); if(retVal == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) { -<<<<<<< HEAD - THROW_EXCEPTION(CommonException, OSFileError) -======= THROW_WIN_FILE_ERROR("Failed to seek in file", mFileName, CommonException, OSFileError); ->>>>>>> 0.12 } #else // ! WIN32 if(::lseek(mOSFileHandle, Offset, ConvertSeekTypeToOSWhence(SeekType)) == -1) { -<<<<<<< HEAD - THROW_EXCEPTION(CommonException, OSFileError) -======= THROW_SYS_FILE_ERROR("Failed to seek in file", mFileName, CommonException, OSFileError); ->>>>>>> 0.12 } #endif // WIN32 @@ -428,14 +371,6 @@ void FileStream::Close() #ifdef WIN32 if(::CloseHandle(mOSFileHandle) == 0) -<<<<<<< HEAD -#else - if(::close(mOSFileHandle) != 0) -#endif - { - THROW_EXCEPTION(CommonException, OSFileCloseError) - } -======= { THROW_WIN_FILE_ERROR("Failed to close file", mFileName, CommonException, OSFileCloseError); @@ -447,7 +382,6 @@ void FileStream::Close() CommonException, OSFileCloseError); } #endif // WIN32 ->>>>>>> 0.12 mOSFileHandle = INVALID_FILE; mIsEOF = true; @@ -478,11 +412,7 @@ bool FileStream::StreamDataLeft() // -------------------------------------------------------------------------- bool FileStream::StreamClosed() { -<<<<<<< HEAD - return mIsEOF; -======= return (mOSFileHandle == INVALID_FILE); ->>>>>>> 0.12 } // -------------------------------------------------------------------------- diff --git a/lib/common/IOStreamGetLine.cpp b/lib/common/IOStreamGetLine.cpp index 9a40f3eb..ef8930b8 100644 --- a/lib/common/IOStreamGetLine.cpp +++ b/lib/common/IOStreamGetLine.cpp @@ -13,16 +13,6 @@ #include "MemLeakFindOn.h" -<<<<<<< HEAD -// utility whitespace function -inline bool iw(int c) -{ - return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded -} - - -======= ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Function @@ -32,16 +22,7 @@ inline bool iw(int c) // // -------------------------------------------------------------------------- IOStreamGetLine::IOStreamGetLine(IOStream &Stream) -<<<<<<< HEAD - : mrStream(Stream), - mLineNumber(0), - mBufferBegin(0), - mBytesInBuffer(0), - mPendingEOF(false), - mEOF(false) -======= : mrStream(Stream) ->>>>>>> 0.12 { } @@ -73,120 +54,6 @@ IOStreamGetLine::~IOStreamGetLine() // -------------------------------------------------------------------------- bool IOStreamGetLine::GetLine(std::string &rOutput, bool Preprocess, int Timeout) { -<<<<<<< HEAD - // EOF? - if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)} - - // Initialise string to stored into - std::string r(mPendingString); - mPendingString.erase(); - - bool foundLineEnd = false; - - while(!foundLineEnd && !mEOF) - { - // Use any bytes left in the buffer - while(mBufferBegin < mBytesInBuffer) - { - int c = mBuffer[mBufferBegin++]; - if(c == '\r') - { - // Ignore nasty Windows line ending extra chars - } - else if(c == '\n') - { - // Line end! - foundLineEnd = true; - break; - } - else - { - // Add to string - r += c; - } - - // Implicit line ending at EOF - if(mBufferBegin >= mBytesInBuffer && mPendingEOF) - { - foundLineEnd = true; - } - } - - // Check size - if(r.size() > IOSTREAMGETLINE_MAX_LINE_SIZE) - { - THROW_EXCEPTION(CommonException, GetLineTooLarge) - } - - // Read more in? - if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF) - { - int bytes = mrStream.Read(mBuffer, sizeof(mBuffer), Timeout); - - // Adjust buffer info - mBytesInBuffer = bytes; - mBufferBegin = 0; - - // EOF / closed? - if(!mrStream.StreamDataLeft()) - { - mPendingEOF = true; - } - - // No data returned? - if(bytes == 0 && mrStream.StreamDataLeft()) - { - // store string away - mPendingString = r; - // Return false; - return false; - } - } - - // EOF? - if(mPendingEOF && mBufferBegin >= mBytesInBuffer) - { - // File is EOF, and now we've depleted the buffer completely, so tell caller as well. - mEOF = true; - } - } - - if(!Preprocess) - { - rOutput = r; - return true; - } - else - { - // Check for comment char, but char before must be whitespace - int end = 0; - int size = r.size(); - while(end < size) - { - if(r[end] == '#' && (end == 0 || (iw(r[end-1])))) - { - break; - } - end++; - } - - // Remove whitespace - int begin = 0; - while(begin < size && iw(r[begin])) - { - begin++; - } - if(!iw(r[end])) end--; - while(end > begin && iw(r[end])) - { - end--; - } - - // Return a sub string - rOutput = r.substr(begin, end - begin + 1); - return true; - } -======= return GetLineInternal(rOutput, Preprocess, Timeout); } @@ -211,7 +78,6 @@ int IOStreamGetLine::ReadMore(int Timeout) } return bytes; ->>>>>>> 0.12 } diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h index 1693e8ac..c4289073 100644 --- a/lib/common/IOStreamGetLine.h +++ b/lib/common/IOStreamGetLine.h @@ -12,24 +12,9 @@ #include <string> -<<<<<<< HEAD -#include "IOStream.h" - -#ifdef BOX_RELEASE_BUILD - #define IOSTREAMGETLINE_BUFFER_SIZE 1024 -#else - #define IOSTREAMGETLINE_BUFFER_SIZE 4 -#endif - -// Just a very large upper bound for line size to avoid -// people sending lots of data over sockets and causing memory problems. -#define IOSTREAMGETLINE_MAX_LINE_SIZE (1024*256) - -======= #include "GetLine.h" #include "IOStream.h" ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Class @@ -38,67 +23,38 @@ // Created: 2003/07/24 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -class IOStreamGetLine -{ -public: - IOStreamGetLine(IOStream &Stream); - ~IOStreamGetLine(); -======= class IOStreamGetLine : public GetLine { public: IOStreamGetLine(IOStream &Stream); virtual ~IOStreamGetLine(); ->>>>>>> 0.12 private: IOStreamGetLine(const IOStreamGetLine &rToCopy); public: bool GetLine(std::string &rOutput, bool Preprocess = false, int Timeout = IOStream::TimeOutInfinite); -<<<<<<< HEAD - bool IsEOF() {return mEOF;} - int GetLineNumber() {return mLineNumber;} -======= ->>>>>>> 0.12 // Call to detach, setting file pointer correctly to last bit read. // Only works for lseek-able file descriptors. void DetachFile(); -<<<<<<< HEAD -======= virtual bool IsStreamDataLeft() { return mrStream.StreamDataLeft(); } ->>>>>>> 0.12 // For doing interesting stuff with the remaining data... // Be careful with this! const void *GetBufferedData() const {return mBuffer + mBufferBegin;} int GetSizeOfBufferedData() const {return mBytesInBuffer - mBufferBegin;} void IgnoreBufferedData(int BytesToIgnore); IOStream &GetUnderlyingStream() {return mrStream;} -<<<<<<< HEAD - -private: - char mBuffer[IOSTREAMGETLINE_BUFFER_SIZE]; - IOStream &mrStream; - int mLineNumber; - int mBufferBegin; - int mBytesInBuffer; - bool mPendingEOF; - bool mEOF; - std::string mPendingString; -======= protected: int ReadMore(int Timeout = IOStream::TimeOutInfinite); private: IOStream &mrStream; ->>>>>>> 0.12 }; #endif // IOSTREAMGETLINE__H diff --git a/lib/common/Logging.cpp b/lib/common/Logging.cpp index 4a7e5e0a..7ce0dd3f 100644 --- a/lib/common/Logging.cpp +++ b/lib/common/Logging.cpp @@ -22,12 +22,9 @@ #ifdef HAVE_UNISTD_H #include <unistd.h> #endif -<<<<<<< HEAD -======= #ifdef WIN32 #include <process.h> #endif ->>>>>>> 0.12 #include <cstring> #include <iomanip> @@ -49,15 +46,12 @@ Log::Level Logging::sGlobalLevel = Log::EVERYTHING; Logging Logging::sGlobalLogging; //automatic initialisation std::string Logging::sProgramName; -<<<<<<< HEAD -======= HideSpecificExceptionGuard::SuppressedExceptions_t HideSpecificExceptionGuard::sSuppressedExceptions; int Logging::Guard::sGuardCount = 0; Log::Level Logging::Guard::sOriginalLevel = Log::INVALID; ->>>>>>> 0.12 Logging::Logging() { ASSERT(!spConsole); @@ -259,15 +253,12 @@ Logger::~Logger() Logging::Remove(this); } -<<<<<<< HEAD -======= bool Logger::IsEnabled(Log::Level level) { return Logging::IsEnabled(level) && (int)mCurrentLevel >= (int)level; } ->>>>>>> 0.12 bool Console::sShowTime = false; bool Console::sShowTimeMicros = false; bool Console::sShowTag = false; @@ -367,13 +358,6 @@ bool Console::Log(Log::Level level, const std::string& rFile, #ifdef WIN32 std::string output = buf.str(); -<<<<<<< HEAD - ConvertUtf8ToConsole(output.c_str(), output); - fprintf(target, "%s\n", output.c_str()); - #else - fprintf(target, "%s\n", buf.str().c_str()); - #endif -======= if(ConvertUtf8ToConsole(output.c_str(), output) == false) { fprintf(target, "%s (and failed to convert to console encoding)\n", @@ -388,7 +372,6 @@ bool Console::Log(Log::Level level, const std::string& rFile, #endif fflush(target); ->>>>>>> 0.12 return true; } @@ -487,8 +470,6 @@ int Syslog::GetNamedFacility(const std::string& rFacility) bool FileLogger::Log(Log::Level Level, const std::string& rFile, int line, std::string& rMessage) { -<<<<<<< HEAD -======= if (mLogFile.StreamClosed()) { /* skip this logger to allow logging failure to open @@ -496,19 +477,14 @@ bool FileLogger::Log(Log::Level Level, const std::string& rFile, return true; } ->>>>>>> 0.12 if (Level > GetLevel()) { return true; } /* avoid infinite loop if this throws an exception */ -<<<<<<< HEAD - Logging::Remove(this); -======= Log::Level oldLevel = GetLevel(); Filter(Log::NOTHING); ->>>>>>> 0.12 std::ostringstream buf; buf << FormatTime(GetCurrentBoxTime(), true, false); @@ -548,12 +524,8 @@ bool FileLogger::Log(Log::Level Level, const std::string& rFile, mLogFile.Write(output.c_str(), output.length()); -<<<<<<< HEAD - Logging::Add(this); -======= // no infinite loop, reset to saved logging level Filter(oldLevel); ->>>>>>> 0.12 return true; } @@ -577,8 +549,6 @@ std::string PrintEscapedBinaryData(const std::string& rInput) return output.str(); } -<<<<<<< HEAD -======= bool HideSpecificExceptionGuard::IsHidden(int type, int subtype) { @@ -594,4 +564,3 @@ bool HideSpecificExceptionGuard::IsHidden(int type, int subtype) return false; } ->>>>>>> 0.12 diff --git a/lib/common/Logging.h b/lib/common/Logging.h index bc0203bb..1074b7c3 100644 --- a/lib/common/Logging.h +++ b/lib/common/Logging.h @@ -10,11 +10,8 @@ #ifndef LOGGING__H #define LOGGING__H -<<<<<<< HEAD -======= #include <assert.h> ->>>>>>> 0.12 #include <cerrno> #include <cstring> #include <iomanip> @@ -46,33 +43,6 @@ if (Logging::IsEnabled(Log::TRACE)) \ { BOX_LOG(Log::TRACE, stuff) } -<<<<<<< HEAD -#define BOX_SYS_ERROR(stuff) \ - stuff << ": " << std::strerror(errno) << " (" << errno << ")" - -#define BOX_LOG_SYS_WARNING(stuff) \ - BOX_WARNING(BOX_SYS_ERROR(stuff)) -#define BOX_LOG_SYS_ERROR(stuff) \ - BOX_ERROR(BOX_SYS_ERROR(stuff)) -#define BOX_LOG_SYS_FATAL(stuff) \ - BOX_FATAL(BOX_SYS_ERROR(stuff)) - -#define LOG_AND_THROW_ERROR(message, filename, exception, subtype) \ - BOX_LOG_SYS_ERROR(message << ": " << filename); \ - THROW_EXCEPTION_MESSAGE(exception, subtype, \ - BOX_SYS_ERROR(message << ": " << filename)); - -inline std::string GetNativeErrorMessage() -{ -#ifdef WIN32 - return GetErrorMessage(GetLastError()); -#else - std::ostringstream _box_log_line; - _box_log_line << std::strerror(errno) << " (" << errno << ")"; - return _box_log_line.str(); -#endif -} -======= #define BOX_SYS_ERRNO_MESSAGE(error_number, stuff) \ stuff << ": " << std::strerror(error_number) << \ " (" << error_number << ")" @@ -113,7 +83,6 @@ inline std::string GetNativeErrorMessage() #define THROW_FILE_ERROR(message, filename, exception, subtype) \ THROW_EXCEPTION_MESSAGE(exception, subtype, \ BOX_FILE_MESSAGE(filename, message)) ->>>>>>> 0.12 #ifdef WIN32 #define BOX_LOG_WIN_ERROR(stuff) \ @@ -126,8 +95,6 @@ inline std::string GetNativeErrorMessage() BOX_WARNING(stuff << ": " << GetErrorMessage(number)) #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_WIN_ERROR(stuff) #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_WIN_WARNING(stuff) -<<<<<<< HEAD -======= #define BOX_WIN_ERRNO_MESSAGE(error_number, stuff) \ stuff << ": " << GetErrorMessage(error_number) #define THROW_WIN_ERROR_NUMBER(message, error_number, exception, subtype) \ @@ -139,17 +106,11 @@ inline std::string GetNativeErrorMessage() #define THROW_WIN_FILE_ERROR(message, filename, exception, subtype) \ THROW_WIN_FILE_ERRNO(message, filename, GetLastError(), \ exception, subtype) ->>>>>>> 0.12 #else #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_SYS_ERROR(stuff) #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_SYS_WARNING(stuff) #endif -<<<<<<< HEAD -#define BOX_LOG_SOCKET_ERROR(_type, _name, _port, stuff) \ - BOX_LOG_NATIVE_ERROR(stuff << " (type " << _type << ", name " << \ - _name << ", port " << _port << ")") -======= #ifdef WIN32 # define BOX_LOG_SOCKET_ERROR(_type, _name, _port, stuff) \ BOX_LOG_WIN_ERROR_NUMBER(stuff << " (type " << _type << ", name " << \ @@ -159,7 +120,6 @@ inline std::string GetNativeErrorMessage() BOX_LOG_NATIVE_ERROR(stuff << " (type " << _type << ", name " << \ _name << ", port " << _port << ")") #endif ->>>>>>> 0.12 #define BOX_FORMAT_HEX32(number) \ std::hex << \ @@ -184,15 +144,12 @@ inline std::string GetNativeErrorMessage() std::setw(6) << \ timespec.tv_usec -<<<<<<< HEAD -======= #define BOX_FORMAT_MICROSECONDS(t) \ (int)((t) / 1000000) << "." << \ std::setw(6) << \ std::setfill('0') << \ (int)((t) % 1000000) << " seconds" ->>>>>>> 0.12 #undef ERROR namespace Log @@ -240,10 +197,6 @@ class Logger virtual const char* GetType() = 0; Log::Level GetLevel() { return mCurrentLevel; } -<<<<<<< HEAD - - virtual void SetProgramName(const std::string& rProgramName) = 0; -======= bool IsEnabled(Log::Level level); virtual void SetProgramName(const std::string& rProgramName) = 0; @@ -265,7 +218,6 @@ class Logger mLogger.Filter(mOldLevel); } }; ->>>>>>> 0.12 }; // -------------------------------------------------------------------------- @@ -373,42 +325,29 @@ class Logging static void SetProgramName(const std::string& rProgramName); static std::string GetProgramName() { return sProgramName; } static void SetFacility(int facility); -<<<<<<< HEAD -======= static Console& GetConsole() { return *spConsole; } static Syslog& GetSyslog() { return *spSyslog; } ->>>>>>> 0.12 class Guard { private: Log::Level mOldLevel; -<<<<<<< HEAD -======= static int sGuardCount; static Log::Level sOriginalLevel; ->>>>>>> 0.12 public: Guard(Log::Level newLevel) { mOldLevel = Logging::GetGlobalLevel(); -<<<<<<< HEAD -======= if(sGuardCount == 0) { sOriginalLevel = mOldLevel; } sGuardCount++; ->>>>>>> 0.12 Logging::SetGlobalLevel(newLevel); } ~Guard() { -<<<<<<< HEAD - Logging::SetGlobalLevel(mOldLevel); - } -======= sGuardCount--; Logging::SetGlobalLevel(mOldLevel); } @@ -420,7 +359,6 @@ class Logging return IsActive() && (int)sOriginalLevel >= (int)originalLevel; } ->>>>>>> 0.12 }; class Tagger @@ -429,11 +367,6 @@ class Logging std::string mOldTag; public: -<<<<<<< HEAD - Tagger(const std::string& rTempTag) - { - mOldTag = Logging::GetProgramName(); -======= Tagger() : mOldTag(Logging::GetProgramName()) { @@ -441,21 +374,17 @@ class Logging Tagger(const std::string& rTempTag) : mOldTag(Logging::GetProgramName()) { ->>>>>>> 0.12 Logging::SetProgramName(mOldTag + " " + rTempTag); } ~Tagger() { Logging::SetProgramName(mOldTag); } -<<<<<<< HEAD -======= void Change(const std::string& newTempTag) { Logging::SetProgramName(mOldTag + " " + newTempTag); } ->>>>>>> 0.12 }; }; @@ -498,8 +427,6 @@ class HideExceptionMessageGuard bool mOldHiddenState; }; -<<<<<<< HEAD -======= class HideSpecificExceptionGuard { private: @@ -524,7 +451,6 @@ class HideSpecificExceptionGuard static bool IsHidden(int type, int subtype); }; ->>>>>>> 0.12 std::string PrintEscapedBinaryData(const std::string& rInput); #endif // LOGGING__H diff --git a/lib/common/MainHelper.h b/lib/common/MainHelper.h index 453b5b5f..3c6e9ff0 100644 --- a/lib/common/MainHelper.h +++ b/lib/common/MainHelper.h @@ -12,16 +12,12 @@ #include <stdio.h> -<<<<<<< HEAD -#include "BoxException.h" -======= #ifdef NEED_BOX_VERSION_H # include "BoxVersion.h" #endif #include "BoxException.h" #include "Logging.h" ->>>>>>> 0.12 #define MAINHELPER_START \ if(argc == 2 && ::strcmp(argv[1], "--version") == 0) \ @@ -29,18 +25,6 @@ MEMLEAKFINDER_INIT \ MEMLEAKFINDER_START \ try { -<<<<<<< HEAD -#define MAINHELPER_END \ - } catch(BoxException &e) { \ - printf("Exception: %s (%d/%d)\n", e.what(), e.GetType(), e.GetSubType()); \ - return 1; \ - } catch(std::exception &e) { \ - printf("Exception: %s\n", e.what()); \ - return 1; \ - } catch(...) { \ - printf("Exception: <UNKNOWN>\n"); \ - return 1; } -======= #define MAINHELPER_END \ } catch(std::exception &e) { \ @@ -50,7 +34,6 @@ BOX_FATAL("UNKNOWN"); \ return 1; \ } ->>>>>>> 0.12 #ifdef BOX_MEMORY_LEAK_TESTING #define MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, marker) \ diff --git a/lib/common/MemBlockStream.cpp b/lib/common/MemBlockStream.cpp index 554fc6ae..3a43a304 100644 --- a/lib/common/MemBlockStream.cpp +++ b/lib/common/MemBlockStream.cpp @@ -22,8 +22,6 @@ // // Function // Name: MemBlockStream::MemBlockStream() -<<<<<<< HEAD -======= // Purpose: Constructor with no contents // Created: 2012/11/07 // @@ -38,7 +36,6 @@ MemBlockStream::MemBlockStream() // // Function // Name: MemBlockStream::MemBlockStream() ->>>>>>> 0.12 // Purpose: Constructor (doesn't copy block, careful with lifetimes) // Created: 2003/09/05 // @@ -86,10 +83,6 @@ MemBlockStream::MemBlockStream(const CollectInBufferStream &rBuffer) ASSERT(mBytesInBuffer >= 0); } -<<<<<<< HEAD - -======= ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Function diff --git a/lib/common/MemBlockStream.h b/lib/common/MemBlockStream.h index ed0bf4ff..5234525b 100644 --- a/lib/common/MemBlockStream.h +++ b/lib/common/MemBlockStream.h @@ -27,10 +27,7 @@ class CollectInBufferStream; class MemBlockStream : public IOStream { public: -<<<<<<< HEAD -======= MemBlockStream(); ->>>>>>> 0.12 MemBlockStream(const void *pBuffer, int Size); MemBlockStream(const StreamableMemBlock &rBlock); MemBlockStream(const CollectInBufferStream &rBuffer); @@ -45,11 +42,8 @@ public: virtual void Seek(pos_type Offset, int SeekType); virtual bool StreamDataLeft(); virtual bool StreamClosed(); -<<<<<<< HEAD -======= virtual const void* GetBuffer() const { return mpBuffer; } virtual int GetSize() const { return mBytesInBuffer; } ->>>>>>> 0.12 private: const char *mpBuffer; diff --git a/lib/common/MemLeakFinder.h b/lib/common/MemLeakFinder.h index e4e9a1ae..1a2cf90c 100644 --- a/lib/common/MemLeakFinder.h +++ b/lib/common/MemLeakFinder.h @@ -28,10 +28,7 @@ class MemLeakSuppressionGuard extern "C" { void *memleakfinder_malloc(size_t size, const char *file, int line); -<<<<<<< HEAD -======= void *memleakfinder_calloc(size_t blocks, size_t size, const char *file, int line); ->>>>>>> 0.12 void *memleakfinder_realloc(void *ptr, size_t size); void memleakfinder_free(void *ptr); } @@ -40,11 +37,8 @@ void memleakfinder_init(); int memleakfinder_numleaks(); -<<<<<<< HEAD -======= void memleakfinder_report_usage_summary(); ->>>>>>> 0.12 void memleakfinder_reportleaks(); void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext); @@ -63,10 +57,7 @@ void *operator new[](size_t size, const char *file, int line); // define the malloc functions now, if required #ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__) -<<<<<<< HEAD -======= #define calloc(X, Y) memleakfinder_calloc(X, Y, __FILE__, __LINE__) ->>>>>>> 0.12 #define realloc memleakfinder_realloc #define free memleakfinder_free #define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED diff --git a/lib/common/StreamableMemBlock.cpp b/lib/common/StreamableMemBlock.cpp index ce2e1f98..b376f037 100644 --- a/lib/common/StreamableMemBlock.cpp +++ b/lib/common/StreamableMemBlock.cpp @@ -3,10 +3,7 @@ // File // Name: StreamableMemBlock.cpp // Purpose: Memory blocks which can be loaded and saved from streams -<<<<<<< HEAD -======= // with a header indicating the size of the block. ->>>>>>> 0.12 // Created: 2003/09/05 // // -------------------------------------------------------------------------- diff --git a/lib/common/StreamableMemBlock.h b/lib/common/StreamableMemBlock.h index 3fcc62b3..6c1c5ab8 100644 --- a/lib/common/StreamableMemBlock.h +++ b/lib/common/StreamableMemBlock.h @@ -2,12 +2,8 @@ // // File // Name: StreamableMemBlock.h -<<<<<<< HEAD -// Purpose: Memory blocks which can be loaded and saved from streams -======= // Purpose: Memory blocks which can be loaded and saved from streams, // with a header indicating the size of the block. ->>>>>>> 0.12 // Created: 2003/09/05 // // -------------------------------------------------------------------------- diff --git a/lib/common/TemporaryDirectory.h b/lib/common/TemporaryDirectory.h deleted file mode 100644 index 9d52ecd9..00000000 --- a/lib/common/TemporaryDirectory.h +++ /dev/null @@ -1,46 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: TemporaryDirectory.h -// Purpose: Location of temporary directory -// Created: 2003/10/13 -// -// -------------------------------------------------------------------------- - -#ifndef TEMPORARYDIRECTORY__H -#define TEMPORARYDIRECTORY__H - -#include <string> - -#ifdef WIN32 - #include <windows.h> -#endif - -// Prefix name with Box to avoid clashing with OS API names -std::string BoxGetTemporaryDirectoryName() -{ -#ifdef WIN32 - // http://msdn.microsoft.com/library/default.asp? - // url=/library/en-us/fileio/fs/creating_and_using_a_temporary_file.asp - - DWORD dwRetVal; - char lpPathBuffer[1024]; - DWORD dwBufSize = sizeof(lpPathBuffer); - - // Get the temp path. - dwRetVal = GetTempPath(dwBufSize, // length of the buffer - lpPathBuffer); // buffer for path - if (dwRetVal > dwBufSize) - { - THROW_EXCEPTION(CommonException, TempDirPathTooLong) - } - - return std::string(lpPathBuffer); -#elif defined TEMP_DIRECTORY_NAME - return std::string(TEMP_DIRECTORY_NAME); -#else - #error non-static temporary directory names not supported yet -#endif -} - -#endif // TEMPORARYDIRECTORY__H diff --git a/lib/common/Test.cpp b/lib/common/Test.cpp index b627ac18..de87c465 100644 --- a/lib/common/Test.cpp +++ b/lib/common/Test.cpp @@ -21,10 +21,7 @@ #include <unistd.h> #endif -<<<<<<< HEAD -======= #include "BoxTime.h" ->>>>>>> 0.12 #include "Test.h" bool TestFileExists(const char *Filename) @@ -47,17 +44,10 @@ bool TestDirExists(const char *Filename) } // -1 if doesn't exist -<<<<<<< HEAD -int TestGetFileSize(const char *Filename) -{ - EMU_STRUCT_STAT st; - if(EMU_STAT(Filename, &st) == 0) -======= int TestGetFileSize(const std::string& Filename) { EMU_STRUCT_STAT st; if(EMU_STAT(Filename.c_str(), &st) == 0) ->>>>>>> 0.12 { return st.st_size; } @@ -462,40 +452,6 @@ void wait_for_operation(int seconds, const char* message) void safe_sleep(int seconds) { -<<<<<<< HEAD - BOX_TRACE("sleeping for " << seconds << " seconds"); - -#ifdef WIN32 - Sleep(seconds * 1000); -#else - struct timespec ts; - memset(&ts, 0, sizeof(ts)); - ts.tv_sec = seconds; - ts.tv_nsec = 0; - while (nanosleep(&ts, &ts) == -1 && errno == EINTR) - { - // FIXME evil hack for OSX, where ts.tv_sec contains - // a negative number interpreted as unsigned 32-bit - // when nanosleep() returns later than expected. - - int32_t secs = (int32_t) ts.tv_sec; - int64_t remain_ns = (secs * 1000000000) + ts.tv_nsec; - - if (remain_ns < 0) - { - BOX_WARNING("nanosleep interrupted " << - ((float)(0 - remain_ns) / 1000000000) << - " secs late"); - return; - } - - BOX_TRACE("nanosleep interrupted with " << - (remain_ns / 1000000000) << " secs remaining, " - "sleeping again"); - } -#endif -======= ShortSleep(SecondsToBoxTime(seconds), true); ->>>>>>> 0.12 } diff --git a/lib/common/Test.h b/lib/common/Test.h index a09a64af..f318c811 100644 --- a/lib/common/Test.h +++ b/lib/common/Test.h @@ -51,33 +51,6 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; #define TEST_THAT_ABORTONFAIL(condition) {if(!(condition)) TEST_ABORT_WITH_MESSAGE("Condition [" #condition "] failed")} // NOTE: The 0- bit is to allow this to work with stuff which has negative constants for flags (eg ConnectionException) -<<<<<<< HEAD -#define TEST_CHECK_THROWS(statement, excepttype, subtype) \ - { \ - bool didthrow = false; \ - HideExceptionMessageGuard hide; \ - try \ - { \ - statement; \ - } \ - catch(excepttype &e) \ - { \ - if(e.GetSubType() != ((unsigned int)excepttype::subtype) \ - && e.GetSubType() != (unsigned int)(0-excepttype::subtype)) \ - { \ - throw; \ - } \ - didthrow = true; \ - } \ - catch(...) \ - { \ - throw; \ - } \ - if(!didthrow) \ - { \ - TEST_FAIL_WITH_MESSAGE("Didn't throw exception " #excepttype "(" #subtype ")") \ - } \ -======= #define TEST_CHECK_THROWS(statement, excepttype, subtype) \ { \ bool didthrow = false; \ @@ -105,7 +78,6 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; { \ TEST_FAIL_WITH_MESSAGE("Didn't throw exception " #excepttype "(" #subtype ")") \ } \ ->>>>>>> 0.12 } // utility macro for comparing two strings in a line @@ -121,11 +93,7 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; \ if(_exp_str != _found_str) \ { \ -<<<<<<< HEAD - BOX_WARNING("Expected <" << _exp_str << "> but found <" << \ -======= BOX_ERROR("Expected <" << _exp_str << "> but found <" << \ ->>>>>>> 0.12 _found_str << ">"); \ \ std::ostringstream _oss3; \ @@ -161,34 +129,22 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; } \ } -<<<<<<< HEAD - -======= ->>>>>>> 0.12 // utility macro for testing a line #define TEST_LINE(_condition, _line) \ TEST_THAT(_condition); \ if (!(_condition)) \ { \ -<<<<<<< HEAD - printf("Test failed on <%s>\n", _line.c_str()); \ -======= std::ostringstream _ossl; \ _ossl << _line; \ std::string _line_str = _ossl.str(); \ printf("Test failed on <%s>\n", _line_str.c_str()); \ ->>>>>>> 0.12 } bool TestFileExists(const char *Filename); bool TestDirExists(const char *Filename); // -1 if doesn't exist -<<<<<<< HEAD -int TestGetFileSize(const char *Filename); -======= int TestGetFileSize(const std::string& Filename); ->>>>>>> 0.12 std::string ConvertPaths(const std::string& rOriginal); int RunCommand(const std::string& rCommandLine); bool ServerIsAlive(int pid); diff --git a/lib/common/Timer.cpp b/lib/common/Timer.cpp index 390ddf8e..ad6b5e8d 100644 --- a/lib/common/Timer.cpp +++ b/lib/common/Timer.cpp @@ -8,14 +8,6 @@ // // -------------------------------------------------------------------------- -<<<<<<< HEAD -#ifdef WIN32 - #define _WIN32_WINNT 0x0500 -#endif - -#include "Box.h" - -======= #include "Box.h" #ifdef WIN32 @@ -26,7 +18,6 @@ # endif #endif ->>>>>>> 0.12 #include <signal.h> #include <cstring> @@ -39,11 +30,7 @@ std::vector<Timer*>* Timers::spTimers = NULL; bool Timers::sRescheduleNeeded = false; #define TIMER_ID "timer " << mName << " (" << this << ") " -<<<<<<< HEAD -#define TIMER_ID_OF(t) "timer " << (t).GetName() << " (" << &(t) << ") " -======= #define TIMER_ID_OF(t) "timer " << (t).GetName() << " (" << &(t) << ")" ->>>>>>> 0.12 typedef void (*sighandler_t)(int); @@ -132,10 +119,7 @@ void Timers::Add(Timer& rTimer) { ASSERT(spTimers); ASSERT(&rTimer); -<<<<<<< HEAD -======= BOX_TRACE(TIMER_ID_OF(rTimer) " added to global queue, rescheduling"); ->>>>>>> 0.12 spTimers->push_back(&rTimer); Reschedule(); } @@ -153,10 +137,7 @@ void Timers::Remove(Timer& rTimer) { ASSERT(spTimers); ASSERT(&rTimer); -<<<<<<< HEAD -======= BOX_TRACE(TIMER_ID_OF(rTimer) " removed from global queue, rescheduling"); ->>>>>>> 0.12 bool restart = true; while (restart) @@ -191,13 +172,6 @@ void Timers::RescheduleIfNeeded() } } -<<<<<<< HEAD -#define FORMAT_MICROSECONDS(t) \ - (int)(t / 1000000) << "." << \ - (int)(t % 1000000) << " seconds" - -======= ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Function @@ -260,19 +234,9 @@ void Timers::Reschedule() if (timeToExpiry <= 0) { -<<<<<<< HEAD - /* - BOX_TRACE("timer " << *i << " has expired, " - "triggering it"); - */ - BOX_TRACE(TIMER_ID_OF(**i) "has expired, " - "triggering " << - FORMAT_MICROSECONDS(-timeToExpiry) << -======= BOX_TRACE(TIMER_ID_OF(**i) " has expired, " "triggering " << BOX_FORMAT_MICROSECONDS(-timeToExpiry) << ->>>>>>> 0.12 " late"); rTimer.OnExpire(); spTimers->erase(i); @@ -282,13 +246,8 @@ void Timers::Reschedule() else { /* -<<<<<<< HEAD - BOX_TRACE("timer " << *i << " has not " - "expired, triggering in " << -======= BOX_TRACE(TIMER_ID_OF(**i) " has not expired, " "triggering in " << ->>>>>>> 0.12 FORMAT_MICROSECONDS(timeToExpiry) << " seconds"); */ @@ -329,13 +288,8 @@ void Timers::Reschedule() } else { -<<<<<<< HEAD - BOX_TRACE("timer: next event: " << nameOfNextEvent << - " expires in " << FORMAT_MICROSECONDS(timeToNextEvent)); -======= BOX_TRACE("timer: next event: " << nameOfNextEvent << " at " << FormatTime(timeNow + timeToNextEvent, false, true)); ->>>>>>> 0.12 } struct itimerval timeout; @@ -375,11 +329,7 @@ void Timers::SignalHandler(int unused) // -------------------------------------------------------------------------- // // Function -<<<<<<< HEAD -// Name: Timer::Timer(size_t timeoutSecs, -======= // Name: Timer::Timer(size_t timeoutMillis, ->>>>>>> 0.12 // const std::string& rName) // Purpose: Standard timer constructor, takes a timeout in // seconds from now, and an optional name for @@ -388,161 +338,82 @@ void Timers::SignalHandler(int unused) // // -------------------------------------------------------------------------- -<<<<<<< HEAD -Timer::Timer(size_t timeoutSecs, const std::string& rName) -: mExpires(GetCurrentBoxTime() + SecondsToBoxTime(timeoutSecs)), -======= Timer::Timer(size_t timeoutMillis, const std::string& rName) : mExpires(0), ->>>>>>> 0.12 mExpired(false), mName(rName) #ifdef WIN32 , mTimerHandle(INVALID_HANDLE_VALUE) #endif { -<<<<<<< HEAD - #ifndef BOX_RELEASE_BUILD - if (timeoutSecs == 0) - { - BOX_TRACE(TIMER_ID "initialised for " << timeoutSecs << - " secs, will not fire"); - } - else - { - BOX_TRACE(TIMER_ID "initialised for " << timeoutSecs << - " secs, to fire at " << FormatTime(mExpires, false, true)); - } - #endif - - if (timeoutSecs == 0) - { - mExpires = 0; - } - else - { - Timers::Add(*this); - Start(timeoutSecs * MICRO_SEC_IN_SEC_LL); - } -======= Set(timeoutMillis, true /* isInit */); ->>>>>>> 0.12 } // -------------------------------------------------------------------------- // // Function // Name: Timer::Start() -<<<<<<< HEAD -// Purpose: This internal function initialises an OS TimerQueue -// timer on Windows, while on Unixes there is only a -// single global timer, managed by the Timers class, -// so this method does nothing. -======= // Purpose: This internal function recalculates the remaining // time (timeout) from the expiry time, and then calls // Start(timeoutMillis). ->>>>>>> 0.12 // Created: 27/07/2008 // // -------------------------------------------------------------------------- void Timer::Start() { -<<<<<<< HEAD -#ifdef WIN32 -======= ->>>>>>> 0.12 box_time_t timeNow = GetCurrentBoxTime(); int64_t timeToExpiry = mExpires - timeNow; if (timeToExpiry <= 0) { BOX_WARNING(TIMER_ID << "fudging expiry from -" << -<<<<<<< HEAD - FORMAT_MICROSECONDS(-timeToExpiry)) - timeToExpiry = 1; - } - - Start(timeToExpiry); -#endif -======= BOX_FORMAT_MICROSECONDS(-timeToExpiry)) timeToExpiry = 1; } Start(timeToExpiry / MICRO_SEC_IN_MILLI_SEC); ->>>>>>> 0.12 } // -------------------------------------------------------------------------- // // Function -<<<<<<< HEAD -// Name: Timer::Start(int64_t delayInMicros) -// Purpose: This internal function initialises an OS TimerQueue -// timer on Windows, with a specified delay already -// calculated to save us doing it again. Like -// Timer::Start(), on Unixes it does nothing. -======= // Name: Timer::Start(int64_t timeoutMillis) // Purpose: This internal function adds this timer to the global // timer list, and on Windows it initialises an OS // TimerQueue timer for it. ->>>>>>> 0.12 // Created: 27/07/2008 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -void Timer::Start(int64_t delayInMicros) -{ -======= void Timer::Start(int64_t timeoutMillis) { ASSERT(mExpires != 0); Timers::Add(*this); ->>>>>>> 0.12 #ifdef WIN32 // only call me once! ASSERT(mTimerHandle == INVALID_HANDLE_VALUE); -<<<<<<< HEAD - int64_t delayInMillis = delayInMicros / 1000; - -======= ->>>>>>> 0.12 // Windows XP always seems to fire timers up to 20 ms late, // at least on my test laptop. Not critical in practice, but our // tests are precise enough that they will fail if we don't // correct for it. -<<<<<<< HEAD - delayInMillis -= 20; - - // Set a system timer to call our timer routine - if (CreateTimerQueueTimer(&mTimerHandle, NULL, TimerRoutine, - (PVOID)this, delayInMillis, 0, WT_EXECUTEINTIMERTHREAD) -======= timeoutMillis -= 20; // Set a system timer to call our timer routine if (CreateTimerQueueTimer(&mTimerHandle, NULL, TimerRoutine, (PVOID)this, timeoutMillis, 0, WT_EXECUTEINTIMERTHREAD) ->>>>>>> 0.12 == FALSE) { BOX_ERROR(TIMER_ID "failed to create timer: " << GetErrorMessage(GetLastError())); mTimerHandle = INVALID_HANDLE_VALUE; } -<<<<<<< HEAD -======= else { BOX_INFO(TIMER_ID << "set for " << timeoutMillis << " ms"); } ->>>>>>> 0.12 #endif } @@ -550,29 +421,20 @@ void Timer::Start(int64_t timeoutMillis) // // Function // Name: Timer::Stop() -<<<<<<< HEAD -// Purpose: This internal function deletes the associated OS -// TimerQueue timer on Windows, and on Unixes does -// nothing. -======= // Purpose: This internal function removes us from the global // list of timers, resets our expiry time, and on // Windows it deletes the associated OS TimerQueue timer. ->>>>>>> 0.12 // Created: 27/07/2008 // // -------------------------------------------------------------------------- void Timer::Stop() { -<<<<<<< HEAD -======= if (mExpires != 0) { Timers::Remove(*this); } ->>>>>>> 0.12 #ifdef WIN32 if (mTimerHandle != INVALID_HANDLE_VALUE) { @@ -602,12 +464,6 @@ Timer::~Timer() BOX_TRACE(TIMER_ID "destroyed"); #endif -<<<<<<< HEAD - Timers::Remove(*this); - Stop(); -} - -======= Stop(); } @@ -632,7 +488,6 @@ void Timer::LogAssignment(const Timer &From) #endif } ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Function @@ -652,36 +507,10 @@ Timer::Timer(const Timer& rToCopy) , mTimerHandle(INVALID_HANDLE_VALUE) #endif { -<<<<<<< HEAD - #ifndef BOX_RELEASE_BUILD - if (mExpired) - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "already expired, will not fire"); - } - else if (mExpires == 0) - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "no expiry, will not fire"); - } - else - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "to fire at " << - (int)(mExpires / 1000000) << "." << - (int)(mExpires % 1000000)); - } - #endif - - if (!mExpired && mExpires != 0) - { - Timers::Add(*this); -======= LogAssignment(rToCopy); if (!mExpired && mExpires != 0) { ->>>>>>> 0.12 Start(); } } @@ -700,32 +529,8 @@ Timer::Timer(const Timer& rToCopy) Timer& Timer::operator=(const Timer& rToCopy) { -<<<<<<< HEAD - #ifndef BOX_RELEASE_BUILD - if (rToCopy.mExpired) - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "already expired, will not fire"); - } - else if (rToCopy.mExpires == 0) - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "no expiry, will not fire"); - } - else - { - BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " - "to fire at " << - (int)(rToCopy.mExpires / 1000000) << "." << - (int)(rToCopy.mExpires % 1000000)); - } - #endif - - Timers::Remove(*this); -======= LogAssignment(rToCopy); ->>>>>>> 0.12 Stop(); mExpires = rToCopy.mExpires; @@ -734,10 +539,6 @@ Timer& Timer::operator=(const Timer& rToCopy) if (!mExpired && mExpires != 0) { -<<<<<<< HEAD - Timers::Add(*this); -======= ->>>>>>> 0.12 Start(); } @@ -747,8 +548,6 @@ Timer& Timer::operator=(const Timer& rToCopy) // -------------------------------------------------------------------------- // // Function -<<<<<<< HEAD -======= // Name: Timer::Reset(size_t timeoutMillis) // Purpose: Simple reset operation for an existing Timer. Avoids // the need to create a temporary timer just to modify @@ -810,7 +609,6 @@ void Timer::Set(size_t timeoutMillis, bool isInit) // -------------------------------------------------------------------------- // // Function ->>>>>>> 0.12 // Name: Timer::OnExpire() // Purpose: Method called by Timers::Reschedule (on Unixes) // on next poll after timer expires, or from diff --git a/lib/common/Timer.h b/lib/common/Timer.h index d3e46b2a..09be58fa 100644 --- a/lib/common/Timer.h +++ b/lib/common/Timer.h @@ -53,11 +53,7 @@ class Timers class Timer { public: -<<<<<<< HEAD - Timer(size_t timeoutSecs, const std::string& rName = ""); -======= Timer(size_t timeoutMillis, const std::string& rName = ""); ->>>>>>> 0.12 virtual ~Timer(); Timer(const Timer &); Timer &operator=(const Timer &); @@ -71,10 +67,7 @@ public: } const std::string& GetName() const { return mName; } -<<<<<<< HEAD -======= virtual void Reset(size_t timeoutMillis); ->>>>>>> 0.12 private: box_time_t mExpires; @@ -82,15 +75,10 @@ private: std::string mName; void Start(); -<<<<<<< HEAD - void Start(int64_t delayInMicros); - void Stop(); -======= void Start(int64_t timeoutMillis); void Stop(); void LogAssignment(const Timer &From); virtual void Set(size_t timeoutMillis, bool isReset); ->>>>>>> 0.12 #ifdef WIN32 HANDLE mTimerHandle; diff --git a/lib/common/UnixUser.cpp b/lib/common/UnixUser.cpp index 7e69b10a..32acacfa 100644 --- a/lib/common/UnixUser.cpp +++ b/lib/common/UnixUser.cpp @@ -31,21 +31,13 @@ // Created: 21/1/04 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -UnixUser::UnixUser(const char *Username) -======= UnixUser::UnixUser(const std::string& Username) ->>>>>>> 0.12 : mUID(0), mGID(0), mRevertOnDestruction(false) { // Get password info -<<<<<<< HEAD - struct passwd *pwd = ::getpwnam(Username); -======= struct passwd *pwd = ::getpwnam(Username.c_str()); ->>>>>>> 0.12 if(pwd == 0) { THROW_EXCEPTION(CommonException, CouldNotLookUpUsername) diff --git a/lib/common/UnixUser.h b/lib/common/UnixUser.h index 361971b8..dafe6aa7 100644 --- a/lib/common/UnixUser.h +++ b/lib/common/UnixUser.h @@ -13,11 +13,7 @@ class UnixUser { public: -<<<<<<< HEAD - UnixUser(const char *Username); -======= UnixUser(const std::string& Username); ->>>>>>> 0.12 UnixUser(uid_t UID, gid_t GID); ~UnixUser(); private: diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp index 3137d980..decc80e8 100644 --- a/lib/common/Utils.cpp +++ b/lib/common/Utils.cpp @@ -24,11 +24,6 @@ #include <cxxabi.h> #endif -<<<<<<< HEAD -#include "Utils.h" -#include "CommonException.h" -#include "Logging.h" -======= #ifdef HAVE_DLFCN_H #include <dlfcn.h> #endif @@ -40,7 +35,6 @@ #include "CommonException.h" #include "Logging.h" #include "Utils.h" ->>>>>>> 0.12 #include "MemLeakFindOn.h" @@ -87,14 +81,6 @@ void SplitString(const std::string &String, char SplitOn, std::vector<std::strin } #ifdef SHOW_BACKTRACE_ON_EXCEPTION -<<<<<<< HEAD -void DumpStackBacktrace() -{ - void *array[10]; - size_t size = backtrace (array, 10); - char **strings = backtrace_symbols (array, size); - -======= static std::string demangle(const std::string& mangled_name) { #ifdef HAVE_CXXABI_H @@ -159,85 +145,10 @@ void DumpStackBacktrace() { void *array[10]; size_t size = backtrace(array, 10); ->>>>>>> 0.12 BOX_TRACE("Obtained " << size << " stack frames."); for(size_t i = 0; i < size; i++) { -<<<<<<< HEAD - // Demangling code copied from - // cctbx_sources/boost_adaptbx/meta_ext.cpp, BSD license - - std::string mangled_frame = strings[i]; - std::string output_frame = strings[i]; // default - - #ifdef HAVE_CXXABI_H - int start = mangled_frame.find('('); - int end = mangled_frame.find('+', start); - std::string mangled_func = mangled_frame.substr(start + 1, - end - start - 1); - - int status; - -#include "MemLeakFindOff.h" - char* result = abi::__cxa_demangle(mangled_func.c_str(), - NULL, NULL, &status); -#include "MemLeakFindOn.h" - - if (result == NULL) - { - if (status == 0) - { - BOX_WARNING("Demangle failed but no error: " << - mangled_func); - } - else if (status == -1) - { - BOX_WARNING("Demangle failed with " - "memory allocation error: " << - mangled_func); - } - else if (status == -2) - { - // Probably non-C++ name, don't demangle - /* - BOX_WARNING("Demangle failed with " - "with invalid name: " << - mangled_func); - */ - } - else if (status == -3) - { - BOX_WARNING("Demangle failed with " - "with invalid argument: " << - mangled_func); - } - else - { - BOX_WARNING("Demangle failed with " - "with unknown error " << status << - ": " << mangled_func); - } - } - else - { - output_frame = mangled_frame.substr(0, start + 1) + - result + mangled_frame.substr(end); -#include "MemLeakFindOff.h" - std::free(result); -#include "MemLeakFindOn.h" - } - #endif // HAVE_CXXABI_H - - BOX_TRACE("Stack frame " << i << ": " << output_frame); - } - -#include "MemLeakFindOff.h" - std::free (strings); -#include "MemLeakFindOn.h" -} -#endif -======= std::ostringstream output; output << "Stack frame " << i << ": "; @@ -270,7 +181,6 @@ void DumpStackBacktrace() } } #endif // SHOW_BACKTRACE_ON_EXCEPTION ->>>>>>> 0.12 @@ -429,8 +339,6 @@ std::string FormatUsageLineStart(const std::string& rName, return result.str(); } -<<<<<<< HEAD -======= std::string BoxGetTemporaryDirectoryName() { @@ -459,4 +367,3 @@ std::string BoxGetTemporaryDirectoryName() } ->>>>>>> 0.12 diff --git a/lib/common/Utils.h b/lib/common/Utils.h index 8a938981..3134245a 100644 --- a/lib/common/Utils.h +++ b/lib/common/Utils.h @@ -39,11 +39,8 @@ std::string FormatUsageBar(int64_t Blocks, int64_t Bytes, int64_t Max, std::string FormatUsageLineStart(const std::string& rName, bool MachineReadable); -<<<<<<< HEAD -======= std::string BoxGetTemporaryDirectoryName(); ->>>>>>> 0.12 #include "MemLeakFindOff.h" #endif // UTILS__H diff --git a/lib/common/ZeroStream.cpp b/lib/common/ZeroStream.cpp index 26fd62cb..d11ed80c 100644 --- a/lib/common/ZeroStream.cpp +++ b/lib/common/ZeroStream.cpp @@ -152,11 +152,7 @@ void ZeroStream::Close() // -------------------------------------------------------------------------- bool ZeroStream::StreamDataLeft() { -<<<<<<< HEAD - return false; -======= return (BytesLeftToRead() > 0); ->>>>>>> 0.12 } // -------------------------------------------------------------------------- diff --git a/lib/common/makeexception.pl.in b/lib/common/makeexception.pl.in index 593740da..b1b3a8ac 100755 --- a/lib/common/makeexception.pl.in +++ b/lib/common/makeexception.pl.in @@ -191,11 +191,7 @@ unsigned int ${class}Exception::GetSubType() const throw() const char *${class}Exception::what() const throw() { #ifdef EXCEPTION_CODENAMES_EXTENDED -<<<<<<< HEAD - if(mSubType < 0 || mSubType > (sizeof(whats) / sizeof(whats[0]))) -======= if(mSubType > (sizeof(whats) / sizeof(whats[0]))) ->>>>>>> 0.12 { return "${class}"; } diff --git a/lib/crypto/CipherAES.h b/lib/crypto/CipherAES.h index dbcce9e5..d2c9ed65 100644 --- a/lib/crypto/CipherAES.h +++ b/lib/crypto/CipherAES.h @@ -37,8 +37,6 @@ public: // Setup any other parameters virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const; -<<<<<<< HEAD -======= virtual std::string GetCipherName() const { std::ostringstream out; @@ -48,7 +46,6 @@ public: } virtual CipherMode GetCipherMode() const { return mMode; } ->>>>>>> 0.12 private: CipherDescription::CipherMode mMode; const void *mpKey; diff --git a/lib/crypto/CipherBlowfish.h b/lib/crypto/CipherBlowfish.h index 27903a3e..152a265c 100644 --- a/lib/crypto/CipherBlowfish.h +++ b/lib/crypto/CipherBlowfish.h @@ -38,8 +38,6 @@ public: // Setup any other parameters virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const; -<<<<<<< HEAD -======= virtual std::string GetCipherName() const { std::ostringstream out; @@ -49,7 +47,6 @@ public: } virtual CipherMode GetCipherMode() const { return mMode; } ->>>>>>> 0.12 #ifdef HAVE_OLD_SSL CipherDescription *Clone() const; void SetIV(const void *pIV); diff --git a/lib/crypto/CipherContext.cpp b/lib/crypto/CipherContext.cpp index 6621f79a..fd149395 100644 --- a/lib/crypto/CipherContext.cpp +++ b/lib/crypto/CipherContext.cpp @@ -13,10 +13,7 @@ #include "CipherContext.h" #include "CipherDescription.h" #include "CipherException.h" -<<<<<<< HEAD -======= #include "CryptoUtils.h" ->>>>>>> 0.12 #include "Random.h" #include "MemLeakFindOn.h" @@ -30,21 +27,12 @@ // // -------------------------------------------------------------------------- CipherContext::CipherContext() -<<<<<<< HEAD - : mInitialised(false), - mWithinTransform(false), - mPaddingOn(true) -#ifdef HAVE_OLD_SSL - , mFunction(Decrypt), - mpDescription(0) -======= : mInitialised(false), mWithinTransform(false), mPaddingOn(true), mFunction(None) #ifdef HAVE_OLD_SSL , mpDescription(0) ->>>>>>> 0.12 #endif { } @@ -77,8 +65,6 @@ CipherContext::~CipherContext() // -------------------------------------------------------------------------- // // Function -<<<<<<< HEAD -======= // Name: CipherContext::LogError(const std::string& operation) // Purpose: Logs and clears any OpenSSL errors, returning the // most recent error message for use in exception @@ -101,7 +87,6 @@ std::string CipherContext::LogError(const std::string& operation) // -------------------------------------------------------------------------- // // Function ->>>>>>> 0.12 // Name: CipherContext::Init(CipherContext::CipherFunction, const CipherDescription &) // Purpose: Initialises the context, specifying the direction for the encryption, and a // description of the cipher to use, it's keys, etc @@ -120,28 +105,13 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes THROW_EXCEPTION(CipherException, BadArguments) } -<<<<<<< HEAD -======= // Store function for later mFunction = Function; ->>>>>>> 0.12 // Initialise the cipher #ifndef HAVE_OLD_SSL EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does -<<<<<<< HEAD - if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL, Function) != 1) -#else - // Store function for later - mFunction = Function; - - // Use old version of init call - if(EVP_CipherInit(&ctx, rDescription.GetCipher(), NULL, NULL, Function) != 1) -#endif - { - THROW_EXCEPTION(CipherException, EVPInitFailure) -======= if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL, (mFunction == Encrypt) ? 1 : 0) != 1) #else @@ -153,15 +123,11 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, "Failed to initialise " << rDescription.GetFullName() << "cipher: " << LogError("initialising cipher")); ->>>>>>> 0.12 } try { -<<<<<<< HEAD -======= mCipherName = rDescription.GetFullName(); ->>>>>>> 0.12 #ifndef HAVE_OLD_SSL // Let the description set up everything else rDescription.SetupParameters(&ctx); @@ -176,12 +142,9 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes } catch(...) { -<<<<<<< HEAD -======= THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, "Failed to configure " << mCipherName << " cipher: " << LogError("configuring cipher")); ->>>>>>> 0.12 EVP_CIPHER_CTX_cleanup(&ctx); throw; } @@ -242,13 +205,9 @@ void CipherContext::Begin() // Initialise the cipher context again if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) { -<<<<<<< HEAD - THROW_EXCEPTION(CipherException, EVPInitFailure) -======= THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, "Failed to reset " << mCipherName << " cipher: " << LogError("resetting cipher")); ->>>>>>> 0.12 } // Mark as being within a transform @@ -301,13 +260,9 @@ int CipherContext::Transform(void *pOutBuffer, int OutLength, const void *pInBuf int outLength = OutLength; if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) { -<<<<<<< HEAD - THROW_EXCEPTION(CipherException, EVPUpdateFailure) -======= THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure, "Failed to " << GetFunction() << " (update) " << mCipherName << " cipher: " << LogError(GetFunction())); ->>>>>>> 0.12 } return outLength; @@ -353,18 +308,12 @@ int CipherContext::Final(void *pOutBuffer, int OutLength) // Do the transform int outLength = OutLength; #ifndef HAVE_OLD_SSL -<<<<<<< HEAD - if(EVP_CipherFinal_ex(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1) - { - THROW_EXCEPTION(CipherException, EVPFinalFailure) -======= if(EVP_CipherFinal(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1) { mWithinTransform = false; THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure, "Failed to " << GetFunction() << " (final) " << mCipherName << " cipher: " << LogError(GetFunction())); ->>>>>>> 0.12 } #else OldOpenSSLFinal((unsigned char*)pOutBuffer, outLength); @@ -442,12 +391,8 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) } } // Reinitialise the cipher for the next time around -<<<<<<< HEAD - if(EVP_CipherInit(&ctx, mpDescription->GetCipher(), NULL, NULL, mFunction) != 1) -======= if(EVP_CipherInit(&ctx, mpDescription->GetCipher(), NULL, NULL, (mFunction == Encrypt) ? 1 : 0) != 1) ->>>>>>> 0.12 { THROW_EXCEPTION(CipherException, EVPInitFailure) } @@ -545,39 +490,6 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p // Do the entire block int outLength = 0; -<<<<<<< HEAD - try - { - // Update - outLength = OutLength; - if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) - { - THROW_EXCEPTION(CipherException, EVPUpdateFailure) - } - // Finalise - int outLength2 = OutLength - outLength; -#ifndef HAVE_OLD_SSL - if(EVP_CipherFinal_ex(&ctx, ((unsigned char*)pOutBuffer) + outLength, &outLength2) != 1) - { - THROW_EXCEPTION(CipherException, EVPFinalFailure) - } -#else - OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2); -#endif - outLength += outLength2; - } - catch(...) - { - // Finalise the context, so definately ready for the next caller - int outs = OutLength; -#ifndef HAVE_OLD_SSL - EVP_CipherFinal_ex(&ctx, (unsigned char*)pOutBuffer, &outs); -#else - OldOpenSSLFinal((unsigned char*)pOutBuffer, outs); -#endif - throw; - } -======= // Update outLength = OutLength; @@ -601,7 +513,6 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2); #endif outLength += outLength2; ->>>>>>> 0.12 return outLength; } @@ -651,13 +562,9 @@ void CipherContext::SetIV(const void *pIV) // Set IV if(EVP_CipherInit(&ctx, NULL, NULL, (unsigned char *)pIV, -1) != 1) { -<<<<<<< HEAD - THROW_EXCEPTION(CipherException, EVPInitFailure) -======= THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, "Failed to " << GetFunction() << " (set IV) " << mCipherName << " cipher: " << LogError(GetFunction())); ->>>>>>> 0.12 } #ifdef HAVE_OLD_SSL @@ -702,24 +609,7 @@ const void *CipherContext::SetRandomIV(int &rLengthOut) // Generate some random data Random::Generate(mGeneratedIV, ivLen); -<<<<<<< HEAD - - // Set IV - if(EVP_CipherInit(&ctx, NULL, NULL, mGeneratedIV, -1) != 1) - { - THROW_EXCEPTION(CipherException, EVPInitFailure) - } - -#ifdef HAVE_OLD_SSL - // Update description - if(mpDescription != 0) - { - mpDescription->SetIV(mGeneratedIV); - } -#endif -======= SetIV(mGeneratedIV); ->>>>>>> 0.12 // Return the IV and it's length rLengthOut = ivLen; diff --git a/lib/crypto/CipherContext.h b/lib/crypto/CipherContext.h index 092c10d5..93c889d6 100644 --- a/lib/crypto/CipherContext.h +++ b/lib/crypto/CipherContext.h @@ -35,23 +35,15 @@ public: private: CipherContext(const CipherContext &); // no copying CipherContext &operator=(const CipherContext &); // no assignment -<<<<<<< HEAD -======= protected: std::string LogError(const std::string& operation); ->>>>>>> 0.12 public: typedef enum { -<<<<<<< HEAD - Decrypt = 0, - Encrypt = 1 -======= None = 0, Decrypt, Encrypt ->>>>>>> 0.12 } CipherFunction; void Init(CipherContext::CipherFunction Function, const CipherDescription &rDescription); @@ -72,13 +64,10 @@ public: const void *SetRandomIV(int &rLengthOut); void UsePadding(bool Padding = true); -<<<<<<< HEAD -======= const char* GetFunction() const { return (mFunction == Encrypt) ? "encrypt" : "decrypt"; } ->>>>>>> 0.12 #ifdef HAVE_OLD_SSL void OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut); @@ -90,14 +79,9 @@ private: bool mWithinTransform; bool mPaddingOn; uint8_t mGeneratedIV[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH]; -<<<<<<< HEAD -#ifdef HAVE_OLD_SSL - CipherFunction mFunction; -======= CipherFunction mFunction; std::string mCipherName; #ifdef HAVE_OLD_SSL ->>>>>>> 0.12 CipherDescription *mpDescription; #endif }; diff --git a/lib/crypto/CipherDescription.h b/lib/crypto/CipherDescription.h index 15438494..813df2ce 100644 --- a/lib/crypto/CipherDescription.h +++ b/lib/crypto/CipherDescription.h @@ -34,11 +34,7 @@ public: // Return OpenSSL cipher object virtual const EVP_CIPHER *GetCipher() const = 0; -<<<<<<< HEAD - -======= ->>>>>>> 0.12 // Setup any other parameters virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const = 0; @@ -51,8 +47,6 @@ public: Mode_OFB = 3 } CipherMode; -<<<<<<< HEAD -======= virtual std::string GetCipherName() const = 0; virtual CipherMode GetCipherMode() const = 0; virtual std::string GetFullName() const @@ -70,7 +64,6 @@ public: return out.str(); } ->>>>>>> 0.12 #ifdef HAVE_OLD_SSL // For the old version of OpenSSL, we need to be able to store cipher descriptions. virtual CipherDescription *Clone() const = 0; diff --git a/lib/raidfile/RaidFileController.cpp b/lib/raidfile/RaidFileController.cpp index 84c2a520..cf93947f 100644 --- a/lib/raidfile/RaidFileController.cpp +++ b/lib/raidfile/RaidFileController.cpp @@ -164,26 +164,18 @@ RaidFileDiscSet &RaidFileController::GetDiscSet(unsigned int DiscSetNum) { if(DiscSetNum < 0 || DiscSetNum >= mSetList.size()) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, NoSuchDiscSet) -======= THROW_EXCEPTION_MESSAGE(RaidFileException, NoSuchDiscSet, DiscSetNum << " (" << mSetList.size() << " disc sets configured)") ->>>>>>> 0.12 } return mSetList[DiscSetNum]; } -<<<<<<< HEAD - -======= // Overload to make usable in gdb debugger. int RaidFileDiscSet::GetSetNumForWriteFiles(const char* filename) const { return GetSetNumForWriteFiles(std::string(filename)); } ->>>>>>> 0.12 // -------------------------------------------------------------------------- // diff --git a/lib/raidfile/RaidFileController.h b/lib/raidfile/RaidFileController.h index b2b8cb0d..601cca22 100644 --- a/lib/raidfile/RaidFileController.h +++ b/lib/raidfile/RaidFileController.h @@ -49,11 +49,7 @@ public: int GetSetID() const {return mSetID;} int GetSetNumForWriteFiles(const std::string &rFilename) const; -<<<<<<< HEAD - -======= int GetSetNumForWriteFiles(const char* filename) const; ->>>>>>> 0.12 unsigned int GetBlockSize() const {return mBlockSize;} // Is this disc set a non-RAID disc set? (ie files never get transformed to raid storage) diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp index 3a7a5893..bcff54c6 100644 --- a/lib/raidfile/RaidFileRead.cpp +++ b/lib/raidfile/RaidFileRead.cpp @@ -12,14 +12,10 @@ #include <errno.h> #include <fcntl.h> #include <stdarg.h> -<<<<<<< HEAD -#include <unistd.h> -======= #ifdef HAVE_UNISTD_H # include <unistd.h> #endif ->>>>>>> 0.12 #include <sys/stat.h> #include <sys/types.h> @@ -1029,13 +1025,8 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, Filename, &startDisc, &existingFiles, pRevisionID); if(existance == RaidFileUtil::NoFile) { -<<<<<<< HEAD - BOX_ERROR("Expected raidfile " << Filename << " does not exist"); - THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist) -======= THROW_FILE_ERROR("Expected raidfile does not exist", Filename, RaidFileException, RaidFileDoesntExist); ->>>>>>> 0.12 } else if(existance == RaidFileUtil::NonRaid) { @@ -1098,11 +1089,6 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string { stripe2errno = errno; } -<<<<<<< HEAD - if(stripe1errno != 0 || stripe2errno != 0) - { - THROW_EXCEPTION(RaidFileException, ErrorOpeningFileForRead) -======= if(stripe1errno != 0) { @@ -1116,7 +1102,6 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string THROW_SYS_FILE_ERRNO("Failed to open RaidFile", stripe2Filename, stripe2errno, RaidFileException, ErrorOpeningFileForRead); ->>>>>>> 0.12 } // stat stripe 1 to find ('half' of) length... @@ -1135,13 +1120,6 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string length += st.st_size; // Handle errors -<<<<<<< HEAD - if(stripe1errno != 0 || stripe2errno != 0) - { - THROW_EXCEPTION(RaidFileException, OSError) - } - -======= if(stripe1errno != 0) { THROW_SYS_FILE_ERRNO("Failed to stat RaidFile", @@ -1156,7 +1134,6 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string RaidFileException, OSError); } ->>>>>>> 0.12 // Make a nice object to represent this file return std::auto_ptr<RaidFileRead>(new RaidFileRead_Raid(SetNumber, Filename, stripe1, stripe2, -1, length, rdiscSet.GetBlockSize(), false /* actually we don't know */)); } @@ -1283,26 +1260,14 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string ASSERT(sizeof(FileSizeType) == 8); // compiler bug (I think) prevents from using 0 - sizeof(FileSizeType)... if(::lseek(parity, -8 /*(0 - sizeof(FileSizeType))*/, SEEK_END) == -1) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to seek " "in parity RaidFile", parityFilename, RaidFileException, OSError); ->>>>>>> 0.12 } // Read it in if(::read(parity, &parityLastData, sizeof(parityLastData)) != sizeof(parityLastData)) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) - } - // Set back to beginning of file - if(::lseek(parity, 0, SEEK_SET) == -1) - { - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to read " "parity RaidFile", parityFilename, @@ -1316,7 +1281,6 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string "in parity RaidFile", parityFilename, RaidFileException, OSError); ->>>>>>> 0.12 } } @@ -1339,14 +1303,10 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string // to get the size from the FileSizeType value at end of the file. if(::fstat(stripe1, &st) != 0) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to " "stat RaidFile stripe 1", stripe1Filename, RaidFileException, OSError); ->>>>>>> 0.12 } pos_type stripe1Size = st.st_size; // Is size integral? @@ -1380,38 +1340,26 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string ASSERT(btr <= (int)sizeof(FileSizeType)); if(::lseek(stripe1, 0 - btr, SEEK_END) == -1) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to " "seek in RaidFile stripe 1", stripe1Filename, RaidFileException, OSError); ->>>>>>> 0.12 } // Read it in if(::read(stripe1, &stripe1LastData, btr) != btr) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to " "read RaidFile stripe 1", stripe1Filename, RaidFileException, OSError); ->>>>>>> 0.12 } // Set back to beginning of file if(::lseek(stripe1, 0, SEEK_SET) == -1) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to " "seek in RaidFile stripe 1", stripe1Filename, RaidFileException, OSError); ->>>>>>> 0.12 } } // Lovely! @@ -1433,14 +1381,10 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string // Get size of stripe2 file if(::fstat(stripe2, &st) != 0) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to " "stat RaidFile stripe 2", stripe2Filename, RaidFileException, OSError); ->>>>>>> 0.12 } pos_type stripe2Size = st.st_size; @@ -1509,12 +1453,8 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string } } -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, FileIsDamagedNotRecoverable) -======= THROW_FILE_ERROR("Failed to recover RaidFile", Filename, RaidFileException, FileIsDamagedNotRecoverable); ->>>>>>> 0.12 // Avoid compiler warning -- it'll never get here... return std::auto_ptr<RaidFileRead>(); @@ -1648,11 +1588,7 @@ bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirN { // build name std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName); -<<<<<<< HEAD - -======= ->>>>>>> 0.12 // read the contents... DIR *dirHandle = 0; try diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp index cb1f9699..82aeef3d 100644 --- a/lib/raidfile/RaidFileWrite.cpp +++ b/lib/raidfile/RaidFileWrite.cpp @@ -11,12 +11,6 @@ #include <errno.h> #include <fcntl.h> -<<<<<<< HEAD -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> -#include <sys/file.h> -======= #ifdef HAVE_UNISTD_H # include <unistd.h> @@ -28,7 +22,6 @@ #ifdef HAVE_SYS_FILE_H # include <sys/file.h> #endif ->>>>>>> 0.12 #include <stdio.h> #include <string.h> @@ -109,9 +102,6 @@ RaidFileWrite::~RaidFileWrite() { if(mOSFileHandle != -1) { -<<<<<<< HEAD - Discard(); -======= // We must not throw exceptions from the destructor // http://stackoverflow.com/a/130123 try @@ -134,7 +124,6 @@ RaidFileWrite::~RaidFileWrite() BOX_ERROR("Failed to discard RaidFile update " "in destructor: unknown exception"); } ->>>>>>> 0.12 } } @@ -165,45 +154,25 @@ void RaidFileWrite::Open(bool AllowOverwrite) RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename); if(existance != RaidFileUtil::NoFile) { -<<<<<<< HEAD - BOX_ERROR("Attempted to overwrite raidfile " << - mSetNumber << " " << mFilename); - THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile) -======= THROW_FILE_ERROR("Attempted to overwrite raidfile " << mSetNumber, mFilename, RaidFileException, CannotOverwriteExistingFile); ->>>>>>> 0.12 } } // Get the filename for the write file -<<<<<<< HEAD - std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename)); - // Add on a temporary extension - writeFilename += 'X'; - - // Attempt to open - mOSFileHandle = ::open(writeFilename.c_str(), -======= mTempFilename = RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename); // Add on a temporary extension mTempFilename += 'X'; // Attempt to open mOSFileHandle = ::open(mTempFilename.c_str(), ->>>>>>> 0.12 O_WRONLY | O_CREAT | O_BINARY, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); if(mOSFileHandle == -1) { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("Failed to open file: " << writeFilename); - THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFile) -======= THROW_SYS_FILE_ERROR("Failed to open RaidFile", mTempFilename, RaidFileException, ErrorOpeningWriteFile); ->>>>>>> 0.12 } // Get a lock on the write file @@ -223,17 +192,6 @@ void RaidFileWrite::Open(bool AllowOverwrite) if (0) #endif { -<<<<<<< HEAD - // Lock was not obtained. - bool wasLocked = (errno == errnoBlock); - // Close the file - ::close(mOSFileHandle); - mOSFileHandle = -1; - // Report an exception? - if(wasLocked) - { - THROW_EXCEPTION(RaidFileException, FileIsCurrentlyOpenForWriting) -======= int errnoSaved = errno; // Lock was not obtained. @@ -250,27 +208,19 @@ void RaidFileWrite::Open(bool AllowOverwrite) "already locked", mTempFilename, errnoSaved, RaidFileException, FileIsCurrentlyOpenForWriting); ->>>>>>> 0.12 } else { // Random error occured -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERRNO("Failed to lock RaidFile", mTempFilename, errnoSaved, RaidFileException, OSError); ->>>>>>> 0.12 } } // Truncate it to size zero if(::ftruncate(mOSFileHandle, 0) != 0) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFileOnTruncate) -======= int errnoSaved = errno; // Close the file @@ -280,7 +230,6 @@ void RaidFileWrite::Open(bool AllowOverwrite) THROW_SYS_FILE_ERRNO("Failed to truncate RaidFile", mTempFilename, errnoSaved, RaidFileException, ErrorOpeningWriteFileOnTruncate); ->>>>>>> 0.12 } // Done! @@ -306,16 +255,10 @@ void RaidFileWrite::Write(const void *pBuffer, int Length) int written = ::write(mOSFileHandle, pBuffer, Length); if(written != Length) { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("RaidFileWrite failed, Length = " << - Length << ", written = " << written); - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to write to RaidFile (attempted " "to write " << Length << " bytes but managed only " << written << ")", mTempFilename, RaidFileException, OSError); ->>>>>>> 0.12 } } @@ -339,12 +282,8 @@ IOStream::pos_type RaidFileWrite::GetPosition() const off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR); if(p == -1) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to get position in RaidFile", mTempFilename, RaidFileException, OSError); ->>>>>>> 0.12 } return p; @@ -369,12 +308,8 @@ void RaidFileWrite::Seek(IOStream::pos_type SeekTo, int SeekType) // Seek... if(::lseek(mOSFileHandle, SeekTo, ConvertSeekTypeToOSWhence(SeekType)) == -1) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to set position in RaidFile", mTempFilename, RaidFileException, OSError); ->>>>>>> 0.12 } } @@ -396,14 +331,8 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) if (mRefCount == 0) { -<<<<<<< HEAD - BOX_ERROR("Attempted to modify object " << mFilename << - ", which has no references"); - THROW_EXCEPTION(RaidFileException, -======= THROW_FILE_ERROR("Attempted to modify object file with " "no references", mTempFilename, RaidFileException, ->>>>>>> 0.12 RequestedModifyUnreferencedFile); } @@ -414,12 +343,8 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) // Close file... if(::close(mOSFileHandle) != 0) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_WIN_FILE_ERROR("Failed to close RaidFile for rename", mTempFilename, RaidFileException, OSError); ->>>>>>> 0.12 } mOSFileHandle = -1; #endif // WIN32 @@ -433,13 +358,6 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) #ifdef WIN32 // need to delete the target first -<<<<<<< HEAD - if(::unlink(renameTo.c_str()) != 0 && - GetLastError() != ERROR_FILE_NOT_FOUND) - { - BOX_LOG_WIN_ERROR("Failed to delete file: " << renameTo); - THROW_EXCEPTION(RaidFileException, OSError) -======= if(::unlink(renameTo.c_str()) != 0) { DWORD errorNumber = GetLastError(); @@ -448,32 +366,21 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) THROW_WIN_FILE_ERRNO("Failed to delete file", renameTo, errorNumber, RaidFileException, OSError); } ->>>>>>> 0.12 } #endif if(::rename(renameFrom.c_str(), renameTo.c_str()) != 0) { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("Failed to rename file: " << renameFrom << - " to " << renameTo); - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_ERROR("Failed to rename file: " << renameFrom << " to " << renameTo, RaidFileException, OSError); ->>>>>>> 0.12 } #ifndef WIN32 // Close file... if(::close(mOSFileHandle) != 0) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to close committed RaidFile", mTempFilename, RaidFileException, OSError); ->>>>>>> 0.12 } mOSFileHandle = -1; #endif // !WIN32 @@ -520,13 +427,8 @@ void RaidFileWrite::Discard() ::close(mOSFileHandle) != 0) #endif // !WIN32 { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("Failed to delete file: " << writeFilename); - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to delete file", writeFilename, RaidFileException, OSError); ->>>>>>> 0.12 } // reset file handle @@ -827,16 +729,9 @@ void RaidFileWrite::Delete() { if (mRefCount != 0 && mRefCount != -1) { -<<<<<<< HEAD - BOX_ERROR("Attempted to delete object " << mFilename << - " which has " << mRefCount << " references"); - THROW_EXCEPTION(RaidFileException, - RequestedDeleteReferencedFile); -======= THROW_FILE_ERROR("Attempted to delete object with " << mRefCount << " references", mFilename, RaidFileException, RequestedDeleteReferencedFile); ->>>>>>> 0.12 } // Get disc set @@ -847,13 +742,9 @@ void RaidFileWrite::Delete() RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename); if(existance == RaidFileUtil::NoFile) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist) -======= THROW_FILE_ERROR("Attempted to delete object which doesn't " "exist", mFilename, RaidFileException, RaidFileDoesntExist); ->>>>>>> 0.12 } // Get the filename for the write file @@ -892,12 +783,8 @@ void RaidFileWrite::Delete() // Check something happened if(!deletedSomething) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_FILE_ERROR("Failed to delete a RaidFile stripe set", mFilename, RaidFileException, OSError); ->>>>>>> 0.12 } } @@ -968,13 +855,6 @@ void RaidFileWrite::CreateDirectory(const RaidFileDiscSet &rSet, const std::stri if(errno == EEXIST) { // No. Bad things. -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, FileExistsInDirectoryCreation) - } - else - { - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to create " "RaidFile directory", dn, RaidFileException, @@ -985,7 +865,6 @@ void RaidFileWrite::CreateDirectory(const RaidFileDiscSet &rSet, const std::stri THROW_SYS_FILE_ERROR("Failed to create " "RaidFile directory", dn, RaidFileException, OSError); ->>>>>>> 0.12 } } } @@ -1068,12 +947,8 @@ IOStream::pos_type RaidFileWrite::GetFileSize() struct stat st; if(fstat(mOSFileHandle, &st) != 0) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to stat RaidFile", mTempFilename, RaidFileException, OSError); ->>>>>>> 0.12 } return st.st_size; @@ -1102,12 +977,8 @@ IOStream::pos_type RaidFileWrite::GetDiscUsageInBlocks() struct stat st; if(fstat(mOSFileHandle, &st) != 0) { -<<<<<<< HEAD - THROW_EXCEPTION(RaidFileException, OSError) -======= THROW_SYS_FILE_ERROR("Failed to stat RaidFile", mTempFilename, RaidFileException, OSError); ->>>>>>> 0.12 } // Then return calculation diff --git a/lib/raidfile/RaidFileWrite.h b/lib/raidfile/RaidFileWrite.h index 52621676..e2887167 100644 --- a/lib/raidfile/RaidFileWrite.h +++ b/lib/raidfile/RaidFileWrite.h @@ -27,9 +27,6 @@ class RaidFileDiscSet; class RaidFileWrite : public IOStream { public: -<<<<<<< HEAD - RaidFileWrite(int SetNumber, const std::string &Filename); -======= // TODO FIXME we should remove this constructor, and ensure that // anyone who writes to a RaidFile knows what the reference count // is before doing so. That requires supporting regenerating the @@ -42,7 +39,6 @@ public: // snapshots. RaidFileWrite(int SetNumber, const std::string &Filename); ->>>>>>> 0.12 RaidFileWrite(int SetNumber, const std::string &Filename, int refcount); ~RaidFileWrite(); private: @@ -74,11 +70,7 @@ private: private: int mSetNumber; -<<<<<<< HEAD - std::string mFilename; -======= std::string mFilename, mTempFilename; ->>>>>>> 0.12 int mOSFileHandle; int mRefCount; }; diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp index 9b96647b..7419f973 100644 --- a/lib/server/Daemon.cpp +++ b/lib/server/Daemon.cpp @@ -25,19 +25,6 @@ #ifdef WIN32 #include <ws2tcpip.h> -<<<<<<< HEAD -#endif - -#include <iostream> - -#include "Daemon.h" -#include "Configuration.h" -#include "ServerException.h" -#include "Guards.h" -#include "UnixUser.h" -#include "FileModificationTime.h" -#include "Logging.h" -======= #include <process.h> #endif @@ -56,7 +43,6 @@ #include "Logging.h" #include "ServerException.h" #include "UnixUser.h" ->>>>>>> 0.12 #include "Utils.h" #include "MemLeakFindOn.h" @@ -85,10 +71,7 @@ Daemon::Daemon() mKeepConsoleOpenAfterFork(false), #endif mHaveConfigFile(false), -<<<<<<< HEAD -======= mLogFileLevel(Log::INVALID), ->>>>>>> 0.12 mAppName(DaemonName()) { // In debug builds, switch on assert failure logging to syslog @@ -125,25 +108,16 @@ std::string Daemon::GetOptionString() { return "c:" #ifndef WIN32 -<<<<<<< HEAD - "DFK" - #endif - "hkPqQt:TUvVW:"; -======= "DF" #endif "hkKo:O:PqQt:TUvVW:"; ->>>>>>> 0.12 } void Daemon::Usage() { std::cout << DaemonBanner() << "\n" -<<<<<<< HEAD -======= "(built with QDBM " << dpversion << ")\n" ->>>>>>> 0.12 "\n" "Usage: " << mAppName << " [options] [config file]\n" "\n" @@ -156,20 +130,6 @@ void Daemon::Usage() " -F Do not fork into background, but fork to serve multiple clients\n" #endif " -k Keep console open after fork, keep writing log messages to it\n" -<<<<<<< HEAD -#ifndef WIN32 - " -K Stop writing log messages to console while daemon is running\n" - " -P Show process ID (PID) in console output\n" -#endif - " -q Run more quietly, reduce verbosity level by one, can repeat\n" - " -Q Run at minimum verbosity, log nothing\n" - " -v Run more verbosely, increase verbosity level by one, can repeat\n" - " -V Run at maximum verbosity, log everything\n" - " -W <level> Set verbosity to error/warning/notice/info/trace/everything\n" - " -t <tag> Tag console output with specified marker\n" - " -T Timestamp console output\n" - " -U Timestamp console output with microseconds\n"; -======= " -K Stop writing log messages to console while daemon is running\n" " -o <file> Log to a file, defaults to maximum verbosity\n" " -O <level> Set file log verbosity to error/warning/notice/info/trace/everything\n" @@ -183,7 +143,6 @@ void Daemon::Usage() " -V Run at maximum verbosity, log everything to console and system\n" " -W <level> Set verbosity to error/warning/notice/info/trace/everything\n" ; ->>>>>>> 0.12 } // -------------------------------------------------------------------------- @@ -222,8 +181,6 @@ int Daemon::ProcessOption(signed int option) break; #endif // !WIN32 -<<<<<<< HEAD -======= case 'h': { Usage(); @@ -231,7 +188,6 @@ int Daemon::ProcessOption(signed int option) } break; ->>>>>>> 0.12 case 'k': { mKeepConsoleOpenAfterFork = true; @@ -244,12 +200,6 @@ int Daemon::ProcessOption(signed int option) } break; -<<<<<<< HEAD - case 'h': - { - Usage(); - return 2; -======= case 'o': { mLogFile = optarg; @@ -265,7 +215,6 @@ int Daemon::ProcessOption(signed int option) BOX_FATAL("Invalid logging level: " << optarg); return 2; } ->>>>>>> 0.12 } break; @@ -294,8 +243,6 @@ int Daemon::ProcessOption(signed int option) } break; -<<<<<<< HEAD -======= case 't': { Logging::SetProgramName(optarg); @@ -315,7 +262,6 @@ int Daemon::ProcessOption(signed int option) Console::SetShowTimeMicros(true); } break; ->>>>>>> 0.12 case 'v': { @@ -341,39 +287,12 @@ int Daemon::ProcessOption(signed int option) mLogLevel = Logging::GetNamedLevel(optarg); if (mLogLevel == Log::INVALID) { -<<<<<<< HEAD - BOX_FATAL("Invalid logging level"); -======= BOX_FATAL("Invalid logging level: " << optarg); ->>>>>>> 0.12 return 2; } } break; -<<<<<<< HEAD - case 't': - { - Logging::SetProgramName(optarg); - Console::SetShowTag(true); - } - break; - - case 'T': - { - Console::SetShowTime(true); - } - break; - - case 'U': - { - Console::SetShowTime(true); - Console::SetShowTimeMicros(true); - } - break; - -======= ->>>>>>> 0.12 case '?': { BOX_FATAL("Unknown option on command line: " @@ -403,18 +322,6 @@ int Daemon::ProcessOption(signed int option) // Created: 2003/07/29 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) -{ - // Find filename of config file - mConfigFileName = DefaultConfigFile; - mAppName = argv[0]; - - #ifdef BOX_RELEASE_BUILD - mLogLevel = Log::NOTICE; // need an int to do math with - #else - mLogLevel = Log::INFO; // need an int to do math with -======= int Daemon::Main(const std::string& rDefaultConfigFile, int argc, const char *argv[]) { @@ -448,7 +355,6 @@ int Daemon::ProcessOptions(int argc, const char *argv[]) mLogLevel = Log::NOTICE; #else mLogLevel = Log::INFO; ->>>>>>> 0.12 #endif if (argc == 2 && strcmp(argv[1], "/?") == 0) @@ -503,9 +409,6 @@ int Daemon::ProcessOptions(int argc, const char *argv[]) Logging::FilterConsole((Log::Level)mLogLevel); Logging::FilterSyslog ((Log::Level)mLogLevel); -<<<<<<< HEAD - return Main(mConfigFileName); -======= if (mLogFileLevel != Log::INVALID) { mapLogFileLogger.reset( @@ -513,7 +416,6 @@ int Daemon::ProcessOptions(int argc, const char *argv[]) } return 0; ->>>>>>> 0.12 } // -------------------------------------------------------------------------- @@ -535,11 +437,7 @@ bool Daemon::Configure(const std::string& rConfigFileName) try { -<<<<<<< HEAD - if (!FileExists(rConfigFileName.c_str())) -======= if (!FileExists(rConfigFileName)) ->>>>>>> 0.12 { BOX_FATAL("The main configuration file for " << DaemonName() << " was not found: " << @@ -1127,11 +1025,6 @@ void Daemon::SetProcessTitle(const char *format, ...) char title[256]; ::vsnprintf(title, sizeof(title), format, args); -<<<<<<< HEAD - // Set process title - ::setproctitle("%s", title); - -======= #ifdef WIN32 StringCchCatA(title, sizeof(title)," - " PACKAGE_NAME); SetConsoleTitleA(title); @@ -1140,7 +1033,6 @@ void Daemon::SetProcessTitle(const char *format, ...) ::setproctitle("%s", title); #endif ->>>>>>> 0.12 #endif // HAVE_SETPROCTITLE } diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h index f2fa98c3..2718c288 100644 --- a/lib/server/Daemon.h +++ b/lib/server/Daemon.h @@ -40,14 +40,9 @@ private: Daemon(const Daemon &rToCopy); public: -<<<<<<< HEAD - virtual int Main(const char *DefaultConfigFile, int argc, - const char *argv[]); -======= virtual int Main(const std::string& rDefaultConfigFile, int argc, const char *argv[]); virtual int ProcessOptions(int argc, const char *argv[]); ->>>>>>> 0.12 /* override this Main() if you want custom option processing: */ virtual int Main(const std::string &rConfigFile); @@ -105,12 +100,9 @@ private: bool mKeepConsoleOpenAfterFork; bool mHaveConfigFile; int mLogLevel; // need an int to do math with -<<<<<<< HEAD -======= std::string mLogFile; Log::Level mLogFileLevel; std::auto_ptr<FileLogger> mapLogFileLogger; ->>>>>>> 0.12 static Daemon *spDaemon; std::string mAppName; }; diff --git a/lib/server/Protocol.cpp b/lib/server/Protocol.cpp index 6333b1db..382f1c37 100644 --- a/lib/server/Protocol.cpp +++ b/lib/server/Protocol.cpp @@ -11,14 +11,9 @@ #include <sys/types.h> -<<<<<<< HEAD -#include <stdlib.h> -#include <string.h> -======= #include <cstdlib> #include <cstring> #include <cstdio> ->>>>>>> 0.12 #include <new> @@ -50,19 +45,6 @@ // // -------------------------------------------------------------------------- Protocol::Protocol(IOStream &rStream) -<<<<<<< HEAD - : mrStream(rStream), - mHandshakeDone(false), - mMaxObjectSize(PROTOCOL_DEFAULT_MAXOBJSIZE), - mTimeout(PROTOCOL_DEFAULT_TIMEOUT), - mpBuffer(0), - mBufferSize(0), - mReadOffset(-1), - mWriteOffset(-1), - mValidDataSize(-1), - mLastErrorType(NoError), - mLastErrorSubType(NoError) -======= : mrStream(rStream), mHandshakeDone(false), mMaxObjectSize(PROTOCOL_DEFAULT_MAXOBJSIZE), @@ -74,7 +56,6 @@ Protocol::Protocol(IOStream &rStream) mValidDataSize(-1), mLogToSysLog(false), mLogToFile(NULL) ->>>>>>> 0.12 { BOX_TRACE("Send block allocation size is " << PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK); @@ -102,37 +83,6 @@ Protocol::~Protocol() // -------------------------------------------------------------------------- // // Function -<<<<<<< HEAD -// Name: Protocol::GetLastError(int &, int &) -// Purpose: Returns true if there was an error, and type and subtype if there was. -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- -bool Protocol::GetLastError(int &rTypeOut, int &rSubTypeOut) -{ - if(mLastErrorType == NoError) - { - // no error. - return false; - } - - // Return type and subtype in args - rTypeOut = mLastErrorType; - rSubTypeOut = mLastErrorSubType; - - // and unset them - mLastErrorType = NoError; - mLastErrorSubType = NoError; - - return true; -} - - -// -------------------------------------------------------------------------- -// -// Function -======= ->>>>>>> 0.12 // Name: Protocol::Handshake() // Purpose: Handshake with peer (exchange ident strings) // Created: 2003/08/20 @@ -150,11 +100,7 @@ void Protocol::Handshake() PW_Handshake hsSend; ::memset(&hsSend, 0, sizeof(hsSend)); // Copy in ident string -<<<<<<< HEAD - ::strncpy(hsSend.mIdent, GetIdentString(), sizeof(hsSend.mIdent)); -======= ::strncpy(hsSend.mIdent, GetProtocolIdentString(), sizeof(hsSend.mIdent)); ->>>>>>> 0.12 // Send it mrStream.Write(&hsSend, sizeof(hsSend)); @@ -227,11 +173,7 @@ void Protocol::CheckAndReadHdr(void *hdr) // Created: 2003/08/19 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -std::auto_ptr<ProtocolObject> Protocol::Receive() -======= std::auto_ptr<Message> Protocol::ReceiveInternal() ->>>>>>> 0.12 { // Get object header PW_ObjectHeader objHeader; @@ -251,11 +193,7 @@ std::auto_ptr<Message> Protocol::ReceiveInternal() } // Create a blank object -<<<<<<< HEAD - std::auto_ptr<ProtocolObject> obj(MakeProtocolObject(ntohl(objHeader.mObjType))); -======= std::auto_ptr<Message> obj(MakeMessage(ntohl(objHeader.mObjType))); ->>>>>>> 0.12 // Make sure memory is allocated to read it into EnsureBufferAllocated(objSize); @@ -307,11 +245,7 @@ std::auto_ptr<Message> Protocol::ReceiveInternal() // Created: 2003/08/19 // // -------------------------------------------------------------------------- -<<<<<<< HEAD -void Protocol::Send(const ProtocolObject &rObject) -======= void Protocol::SendInternal(const Message &rObject) ->>>>>>> 0.12 { // Check usage if(mValidDataSize != -1 || mWriteOffset != -1 || mReadOffset != -1) @@ -754,8 +688,6 @@ void Protocol::SendStream(IOStream &rStream) // Can't send this using the fixed size header uncertainSize = true; } -<<<<<<< HEAD -======= if(streamSize == 0) { @@ -767,7 +699,6 @@ void Protocol::SendStream(IOStream &rStream) "Sending a stream with a definite size of zero " "is not allowed in the protocol"); } ->>>>>>> 0.12 // Inform sub class InformStreamSending(streamSize); @@ -907,9 +838,6 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) // -------------------------------------------------------------------------- void Protocol::InformStreamReceiving(u_int32_t Size) { -<<<<<<< HEAD - // Do nothing -======= if(GetLogToSysLog()) { if(Size == Protocol::ProtocolStream_SizeUncertain) @@ -930,7 +858,6 @@ void Protocol::InformStreamReceiving(u_int32_t Size) : "Receiving stream, size %d\n", Size); ::fflush(GetLogToFile()); } ->>>>>>> 0.12 } // -------------------------------------------------------------------------- @@ -943,9 +870,6 @@ void Protocol::InformStreamReceiving(u_int32_t Size) // -------------------------------------------------------------------------- void Protocol::InformStreamSending(u_int32_t Size) { -<<<<<<< HEAD - // Do nothing -======= if(GetLogToSysLog()) { if(Size == Protocol::ProtocolStream_SizeUncertain) @@ -966,7 +890,6 @@ void Protocol::InformStreamSending(u_int32_t Size) : "Sending stream, size %d\n", Size); ::fflush(GetLogToFile()); } ->>>>>>> 0.12 } diff --git a/lib/server/Protocol.h b/lib/server/Protocol.h index dc660ad5..42cb0ff8 100644 --- a/lib/server/Protocol.h +++ b/lib/server/Protocol.h @@ -12,22 +12,14 @@ #include <sys/types.h> -<<<<<<< HEAD -class IOStream; -#include "ProtocolObject.h" -======= ->>>>>>> 0.12 #include <memory> #include <vector> #include <string> -<<<<<<< HEAD -======= #include "Message.h" class IOStream; ->>>>>>> 0.12 // default timeout is 15 minutes #define PROTOCOL_DEFAULT_TIMEOUT (15*60*1000) // 16 default maximum object size -- should be enough @@ -50,13 +42,6 @@ public: private: Protocol(const Protocol &rToCopy); -<<<<<<< HEAD -public: - void Handshake(); - std::auto_ptr<ProtocolObject> Receive(); - void Send(const ProtocolObject &rObject); - -======= protected: // Unsafe to make public, as they may allow sending objects // from a different protocol. The derived class prevents this. @@ -65,7 +50,6 @@ protected: public: void Handshake(); ->>>>>>> 0.12 std::auto_ptr<IOStream> ReceiveStream(); void SendStream(IOStream &rStream); @@ -75,11 +59,6 @@ public: UnknownError = 0 }; -<<<<<<< HEAD - bool GetLastError(int &rTypeOut, int &rSubTypeOut); - -======= ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Function @@ -111,11 +90,7 @@ public: // -------------------------------------------------------------------------- void SetMaxObjectSize(unsigned int NewMaxObjSize) {mMaxObjectSize = NewMaxObjSize;} -<<<<<<< HEAD - // For ProtocolObject derived classes -======= // For Message derived classes ->>>>>>> 0.12 void Read(void *Buffer, int Size); void Read(std::string &rOut, int Size); void Read(int64_t &rOut); @@ -196,13 +171,6 @@ public: { ProtocolStream_SizeUncertain = 0xffffffff }; -<<<<<<< HEAD - -protected: - virtual std::auto_ptr<ProtocolObject> MakeProtocolObject(int ObjType) = 0; - virtual const char *GetIdentString() = 0; - void SetError(int Type, int SubType) {mLastErrorType = Type; mLastErrorSubType = SubType;} -======= bool GetLogToSysLog() { return mLogToSysLog; } FILE *GetLogToFile() { return mLogToFile; } void SetLogToSysLog(bool Log = false) {mLogToSysLog = Log;} @@ -212,7 +180,6 @@ protected: virtual std::auto_ptr<Message> MakeMessage(int ObjType) = 0; virtual const char *GetProtocolIdentString() = 0; ->>>>>>> 0.12 void CheckAndReadHdr(void *hdr); // don't use type here to avoid dependency // Will be used for logging @@ -223,10 +190,6 @@ private: void EnsureBufferAllocated(int Size); int SendStreamSendBlock(uint8_t *Block, int BytesInBlock); -<<<<<<< HEAD -private: -======= ->>>>>>> 0.12 IOStream &mrStream; bool mHandshakeDone; unsigned int mMaxObjectSize; @@ -236,17 +199,12 @@ private: int mReadOffset; int mWriteOffset; int mValidDataSize; -<<<<<<< HEAD - int mLastErrorType; - int mLastErrorSubType; -======= bool mLogToSysLog; FILE *mLogToFile; }; class ProtocolContext { ->>>>>>> 0.12 }; #endif // PROTOCOL__H diff --git a/lib/server/ProtocolObject.cpp b/lib/server/ProtocolObject.cpp deleted file mode 100644 index fb09f820..00000000 --- a/lib/server/ProtocolObject.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: ProtocolObject.h -// Purpose: Protocol object base class -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- - -#include "Box.h" -#include "ProtocolObject.h" -#include "CommonException.h" - -#include "MemLeakFindOn.h" - -// -------------------------------------------------------------------------- -// -// Function -// Name: ProtocolObject::ProtocolObject() -// Purpose: Default constructor -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- -ProtocolObject::ProtocolObject() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: ProtocolObject::ProtocolObject() -// Purpose: Destructor -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- -ProtocolObject::~ProtocolObject() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: ProtocolObject::ProtocolObject() -// Purpose: Copy constructor -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- -ProtocolObject::ProtocolObject(const ProtocolObject &rToCopy) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: ProtocolObject::IsError(int &, int &) -// Purpose: Does this represent an error, and if so, what is the type and subtype? -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- -bool ProtocolObject::IsError(int &rTypeOut, int &rSubTypeOut) const -{ - return false; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: ProtocolObject::IsConversationEnd() -// Purpose: Does this command end the conversation? -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- -bool ProtocolObject::IsConversationEnd() const -{ - return false; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: ProtocolObject::GetType() -// Purpose: Return type of the object -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- -int ProtocolObject::GetType() const -{ - // This isn't implemented in the base class! - THROW_EXCEPTION(CommonException, Internal) -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: ProtocolObject::SetPropertiesFromStreamData(Protocol &) -// Purpose: Set the properties of the object from the stream data ready in the Protocol object -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- -void ProtocolObject::SetPropertiesFromStreamData(Protocol &rProtocol) -{ - // This isn't implemented in the base class! - THROW_EXCEPTION(CommonException, Internal) -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: ProtocolObject::WritePropertiesToStreamData(Protocol &) -// Purpose: Write the properties of the object into the stream data in the Protocol object -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- -void ProtocolObject::WritePropertiesToStreamData(Protocol &rProtocol) const -{ - // This isn't implemented in the base class! - THROW_EXCEPTION(CommonException, Internal) -} - - - diff --git a/lib/server/ProtocolObject.h b/lib/server/ProtocolObject.h deleted file mode 100644 index 0a127ab5..00000000 --- a/lib/server/ProtocolObject.h +++ /dev/null @@ -1,41 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: ProtocolObject.h -// Purpose: Protocol object base class -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- - -#ifndef PROTOCOLOBJECT__H -#define PROTOCOLOBJECT__H - -class Protocol; - -// -------------------------------------------------------------------------- -// -// Class -// Name: ProtocolObject -// Purpose: Basic object representation of objects to pass through a Protocol session -// Created: 2003/08/19 -// -// -------------------------------------------------------------------------- -class ProtocolObject -{ -public: - ProtocolObject(); - virtual ~ProtocolObject(); - ProtocolObject(const ProtocolObject &rToCopy); - - // Info about this object - virtual int GetType() const; - virtual bool IsError(int &rTypeOut, int &rSubTypeOut) const; - virtual bool IsConversationEnd() const; - - // reading and writing with Protocol objects - virtual void SetPropertiesFromStreamData(Protocol &rProtocol); - virtual void WritePropertiesToStreamData(Protocol &rProtocol) const; -}; - -#endif // PROTOCOLOBJECT__H - diff --git a/lib/server/SSLLib.cpp b/lib/server/SSLLib.cpp index 6feaae4a..004d2d98 100644 --- a/lib/server/SSLLib.cpp +++ b/lib/server/SSLLib.cpp @@ -18,10 +18,7 @@ #include <wincrypt.h> #endif -<<<<<<< HEAD -======= #include "CryptoUtils.h" ->>>>>>> 0.12 #include "SSLLib.h" #include "ServerException.h" @@ -43,14 +40,9 @@ void SSLLib::Initialise() { if(!::SSL_library_init()) { -<<<<<<< HEAD - LogError("initialising OpenSSL"); - THROW_EXCEPTION(ServerException, SSLLibraryInitialisationError) -======= THROW_EXCEPTION_MESSAGE(ServerException, SSLLibraryInitialisationError, CryptoUtils::LogError("initialising OpenSSL")); ->>>>>>> 0.12 } // More helpful error messages @@ -99,26 +91,3 @@ void SSLLib::Initialise() } -<<<<<<< HEAD -// -------------------------------------------------------------------------- -// -// Function -// Name: SSLLib::LogError(const char *) -// Purpose: Logs an error -// Created: 2003/08/06 -// -// -------------------------------------------------------------------------- -void SSLLib::LogError(const std::string& rErrorDuringAction) -{ - unsigned long errcode; - char errname[256]; // SSL docs say at least 120 bytes - while((errcode = ERR_get_error()) != 0) - { - ::ERR_error_string_n(errcode, errname, sizeof(errname)); - BOX_ERROR("SSL error while " << rErrorDuringAction << ": " << - errname); - } -} - -======= ->>>>>>> 0.12 diff --git a/lib/server/SSLLib.h b/lib/server/SSLLib.h index b679d623..d11c7804 100644 --- a/lib/server/SSLLib.h +++ b/lib/server/SSLLib.h @@ -29,10 +29,6 @@ namespace SSLLib { void Initialise(); -<<<<<<< HEAD - void LogError(const std::string& rErrorDuringAction); -======= ->>>>>>> 0.12 }; #endif // SSLLIB__H diff --git a/lib/server/ServerException.txt b/lib/server/ServerException.txt index f8c558c6..474b4067 100644 --- a/lib/server/ServerException.txt +++ b/lib/server/ServerException.txt @@ -13,11 +13,7 @@ SocketOpenError 10 SocketPollError 11 SocketCloseError 13 SocketNameUNIXPathTooLong 14 -<<<<<<< HEAD -SocketBindError 16 Check the ListenAddresses directive in your config file -- must refer to local IP addresses only -======= SocketBindError 16 Check the ListenAddresses directive (bbstored) or CommandSocket (bbackupd) in your config file -- must refer to local IP addresses (or existing writable path) only ->>>>>>> 0.12 SocketAcceptError 17 ServerStreamBadListenAddrs 18 ServerForkError 19 diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h index 8625832d..a9b56169 100644 --- a/lib/server/ServerStream.h +++ b/lib/server/ServerStream.h @@ -48,8 +48,6 @@ private: ServerStream(const ServerStream &rToCopy) { } -<<<<<<< HEAD -======= std::string mConnectionDetails; @@ -59,7 +57,6 @@ protected: return mConnectionDetails; } ->>>>>>> 0.12 public: virtual const char *DaemonName() const @@ -134,13 +131,10 @@ public: protected: virtual void NotifyListenerIsReady() { } -<<<<<<< HEAD -======= virtual void LogConnectionDetails(std::string details) { BOX_NOTICE("Handling incoming connection from " << details); } ->>>>>>> 0.12 public: virtual void Run2(bool &rChildExit) @@ -256,14 +250,9 @@ public: { // Get the incoming connection // (with zero wait time) -<<<<<<< HEAD - std::string logMessage; - std::auto_ptr<StreamType> connection(psocket->Accept(0, &logMessage)); -======= std::auto_ptr<StreamType> connection( psocket->Accept(0, &mConnectionDetails)); ->>>>>>> 0.12 // Was there one (there should be...) if(connection.get()) @@ -289,10 +278,7 @@ public: // Set up daemon EnterChild(); SetProcessTitle("transaction"); -<<<<<<< HEAD -======= LogConnectionDetails(mConnectionDetails); ->>>>>>> 0.12 // Memory leak test the forked process #ifdef BOX_MEMORY_LEAK_TESTING @@ -310,13 +296,9 @@ public: } // Log it -<<<<<<< HEAD - BOX_NOTICE("Message from child process " << pid << ": " << logMessage); -======= BOX_TRACE("Forked child process " << pid << " to handle connection from " << mConnectionDetails); ->>>>>>> 0.12 } else { diff --git a/lib/server/Socket.cpp b/lib/server/Socket.cpp index 69a11330..f2a4996b 100644 --- a/lib/server/Socket.cpp +++ b/lib/server/Socket.cpp @@ -123,32 +123,8 @@ void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, // -------------------------------------------------------------------------- void Socket::LogIncomingConnection(const struct sockaddr *addr, socklen_t addrlen) { -<<<<<<< HEAD - if(addr == NULL) {THROW_EXCEPTION(CommonException, BadArguments)} - - switch(addr->sa_family) - { - case AF_UNIX: - BOX_INFO("Incoming connection from local (UNIX socket)"); - break; - - case AF_INET: - { - sockaddr_in *a = (sockaddr_in*)addr; - BOX_INFO("Incoming connection from " << - inet_ntoa(a->sin_addr) << " port " << - ntohs(a->sin_port)); - } - break; - - default: - BOX_WARNING("Incoming connection of unknown type"); - break; - } -======= BOX_INFO("Incoming connection from " << IncomingConnectionLogMessage(addr, addrlen)); ->>>>>>> 0.12 } // -------------------------------------------------------------------------- @@ -166,40 +142,25 @@ std::string Socket::IncomingConnectionLogMessage(const struct sockaddr *addr, so switch(addr->sa_family) { case AF_UNIX: -<<<<<<< HEAD - return std::string("Incoming connection from local (UNIX socket)"); -======= return std::string("local (UNIX socket)"); ->>>>>>> 0.12 break; case AF_INET: { -<<<<<<< HEAD - char msg[256]; // more than enough - sockaddr_in *a = (sockaddr_in*)addr; - sprintf(msg, "Incoming connection from %s port %d", inet_ntoa(a->sin_addr), ntohs(a->sin_port)); - return std::string(msg); -======= sockaddr_in *a = (sockaddr_in*)addr; std::ostringstream oss; oss << inet_ntoa(a->sin_addr) << " port " << ntohs(a->sin_port); return oss.str(); ->>>>>>> 0.12 } break; default: -<<<<<<< HEAD - return std::string("Incoming connection of unknown type"); -======= { std::ostringstream oss; oss << "unknown socket type " << addr->sa_family; return oss.str(); } ->>>>>>> 0.12 break; } diff --git a/lib/server/SocketListen.h b/lib/server/SocketListen.h index 635b15e8..39c60ba6 100644 --- a/lib/server/SocketListen.h +++ b/lib/server/SocketListen.h @@ -87,24 +87,16 @@ public: { Close(); } -<<<<<<< HEAD -======= ->>>>>>> 0.12 private: SocketListen(const SocketListen &rToCopy) { } -<<<<<<< HEAD -public: - -======= int mType, mPort; std::string mName; public: ->>>>>>> 0.12 enum { MaxMultipleListenSockets = MaxMultiListenSockets @@ -120,13 +112,8 @@ public: if(::close(mSocketHandle) == -1) #endif { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("Failed to close network " - "socket"); -======= BOX_LOG_SOCKET_ERROR(mType, mName, mPort, "Failed to close network socket"); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, SocketCloseError) } @@ -144,13 +131,10 @@ public: // ------------------------------------------------------------------ void Listen(Socket::Type Type, const char *Name, int Port = 0) { -<<<<<<< HEAD -======= mType = Type; mName = Name; mPort = Port; ->>>>>>> 0.12 if(mSocketHandle != -1) { THROW_EXCEPTION(ServerException, SocketAlreadyOpen); @@ -168,12 +152,8 @@ public: 0 /* let OS choose protocol */); if(mSocketHandle == -1) { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("Failed to create a network socket"); -======= BOX_LOG_SOCKET_ERROR(Type, Name, Port, "Failed to create a network socket"); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, SocketOpenError) } @@ -187,12 +167,8 @@ public: &option, sizeof(option)) == -1) #endif { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("Failed to set socket options"); -======= BOX_LOG_SOCKET_ERROR(Type, Name, Port, "Failed to set socket options"); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, SocketOpenError) } @@ -200,12 +176,6 @@ public: if(::bind(mSocketHandle, &addr.sa_generic, addrLen) == -1 || ::listen(mSocketHandle, ListenBacklog) == -1) { -<<<<<<< HEAD - // Dispose of the socket - ::close(mSocketHandle); - mSocketHandle = -1; - THROW_EXCEPTION(ServerException, SocketBindError) -======= int err_number = errno; BOX_LOG_SOCKET_ERROR(Type, Name, Port, @@ -218,7 +188,6 @@ public: THROW_SYS_FILE_ERRNO("Failed to bind or listen " "on socket", Name, err_number, ServerException, SocketBindError); ->>>>>>> 0.12 } } @@ -271,11 +240,7 @@ public: // signal? if(errno == EINTR) { -<<<<<<< HEAD - BOX_ERROR("Failed to accept " -======= BOX_INFO("Failed to accept " ->>>>>>> 0.12 "connection: interrupted by " "signal"); // return nothing @@ -283,13 +248,8 @@ public: } else { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("Failed to poll " - "connection"); -======= BOX_LOG_SOCKET_ERROR(mType, mName, mPort, "Failed to poll connection"); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, SocketPollError) } @@ -308,12 +268,8 @@ public: // Got socket (or error), unlock (implicit in destruction) if(sock == -1) { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("Failed to accept connection"); -======= BOX_LOG_SOCKET_ERROR(mType, mName, mPort, "Failed to accept connection"); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, SocketAcceptError) } diff --git a/lib/server/SocketStream.cpp b/lib/server/SocketStream.cpp index 7c92ebba..6ef4b8d1 100644 --- a/lib/server/SocketStream.cpp +++ b/lib/server/SocketStream.cpp @@ -154,24 +154,16 @@ void SocketStream::Open(Socket::Type Type, const std::string& rName, int Port) int sockDomain = 0; SocketAllAddr addr; int addrLen = 0; -<<<<<<< HEAD - Socket::NameLookupToSockAddr(addr, sockDomain, Type, rName, Port, addrLen); -======= Socket::NameLookupToSockAddr(addr, sockDomain, Type, rName, Port, addrLen); ->>>>>>> 0.12 // Create the socket mSocketHandle = ::socket(sockDomain, SOCK_STREAM, 0 /* let OS choose protocol */); if(mSocketHandle == INVALID_SOCKET_VALUE) { -<<<<<<< HEAD - BOX_LOG_SYS_ERROR("Failed to create a network socket"); -======= BOX_LOG_SOCKET_ERROR(Type, rName, Port, "Failed to create a network socket"); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, SocketOpenError) } @@ -179,24 +171,11 @@ void SocketStream::Open(Socket::Type Type, const std::string& rName, int Port) if(::connect(mSocketHandle, &addr.sa_generic, addrLen) == -1) { // Dispose of the socket -<<<<<<< HEAD -#ifdef WIN32 - DWORD err = WSAGetLastError(); - ::closesocket(mSocketHandle); - BOX_LOG_WIN_ERROR_NUMBER("Failed to connect to socket " - "(type " << Type << ", name " << rName << - ", port " << Port << ")", err); -#else // !WIN32 - BOX_LOG_SYS_ERROR("Failed to connect to socket (type " << - Type << ", name " << rName << ", port " << Port << - ")"); -======= BOX_LOG_SOCKET_ERROR(Type, rName, Port, "Failed to connect to socket"); #ifdef WIN32 ::closesocket(mSocketHandle); #else // !WIN32 ->>>>>>> 0.12 ::close(mSocketHandle); #endif // WIN32 diff --git a/lib/server/SocketStream.h b/lib/server/SocketStream.h index aa62e4e9..2fb5e391 100644 --- a/lib/server/SocketStream.h +++ b/lib/server/SocketStream.h @@ -51,10 +51,6 @@ public: virtual bool GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut); protected: -<<<<<<< HEAD - tOSSocketHandle GetSocketHandle(); -======= ->>>>>>> 0.12 void MarkAsReadClosed() {mReadClosed = true;} void MarkAsWriteClosed() {mWriteClosed = true;} @@ -72,14 +68,11 @@ public: off_t GetBytesWritten() const {return mBytesWritten;} void ResetCounters() {mBytesRead = mBytesWritten = 0;} bool IsOpened() { return mSocketHandle != INVALID_SOCKET_VALUE; } -<<<<<<< HEAD -======= /** * Only for use by NiceSocketStream! */ tOSSocketHandle GetSocketHandle(); ->>>>>>> 0.12 }; #endif // SOCKETSTREAM__H diff --git a/lib/server/SocketStreamTLS.cpp b/lib/server/SocketStreamTLS.cpp index 6f1cc46a..576b53a2 100644 --- a/lib/server/SocketStreamTLS.cpp +++ b/lib/server/SocketStreamTLS.cpp @@ -19,20 +19,12 @@ #include <poll.h> #endif -<<<<<<< HEAD -#include "SocketStreamTLS.h" -#include "SSLLib.h" -#include "ServerException.h" -#include "TLSContext.h" -#include "BoxTime.h" -======= #include "BoxTime.h" #include "CryptoUtils.h" #include "ServerException.h" #include "SocketStreamTLS.h" #include "SSLLib.h" #include "TLSContext.h" ->>>>>>> 0.12 #include "MemLeakFindOn.h" @@ -133,11 +125,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) mpBIO = ::BIO_new(::BIO_s_socket()); if(mpBIO == 0) { -<<<<<<< HEAD - SSLLib::LogError("creating socket bio"); -======= CryptoUtils::LogError("creating socket bio"); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, TLSAllocationFailed) } @@ -148,11 +136,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) mpSSL = ::SSL_new(rContext.GetRawContext()); if(mpSSL == 0) { -<<<<<<< HEAD - SSLLib::LogError("creating SSL object"); -======= CryptoUtils::LogError("creating SSL object"); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, TLSAllocationFailed) } @@ -220,20 +204,12 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) // Error occured if(IsServer) { -<<<<<<< HEAD - SSLLib::LogError("accepting connection"); -======= CryptoUtils::LogError("accepting connection"); ->>>>>>> 0.12 THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) } else { -<<<<<<< HEAD - SSLLib::LogError("connecting"); -======= CryptoUtils::LogError("connecting"); ->>>>>>> 0.12 THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) } } @@ -360,11 +336,7 @@ int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) break; default: -<<<<<<< HEAD - SSLLib::LogError("reading"); -======= CryptoUtils::LogError("reading"); ->>>>>>> 0.12 THROW_EXCEPTION(ConnectionException, Conn_TLSReadFailed) break; } @@ -429,11 +401,7 @@ void SocketStreamTLS::Write(const void *pBuffer, int NBytes) break; default: -<<<<<<< HEAD - SSLLib::LogError("writing"); -======= CryptoUtils::LogError("writing"); ->>>>>>> 0.12 THROW_EXCEPTION(ConnectionException, Conn_TLSWriteFailed) break; } @@ -475,11 +443,7 @@ void SocketStreamTLS::Shutdown(bool Read, bool Write) if(::SSL_shutdown(mpSSL) < 0) { -<<<<<<< HEAD - SSLLib::LogError("shutting down"); -======= CryptoUtils::LogError("shutting down"); ->>>>>>> 0.12 THROW_EXCEPTION(ConnectionException, Conn_TLSShutdownFailed) } diff --git a/lib/server/TLSContext.cpp b/lib/server/TLSContext.cpp index 1f06d602..341043e9 100644 --- a/lib/server/TLSContext.cpp +++ b/lib/server/TLSContext.cpp @@ -12,11 +12,7 @@ #define TLS_CLASS_IMPLEMENTATION_CPP #include <openssl/ssl.h> -<<<<<<< HEAD -#include "TLSContext.h" -======= #include "CryptoUtils.h" ->>>>>>> 0.12 #include "ServerException.h" #include "SSLLib.h" #include "TLSContext.h" @@ -81,22 +77,14 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c { std::string msg = "loading certificates from "; msg += CertificatesFile; -<<<<<<< HEAD - SSLLib::LogError(msg); -======= CryptoUtils::LogError(msg); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, TLSLoadCertificatesFailed) } if(::SSL_CTX_use_PrivateKey_file(mpContext, PrivateKeyFile, SSL_FILETYPE_PEM) != 1) { std::string msg = "loading private key from "; msg += PrivateKeyFile; -<<<<<<< HEAD - SSLLib::LogError(msg); -======= CryptoUtils::LogError(msg); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, TLSLoadPrivateKeyFailed) } @@ -105,11 +93,7 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c { std::string msg = "loading CA cert from "; msg += TrustedCAsFile; -<<<<<<< HEAD - SSLLib::LogError(msg); -======= CryptoUtils::LogError(msg); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, TLSLoadTrustedCAsFailed) } @@ -121,11 +105,7 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c // Setup allowed ciphers if(::SSL_CTX_set_cipher_list(mpContext, CIPHER_LIST) != 1) { -<<<<<<< HEAD - SSLLib::LogError("setting cipher list to " CIPHER_LIST); -======= CryptoUtils::LogError("setting cipher list to " CIPHER_LIST); ->>>>>>> 0.12 THROW_EXCEPTION(ServerException, TLSSetCiphersFailed) } } diff --git a/lib/server/makeprotocol.pl.in b/lib/server/makeprotocol.pl.in index 9caa970d..a074b435 100755 --- a/lib/server/makeprotocol.pl.in +++ b/lib/server/makeprotocol.pl.in @@ -30,26 +30,6 @@ my %log_display_types = 'string' => ['%s', 'VAR.c_str()'] ); -<<<<<<< HEAD - - -my ($type, $file) = @ARGV; - -if($type ne 'Server' && $type ne 'Client') -{ - die "Neither Server or Client is specified on command line\n"; -} - -open IN, $file or die "Can't open input file $file\n"; - -print "Making $type protocol classes from $file...\n"; - -my @extra_header_files; - -my $implement_syslog = 0; -my $implement_filelog = 0; - -======= if (@ARGV != 1) { die "Usage: $0 <protocol-txt-file>\n"; @@ -63,7 +43,6 @@ print "Making protocol classes from $file...\n"; my @extra_header_files; ->>>>>>> 0.12 # read attributes my %attr; while(<IN>) @@ -75,43 +54,6 @@ while(<IN>) my ($k,$v) = split /\s+/,$l,2; -<<<<<<< HEAD - if($k eq 'ClientType') - { - add_type($v) if $type eq 'Client'; - } - elsif($k eq 'ServerType') - { - add_type($v) if $type eq 'Server'; - } - elsif($k eq 'ImplementLog') - { - my ($log_if_type,$log_type) = split /\s+/,$v; - if($type eq $log_if_type) - { - if($log_type eq 'syslog') - { - $implement_syslog = 1; - } - elsif($log_type eq 'file') - { - $implement_filelog = 1; - } - else - { - printf("ERROR: Unknown log type for implementation: $log_type\n"); - exit(1); - } - } - } - elsif($k eq 'LogTypeToText') - { - my ($log_if_type,$type_name,$printf_format,$arg_template) = split /\s+/,$v; - if($type eq $log_if_type) - { - $log_display_types{$type_name} = [$printf_format,$arg_template] - } -======= if($k eq 'AddType') { add_type($v); @@ -124,7 +66,6 @@ while(<IN>) { my ($type_name,$printf_format,$arg_template) = split /\s+/,$v; $log_display_types{$type_name} = [$printf_format,$arg_template] ->>>>>>> 0.12 } else { @@ -200,11 +141,6 @@ close IN; # open files -<<<<<<< HEAD -my $h_filename = 'autogen_'.$protocol_name.'Protocol'.$type.'.h'; -open CPP,'>autogen_'.$protocol_name.'Protocol'.$type.'.cpp'; -open H,">$h_filename"; -======= my $filename_base = 'autogen_'.$protocol_name.'Protocol'; print "Writing $filename_base.cpp\n"; print "Writing $filename_base.h\n"; @@ -212,7 +148,6 @@ open CPP, "> $filename_base.cpp"; open H, "> $filename_base.h"; my $guardname = uc 'AUTOGEN_'.$protocol_name.'Protocol_H'; ->>>>>>> 0.12 print CPP <<__E; @@ -222,41 +157,16 @@ print CPP <<__E; #include <sstream> -<<<<<<< HEAD -#include "$h_filename" -#include "IOStream.h" - -__E - -if($implement_syslog) -{ - print H <<EOF; -#ifndef WIN32 -#include <syslog.h> -#endif -EOF -} - - -my $guardname = uc 'AUTOGEN_'.$protocol_name.'Protocol'.$type.'_H'; -print H <<__E; - -======= #include "$filename_base.h" #include "IOStream.h" __E print H <<__E; ->>>>>>> 0.12 // Auto-generated file -- do not edit #ifndef $guardname #define $guardname -<<<<<<< HEAD -#include "Protocol.h" -#include "ProtocolObject.h" -======= #include <cstdio> #include <list> @@ -266,128 +176,16 @@ print H <<__E; #include "Protocol.h" #include "Message.h" ->>>>>>> 0.12 #include "ServerException.h" class IOStream; -<<<<<<< HEAD -__E - -if($implement_filelog) -{ - print H qq~#include <stdio.h>\n~; -} -======= __E ->>>>>>> 0.12 # extra headers for(@extra_header_files) { -<<<<<<< HEAD - print H qq~#include "$_"\n~ -} -print H "\n"; - -if($type eq 'Server') -{ - # need utils file for the server - print H '#include "Utils.h"',"\n\n" -} - - -my $derive_objects_from = 'ProtocolObject'; -my $objects_extra_h = ''; -my $objects_extra_cpp = ''; -if($type eq 'Server') -{ - # define the context - print H "class $context_class;\n\n"; - print CPP "#include \"$context_class_inc\"\n\n"; - - # change class we derive the objects from - $derive_objects_from = $protocol_name.'ProtocolObject'; - - $objects_extra_h = <<__E; - virtual std::auto_ptr<ProtocolObject> DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext); -__E - $objects_extra_cpp = <<__E; -std::auto_ptr<ProtocolObject> ${derive_objects_from}::DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext) -{ - THROW_EXCEPTION(ConnectionException, Conn_Protocol_TriedToExecuteReplyCommand) -} -__E -} - -print CPP qq~#include "MemLeakFindOn.h"\n~; - -if($type eq 'Client' && ($implement_syslog || $implement_filelog)) -{ - # change class we derive the objects from - $derive_objects_from = $protocol_name.'ProtocolObjectCl'; -} -if($implement_syslog) -{ - $objects_extra_h .= <<__E; - virtual void LogSysLog(const char *Action) const = 0; -__E -} -if($implement_filelog) -{ - $objects_extra_h .= <<__E; - virtual void LogFile(const char *Action, FILE *file) const = 0; -__E -} - -if($derive_objects_from ne 'ProtocolObject') -{ - # output a definition for the protocol object derived class - print H <<__E; -class ${protocol_name}ProtocolServer; - -class $derive_objects_from : public ProtocolObject -{ -public: - $derive_objects_from(); - virtual ~$derive_objects_from(); - $derive_objects_from(const $derive_objects_from &rToCopy); - -$objects_extra_h -}; -__E - - # and some cpp definitions - print CPP <<__E; -${derive_objects_from}::${derive_objects_from}() -{ -} -${derive_objects_from}::~${derive_objects_from}() -{ -} -${derive_objects_from}::${derive_objects_from}(const $derive_objects_from &rToCopy) -{ -} -$objects_extra_cpp -__E -} - - - -my $classname_base = $protocol_name.'Protocol'.$type; - -# output the classes -for my $cmd (@cmd_list) -{ - print H <<__E; -class $classname_base$cmd : public $derive_objects_from -{ -public: - $classname_base$cmd(); - $classname_base$cmd(const $classname_base$cmd &rToCopy); - ~$classname_base$cmd(); -======= print H qq@#include "$_"\n@; } @@ -473,17 +271,13 @@ public: $cmd_class(); $cmd_class(const $cmd_class &rToCopy); ~$cmd_class(); ->>>>>>> 0.12 int GetType() const; enum { TypeID = $cmd_id{$cmd} }; __E -<<<<<<< HEAD -======= ->>>>>>> 0.12 # constants if(exists $cmd_constants{$cmd}) { @@ -491,82 +285,18 @@ __E print H join(",\n\t\t",@{$cmd_constants{$cmd}}); print H "\n\t};\n"; } -<<<<<<< HEAD -======= ->>>>>>> 0.12 # flags if(obj_is_type($cmd,'EndsConversation')) { print H "\tbool IsConversationEnd() const;\n"; } -<<<<<<< HEAD -======= ->>>>>>> 0.12 if(obj_is_type($cmd,'IsError')) { print H "\tbool IsError(int &rTypeOut, int &rSubTypeOut) const;\n"; print H "\tstd::string GetMessage() const;\n"; } -<<<<<<< HEAD - if($type eq 'Server' && obj_is_type($cmd, 'Command')) - { - print H "\tstd::auto_ptr<ProtocolObject> DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext); // IMPLEMENT THIS\n" - } - - # want to be able to read from streams? - my $read_from_streams = (obj_is_type($cmd,'Command') && $type eq 'Server') || (obj_is_type($cmd,'Reply') && $type eq 'Client'); - my $write_to_streams = (obj_is_type($cmd,'Command') && $type eq 'Client') || (obj_is_type($cmd,'Reply') && $type eq 'Server'); - - if($read_from_streams) - { - print H "\tvoid SetPropertiesFromStreamData(Protocol &rProtocol);\n"; - - # write Get functions - for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) - { - my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); - - print H "\t".translate_type_to_arg_type($ty)." Get$nm() {return m$nm;}\n"; - } - } - my $param_con_args = ''; - if($write_to_streams) - { - # extra constructor? - if($#{$cmd_contents{$cmd}} >= 0) - { - my @a; - for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) - { - my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); - - push @a,translate_type_to_arg_type($ty)." $nm"; - } - $param_con_args = join(', ',@a); - print H "\t$classname_base$cmd(".$param_con_args.");\n"; - } - print H "\tvoid WritePropertiesToStreamData(Protocol &rProtocol) const;\n"; - # set functions - for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) - { - my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); - - print H "\tvoid Set$nm(".translate_type_to_arg_type($ty)." $nm) {m$nm = $nm;}\n"; - } - } - - if($implement_syslog) - { - print H "\tvirtual void LogSysLog(const char *Action) const;\n"; - } - if($implement_filelog) - { - print H "\tvirtual void LogFile(const char *Action, FILE *file) const;\n"; - } - -======= if(obj_is_type($cmd, 'Command')) { @@ -613,7 +343,6 @@ __E print H "\tvirtual void LogSysLog(const char *Action) const;\n"; print H "\tvirtual void LogFile(const char *Action, FILE *file) const;\n"; print H "\tvirtual std::string ToString() const;\n"; ->>>>>>> 0.12 # write member variables and setup for cpp file my @def_constructor_list; @@ -647,20 +376,6 @@ __E my $param_con_vars = join(",\n\t ",@param_constructor_list); $param_con_vars = "\n\t: ".$param_con_vars if $param_con_vars ne ''; -<<<<<<< HEAD - my $class = "$classname_base$cmd".'::'; - print CPP <<__E; -$class$classname_base$cmd()$def_con_vars -{ -} -$class$classname_base$cmd(const $classname_base$cmd &rToCopy)$copy_con_vars -{ -} -$class~$classname_base$cmd() -{ -} -int ${class}GetType() const -======= print CPP <<__E; $cmd_class\::$cmd_class()$def_con_vars { @@ -672,56 +387,10 @@ $cmd_class\::~$cmd_class() { } int $cmd_class\::GetType() const ->>>>>>> 0.12 { return $cmd_id{$cmd}; } __E -<<<<<<< HEAD - if($read_from_streams) - { - print CPP "void ${class}SetPropertiesFromStreamData(Protocol &rProtocol)\n{\n"; - for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) - { - my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); - if($ty =~ m/\Avector/) - { - print CPP "\trProtocol.ReadVector(m$nm);\n"; - } - else - { - print CPP "\trProtocol.Read(m$nm);\n"; - } - } - print CPP "}\n"; - } - if($write_to_streams) - { - # implement extra constructor? - if($param_con_vars ne '') - { - print CPP "$class$classname_base$cmd($param_con_args)$param_con_vars\n{\n}\n"; - } - print CPP "void ${class}WritePropertiesToStreamData(Protocol &rProtocol) const\n{\n"; - for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) - { - my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); - if($ty =~ m/\Avector/) - { - print CPP "\trProtocol.WriteVector(m$nm);\n"; - } - else - { - print CPP "\trProtocol.Write(m$nm);\n"; - } - } - print CPP "}\n"; - } - if(obj_is_type($cmd,'EndsConversation')) - { - print CPP "bool ${class}IsConversationEnd() const\n{\n\treturn true;\n}\n"; - } -======= print CPP "void $cmd_class\::SetPropertiesFromStreamData(Protocol &rProtocol)\n{\n"; for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) { @@ -762,27 +431,18 @@ __E print CPP "bool $cmd_class\::IsConversationEnd() const\n{\n\treturn true;\n}\n"; } ->>>>>>> 0.12 if(obj_is_type($cmd,'IsError')) { # get parameters my ($mem_type,$mem_subtype) = split /,/,obj_get_type_params($cmd,'IsError'); print CPP <<__E; -<<<<<<< HEAD -bool ${class}IsError(int &rTypeOut, int &rSubTypeOut) const -======= bool $cmd_class\::IsError(int &rTypeOut, int &rSubTypeOut) const ->>>>>>> 0.12 { rTypeOut = m$mem_type; rSubTypeOut = m$mem_subtype; return true; } -<<<<<<< HEAD -std::string ${class}GetMessage() const -======= std::string $cmd_class\::GetMessage() const ->>>>>>> 0.12 { switch(m$mem_subtype) { @@ -806,149 +466,6 @@ __E __E } -<<<<<<< HEAD - if($implement_syslog) - { - my ($log) = make_log_strings_framework($cmd); - print CPP <<__E; -void ${class}LogSysLog(const char *Action) const -{ - BOX_TRACE($log); -} -__E - } - if($implement_filelog) - { - my ($log) = make_log_strings_framework($cmd); - print CPP <<__E; -void ${class}LogFile(const char *Action, FILE *File) const -{ - std::ostringstream oss; - oss << $log; - ::fprintf(File, "%s\\n", oss.str().c_str()); - ::fflush(File); -} -__E - } -} - -# finally, the protocol object itself -print H <<__E; -class $classname_base : public Protocol -{ -public: - $classname_base(IOStream &rStream); - virtual ~$classname_base(); - - std::auto_ptr<$derive_objects_from> Receive(); - void Send(const ${derive_objects_from} &rObject); -__E -if($implement_syslog) -{ - print H "\tvoid SetLogToSysLog(bool Log = false) {mLogToSysLog = Log;}\n"; -} -if($implement_filelog) -{ - print H "\tvoid SetLogToFile(FILE *File = 0) {mLogToFile = File;}\n"; -} -if($type eq 'Server') -{ - # need to put in the conversation function - print H "\tvoid DoServer($context_class &rContext);\n\n"; - # and the send vector thing - print H "\tvoid SendStreamAfterCommand(IOStream *pStream);\n\n"; -} -if($type eq 'Client') -{ - # add plain object taking query functions - my $with_params; - for my $cmd (@cmd_list) - { - if(obj_is_type($cmd,'Command')) - { - my $has_stream = obj_is_type($cmd,'StreamWithCommand'); - my $argextra = $has_stream?', IOStream &rStream':''; - my $queryextra = $has_stream?', rStream':''; - my $reply = obj_get_type_params($cmd,'Command'); - print H "\tstd::auto_ptr<$classname_base$reply> Query(const $classname_base$cmd &rQuery$argextra);\n"; - my @a; - my @na; - for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) - { - my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); - push @a,translate_type_to_arg_type($ty)." $nm"; - push @na,"$nm"; - } - my $ar = join(', ',@a); - my $nar = join(', ',@na); - $nar = "($nar)" if $nar ne ''; - - $with_params .= "\tinline std::auto_ptr<$classname_base$reply> Query$cmd($ar$argextra)\n\t{\n"; - $with_params .= "\t\t$classname_base$cmd send$nar;\n"; - $with_params .= "\t\treturn Query(send$queryextra);\n"; - $with_params .= "\t}\n"; - } - } - # quick hack to correct bad argument lists for commands with zero paramters but with streams - $with_params =~ s/\(, /(/g; - print H "\n",$with_params,"\n"; -} -print H <<__E; -private: - $classname_base(const $classname_base &rToCopy); -__E -if($type eq 'Server') -{ - # need to put the streams to send vector - print H "\tstd::vector<IOStream*> mStreamsToSend;\n\tvoid DeleteStreamsToSend();\n"; -} - -if($implement_filelog || $implement_syslog) -{ - print H <<__E; - virtual void InformStreamReceiving(u_int32_t Size); - virtual void InformStreamSending(u_int32_t Size); -__E -} - -if($implement_syslog) -{ - print H "private:\n\tbool mLogToSysLog;\n"; -} -if($implement_filelog) -{ - print H "private:\n\tFILE *mLogToFile;\n"; -} -print H <<__E; - -protected: - virtual std::auto_ptr<ProtocolObject> MakeProtocolObject(int ObjType); - virtual const char *GetIdentString(); -}; - -__E - -my $constructor_extra = ''; -$constructor_extra .= ', mLogToSysLog(false)' if $implement_syslog; -$constructor_extra .= ', mLogToFile(0)' if $implement_filelog; - -my $destructor_extra = ($type eq 'Server')?"\n\tDeleteStreamsToSend();":''; - -my $prefix = $classname_base.'::'; -print CPP <<__E; -$prefix$classname_base(IOStream &rStream) - : Protocol(rStream)$constructor_extra -{ -} -$prefix~$classname_base() -{$destructor_extra -} -const char *${prefix}GetIdentString() -{ - return "$ident_string"; -} -std::auto_ptr<ProtocolObject> ${prefix}MakeProtocolObject(int ObjType) -======= my ($log) = make_log_strings_framework($cmd); print CPP <<__E; std::string $cmd_class\::ToString() const @@ -1388,25 +905,11 @@ __E # write receive and send functions print CPP <<__E; std::auto_ptr<Message> $server_or_client_class\::MakeMessage(int ObjType) ->>>>>>> 0.12 { switch(ObjType) { __E -<<<<<<< HEAD -# do objects within this -for my $cmd (@cmd_list) -{ - print CPP <<__E; - case $cmd_id{$cmd}: - return std::auto_ptr<ProtocolObject>(new $classname_base$cmd); - break; -__E -} - -print CPP <<__E; -======= # do objects within this for my $cmd (@cmd_list) { @@ -1418,40 +921,11 @@ __E } print CPP <<__E; ->>>>>>> 0.12 default: THROW_EXCEPTION(ConnectionException, Conn_Protocol_UnknownCommandRecieved) } } __E -<<<<<<< HEAD -# write receive and send functions -print CPP <<__E; -std::auto_ptr<$derive_objects_from> ${prefix}Receive() -{ - std::auto_ptr<${derive_objects_from}> preply((${derive_objects_from}*)(Protocol::Receive().release())); - -__E - if($implement_syslog) - { - print CPP <<__E; - if(mLogToSysLog) - { - preply->LogSysLog("Receive"); - } -__E - } - if($implement_filelog) - { - print CPP <<__E; - if(mLogToFile != 0) - { - preply->LogFile("Receive", mLogToFile); - } -__E - } -print CPP <<__E; -======= if(not $writing_local) { @@ -1470,45 +944,10 @@ std::auto_ptr<$message_base_class> $server_or_client_class\::Receive() { preply->LogFile("Receive", GetLogToFile()); } ->>>>>>> 0.12 return preply; } -<<<<<<< HEAD -void ${prefix}Send(const ${derive_objects_from} &rObject) -{ -__E - if($implement_syslog) - { - print CPP <<__E; - if(mLogToSysLog) - { - rObject.LogSysLog("Send"); - } -__E - } - if($implement_filelog) - { - print CPP <<__E; - if(mLogToFile != 0) - { - rObject.LogFile("Send", mLogToFile); - } -__E - } - -print CPP <<__E; - Protocol::Send(rObject); -} - -__E -# write server function? -if($type eq 'Server') -{ - print CPP <<__E; -void ${prefix}DoServer($context_class &rContext) -======= void $server_or_client_class\::Send(const $message_base_class &rObject) { if(GetLogToSysLog()) @@ -1532,7 +971,6 @@ __E { print CPP <<__E; void $server_or_client_class\::DoServer($context_class &rContext) ->>>>>>> 0.12 { // Handshake with client Handshake(); @@ -1542,22 +980,6 @@ void $server_or_client_class\::DoServer($context_class &rContext) while(inProgress) { // Get an object from the conversation -<<<<<<< HEAD - std::auto_ptr<${derive_objects_from}> pobj(Receive()); - - // Run the command - std::auto_ptr<${derive_objects_from}> preply((${derive_objects_from}*)(pobj->DoCommand(*this, rContext).release())); - - // Send the reply - Send(*(preply.get())); - - // Send any streams - for(unsigned int s = 0; s < mStreamsToSend.size(); s++) - { - // Send the streams - SendStream(*mStreamsToSend[s]); - } -======= std::auto_ptr<$message_base_class> pobj = Receive(); // Run the command @@ -1574,7 +996,6 @@ void $server_or_client_class\::DoServer($context_class &rContext) SendStream(**i); } ->>>>>>> 0.12 // Delete these streams DeleteStreamsToSend(); @@ -1586,122 +1007,6 @@ void $server_or_client_class\::DoServer($context_class &rContext) } } -<<<<<<< HEAD -void ${prefix}SendStreamAfterCommand(IOStream *pStream) -{ - ASSERT(pStream != NULL); - mStreamsToSend.push_back(pStream); -} - -void ${prefix}DeleteStreamsToSend() -{ - for(std::vector<IOStream*>::iterator i(mStreamsToSend.begin()); i != mStreamsToSend.end(); ++i) - { - delete (*i); - } - mStreamsToSend.clear(); -} - -__E -} - -# write logging functions? -if($implement_filelog || $implement_syslog) -{ - my ($fR,$fS); - - if($implement_syslog) - { - $fR .= <<__E; - if(mLogToSysLog) - { - if(Size==Protocol::ProtocolStream_SizeUncertain) - { - BOX_TRACE("Receiving stream, size uncertain"); - } - else - { - BOX_TRACE("Receiving stream, size " << Size); - } - } -__E - - $fS .= <<__E; - if(mLogToSysLog) - { - if(Size==Protocol::ProtocolStream_SizeUncertain) - { - BOX_TRACE("Sending stream, size uncertain"); - } - else - { - BOX_TRACE("Sending stream, size " << Size); - } - } -__E - } - - if($implement_filelog) - { - $fR .= <<__E; - if(mLogToFile) - { - ::fprintf(mLogToFile, - (Size==Protocol::ProtocolStream_SizeUncertain) - ?"Receiving stream, size uncertain\\n" - :"Receiving stream, size %d\\n", Size); - ::fflush(mLogToFile); - } -__E - $fS .= <<__E; - if(mLogToFile) - { - ::fprintf(mLogToFile, - (Size==Protocol::ProtocolStream_SizeUncertain) - ?"Sending stream, size uncertain\\n" - :"Sending stream, size %d\\n", Size); - ::fflush(mLogToFile); - } -__E - } - - print CPP <<__E; - -void ${prefix}InformStreamReceiving(u_int32_t Size) -{ -$fR} - -void ${prefix}InformStreamSending(u_int32_t Size) -{ -$fS} - -__E -} - - -# write client Query functions? -if($type eq 'Client') -{ - for my $cmd (@cmd_list) - { - if(obj_is_type($cmd,'Command')) - { - my $reply = obj_get_type_params($cmd,'Command'); - my $reply_id = $cmd_id{$reply}; - my $has_stream = obj_is_type($cmd,'StreamWithCommand'); - my $argextra = $has_stream?', IOStream &rStream':''; - my $send_stream_extra = ''; - if($has_stream) - { - $send_stream_extra = <<__E; - - // Send stream after the command - SendStream(rStream); -__E - } - print CPP <<__E; -std::auto_ptr<$classname_base$reply> ${classname_base}::Query(const $classname_base$cmd &rQuery$argextra) -======= __E } @@ -1734,44 +1039,10 @@ __E print CPP <<__E; std::auto_ptr<$reply_class> $server_or_client_class\::Query(const $request_class &rQuery$argextra) ->>>>>>> 0.12 { // Send query Send(rQuery); $send_stream_extra -<<<<<<< HEAD - // Wait for the reply - std::auto_ptr<${derive_objects_from}> preply(Receive().release()); - - if(preply->GetType() == $reply_id) - { - // Correct response - return std::auto_ptr<$classname_base$reply>(($classname_base$reply*)preply.release()); - } - else - { - // Set protocol error - int type, subType; - if(preply->IsError(type, subType)) - { - SetError(type, subType); - BOX_WARNING("$cmd command failed: received error " << - ((${classname_base}Error&)*preply).GetMessage()); - } - else - { - SetError(Protocol::UnknownError, Protocol::UnknownError); - BOX_WARNING("$cmd command failed: received " - "unexpected response type " << - preply->GetType()); - } - - // Throw an exception - THROW_EXCEPTION(ConnectionException, Conn_Protocol_UnexpectedReply) - } -} -__E -======= // Wait for the reply std::auto_ptr<$message_base_class> preply = Receive(); @@ -1808,16 +1079,10 @@ std::auto_ptr<$reply_class> $server_or_client_class\::Query(const $request_class __E } } ->>>>>>> 0.12 } } } -<<<<<<< HEAD - - -======= ->>>>>>> 0.12 print H <<__E; #endif // $guardname @@ -1827,12 +1092,7 @@ __E close H; close CPP; -<<<<<<< HEAD - -sub obj_is_type -======= sub obj_is_type ($$) ->>>>>>> 0.12 { my ($c,$ty) = @_; for(@{$cmd_attributes{$c}}) @@ -1886,43 +1146,6 @@ sub translate_type_to_member_type return $typename } -<<<<<<< HEAD -sub make_log_strings -{ - my ($cmd) = @_; - - my @str; - my @arg; - for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2) - { - my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]); - - if(exists $log_display_types{$ty}) - { - # need to translate it - my ($format,$arg) = @{$log_display_types{$ty}}; - $arg =~ s/VAR/m$nm/g; - - if ($format eq "0x%llx" and $target_windows) - { - $format = "0x%I64x"; - $arg = "(uint64_t)$arg"; - } - - push @str,$format; - push @arg,$arg; - } - else - { - # is opaque - push @str,'OPAQUE'; - } - } - return ($cmd.'('.join(',',@str).')', join(',','',@arg)); -} - -======= ->>>>>>> 0.12 sub make_log_strings_framework { my ($cmd) = @_; @@ -1939,11 +1162,7 @@ sub make_log_strings_framework my ($format,$arg) = @{$log_display_types{$ty}}; $arg =~ s/VAR/m$nm/g; -<<<<<<< HEAD - if ($format eq '\\"%s\\"') -======= if ($format eq '"%s"') ->>>>>>> 0.12 { $arg = "\"\\\"\" << $arg << \"\\\"\""; } @@ -1970,11 +1189,7 @@ sub make_log_strings_framework } } -<<<<<<< HEAD - my $log_cmd = "Action << \" $cmd(\" "; -======= my $log_cmd = '"'.$cmd.'(" '; ->>>>>>> 0.12 foreach my $arg (@args) { $arg = "<< $arg "; @@ -1984,7 +1199,3 @@ sub make_log_strings_framework return $log_cmd; } -<<<<<<< HEAD - -======= ->>>>>>> 0.12 diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp index 9b5159cf..ef237671 100644 --- a/lib/win32/emu.cpp +++ b/lib/win32/emu.cpp @@ -1,11 +1,5 @@ // Box Backup Win32 native port by Nick Knight -<<<<<<< HEAD -// Need at least 0x0500 to use GetFileSizeEx on Cygwin/MinGW -#define WINVER 0x0500 - -======= ->>>>>>> 0.12 #include "emu.h" #ifdef WIN32 @@ -85,12 +79,6 @@ bool EnableBackupRights() return true; } -<<<<<<< HEAD -// forward declaration -char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage); - -======= ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Function @@ -252,11 +240,7 @@ char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) { ::syslog(LOG_WARNING, "Failed to convert wide string to narrow: " -<<<<<<< HEAD - "error %d", GetLastError()); -======= "%s", GetErrorMessage(GetLastError()).c_str()); ->>>>>>> 0.12 errno = EINVAL; return NULL; } @@ -288,11 +272,7 @@ char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) { ::syslog(LOG_WARNING, "Failed to convert wide string to narrow: " -<<<<<<< HEAD - "error %i", GetLastError()); -======= "%s", GetErrorMessage(GetLastError()).c_str()); ->>>>>>> 0.12 errno = EACCES; delete [] buffer; return NULL; @@ -301,8 +281,6 @@ char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) return buffer; } -<<<<<<< HEAD -======= bool ConvertFromWideString(const std::wstring& rInput, std::string* pOutput, unsigned int codepage) { @@ -366,7 +344,6 @@ bool ConvertFromWideString(const std::wstring& rInput, return true; } ->>>>>>> 0.12 // -------------------------------------------------------------------------- // // Function @@ -471,9 +448,6 @@ std::string ConvertPathToAbsoluteUnicode(const char *pFileName) return tmpStr; } -<<<<<<< HEAD - if (filename.length() > 2 && filename[0] == '\\' && -======= if (filename.length() > 4 && filename[0] == '\\' && filename[1] == '\\' && filename[2] == '?' && filename[3] == '\\') @@ -483,7 +457,6 @@ std::string ConvertPathToAbsoluteUnicode(const char *pFileName) tmpStr = ""; } else if (filename.length() > 2 && filename[0] == '\\' && ->>>>>>> 0.12 filename[1] == '\\') { tmpStr += "UNC\\"; @@ -493,27 +466,16 @@ std::string ConvertPathToAbsoluteUnicode(const char *pFileName) } else if (filename.length() >= 1 && filename[0] == '\\') { -<<<<<<< HEAD - // root directory of current drive. -======= // starts with \, i.e. root directory of current drive. ->>>>>>> 0.12 tmpStr = wd; tmpStr.resize(2); // drive letter and colon } else if (filename.length() >= 2 && filename[1] != ':') { -<<<<<<< HEAD - // Must be relative. We need to get the - // current directory to make it absolute. - tmpStr += wd; - if (tmpStr[tmpStr.length()] != '\\') -======= // Must be a relative path. We need to get the // current directory to make it absolute. tmpStr += wd; if (tmpStr[tmpStr.length()-1] != '\\') ->>>>>>> 0.12 { tmpStr += '\\'; } @@ -542,11 +504,7 @@ std::string ConvertPathToAbsoluteUnicode(const char *pFileName) ""); } -<<<<<<< HEAD - i = lastSlash; -======= i = lastSlash - 1; ->>>>>>> 0.12 } } @@ -761,11 +719,7 @@ int emu_fstat(HANDLE hdir, struct emu_stat * st) { conv.HighPart = fi.nFileSizeHigh; conv.LowPart = fi.nFileSizeLow; -<<<<<<< HEAD - st->st_size = (_off_t)conv.QuadPart; -======= st->st_size = conv.QuadPart; ->>>>>>> 0.12 } // at the mo @@ -1099,11 +1053,7 @@ DIR *opendir(const char *name) std::string dirName(name); //append a '\' win32 findfirst is sensitive to this -<<<<<<< HEAD - if ( dirName[dirName.size()] != '\\' || dirName[dirName.size()] != '/' ) -======= if (dirName[dirName.size()-1] != '\\' || dirName[dirName.size()-1] != '/') ->>>>>>> 0.12 { dirName += '\\'; } @@ -1127,26 +1077,16 @@ DIR *opendir(const char *name) return NULL; } -<<<<<<< HEAD - pDir->fd = _wfindfirst((const wchar_t*)pDir->name, &(pDir->info)); - - if (pDir->fd == -1) -======= pDir->fd = FindFirstFileW(pDir->name, &pDir->info); DWORD tmp = GetLastError(); if (pDir->fd == INVALID_HANDLE_VALUE) ->>>>>>> 0.12 { delete [] pDir->name; delete pDir; return NULL; } -<<<<<<< HEAD - -======= ->>>>>>> 0.12 pDir->result.d_name = 0; return pDir; } @@ -1169,17 +1109,6 @@ struct dirent *readdir(DIR *dp) { struct dirent *den = NULL; -<<<<<<< HEAD - if (dp && dp->fd != -1) - { - if (!dp->result.d_name || - _wfindnext(dp->fd, &dp->info) != -1) - { - den = &dp->result; - std::wstring input(dp->info.name); - memset(tempbuff, 0, sizeof(tempbuff)); - WideCharToMultiByte(CP_UTF8, 0, dp->info.name, -======= if (dp && dp->fd != INVALID_HANDLE_VALUE) { // first time around, when dp->result.d_name == NULL, use @@ -1192,20 +1121,10 @@ struct dirent *readdir(DIR *dp) std::wstring input(dp->info.cFileName); memset(tempbuff, 0, sizeof(tempbuff)); WideCharToMultiByte(CP_UTF8, 0, dp->info.cFileName, ->>>>>>> 0.12 -1, &tempbuff[0], sizeof (tempbuff), NULL, NULL); //den->d_name = (char *)dp->info.name; den->d_name = &tempbuff[0]; -<<<<<<< HEAD - if (dp->info.attrib & FILE_ATTRIBUTE_DIRECTORY) - { - den->d_type = S_IFDIR; - } - else - { - den->d_type = S_IFREG; -======= den->d_type = dp->info.dwFileAttributes; } else // FindNextFileW failed @@ -1221,7 +1140,6 @@ struct dirent *readdir(DIR *dp) else { errno = ENOSYS; ->>>>>>> 0.12 } } } @@ -1229,10 +1147,7 @@ struct dirent *readdir(DIR *dp) { errno = EBADF; } -<<<<<<< HEAD -======= ->>>>>>> 0.12 return den; } catch (...) @@ -1254,14 +1169,6 @@ int closedir(DIR *dp) { try { -<<<<<<< HEAD - int finres = -1; - if (dp) - { - if(dp->fd != -1) - { - finres = _findclose(dp->fd); -======= BOOL finres = false; if (dp) @@ -1269,21 +1176,12 @@ int closedir(DIR *dp) if(dp->fd != INVALID_HANDLE_VALUE) { finres = FindClose(dp->fd); ->>>>>>> 0.12 } delete [] dp->name; delete dp; } -<<<<<<< HEAD - if (finres == -1) // errors go to EBADF - { - errno = EBADF; - } - - return finres; -======= if (finres == FALSE) // errors go to EBADF { winerrno = GetLastError(); @@ -1291,7 +1189,6 @@ int closedir(DIR *dp) } return (finres == TRUE) ? 0 : -1; ->>>>>>> 0.12 } catch (...) { diff --git a/lib/win32/emu.h b/lib/win32/emu.h index bc91e2ba..bf408050 100644 --- a/lib/win32/emu.h +++ b/lib/win32/emu.h @@ -1,10 +1,7 @@ // emulates unix syscalls to win32 functions -<<<<<<< HEAD -======= #include "emu_winver.h" ->>>>>>> 0.12 #ifdef WIN32 #define EMU_STRUCT_STAT struct emu_stat #define EMU_STAT emu_stat @@ -20,8 +17,6 @@ #if ! defined EMU_INCLUDE && defined WIN32 #define EMU_INCLUDE -<<<<<<< HEAD -======= // Need feature detection macros below #include "../common/BoxConfig.h" @@ -30,7 +25,6 @@ # define __MINGW_FEATURES__ 0 #endif ->>>>>>> 0.12 // basic types, may be required by other headers since we // don't include sys/types.h @@ -58,17 +52,6 @@ typedef unsigned int pid_t; #endif -<<<<<<< HEAD -// set up to include the necessary parts of Windows headers - -#define WIN32_LEAN_AND_MEAN - -#ifndef __MSVCRT_VERSION__ -#define __MSVCRT_VERSION__ 0x0601 -#endif - -======= ->>>>>>> 0.12 // Windows headers #include <winsock2.h> @@ -239,28 +222,15 @@ inline int strcasecmp(const char *s1, const char *s2) struct dirent { char *d_name; -<<<<<<< HEAD - unsigned long d_type; -======= DWORD d_type; // file attributes ->>>>>>> 0.12 }; struct DIR { -<<<<<<< HEAD - intptr_t fd; // filedescriptor - // struct _finddata_t info; - struct _wfinddata_t info; - // struct _finddata_t info; - struct dirent result; // d_name (first time null) - wchar_t *name; // null-terminated byte string -======= HANDLE fd; // the HANDLE returned by FindFirstFile WIN32_FIND_DATAW info; struct dirent result; // d_name (first time null) wchar_t* name; // null-terminated byte string ->>>>>>> 0.12 }; DIR *opendir(const char *name); @@ -316,11 +286,6 @@ inline unsigned int sleep(unsigned int secs) } #define INFTIM -1 -<<<<<<< HEAD -#define POLLIN 0x1 -#define POLLERR 0x8 -#define POLLOUT 0x4 -======= #ifndef POLLIN # define POLLIN 0x1 @@ -333,7 +298,6 @@ inline unsigned int sleep(unsigned int secs) #ifndef POLLOUT # define POLLOUT 0x4 #endif ->>>>>>> 0.12 #define SHUT_RDWR SD_BOTH #define SHUT_RD SD_RECEIVE @@ -352,14 +316,6 @@ inline int ioctl(SOCKET sock, int flag, int * something) return 0; } -<<<<<<< HEAD -extern "C" inline int getpid() -{ - return (int)GetCurrentProcessId(); -} - -======= ->>>>>>> 0.12 inline int waitpid(pid_t pid, int *status, int) { return 0; @@ -399,21 +355,12 @@ int emu_chmod (const char* pName, mode_t mode); char* emu_getcwd (char* pBuffer, int BufSize); int emu_rename (const char* pOldName, const char* pNewName); -<<<<<<< HEAD -#define chdir(directory) emu_chdir (directory) -#define mkdir(path, mode) emu_mkdir (path) -#define unlink(file) emu_unlink (file) -#define utimes(buffer, times) emu_utimes (buffer, times) -#define chmod(file, mode) emu_chmod (file, mode) -#define getcwd(buffer, size) emu_getcwd (buffer, size) -======= #define chdir(directory) emu_chdir (directory) #define mkdir(path, mode) emu_mkdir (path) #define unlink(file) emu_unlink (file) #define utimes(buffer, times) emu_utimes (buffer, times) #define chmod(file, mode) emu_chmod (file, mode) #define getcwd(buffer, size) emu_getcwd (buffer, size) ->>>>>>> 0.12 #define rename(oldname, newname) emu_rename (oldname, newname) // Not safe to replace stat/fstat/lstat on mingw at least, as struct stat @@ -453,13 +400,10 @@ bool ConvertFromUtf8 (const std::string& rSource, std::string& rDest, int destCodePage); bool ConvertUtf8ToConsole(const std::string& rSource, std::string& rDest); bool ConvertConsoleToUtf8(const std::string& rSource, std::string& rDest); -<<<<<<< HEAD -======= char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage); bool ConvertFromWideString(const std::wstring& rInput, std::string* pOutput, unsigned int codepage); std::string ConvertPathToAbsoluteUnicode(const char *pFileName); ->>>>>>> 0.12 // Utility function which returns a default config file name, // based on the path of the current executable. @@ -473,8 +417,6 @@ std::string GetErrorMessage(DWORD errorCode); // relatively recent C runtime lib int console_read(char* pBuffer, size_t BufferSize); -<<<<<<< HEAD -======= // Defined thus by MinGW, but missing from MSVC // [http://curl.haxx.se/mail/lib-2004-11/0260.html] // note: chsize() doesn't work over 2GB: @@ -487,7 +429,6 @@ int console_read(char* pBuffer, size_t BufferSize); } #endif ->>>>>>> 0.12 #ifdef _MSC_VER /* disable certain compiler warnings to be able to actually see the show-stopper ones */ #pragma warning(disable:4101) // unreferenced local variable diff --git a/lib/win32/getopt_long.cpp b/lib/win32/getopt_long.cpp index a508d22a..31695aa0 100755 --- a/lib/win32/getopt_long.cpp +++ b/lib/win32/getopt_long.cpp @@ -58,10 +58,7 @@ */
// #include "Box.h"
-<<<<<<< HEAD -======= #include "emu.h"
->>>>>>> 0.12 #include <errno.h>
#include <stdarg.h>
|