diff options
Diffstat (limited to 'lib/backupstore')
35 files changed, 4296 insertions, 1622 deletions
diff --git a/lib/backupstore/BackgroundTask.h b/lib/backupstore/BackgroundTask.h new file mode 100644 index 00000000..bae9162f --- /dev/null +++ b/lib/backupstore/BackgroundTask.h @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackgroundTask.h +// Purpose: Declares the BackgroundTask interface. +// Created: 2014/04/07 +// +// -------------------------------------------------------------------------- + +#ifndef BACKGROUNDTASK__H +#define BACKGROUNDTASK__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackgroundTask +// Purpose: Provides a RunBackgroundTask() method which allows +// background tasks such as polling the command socket +// to happen while a file is being uploaded. If it +// returns false, the current task should be aborted. +// Created: 2014/04/07 +// +// -------------------------------------------------------------------------- +class BackgroundTask +{ + public: + enum State { + Unknown = 0, + Scanning_Dirs, + Searching_Blocks, + Uploading_Full, + Uploading_Patch, + }; + virtual ~BackgroundTask() { } + virtual bool RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum) = 0; +}; + +#endif // BACKGROUNDTASK__H diff --git a/lib/backupstore/BackupAccountControl.cpp b/lib/backupstore/BackupAccountControl.cpp new file mode 100644 index 00000000..331ef841 --- /dev/null +++ b/lib/backupstore/BackupAccountControl.cpp @@ -0,0 +1,267 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupAccountControl.cpp +// Purpose: Client-side account management for Amazon S3 stores +// Created: 2015/06/27 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <climits> +#include <iostream> + +#include "autogen_CommonException.h" +#include "autogen_BackupStoreException.h" +#include "BackupAccountControl.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreInfo.h" +#include "Configuration.h" +#include "HTTPResponse.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +void BackupAccountControl::CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit) +{ + if(SoftLimit > HardLimit) + { + BOX_FATAL("Soft limit must be less than the hard limit."); + exit(1); + } + if(SoftLimit > ((HardLimit * MAX_SOFT_LIMIT_SIZE) / 100)) + { + BOX_WARNING("We recommend setting the soft limit below " << + MAX_SOFT_LIMIT_SIZE << "% of the hard limit, or " << + HumanReadableSize((HardLimit * MAX_SOFT_LIMIT_SIZE) + / 100) << " in this case."); + } +} + +int64_t BackupAccountControl::SizeStringToBlocks(const char *string, int blockSize) +{ + // Get number + char *endptr = (char*)string; + int64_t number = strtol(string, &endptr, 0); + if(endptr == string || number == LONG_MIN || number == LONG_MAX) + { + BOX_FATAL("'" << string << "' is not a valid number."); + exit(1); + } + + // Check units + switch(*endptr) + { + case 'M': + case 'm': + // Units: Mb + return (number * 1024*1024) / blockSize; + break; + + case 'G': + case 'g': + // Units: Gb + return (number * 1024*1024*1024) / blockSize; + break; + + case 'B': + case 'b': + // Units: Blocks + // Easy! Just return the number specified. + return number; + break; + + default: + BOX_FATAL(string << " has an invalid units specifier " + "(use B for blocks, M for MB, G for GB, eg 2GB)"); + exit(1); + break; + } +} + +std::string BackupAccountControl::BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int BlockSize) +{ + return FormatUsageBar(Blocks, Blocks * BlockSize, MaxBlocks * BlockSize, + mMachineReadableOutput); +} + +int BackupAccountControl::PrintAccountInfo(const BackupStoreInfo& info, + int BlockSize) +{ + // Then print out lots of info + std::cout << FormatUsageLineStart("Account ID", mMachineReadableOutput) << + BOX_FORMAT_ACCOUNT(info.GetAccountID()) << std::endl; + std::cout << FormatUsageLineStart("Account Name", mMachineReadableOutput) << + info.GetAccountName() << std::endl; + std::cout << FormatUsageLineStart("Last object ID", mMachineReadableOutput) << + BOX_FORMAT_OBJECTID(info.GetLastObjectIDUsed()) << std::endl; + std::cout << FormatUsageLineStart("Used", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksUsed(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Current files", + mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInCurrentFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Old files", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInOldFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Deleted files", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInDeletedFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Directories", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInDirectories(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Soft limit", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksSoftLimit(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Hard limit", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksHardLimit(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Client store marker", mMachineReadableOutput) << + info.GetClientStoreMarker() << std::endl; + std::cout << FormatUsageLineStart("Current Files", mMachineReadableOutput) << + info.GetNumCurrentFiles() << std::endl; + std::cout << FormatUsageLineStart("Old Files", mMachineReadableOutput) << + info.GetNumOldFiles() << std::endl; + std::cout << FormatUsageLineStart("Deleted Files", mMachineReadableOutput) << + info.GetNumDeletedFiles() << std::endl; + std::cout << FormatUsageLineStart("Directories", mMachineReadableOutput) << + info.GetNumDirectories() << std::endl; + std::cout << FormatUsageLineStart("Enabled", mMachineReadableOutput) << + (info.IsAccountEnabled() ? "yes" : "no") << std::endl; + + return 0; +} + +S3BackupAccountControl::S3BackupAccountControl(const Configuration& config, + bool machineReadableOutput) +: BackupAccountControl(config, machineReadableOutput) +{ + if(!mConfig.SubConfigurationExists("S3Store")) + { + THROW_EXCEPTION_MESSAGE(CommonException, + InvalidConfiguration, + "The S3Store configuration subsection is required " + "when S3Store mode is enabled"); + } + const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); + + mBasePath = s3config.GetKeyValue("BasePath"); + if(mBasePath.size() == 0) + { + mBasePath = "/"; + } + else + { + if(mBasePath[0] != '/' || mBasePath[mBasePath.size() - 1] != '/') + { + THROW_EXCEPTION_MESSAGE(CommonException, + InvalidConfiguration, + "If S3Store.BasePath is not empty then it must start and " + "end with a slash, e.g. '/subdir/', but it currently does not."); + } + } + + mapS3Client.reset(new S3Client( + s3config.GetKeyValue("HostName"), + s3config.GetKeyValueInt("Port"), + s3config.GetKeyValue("AccessKey"), + s3config.GetKeyValue("SecretKey"))); + + mapFileSystem.reset(new S3BackupFileSystem(mConfig, mBasePath, *mapS3Client)); +} + +std::string S3BackupAccountControl::GetFullURL(const std::string ObjectPath) const +{ + const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); + return std::string("http://") + s3config.GetKeyValue("HostName") + ":" + + s3config.GetKeyValue("Port") + GetFullPath(ObjectPath); +} + +int S3BackupAccountControl::CreateAccount(const std::string& name, int32_t SoftLimit, + int32_t HardLimit) +{ + // Try getting the info file. If we get a 200 response then it already + // exists, and we should bail out. If we get a 404 then it's safe to + // continue. Otherwise something else is wrong and we should bail out. + std::string info_url = GetFullURL(S3_INFO_FILE_NAME); + + HTTPResponse response = GetObject(S3_INFO_FILE_NAME); + if(response.GetResponseCode() == HTTPResponse::Code_OK) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, AccountAlreadyExists, + "The BackupStoreInfo file already exists at this URL: " << + info_url); + } + + if(response.GetResponseCode() != HTTPResponse::Code_NotFound) + { + mapS3Client->CheckResponse(response, std::string("Failed to check for an " + "existing BackupStoreInfo file at this URL: ") + info_url); + } + + BackupStoreInfo info(0, // fake AccountID for S3 stores + info_url, // FileName, + SoftLimit, HardLimit); + info.SetAccountName(name); + + // And an empty directory + BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); + int64_t rootDirSize = mapFileSystem->PutDirectory(rootDir); + + // Update the store info to reflect the size of the root directory + info.ChangeBlocksUsed(rootDirSize); + info.ChangeBlocksInDirectories(rootDirSize); + info.AdjustNumDirectories(1); + int64_t id = info.AllocateObjectID(); + ASSERT(id == BACKUPSTORE_ROOT_DIRECTORY_ID); + + CollectInBufferStream out; + info.Save(out); + out.SetForReading(); + + response = PutObject(S3_INFO_FILE_NAME, out); + mapS3Client->CheckResponse(response, std::string("Failed to upload the new BackupStoreInfo " + "file to this URL: ") + info_url); + + // Now get the file again, to check that it really worked. + response = GetObject(S3_INFO_FILE_NAME); + mapS3Client->CheckResponse(response, std::string("Failed to download the new BackupStoreInfo " + "file that we just created: ") + info_url); + + return 0; +} + +std::string S3BackupFileSystem::GetDirectoryURI(int64_t ObjectID) +{ + std::ostringstream out; + out << mBasePath << "dirs/" << BOX_FORMAT_OBJECTID(ObjectID) << ".dir"; + return out.str(); +} + +std::auto_ptr<HTTPResponse> S3BackupFileSystem::GetDirectory(BackupStoreDirectory& rDir) +{ + std::string uri = GetDirectoryURI(rDir.GetObjectID()); + HTTPResponse response = mrClient.GetObject(uri); + mrClient.CheckResponse(response, + std::string("Failed to download directory: ") + uri); + return std::auto_ptr<HTTPResponse>(new HTTPResponse(response)); +} + +int S3BackupFileSystem::PutDirectory(BackupStoreDirectory& rDir) +{ + CollectInBufferStream out; + rDir.WriteToStream(out); + out.SetForReading(); + + std::string uri = GetDirectoryURI(rDir.GetObjectID()); + HTTPResponse response = mrClient.PutObject(uri, out); + mrClient.CheckResponse(response, + std::string("Failed to upload directory: ") + uri); + + int blocks = (out.GetSize() + S3_NOTIONAL_BLOCK_SIZE - 1) / S3_NOTIONAL_BLOCK_SIZE; + return blocks; +} + diff --git a/lib/backupstore/BackupAccountControl.h b/lib/backupstore/BackupAccountControl.h new file mode 100644 index 00000000..00118ec2 --- /dev/null +++ b/lib/backupstore/BackupAccountControl.h @@ -0,0 +1,91 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupAccountControl.h +// Purpose: Client-side account management for Amazon S3 stores +// Created: 2015/06/27 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPACCOUNTCONTROL__H +#define BACKUPACCOUNTCONTROL__H + +#include <string> + +#include "BackupStoreAccountDatabase.h" +#include "HTTPResponse.h" +#include "S3Client.h" + +class BackupStoreDirectory; +class BackupStoreInfo; +class Configuration; + +class BackupAccountControl +{ +protected: + const Configuration& mConfig; + bool mMachineReadableOutput; + +public: + BackupAccountControl(const Configuration& config, + bool machineReadableOutput = false) + : mConfig(config), + mMachineReadableOutput(machineReadableOutput) + { } + void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit); + int64_t SizeStringToBlocks(const char *string, int BlockSize); + std::string BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int BlockSize); + int PrintAccountInfo(const BackupStoreInfo& info, int BlockSize); +}; + +class S3BackupFileSystem +{ +private: + std::string mBasePath; + S3Client& mrClient; +public: + S3BackupFileSystem(const Configuration& config, const std::string& BasePath, + S3Client& rClient) + : mBasePath(BasePath), + mrClient(rClient) + { } + std::string GetDirectoryURI(int64_t ObjectID); + std::auto_ptr<HTTPResponse> GetDirectory(BackupStoreDirectory& rDir); + int PutDirectory(BackupStoreDirectory& rDir); +}; + +class S3BackupAccountControl : public BackupAccountControl +{ +private: + std::string mBasePath; + std::auto_ptr<S3Client> mapS3Client; + std::auto_ptr<S3BackupFileSystem> mapFileSystem; +public: + S3BackupAccountControl(const Configuration& config, + bool machineReadableOutput = false); + std::string GetFullPath(const std::string ObjectPath) const + { + return mBasePath + ObjectPath; + } + std::string GetFullURL(const std::string ObjectPath) const; + int CreateAccount(const std::string& name, int32_t SoftLimit, int32_t HardLimit); + int GetBlockSize() { return 4096; } + HTTPResponse GetObject(const std::string& name) + { + return mapS3Client->GetObject(GetFullPath(name)); + } + HTTPResponse PutObject(const std::string& name, IOStream& rStreamToSend, + const char* pContentType = NULL) + { + return mapS3Client->PutObject(GetFullPath(name), rStreamToSend, + pContentType); + } +}; + +// max size of soft limit as percent of hard limit +#define MAX_SOFT_LIMIT_SIZE 97 +#define S3_INFO_FILE_NAME "boxbackup.info" +#define S3_NOTIONAL_BLOCK_SIZE 1048576 + +#endif // BACKUPACCOUNTCONTROL__H + diff --git a/lib/backupstore/BackupClientFileAttributes.cpp b/lib/backupstore/BackupClientFileAttributes.cpp index d76432ba..37140301 100644 --- a/lib/backupstore/BackupClientFileAttributes.cpp +++ b/lib/backupstore/BackupClientFileAttributes.cpp @@ -55,20 +55,20 @@ BEGIN_STRUCTURE_PACKING_FOR_WIRE typedef struct { int32_t AttributeType; - u_int32_t UID; - u_int32_t GID; - u_int64_t ModificationTime; - u_int64_t AttrModificationTime; - u_int32_t UserDefinedFlags; - u_int32_t FileGenerationNumber; - u_int16_t Mode; + uint32_t UID; + uint32_t GID; + uint64_t ModificationTime; + uint64_t AttrModificationTime; + uint32_t UserDefinedFlags; + uint32_t FileGenerationNumber; + uint16_t Mode; // Symbolic link filename may follow // Extended attribute (xattr) information may follow, format is: - // u_int32_t Size of extended attribute block (excluding this word) + // uint32_t Size of extended attribute block (excluding this word) // For each of NumberOfAttributes (sorted by AttributeName): - // u_int16_t AttributeNameLength + // uint16_t AttributeNameLength // char AttributeName[AttributeNameLength] - // u_int32_t AttributeValueLength + // uint32_t AttributeValueLength // unsigned char AttributeValue[AttributeValueLength] // AttributeName is 0 terminated, AttributeValue is not (and may be binary data) } attr_StreamFormat; @@ -117,7 +117,7 @@ namespace BackupClientFileAttributes::BackupClientFileAttributes() : mpClearAttributes(0) { - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); + ASSERT(sizeof(uint64_t) == sizeof(box_time_t)); } // -------------------------------------------------------------------------- @@ -131,7 +131,7 @@ BackupClientFileAttributes::BackupClientFileAttributes() BackupClientFileAttributes::BackupClientFileAttributes(const EMU_STRUCT_STAT &st) : mpClearAttributes(0) { - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); + ASSERT(sizeof(uint64_t) == sizeof(box_time_t)); StreamableMemBlock *pnewAttr = new StreamableMemBlock; FillAttributes(*pnewAttr, (const char *)NULL, st, true); @@ -411,7 +411,7 @@ void BackupClientFileAttributes::ReadAttributes(const std::string& Filename, // __time64_t winTime = BoxTimeToSeconds( // pnewAttr->ModificationTime); - u_int64_t modTime = box_ntoh64(pattr->ModificationTime); + uint64_t modTime = box_ntoh64(pattr->ModificationTime); box_time_t modSecs = BoxTimeToSeconds(modTime); __time64_t winTime = modSecs; @@ -545,7 +545,7 @@ void BackupClientFileAttributes::FillAttributesLink( void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const std::string& Filename) { -#ifdef HAVE_SYS_XATTR_H +#if defined HAVE_LLISTXATTR && defined HAVE_LGETXATTR int listBufferSize = 10000; char* list = new char[listBufferSize]; @@ -582,30 +582,30 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc // Leave space for attr block size later int xattrBlockSizeOffset = xattrSize; - xattrSize += sizeof(u_int32_t); + xattrSize += sizeof(uint32_t); // Loop for each attribute for(std::vector<std::string>::const_iterator attrKeyI = attrKeys.begin(); attrKeyI!=attrKeys.end(); ++attrKeyI) { std::string attrKey(*attrKeyI); - if(xattrSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t)>static_cast<unsigned int>(xattrBufferSize)) + if(xattrSize+sizeof(uint16_t)+attrKey.size()+1+sizeof(uint32_t)>static_cast<unsigned int>(xattrBufferSize)) { - xattrBufferSize = (xattrBufferSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t))*2; + xattrBufferSize = (xattrBufferSize+sizeof(uint16_t)+attrKey.size()+1+sizeof(uint32_t))*2; outputBlock.ResizeBlock(xattrBufferSize); buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); } // Store length and text for attibute name - u_int16_t keyLength = htons(attrKey.size()+1); - std::memcpy(buffer+xattrSize, &keyLength, sizeof(u_int16_t)); - xattrSize += sizeof(u_int16_t); + uint16_t keyLength = htons(attrKey.size()+1); + std::memcpy(buffer+xattrSize, &keyLength, sizeof(uint16_t)); + xattrSize += sizeof(uint16_t); std::memcpy(buffer+xattrSize, attrKey.c_str(), attrKey.size()+1); xattrSize += attrKey.size()+1; // Leave space for value size int valueSizeOffset = xattrSize; - xattrSize += sizeof(u_int32_t); + xattrSize += sizeof(uint32_t); // Find size of attribute (must call with buffer and length 0 on some platforms, // as -1 is returned if the data doesn't fit.) @@ -642,21 +642,30 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc xattrSize += valueSize; // Fill in value size - u_int32_t valueLength = htonl(valueSize); - std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(u_int32_t)); + uint32_t valueLength = htonl(valueSize); + std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(uint32_t)); } // Fill in attribute block size - u_int32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(u_int32_t)); - std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(u_int32_t)); + uint32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(uint32_t)); + std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(uint32_t)); outputBlock.ResizeBlock(xattrSize); } else if(listSize<0) { - if(errno == EOPNOTSUPP || errno == EACCES) + if(errno == EOPNOTSUPP || errno == EACCES +#if HAVE_DECL_ENOTSUP + // NetBSD uses ENOTSUP instead + // https://mail-index.netbsd.org/tech-kern/2011/12/13/msg012185.html + || errno == ENOTSUP +#endif + ) { - // fail silently + // Not supported by OS, or not on this filesystem + BOX_TRACE(BOX_SYS_ERRNO_MESSAGE(errno, + BOX_FILE_MESSAGE(Filename, "Failed to " + "list extended attributes"))); } else if(errno == ERANGE) { @@ -664,12 +673,17 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc "attributes of '" << Filename << "': " "buffer too small, not backed up"); } + else if(errno == ENOENT) + { + BOX_ERROR("Failed to list extended " + "attributes of '" << Filename << "': " + "file no longer exists"); + } else { - BOX_LOG_SYS_ERROR("Failed to list extended " - "attributes of '" << Filename << "', " - "not backed up"); - THROW_EXCEPTION(CommonException, OSFileError); + THROW_SYS_FILE_ERROR("Failed to list extended " + "attributes for unknown reason", Filename, + CommonException, OSFileError); } } } @@ -679,7 +693,7 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc throw; } delete[] list; -#endif +#endif // defined HAVE_LLISTXATTR && defined HAVE_LGETXATTR } // -------------------------------------------------------------------------- @@ -839,7 +853,7 @@ void BackupClientFileAttributes::WriteAttributes(const std::string& Filename, #endif } - if(static_cast<int>(xattrOffset+sizeof(u_int32_t))<=mpClearAttributes->GetSize()) + if(static_cast<int>(xattrOffset+sizeof(uint32_t))<=mpClearAttributes->GetSize()) { WriteExtendedAttr(Filename, xattrOffset); } @@ -978,13 +992,13 @@ void BackupClientFileAttributes::EnsureClearAvailable() const // -------------------------------------------------------------------------- void BackupClientFileAttributes::WriteExtendedAttr(const std::string& Filename, int xattrOffset) const { -#ifdef HAVE_SYS_XATTR_H +#if defined HAVE_LSETXATTR const char* buffer = static_cast<char*>(mpClearAttributes->GetBuffer()); - u_int32_t xattrBlockLength = 0; - std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(u_int32_t)); + uint32_t xattrBlockLength = 0; + std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(uint32_t)); int xattrBlockSize = ntohl(xattrBlockLength); - xattrOffset += sizeof(u_int32_t); + xattrOffset += sizeof(uint32_t); int xattrEnd = xattrOffset+xattrBlockSize; if(xattrEnd>mpClearAttributes->GetSize()) @@ -995,18 +1009,18 @@ void BackupClientFileAttributes::WriteExtendedAttr(const std::string& Filename, while(xattrOffset<xattrEnd) { - u_int16_t keyLength = 0; - std::memcpy(&keyLength, buffer+xattrOffset, sizeof(u_int16_t)); + uint16_t keyLength = 0; + std::memcpy(&keyLength, buffer+xattrOffset, sizeof(uint16_t)); int keySize = ntohs(keyLength); - xattrOffset += sizeof(u_int16_t); + xattrOffset += sizeof(uint16_t); const char* key = buffer+xattrOffset; xattrOffset += keySize; - u_int32_t valueLength = 0; - std::memcpy(&valueLength, buffer+xattrOffset, sizeof(u_int32_t)); + uint32_t valueLength = 0; + std::memcpy(&valueLength, buffer+xattrOffset, sizeof(uint32_t)); int valueSize = ntohl(valueLength); - xattrOffset += sizeof(u_int32_t); + xattrOffset += sizeof(uint32_t); // FIXME: Warn on EOPNOTSUPP if(::lsetxattr(Filename.c_str(), key, buffer+xattrOffset, diff --git a/lib/backupstore/BackupCommands.cpp b/lib/backupstore/BackupCommands.cpp index 318ce55a..22ef0215 100644 --- a/lib/backupstore/BackupCommands.cpp +++ b/lib/backupstore/BackupCommands.cpp @@ -52,11 +52,66 @@ return PROTOCOL_ERROR(Err_SessionReadOnly); \ } + // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolVersion::DoCommand(Protocol &, BackupStoreContext &) -// Purpose: Return the current version, or an error if the requested version isn't allowed +// Name: BackupProtocolMessage::HandleException(BoxException& e) +// Purpose: Return an error message appropriate to the passed-in +// exception, or rethrow it. +// Created: 2014/09/14 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupProtocolMessage> BackupProtocolReplyable::HandleException(BoxException& e) const +{ + if(e.GetType() == RaidFileException::ExceptionType && + e.GetSubType() == RaidFileException::RaidFileDoesntExist) + { + return PROTOCOL_ERROR(Err_DoesNotExist); + } + else if (e.GetType() == BackupStoreException::ExceptionType) + { + if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify) + { + return PROTOCOL_ERROR(Err_FileDoesNotVerify); + } + else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit) + { + return PROTOCOL_ERROR(Err_StorageLimitExceeded); + } + else if(e.GetSubType() == BackupStoreException::MultiplyReferencedObject) + { + return PROTOCOL_ERROR(Err_MultiplyReferencedObject); + } + else if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory) + { + return PROTOCOL_ERROR(Err_DoesNotExistInDirectory); + } + else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory) + { + return PROTOCOL_ERROR(Err_TargetNameExists); + } + else if(e.GetSubType() == BackupStoreException::ObjectDoesNotExist) + { + return PROTOCOL_ERROR(Err_DoesNotExist); + } + else if(e.GetSubType() == BackupStoreException::PatchChainInfoBadInDirectory) + { + return PROTOCOL_ERROR(Err_PatchConsistencyError); + } + } + + throw; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolVersion::DoCommand(Protocol &, +// BackupStoreContext &) +// Purpose: Return the current version, or an error if the +// requested version isn't allowed // Created: 2003/08/20 // // -------------------------------------------------------------------------- @@ -93,7 +148,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc // and that the client actually has an account on this machine if(mClientID != rContext.GetClientID()) { - BOX_WARNING("Failed login from client ID " << + BOX_WARNING("Failed login from client ID " << BOX_FORMAT_ACCOUNT(mClientID) << ": " "wrong certificate for this account"); return PROTOCOL_ERROR(Err_BadLogin); @@ -101,7 +156,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc if(!rContext.GetClientHasAccount()) { - BOX_WARNING("Failed login from client ID " << + BOX_WARNING("Failed login from client ID " << BOX_FORMAT_ACCOUNT(mClientID) << ": " "no such account on this server"); return PROTOCOL_ERROR(Err_BadLogin); @@ -117,17 +172,17 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc BOX_FORMAT_ACCOUNT(mClientID)); return PROTOCOL_ERROR(Err_CannotLockStoreForWriting); } - + // Debug: check we got the lock ASSERT(!rContext.SessionIsReadOnly()); } - + // Load the store info rContext.LoadStoreInfo(); if(!rContext.GetBackupStoreInfo().IsAccountEnabled()) { - BOX_WARNING("Refused login from disabled client ID " << + BOX_WARNING("Refused login from disabled client ID " << BOX_FORMAT_ACCOUNT(mClientID)); return PROTOCOL_ERROR(Err_DisabledAccount); } @@ -137,9 +192,9 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc // Mark the next phase rContext.SetPhase(BackupStoreContext::Phase_Commands); - + // Log login - BOX_NOTICE("Login from Client ID " << + BOX_NOTICE("Login from Client ID " << BOX_FORMAT_ACCOUNT(mClientID) << " " "(name=" << rContext.GetAccountName() << "): " << (((mFlags & Flags_ReadOnly) != Flags_ReadOnly) @@ -164,14 +219,15 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc // -------------------------------------------------------------------------- std::auto_ptr<BackupProtocolMessage> BackupProtocolFinished::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const { - BOX_NOTICE("Session finished for Client ID " << + // can be called in any phase + + BOX_NOTICE("Session finished for Client ID " << BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << " " "(name=" << rContext.GetAccountName() << ")"); // Let the context know about it rContext.ReceivedFinishCommand(); - // can be called in any phase return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolFinished); } @@ -191,26 +247,15 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolListDirectory::DoCommand(Back // Store the listing to a stream std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream); - try - { - // Ask the context for a directory - const BackupStoreDirectory &rdir( - rContext.GetDirectory(mObjectID)); - rdir.WriteToStream(*stream, mFlagsMustBeSet, - mFlagsNotToBeSet, mSendAttributes, - false /* never send dependency info to the client */); - } - catch (RaidFileException &e) - { - if (e.GetSubType() == RaidFileException::RaidFileDoesntExist) - { - return PROTOCOL_ERROR(Err_DoesNotExist); - } - throw; - } + // Ask the context for a directory + const BackupStoreDirectory &rdir( + rContext.GetDirectory(mObjectID)); + rdir.WriteToStream(*stream, mFlagsMustBeSet, + mFlagsNotToBeSet, mSendAttributes, + false /* never send dependency info to the client */); stream->SetForReading(); - + // Get the protocol to send the stream rProtocol.SendStreamAfterCommand(static_cast< std::auto_ptr<IOStream> > (stream)); @@ -226,7 +271,9 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolListDirectory::DoCommand(Back // Created: 2003/09/02 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -237,7 +284,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand(BackupPr { return hookResult; } - + // Check that the diff from file actually exists, if it's specified if(mDiffFromFileID != 0) { @@ -247,35 +294,13 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand(BackupPr return PROTOCOL_ERROR(Err_DiffFromFileDoesNotExist); } } - - // A stream follows, which contains the file - std::auto_ptr<IOStream> filestream(rProtocol.ReceiveStream()); - + // Ask the context to store it - int64_t id = 0; - try - { - id = rContext.AddFile(*filestream, mDirectoryObjectID, - mModificationTime, mAttributesHash, mDiffFromFileID, - mFilename, - true /* mark files with same name as old versions */); - } - catch(BackupStoreException &e) - { - if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify) - { - return PROTOCOL_ERROR(Err_FileDoesNotVerify); - } - else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit) - { - return PROTOCOL_ERROR(Err_StorageLimitExceeded); - } - else - { - throw; - } - } - + int64_t id = rContext.AddFile(rDataStream, mDirectoryObjectID, + mModificationTime, mAttributesHash, mDiffFromFileID, + mFilename, + true /* mark files with same name as old versions */); + // Tell the caller what the file ID was return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(id)); } @@ -298,7 +323,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObject::DoCommand(BackupPr // Check the object exists if(!rContext.ObjectExists(mObjectID)) { - return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(NoObject)); + return PROTOCOL_ERROR(Err_DoesNotExist); } // Open the object @@ -315,7 +340,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObject::DoCommand(BackupPr // // Function // Name: BackupProtocolGetFile::DoCommand(Protocol &, BackupStoreContext &) -// Purpose: Command to get an file object from the server -- may have to do a bit of +// Purpose: Command to get an file object from the server -- may have to do a bit of // work to get the object. // Created: 2003/09/03 // @@ -357,13 +382,13 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt en = rdir.FindEntryByID(id); if(en == 0) { - BOX_ERROR("Object " << + BOX_ERROR("Object " << BOX_FORMAT_OBJECTID(mObjectID) << - " in dir " << + " in dir " << BOX_FORMAT_OBJECTID(mInDirectory) << " for account " << BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << - " references object " << + " references object " << BOX_FORMAT_OBJECTID(id) << " which does not exist in dir"); return PROTOCOL_ERROR(Err_PatchConsistencyError); @@ -371,89 +396,73 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt id = en->GetDependsNewer(); } while(en != 0 && id != 0); - + // OK! The last entry in the chain is the full file, the others are patches back from it. // Open the last one, which is the current from file std::auto_ptr<IOStream> from(rContext.OpenObject(patchChain[patchChain.size() - 1])); - + // Then, for each patch in the chain, do a combine for(int p = ((int)patchChain.size()) - 2; p >= 0; --p) { // ID of patch int64_t patchID = patchChain[p]; - + // Open it a couple of times std::auto_ptr<IOStream> diff(rContext.OpenObject(patchID)); std::auto_ptr<IOStream> diff2(rContext.OpenObject(patchID)); - + // Choose a temporary filename for the result of the combination std::ostringstream fs; - fs << rContext.GetStoreRoot() << ".recombinetemp." << p; - std::string tempFn = + fs << rContext.GetAccountRoot() << ".recombinetemp." << p; + std::string tempFn = RaidFileController::DiscSetPathToFileSystemPath( rContext.GetStoreDiscSet(), fs.str(), p + 16); - + // Open the temporary file - std::auto_ptr<IOStream> combined; - try - { - { - // Write nastily to allow this to work with gcc 2.x - std::auto_ptr<IOStream> t( - new InvisibleTempFileStream( - tempFn.c_str(), - O_RDWR | O_CREAT | - O_EXCL | O_BINARY | - O_TRUNC)); - combined = t; - } - } - catch(...) - { - // Make sure it goes - ::unlink(tempFn.c_str()); - throw; - } - + std::auto_ptr<IOStream> combined( + new InvisibleTempFileStream( + tempFn, O_RDWR | O_CREAT | O_EXCL | + O_BINARY | O_TRUNC)); + // Do the combining BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined); - + // Move to the beginning of the combined file combined->Seek(0, IOStream::SeekType_Absolute); - + // Then shuffle round for the next go if (from.get()) from->Close(); from = combined; } - + // Now, from contains a nice file to send to the client. Reorder it { // Write nastily to allow this to work with gcc 2.x std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */)); stream = t; } - + // Release from file to avoid double deletion from.release(); } else { // Simple case: file already exists on disc ready to go - + // Open the object std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID)); BufferedStream buf(*object); - + // Verify it if(!BackupStoreFile::VerifyEncodedFileFormat(buf)) { return PROTOCOL_ERROR(Err_FileDoesNotVerify); } - + // Reset stream -- seek to beginning object->Seek(0, IOStream::SeekType_Absolute); - + // Reorder the stream/file into stream order { // Write nastily to allow this to work with gcc 2.x @@ -461,15 +470,15 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt stream = t; } - // Object will be deleted when the stream is deleted, - // so can release the object auto_ptr here to avoid + // Object will be deleted when the stream is deleted, + // so can release the object auto_ptr here to avoid // premature deletion object.release(); } // Stream the reordered stream to the peer rProtocol.SendStreamAfterCommand(stream); - + // Tell the caller what the file was return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(mObjectID)); } @@ -483,18 +492,38 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt // Created: 2003/09/04 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const +{ + return BackupProtocolCreateDirectory2(mContainingDirectoryID, + mAttributesModTime, 0 /* ModificationTime */, + mDirectoryName).DoCommand(rProtocol, rContext, rDataStream); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolCreateDirectory2::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Create directory command, with a specific +// modification time. +// Created: 2014/02/11 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory2::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - - // Get the stream containing the attributes - std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); - // Collect the attributes -- do this now so no matter what the outcome, + + // Collect the attributes -- do this now so no matter what the outcome, // the data has been absorbed. StreamableMemBlock attr; - attr.Set(*attrstream, rProtocol.GetTimeout()); - + attr.Set(rDataStream, rProtocol.GetTimeout()); + // Check to see if the hard limit has been exceeded if(rContext.HardLimitExceeded()) { @@ -503,8 +532,10 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand(Ba } bool alreadyExists = false; - int64_t id = rContext.AddDirectory(mContainingDirectoryID, mDirectoryName, attr, mAttributesModTime, alreadyExists); - + int64_t id = rContext.AddDirectory(mContainingDirectoryID, + mDirectoryName, attr, mAttributesModTime, mModificationTime, + alreadyExists); + if(alreadyExists) { return PROTOCOL_ERROR(Err_DirectoryAlreadyExists); @@ -524,17 +555,17 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand(Ba // Created: 2003/09/06 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolChangeDirAttributes::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> BackupProtocolChangeDirAttributes::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - // Get the stream containing the attributes - std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); - // Collect the attributes -- do this now so no matter what the outcome, + // Collect the attributes -- do this now so no matter what the outcome, // the data has been absorbed. StreamableMemBlock attr; - attr.Set(*attrstream, rProtocol.GetTimeout()); + attr.Set(rDataStream, rProtocol.GetTimeout()); // Get the context to do it's magic rContext.ChangeDirAttributes(mObjectID, attr, mAttributesModTime); @@ -552,17 +583,18 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolChangeDirAttributes::DoComman // Created: 2003/09/06 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolSetReplacementFileAttributes::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> +BackupProtocolSetReplacementFileAttributes::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - // Get the stream containing the attributes - std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); - // Collect the attributes -- do this now so no matter what the outcome, + // Collect the attributes -- do this now so no matter what the outcome, // the data has been absorbed. StreamableMemBlock attr; - attr.Set(*attrstream, rProtocol.GetTimeout()); + attr.Set(rDataStream, rProtocol.GetTimeout()); // Get the context to do it's magic int64_t objectID = 0; @@ -577,7 +609,6 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolSetReplacementFileAttributes: } - // -------------------------------------------------------------------------- // // Function @@ -644,19 +675,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolDeleteDirectory::DoCommand(Ba } // Context handles this - try - { - rContext.DeleteDirectory(mObjectID); - } - catch (BackupStoreException &e) - { - if(e.GetSubType() == BackupStoreException::MultiplyReferencedObject) - { - return PROTOCOL_ERROR(Err_MultiplyReferencedObject); - } - - throw; - } + rContext.DeleteDirectory(mObjectID); // return the object ID return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(mObjectID)); @@ -722,29 +741,11 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolMoveObject::DoCommand(BackupP { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - + // Let context do this, but modify error reporting on exceptions... - try - { - rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory, - mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName, - (mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject); - } - catch(BackupStoreException &e) - { - if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory) - { - return PROTOCOL_ERROR(Err_DoesNotExist); - } - else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory) - { - return PROTOCOL_ERROR(Err_TargetNameExists); - } - else - { - throw; - } - } + rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory, + mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName, + (mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject); // Return the object ID return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(mObjectID)); @@ -762,21 +763,21 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolMoveObject::DoCommand(BackupP std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const { CHECK_PHASE(Phase_Commands) - + // Create a stream for the list of filenames std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream); // Object and directory IDs int64_t objectID = mObjectID; int64_t dirID = mContainingDirectoryID; - + // Data to return in the reply int32_t numNameElements = 0; int16_t objectFlags = 0; int64_t modTime = 0; uint64_t attrModHash = 0; bool haveModTimes = false; - + do { // Check the directory really exists @@ -786,7 +787,28 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back } // Load up the directory - const BackupStoreDirectory &rdir(rContext.GetDirectory(dirID)); + const BackupStoreDirectory *pDir; + + try + { + pDir = &rContext.GetDirectory(dirID); + } + catch(BackupStoreException &e) + { + if(e.GetSubType() == BackupStoreException::ObjectDoesNotExist) + { + // If this can't be found, then there is a problem... + // tell the caller it can't be found. + return std::auto_ptr<BackupProtocolMessage>( + new BackupProtocolObjectName( + BackupProtocolObjectName::NumNameElements_ObjectDoesntExist, + 0, 0, 0)); + } + + throw; + } + + const BackupStoreDirectory& rdir(*pDir); // Find the element in this directory and store it's name if(objectID != ObjectID_DirectoryOnly) @@ -799,13 +821,13 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back // Abort! return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolObjectName(BackupProtocolObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0)); } - + // Store flags? if(objectFlags == 0) { objectFlags = en->GetFlags(); } - + // Store modification times? if(!haveModTimes) { @@ -813,14 +835,14 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back attrModHash = en->GetAttributesHash(); haveModTimes = true; } - + // Store the name in the stream en->GetName().WriteToStream(*stream); - + // Count of name elements ++numNameElements; } - + // Setup for next time round objectID = dirID; dirID = rdir.GetContainerID(); @@ -831,7 +853,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back if(numNameElements > 0) { // Get the stream ready to go - stream->SetForReading(); + stream->SetForReading(); // Tell the protocol to send the stream rProtocol.SendStreamAfterCommand(static_cast< std::auto_ptr<IOStream> >(stream)); } @@ -856,10 +878,10 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByID::DoCommand( // Open the file std::auto_ptr<IOStream> stream(rContext.OpenObject(mObjectID)); - + // Move the file pointer to the block index BackupStoreFile::MoveStreamPositionToBlockIndex(*stream); - + // Return the stream to the client rProtocol.SendStreamAfterCommand(stream); @@ -882,7 +904,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByName::DoComman // Get the directory const BackupStoreDirectory &dir(rContext.GetDirectory(mInDirectory)); - + // Find the latest object ID within it which has the same name int64_t objectID = 0; BackupStoreDirectory::Iterator i(dir); @@ -898,7 +920,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByName::DoComman } } } - + // Found anything? if(objectID == 0) { @@ -908,10 +930,10 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByName::DoComman // Open the file std::auto_ptr<IOStream> stream(rContext.OpenObject(objectID)); - + // Move the file pointer to the block index BackupStoreFile::MoveStreamPositionToBlockIndex(*stream); - + // Return the stream to the client rProtocol.SendStreamAfterCommand(stream); @@ -934,11 +956,11 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetAccountUsage::DoCommand(Ba // Get store info from context const BackupStoreInfo &rinfo(rContext.GetBackupStoreInfo()); - + // Find block size RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(rinfo.GetDiscSetNumber())); - + // Return info return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolAccountUsage( rinfo.GetBlocksUsed(), @@ -968,3 +990,48 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetIsAlive::DoCommand(BackupP // return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolIsAlive()); } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolGetAccountUsage2::DoCommand(BackupProtocolReplyable &, BackupStoreContext &) +// Purpose: Return the amount of disc space used +// Created: 26/12/13 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupProtocolMessage> BackupProtocolGetAccountUsage2::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +{ + CHECK_PHASE(Phase_Commands) + + // Get store info from context + const BackupStoreInfo &info(rContext.GetBackupStoreInfo()); + + // Find block size + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(info.GetDiscSetNumber())); + + // Return info + BackupProtocolAccountUsage2* usage = new BackupProtocolAccountUsage2(); + std::auto_ptr<BackupProtocolMessage> reply(usage); + #define COPY(name) usage->Set ## name (info.Get ## name ()) + COPY(AccountName); + usage->SetAccountEnabled(info.IsAccountEnabled()); + COPY(ClientStoreMarker); + usage->SetBlockSize(rdiscSet.GetBlockSize()); + COPY(LastObjectIDUsed); + COPY(BlocksUsed); + COPY(BlocksInCurrentFiles); + COPY(BlocksInOldFiles); + COPY(BlocksInDeletedFiles); + COPY(BlocksInDirectories); + COPY(BlocksSoftLimit); + COPY(BlocksHardLimit); + COPY(NumCurrentFiles); + COPY(NumOldFiles); + COPY(NumDeletedFiles); + COPY(NumDirectories); + #undef COPY + + return reply; +} diff --git a/lib/backupstore/BackupConstants.h b/lib/backupstore/BackupConstants.h index 19d06a15..195dc621 100644 --- a/lib/backupstore/BackupConstants.h +++ b/lib/backupstore/BackupConstants.h @@ -13,6 +13,9 @@ // 15 minutes to timeout (milliseconds) #define BACKUP_STORE_TIMEOUT (15*60*1000) +// Time to wait for retry after a backup error +#define BACKUP_ERROR_RETRY_SECONDS 100 + // Should the store daemon convert files to Raid immediately? #define BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY true diff --git a/lib/backupstore/BackupProtocol.h b/lib/backupstore/BackupProtocol.h new file mode 100644 index 00000000..d9070c73 --- /dev/null +++ b/lib/backupstore/BackupProtocol.h @@ -0,0 +1,71 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupProtocol.h +// Purpose: A thin wrapper around autogen_BackupProtocol.h +// Created: 2014/01/05 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPPROTOCOL__H +#define BACKUPPROTOCOL__H + +#include <autogen_BackupProtocol.h> +#include <BackupStoreConstants.h> +#include <BackupStoreContext.h> + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupProtocolLocal2 +// Purpose: BackupProtocolLocal with a few more IQ points +// Created: 2014/09/20 +// +// -------------------------------------------------------------------------- +class BackupProtocolLocal2 : public BackupProtocolLocal +{ +private: + BackupStoreContext mContext; + int32_t mAccountNumber; + bool mReadOnly; + +protected: + BackupStoreContext& GetContext() { return mContext; } + +public: + BackupProtocolLocal2(int32_t AccountNumber, + const std::string& ConnectionDetails, + const std::string& AccountRootDir, int DiscSetNumber, + bool ReadOnly) + // This is rather ugly: the BackupProtocolLocal constructor must not + // touch the Context, because it's not initialised yet! + : BackupProtocolLocal(mContext), + mContext(AccountNumber, (HousekeepingInterface *)NULL, + ConnectionDetails), + mAccountNumber(AccountNumber), + mReadOnly(ReadOnly) + { + mContext.SetClientHasAccount(AccountRootDir, DiscSetNumber); + QueryVersion(BACKUP_STORE_SERVER_VERSION); + QueryLogin(AccountNumber, + ReadOnly ? BackupProtocolLogin::Flags_ReadOnly : 0); + } + virtual ~BackupProtocolLocal2() { } + + std::auto_ptr<BackupProtocolFinished> Query(const BackupProtocolFinished &rQuery) + { + std::auto_ptr<BackupProtocolFinished> finished = + BackupProtocolLocal::Query(rQuery); + mContext.ReleaseWriteLock(); + return finished; + } + + void Reopen() + { + QueryVersion(BACKUP_STORE_SERVER_VERSION); + QueryLogin(mAccountNumber, + mReadOnly ? BackupProtocolLogin::Flags_ReadOnly : 0); + } +}; + +#endif // BACKUPPROTOCOL__H diff --git a/lib/backupstore/backupprotocol.txt b/lib/backupstore/BackupProtocol.txt index aa987e70..5921d009 100644 --- a/lib/backupstore/backupprotocol.txt +++ b/lib/backupstore/BackupProtocol.txt @@ -7,6 +7,7 @@ IdentString Box-Backup:v=C ServerContextClass BackupStoreContext BackupStoreContext.h AddType Filename BackupStoreFilenameClear BackupStoreFilenameClear.h +AddType String std::string ImplementLog Server syslog ImplementLog Client syslog @@ -76,8 +77,7 @@ SetClientStoreMarker 6 Command(Success) GetObject 10 Command(Success) int64 ObjectID - CONSTANT NoObject 0 - # reply has stream following, if ObjectID != NoObject + # reply has stream following (if successful) MoveObject 11 Command(Success) @@ -123,6 +123,14 @@ CreateDirectory 20 Command(Success) StreamWithCommand # stream following containing attributes +CreateDirectory2 46 Command(Success) StreamWithCommand + int64 ContainingDirectoryID + int64 AttributesModTime + int64 ModificationTime + Filename DirectoryName + # stream following containing attributes + + ListDirectory 21 Command(Success) int64 ObjectID int16 FlagsMustBeSet @@ -151,6 +159,7 @@ ChangeDirAttributes 22 Command(Success) StreamWithCommand DeleteDirectory 23 Command(Success) int64 ObjectID + UndeleteDirectory 24 Command(Success) int64 ObjectID # may not have exactly the desired effect if files within in have been deleted before the directory was deleted. @@ -233,3 +242,25 @@ GetIsAlive 42 Command(IsAlive) IsAlive 43 Reply # no data members +GetAccountUsage2 44 Command(AccountUsage2) + # no data members + +AccountUsage2 45 Reply + String AccountName + bool AccountEnabled + int64 ClientStoreMarker + int32 BlockSize + int64 LastObjectIDUsed + int64 BlocksUsed + int64 BlocksInCurrentFiles + int64 BlocksInOldFiles + int64 BlocksInDeletedFiles + int64 BlocksInDirectories + int64 BlocksSoftLimit + int64 BlocksHardLimit + int64 NumCurrentFiles + int64 NumOldFiles + int64 NumDeletedFiles + int64 NumDirectories + +# 46 is CreateDirectory2 diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp index 201491a3..c5f012fc 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.cpp +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -247,7 +247,8 @@ void BackupStoreAccountDatabase::Write() { // Write out the entry char line[256]; // more than enough for a couple of integers in string form - int s = ::sprintf(line, "%x:%d\n", i->second.GetID(), i->second.GetDiscSet()); + int s = ::snprintf(line, sizeof(line), "%x:%d\n", + i->second.GetID(), i->second.GetDiscSet()); if(::write(file, line, s) != s) { THROW_EXCEPTION(CommonException, OSFileError) diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp index 18500fc1..7955b3c4 100644 --- a/lib/backupstore/BackupStoreAccounts.cpp +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -9,19 +9,29 @@ #include "Box.h" -#include <stdio.h> +#include <algorithm> +#include <climits> +#include <cstdio> +#include <cstring> +#include <iostream> #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" +#include "BackupStoreCheck.h" +#include "BackupStoreConfigVerify.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BackupStoreInfo.h" #include "BackupStoreRefCountDatabase.h" #include "BoxPortsAndFiles.h" +#include "HousekeepStoreAccount.h" +#include "NamedLock.h" +#include "RaidFileController.h" #include "RaidFileWrite.h" #include "StoreStructure.h" #include "UnixUser.h" +#include "Utils.h" #include "MemLeakFindOn.h" @@ -110,10 +120,7 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, info->Save(); // Create the refcount database - BackupStoreRefCountDatabase::CreateNew(Entry); - std::auto_ptr<BackupStoreRefCountDatabase> refcount( - BackupStoreRefCountDatabase::Load(Entry, false)); - refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID); + BackupStoreRefCountDatabase::Create(Entry)->Commit(); } // As the original user... @@ -151,9 +158,9 @@ void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, i std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) { char accid[64]; // big enough! - ::sprintf(accid, "%08x" DIRECTORY_SEPARATOR, ID); - return std::string(std::string(BOX_RAIDFILE_ROOT_BBSTORED - DIRECTORY_SEPARATOR) + accid); + ::snprintf(accid, sizeof(accid) - 1, "%08x" DIRECTORY_SEPARATOR, ID); + return std::string(BOX_RAIDFILE_ROOT_BBSTORED DIRECTORY_SEPARATOR) + + accid; } @@ -198,6 +205,391 @@ void BackupStoreAccounts::LockAccount(int32_t ID, NamedLock& rNamedLock) { THROW_EXCEPTION_MESSAGE(BackupStoreException, CouldNotLockStoreAccount, "Failed to get exclusive " - "lock on account " << ID); + "lock on account " << BOX_FORMAT_ACCOUNT(ID)); } } + +int BackupStoreAccountsControl::BlockSizeOfDiscSet(int discSetNum) +{ + // Get controller, check disc set number + RaidFileController &controller(RaidFileController::GetController()); + if(discSetNum < 0 || discSetNum >= controller.GetNumDiscSets()) + { + BOX_FATAL("Disc set " << discSetNum << " does not exist."); + exit(1); + } + + // Return block size + return controller.GetDiscSet(discSetNum).GetBlockSize(); +} + +int BackupStoreAccountsControl::SetLimit(int32_t ID, const char *SoftLimitStr, + const char *HardLimitStr) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change limits."); + return 1; + } + + // Load the info + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, + discSetNum, false /* Read/Write */)); + + // Change the limits + int blocksize = BlockSizeOfDiscSet(discSetNum); + int64_t softlimit = SizeStringToBlocks(SoftLimitStr, blocksize); + int64_t hardlimit = SizeStringToBlocks(HardLimitStr, blocksize); + CheckSoftHardLimits(softlimit, hardlimit); + info->ChangeLimits(softlimit, hardlimit); + + // Save + info->Save(); + + BOX_NOTICE("Limits on account " << BOX_FORMAT_ACCOUNT(ID) << + " changed to " << softlimit << " soft, " << + hardlimit << " hard."); + + return 0; +} + +int BackupStoreAccountsControl::SetAccountName(int32_t ID, const std::string& rNewAccountName) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change name."); + return 1; + } + + // Load the info + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, false /* Read/Write */)); + + info->SetAccountName(rNewAccountName); + + // Save + info->Save(); + + BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << + " name changed to " << rNewAccountName); + + return 0; +} + +int BackupStoreAccountsControl::PrintAccountInfo(int32_t ID) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + + if(!OpenAccount(ID, rootDir, discSetNum, user, + NULL /* no write lock needed for this read-only operation */)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to display info."); + return 1; + } + + // Load it in + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, true /* ReadOnly */)); + + return BackupAccountControl::PrintAccountInfo(*info, + BlockSizeOfDiscSet(discSetNum)); +} + +int BackupStoreAccountsControl::SetAccountEnabled(int32_t ID, bool enabled) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change enabled flag."); + return 1; + } + + // Load it in + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, false /* ReadOnly */)); + info->SetAccountEnabled(enabled); + info->Save(); + return 0; +} + +int BackupStoreAccountsControl::DeleteAccount(int32_t ID, bool AskForConfirmation) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + // Obtain a write lock, as the daemon user + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for deletion."); + return 1; + } + + // Check user really wants to do this + if(AskForConfirmation) + { + BOX_WARNING("Really delete account " << + BOX_FORMAT_ACCOUNT(ID) << "? (type 'yes' to confirm)"); + char response[256]; + if(::fgets(response, sizeof(response), stdin) == 0 || ::strcmp(response, "yes\n") != 0) + { + BOX_NOTICE("Deletion cancelled."); + return 0; + } + } + + // Back to original user, but write lock is maintained + user.reset(); + + std::auto_ptr<BackupStoreAccountDatabase> db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Delete from account database + db->DeleteEntry(ID); + + // Write back to disc + db->Write(); + + // Remove the store files... + + // First, become the user specified in the config file + std::string username; + { + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Become the right user + if(!username.empty()) + { + // Username specified, change... + user.reset(new UnixUser(username)); + user->ChangeProcessUser(true /* temporary */); + // Change will be undone when user goes out of scope + } + + // Secondly, work out which directories need wiping + std::vector<std::string> toDelete; + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet discSet(rcontroller.GetDiscSet(discSetNum)); + for(RaidFileDiscSet::const_iterator i(discSet.begin()); i != discSet.end(); ++i) + { + if(std::find(toDelete.begin(), toDelete.end(), *i) == toDelete.end()) + { + toDelete.push_back((*i) + DIRECTORY_SEPARATOR + rootDir); + } + } + + // NamedLock will throw an exception if it can't delete the lockfile, + // which it can't if it doesn't exist. Now that we've deleted the account, + // nobody can open it anyway, so it's safe to unlock. + writeLock.ReleaseLock(); + + int retcode = 0; + + // Thirdly, delete the directories... + for(std::vector<std::string>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) + { + BOX_NOTICE("Deleting store directory " << (*d) << "..."); + // Just use the rm command to delete the files +#ifdef WIN32 + std::string cmd("rmdir /s/q "); + std::string dir = *d; + + // rmdir doesn't understand forward slashes, so replace them all. + for(std::string::iterator i = dir.begin(); i != dir.end(); i++) + { + if(*i == '/') + { + *i = '\\'; + } + } + cmd += dir; +#else + std::string cmd("rm -rf "); + cmd += *d; +#endif + // Run command + if(::system(cmd.c_str()) != 0) + { + BOX_ERROR("Failed to delete files in " << (*d) << + ", delete them manually."); + retcode = 1; + } + } + + // Success! + return retcode; +} + +bool BackupStoreAccountsControl::OpenAccount(int32_t ID, std::string &rRootDirOut, + int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock) +{ + // Load in the account database + std::auto_ptr<BackupStoreAccountDatabase> db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Exists? + if(!db->EntryExists(ID)) + { + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " does not exist."); + return false; + } + + // Get info from the database + BackupStoreAccounts acc(*db); + acc.GetAccountRoot(ID, rRootDirOut, rDiscSetOut); + + // Get the user under which the daemon runs + std::string username; + { + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Become the right user + if(!username.empty()) + { + // Username specified, change... + apUser.reset(new UnixUser(username)); + apUser->ChangeProcessUser(true /* temporary */); + // Change will be undone when apUser goes out of scope + // in the caller. + } + + if(pLock) + { + acc.LockAccount(ID, *pLock); + } + + return true; +} + +int BackupStoreAccountsControl::CheckAccount(int32_t ID, bool FixErrors, bool Quiet, + bool ReturnNumErrorsFound) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, + FixErrors ? &writeLock : NULL)) // don't need a write lock if not making changes + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for checking."); + return 1; + } + + // Check it + BackupStoreCheck check(rootDir, discSetNum, ID, FixErrors, Quiet); + check.Check(); + + if(ReturnNumErrorsFound) + { + return check.GetNumErrorsFound(); + } + else + { + return check.ErrorsFound() ? 1 : 0; + } +} + +int BackupStoreAccountsControl::CreateAccount(int32_t ID, int32_t DiscNumber, + int32_t SoftLimit, int32_t HardLimit) +{ + // Load in the account database + std::auto_ptr<BackupStoreAccountDatabase> db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Already exists? + if(db->EntryExists(ID)) + { + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " already exists."); + return 1; + } + + // Get the user under which the daemon runs + std::string username; + { + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Create it. + BackupStoreAccounts acc(*db); + acc.Create(ID, DiscNumber, SoftLimit, HardLimit, username); + + BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << " created."); + + return 0; +} + +int BackupStoreAccountsControl::HousekeepAccountNow(int32_t ID) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + + if(!OpenAccount(ID, rootDir, discSetNum, user, + NULL /* housekeeping locks the account itself */)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for housekeeping."); + return 1; + } + + HousekeepStoreAccount housekeeping(ID, rootDir, discSetNum, NULL); + bool success = housekeeping.DoHousekeeping(); + + if(!success) + { + BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(ID) + << " for housekeeping: perhaps a client is " + "still connected?"); + return 1; + } + else + { + BOX_TRACE("Finished housekeeping on account " << + BOX_FORMAT_ACCOUNT(ID)); + return 0; + } +} + diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h index 3163f15c..bcc3cf1c 100644 --- a/lib/backupstore/BackupStoreAccounts.h +++ b/lib/backupstore/BackupStoreAccounts.h @@ -13,6 +13,7 @@ #include <string> #include "BackupStoreAccountDatabase.h" +#include "BackupAccountControl.h" #include "NamedLock.h" // -------------------------------------------------------------------------- @@ -51,5 +52,34 @@ private: BackupStoreAccountDatabase &mrDatabase; }; +class Configuration; +class UnixUser; + +class BackupStoreAccountsControl : public BackupAccountControl +{ +public: + BackupStoreAccountsControl(const Configuration& config, + bool machineReadableOutput = false) + : BackupAccountControl(config, machineReadableOutput) + { } + int BlockSizeOfDiscSet(int discSetNum); + bool OpenAccount(int32_t ID, std::string &rRootDirOut, + int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock); + int SetLimit(int32_t ID, const char *SoftLimitStr, + const char *HardLimitStr); + int SetAccountName(int32_t ID, const std::string& rNewAccountName); + int PrintAccountInfo(int32_t ID); + int SetAccountEnabled(int32_t ID, bool enabled); + int DeleteAccount(int32_t ID, bool AskForConfirmation); + int CheckAccount(int32_t ID, bool FixErrors, bool Quiet, + bool ReturnNumErrorsFound = false); + int CreateAccount(int32_t ID, int32_t DiscNumber, int32_t SoftLimit, + int32_t HardLimit); + int HousekeepAccountNow(int32_t ID); +}; + +// max size of soft limit as percent of hard limit +#define MAX_SOFT_LIMIT_SIZE 97 + #endif // BACKUPSTOREACCOUNTS__H diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index f2302337..b53ebf6d 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -17,11 +17,13 @@ #endif #include "autogen_BackupStoreException.h" +#include "BackupStoreAccountDatabase.h" #include "BackupStoreCheck.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreFile.h" #include "BackupStoreObjectMagic.h" +#include "BackupStoreRefCountDatabase.h" #include "RaidFileController.h" #include "RaidFileException.h" #include "RaidFileRead.h" @@ -58,7 +60,7 @@ BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNum mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), mBlocksInDirectories(0), - mNumFiles(0), + mNumCurrentFiles(0), mNumOldFiles(0), mNumDeletedFiles(0), mNumDirectories(0) @@ -78,6 +80,24 @@ BackupStoreCheck::~BackupStoreCheck() { // Clean up FreeInfo(); + + // Avoid an exception if we forget to discard mapNewRefs + if (mapNewRefs.get()) + { + // Discard() can throw exception, but destructors aren't supposed to do that, so + // just catch and log them. + try + { + mapNewRefs->Discard(); + } + catch(BoxException &e) + { + BOX_ERROR("Error while destroying BackupStoreCheck: discarding " + "the refcount database threw an exception: " << e.what()); + } + + mapNewRefs.reset(); + } } @@ -92,15 +112,21 @@ BackupStoreCheck::~BackupStoreCheck() // -------------------------------------------------------------------------- void BackupStoreCheck::Check() { - std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); - ASSERT(FileExists(writeLockFilename)); + if(mFixErrors) + { + std::string writeLockFilename; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); + ASSERT(FileExists(writeLockFilename)); + } if(!mQuiet && mFixErrors) { - BOX_NOTICE("Will fix errors encountered during checking."); + BOX_INFO("Will fix errors encountered during checking."); } + BackupStoreAccountDatabase::Entry account(mAccountID, mDiscSetNumber); + mapNewRefs = BackupStoreRefCountDatabase::Create(account); + // Phase 1, check objects if(!mQuiet) { @@ -109,14 +135,14 @@ void BackupStoreCheck::Check() BOX_INFO("Phase 1, check objects..."); } CheckObjects(); - + // Phase 2, check directories if(!mQuiet) { BOX_INFO("Phase 2, check directories..."); } CheckDirectories(); - + // Phase 3, check root if(!mQuiet) { @@ -138,16 +164,38 @@ void BackupStoreCheck::Check() } FixDirsWithWrongContainerID(); FixDirsWithLostDirs(); - + // Phase 6, regenerate store info if(!mQuiet) { BOX_INFO("Phase 6, regenerate store info..."); } WriteNewStoreInfo(); - -// DUMP_OBJECT_INFO - + + try + { + std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs = + BackupStoreRefCountDatabase::Load(account, false); + mNumberErrorsFound += mapNewRefs->ReportChangesTo(*apOldRefs); + } + catch(BoxException &e) + { + BOX_WARNING("Reference count database was missing or " + "corrupted, cannot check it for errors."); + mNumberErrorsFound++; + } + + // force file to be saved and closed before releasing the lock below + if(mFixErrors) + { + mapNewRefs->Commit(); + } + else + { + mapNewRefs->Discard(); + } + mapNewRefs.reset(); + if(mNumberErrorsFound > 0) { BOX_WARNING("Finished checking store account ID " << @@ -250,16 +298,18 @@ void BackupStoreCheck::CheckObjects() { // Make sure the starting root dir doesn't end with '/'. std::string start(mStoreRoot); - if(start.size() > 0 && start[start.size() - 1] == '/') + if(start.size() > 0 && ( + start[start.size() - 1] == '/' || + start[start.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR)) { start.resize(start.size() - 1); } - - maxDir = CheckObjectsScanDir(0, 1, mStoreRoot); + + maxDir = CheckObjectsScanDir(0, 1, start); BOX_TRACE("Max dir starting ID is " << BOX_FORMAT_OBJECTID(maxDir)); } - + // Then go through and scan all the objects within those directories for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH)) { @@ -287,24 +337,25 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const // If any of the directories is missing, create it. RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); - + if(!rdiscSet.IsNonRaidSet()) { unsigned int numDiscs = rdiscSet.size(); - + for(unsigned int l = 0; l < numDiscs; ++l) { // build name std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName); - struct stat st; + EMU_STRUCT_STAT st; - if(stat(dn.c_str(), &st) != 0 && errno == ENOENT) + if(EMU_STAT(dn.c_str(), &st) != 0 && + errno == ENOENT) { if(mkdir(dn.c_str(), 0755) != 0) { THROW_SYS_FILE_ERROR("Failed to " "create missing RaidFile " - "directory", dn, + "directory", dn, RaidFileException, OSError); } } @@ -334,7 +385,7 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const else { BOX_ERROR("Spurious or invalid directory " << - rDirName << DIRECTORY_SEPARATOR << + rDirName << DIRECTORY_SEPARATOR << (*i) << " found, " << (mFixErrors?"deleting":"delete manually")); ++mNumberErrorsFound; @@ -361,11 +412,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) std::string dirName; StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */); // Check expectations - ASSERT(dirName.size() > 4 && + ASSERT(dirName.size() > 4 && dirName[dirName.size() - 4] == DIRECTORY_SEPARATOR_ASCHAR); // Remove the filename from it dirName.resize(dirName.size() - 4); // four chars for "/o00" - + // Check directory exists if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName)) { @@ -377,14 +428,14 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) std::vector<std::string> files; RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName, RaidFileRead::DirReadType_FilesOnly, files); - + // Array of things present bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)]; for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l) { idsPresent[l] = false; } - + // Parse each entry, building up a list of object IDs which are present in the dir. // This is done so that whatever order is retured from the directory, objects are scanned // in order. @@ -405,7 +456,8 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) fileOK = false; } // info and refcount databases are OK in the root directory - else if(*i == "info" || *i == "refcount.db") + else if(*i == "info" || *i == "refcount.db" || + *i == "refcount.rdb" || *i == "refcount.rdbX") { fileOK = true; } @@ -413,11 +465,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) { fileOK = false; } - + if(!fileOK) { // Unexpected or bad file, delete it - BOX_ERROR("Spurious file " << dirName << + BOX_ERROR("Spurious file " << dirName << DIRECTORY_SEPARATOR << (*i) << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; @@ -428,7 +480,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) } } } - + // Check all the objects found in this directory for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i) { @@ -436,7 +488,8 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) { // Check the object is OK, and add entry char leaf[8]; - ::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i); + ::snprintf(leaf, sizeof(leaf), + DIRECTORY_SEPARATOR "o%02x", i); if(!CheckAndAddObject(StartID | i, dirName + leaf)) { // File was bad, delete it @@ -480,7 +533,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, std::auto_ptr<RaidFileRead> file( RaidFileRead::Open(mDiscSetNumber, rFilename)); size = file->GetDiscUsageInBlocks(); - + // Read in first four bytes -- don't have to worry about // retrying if not all bytes read as is RaidFile uint32_t signature; @@ -491,7 +544,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, } // Seek back to beginning file->Seek(0, IOStream::SeekType_Absolute); - + // Then... check depending on the type switch(ntohl(signature)) { @@ -519,7 +572,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, // Error caught, not a good file then, let it be deleted return false; } - + // Got a container ID? (ie check was successful) if(containerID == -1) { @@ -538,13 +591,13 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, // If it looks like a good object, and it's non-RAID, and // this is a RAID set, then convert it to RAID. - + RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); if(!rdiscSet.IsNonRaidSet()) { // See if the file exists - RaidFileUtil::ExistType existance = + RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, rFilename); if(existance == RaidFileUtil::NonRaid) { @@ -565,7 +618,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, if(mFixErrors) { std::auto_ptr<RaidFileRead> read( - RaidFileRead::Open(mDiscSetNumber, + RaidFileRead::Open(mDiscSetNumber, rFilename)); RaidFileWrite write(mDiscSetNumber, rFilename); write.Open(true /* overwrite */); @@ -575,7 +628,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, } } } - + // Report success return true; } @@ -636,7 +689,7 @@ int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream) // Wrong object ID return -1; } - + // Return container ID return dir.GetContainerID(); } @@ -669,7 +722,7 @@ void BackupStoreCheck::CheckDirectories() { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; - + for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); @@ -703,9 +756,9 @@ void BackupStoreCheck::CheckDirectories() BOX_FORMAT_OBJECTID(pblock->mID[e]) << " was OK after fixing"); } - + if(isModified && mFixErrors) - { + { BOX_WARNING("Writing modified directory to disk: " << BOX_FORMAT_OBJECTID(pblock->mID[e])); RaidFileWrite fixed(mDiscSetNumber, filename); @@ -714,64 +767,7 @@ void BackupStoreCheck::CheckDirectories() fixed.Commit(true /* convert to raid representation now */); } - // Count valid entries - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next()) != 0) - { - int32_t iIndex; - IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); - - ASSERT(piBlock != 0 || - mDirsWhichContainLostDirs.find(en->GetObjectID()) - != mDirsWhichContainLostDirs.end()); - if (piBlock) - { - // Normally it would exist and this - // check would not be necessary, but - // we might have missing directories - // that we will recreate later. - // cf mDirsWhichContainLostDirs. - uint8_t iflags = GetFlags(piBlock, iIndex); - SetFlags(piBlock, iIndex, iflags | Flags_IsContained); - } - - if(en->IsDir()) - { - mNumDirectories++; - } - else if(!en->IsFile()) - { - BOX_TRACE("Not counting object " << - BOX_FORMAT_OBJECTID(en->GetObjectID()) << - " with flags " << en->GetFlags()); - } - else // it's a good file, add to sizes - if(en->IsOld() && en->IsDeleted()) - { - BOX_WARNING("File " << - BOX_FORMAT_OBJECTID(en->GetObjectID()) << - " is both old and deleted, " - "this should not happen!"); - } - else if(en->IsOld()) - { - mNumFiles++; - mNumOldFiles++; - mBlocksInOldFiles += en->GetSizeInBlocks(); - } - else if(en->IsDeleted()) - { - mNumFiles++; - mNumDeletedFiles++; - mBlocksInDeletedFiles += en->GetSizeInBlocks(); - } - else - { - mNumFiles++; - mBlocksInCurrentFiles += en->GetSizeInBlocks(); - } - } + CountDirectoryEntries(dir); } } } @@ -798,8 +794,8 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) ++mNumberErrorsFound; isModified = true; } - - // Go through, and check that everything in that directory exists and is valid + + // Go through, and check that every entry exists and is valid BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) @@ -824,14 +820,14 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) { // Just remove the entry badEntry = true; - BOX_ERROR("Directory ID " << + BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(dir.GetObjectID()) << - " references object " << + " references object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " which does not exist."); ++mNumberErrorsFound; } - + // Is this entry worth keeping? if(badEntry) { @@ -846,16 +842,16 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) { BOX_ERROR("Removing directory entry " << BOX_FORMAT_OBJECTID(*d) << " from " - "directory " << + "directory " << BOX_FORMAT_OBJECTID(dir.GetObjectID())); ++mNumberErrorsFound; dir.DeleteEntry(*d); } - + // Mark as modified restart = true; isModified = true; - + // Errors found } } @@ -863,6 +859,84 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) return isModified; } +// Count valid remaining entries and the number of blocks in them. +void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir) +{ + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + int32_t iIndex; + IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); + bool badEntry = false; + bool wasAlreadyContained = false; + + ASSERT(piBlock != 0 || + mDirsWhichContainLostDirs.find(en->GetObjectID()) + != mDirsWhichContainLostDirs.end()); + + if (piBlock) + { + // Normally it would exist and this + // check would not be necessary, but + // we might have missing directories + // that we will recreate later. + // cf mDirsWhichContainLostDirs. + uint8_t iflags = GetFlags(piBlock, iIndex); + wasAlreadyContained = (iflags & Flags_IsContained); + SetFlags(piBlock, iIndex, iflags | Flags_IsContained); + } + + if(wasAlreadyContained) + { + // don't double-count objects that are + // contained by another directory as well. + } + else if(en->IsDir()) + { + mNumDirectories++; + } + else if(!en->IsFile()) + { + BOX_TRACE("Not counting object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " with flags " << en->GetFlags()); + } + else // it's a file + { + // Add to sizes? + // If piBlock was zero, then wasAlreadyContained + // might be uninitialized; but we only process + // files here, and if a file's piBlock was zero + // then badEntry would be set above, so we + // wouldn't be here. + ASSERT(!badEntry) + + // It can be both old and deleted. + // If neither, then it's current. + if(en->IsDeleted()) + { + mNumDeletedFiles++; + mBlocksInDeletedFiles += en->GetSizeInBlocks(); + } + + if(en->IsOld()) + { + mNumOldFiles++; + mBlocksInOldFiles += en->GetSizeInBlocks(); + } + + if(!en->IsDeleted() && !en->IsOld()) + { + mNumCurrentFiles++; + mBlocksInCurrentFiles += en->GetSizeInBlocks(); + } + } + + mapNewRefs->AddReference(en->GetObjectID()); + } +} + bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified) { @@ -871,8 +945,7 @@ bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, ASSERT(piBlock != 0); uint8_t iflags = GetFlags(piBlock, IndexInDirBlock); - bool badEntry = false; - + // Is the type the same? if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir()) { @@ -882,73 +955,67 @@ bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, " references object " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " which has a different type than expected."); - badEntry = true; ++mNumberErrorsFound; + return false; // remove this entry } + // Check that the entry is not already contained. - else if(iflags & Flags_IsContained) + if(iflags & Flags_IsContained) { BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(DirectoryID) << " references object " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " which is already contained."); - badEntry = true; ++mNumberErrorsFound; + return false; // remove this entry } - else + + // Not already contained by another directory. + // Don't set the flag until later, after we finish repairing + // the directory and removing all bad entries. + + // Check that the container ID of the object is correct + if(piBlock->mContainer[IndexInDirBlock] != DirectoryID) { - // Not already contained by another directory. - // Don't set the flag until later, after we finish repairing - // the directory and removing all bad entries. - - // Check that the container ID of the object is correct - if(piBlock->mContainer[IndexInDirBlock] != DirectoryID) + // Needs fixing... + if(iflags & Flags_IsDir) { - // Needs fixing... - if(iflags & Flags_IsDir) - { - // Add to will fix later list - BOX_ERROR("Directory ID " << - BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) - << " has wrong container ID."); - mDirsWithWrongContainerID.push_back(rEntry.GetObjectID()); - ++mNumberErrorsFound; - } - else - { - // This is OK for files, they might move - BOX_NOTICE("File ID " << - BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) - << " has different container ID, " - "probably moved"); - } - - // Fix entry for now - piBlock->mContainer[IndexInDirBlock] = DirectoryID; + // Add to will fix later list + BOX_ERROR("Directory ID " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) + << " has wrong container ID."); + mDirsWithWrongContainerID.push_back(rEntry.GetObjectID()); + ++mNumberErrorsFound; + } + else + { + // This is OK for files, they might move + BOX_INFO("File ID " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) + << " has different container ID, " + "probably moved"); } + + // Fix entry for now + piBlock->mContainer[IndexInDirBlock] = DirectoryID; } - - // Check the object size, if it's OK and a file - if(!badEntry && !rEntry.IsDir()) + + // Check the object size + if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock]) { - if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock]) - { - // Wrong size, correct it. - rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]); + // Wrong size, correct it. + BOX_ERROR("Directory " << BOX_FORMAT_OBJECTID(DirectoryID) << + " entry for " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << + " has wrong size " << rEntry.GetSizeInBlocks() << + ", should be " << piBlock->mObjectSizeInBlocks[IndexInDirBlock]); - // Mark as changed - rIsModified = true; - ++mNumberErrorsFound; + rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]); - // Tell user - BOX_ERROR("Directory ID " << - BOX_FORMAT_OBJECTID(DirectoryID) << - " has wrong size for object " << - BOX_FORMAT_OBJECTID(rEntry.GetObjectID())); - } + // Mark as changed + rIsModified = true; + ++mNumberErrorsFound; } - return !badEntry; + return true; // don't delete this entry } - diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h index 178a873a..5353c968 100644 --- a/lib/backupstore/BackupStoreCheck.h +++ b/lib/backupstore/BackupStoreCheck.h @@ -20,6 +20,7 @@ class IOStream; class BackupStoreFilename; +class BackupStoreRefCountDatabase; /* @@ -130,8 +131,9 @@ private: bool CheckDirectory(BackupStoreDirectory& dir); bool CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified); + void CountDirectoryEntries(BackupStoreDirectory& dir); int64_t CheckFile(int64_t ObjectID, IOStream &rStream); - int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); + int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); // Fixing functions bool TryToRecreateDirectory(int64_t MissingDirectoryID); @@ -195,6 +197,9 @@ private: // Set of extra directories added std::set<BackupStoreCheck_ID_t> mDirsAdded; + + // The refcount database, being reconstructed as the check/fix progresses + std::auto_ptr<BackupStoreRefCountDatabase> mapNewRefs; // Misc stuff int32_t mLostDirNameSerial; @@ -206,7 +211,7 @@ private: int64_t mBlocksInOldFiles; int64_t mBlocksInDeletedFiles; int64_t mBlocksInDirectories; - int64_t mNumFiles; + int64_t mNumCurrentFiles; int64_t mNumOldFiles; int64_t mNumDeletedFiles; int64_t mNumDirectories; diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp index 90e21e7f..13831a09 100644 --- a/lib/backupstore/BackupStoreCheck2.cpp +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -20,6 +20,7 @@ #include "BackupStoreFileWire.h" #include "BackupStoreInfo.h" #include "BackupStoreObjectMagic.h" +#include "BackupStoreRefCountDatabase.h" #include "MemBlockStream.h" #include "RaidFileRead.h" #include "RaidFileWrite.h" @@ -40,7 +41,7 @@ void BackupStoreCheck::CheckRoot() { int32_t index = 0; IDBlock *pblock = LookupID(BACKUPSTORE_ROOT_DIRECTORY_ID, index); - + if(pblock != 0) { // Found it. Which is lucky. Mark it as contained. @@ -49,9 +50,9 @@ void BackupStoreCheck::CheckRoot() else { BOX_WARNING("Root directory doesn't exist"); - + ++mNumberErrorsFound; - + if(mFixErrors) { // Create a new root directory @@ -78,7 +79,7 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain } BackupStoreDirectory dir(DirectoryID, ContainingDirID); - + // Serialise to disc std::string filename; StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); @@ -87,10 +88,10 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain dir.WriteToStream(obj); int64_t size = obj.GetDiscUsageInBlocks(); obj.Commit(true /* convert to raid now */); - + // Record the fact we've done this mDirsAdded.insert(DirectoryID); - + // Add to sizes mBlocksUsed += size; mBlocksInDirectories += size; @@ -131,15 +132,16 @@ void BackupStoreCheck::CheckUnattachedObjects() { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; - + for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); if((flags & Flags_IsContained) == 0) { // Unattached object... + int64_t ObjectID = pblock->mID[e]; BOX_ERROR("Object " << - BOX_FORMAT_OBJECTID(pblock->mID[e]) << + BOX_FORMAT_OBJECTID(ObjectID) << " is unattached."); ++mNumberErrorsFound; @@ -149,6 +151,8 @@ void BackupStoreCheck::CheckUnattachedObjects() if((flags & Flags_IsDir) == Flags_IsDir) { // Directory. Just put into lost and found. + // (It doesn't contain its filename, so we + // can't recreate the entry in the parent) putIntoDirectoryID = GetLostAndFoundDirID(); } else @@ -157,7 +161,9 @@ void BackupStoreCheck::CheckUnattachedObjects() { int64_t diffFromObjectID = 0; std::string filename; - StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */); + StoreStructure::MakeObjectFilename(ObjectID, + mStoreRoot, mDiscSetNumber, filename, + false /* don't attempt to make sure the dir exists */); // The easiest way to do this is to verify it again. Not such a bad penalty, because // this really shouldn't be done very often. @@ -170,20 +176,22 @@ void BackupStoreCheck::CheckUnattachedObjects() // Just delete it to be safe. if(diffFromObjectID != 0) { - BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached, and is a patch. Deleting, cannot reliably recover."); - + BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(ObjectID) << " is unattached, and is a patch. Deleting, cannot reliably recover."); + // Delete this object instead if(mFixErrors) { RaidFileWrite del(mDiscSetNumber, filename); del.Delete(); } - + + mBlocksUsed -= pblock->mObjectSizeInBlocks[e]; + // Move on to next item continue; } } - + // Files contain their original filename, so perhaps the orginal directory still exists, // or we can infer the existance of a directory? // Look for a matching entry in the mDirsWhichContainLostDirs map. @@ -249,9 +257,10 @@ void BackupStoreCheck::CheckUnattachedObjects() } // Add it to the directory - pFixer->InsertObject(pblock->mID[e], + pFixer->InsertObject(ObjectID, ((flags & Flags_IsDir) == Flags_IsDir), lostDirNameSerial); + mapNewRefs->AddReference(ObjectID); } } } @@ -284,7 +293,7 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) // Not a missing directory, can't recreate. return false; } - + // Can recreate this! Wooo! if(!mFixErrors) { @@ -297,12 +306,12 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) BOX_WARNING("Recreating missing directory " << BOX_FORMAT_OBJECTID(MissingDirectoryID)); - + // Create a blank directory BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */); // Note that this directory already contains a directory entry pointing to // this dir, so it doesn't have to be added. - + // Serialise to disc std::string filename; StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); @@ -310,10 +319,10 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) root.Open(false /* don't allow overwriting */); dir.WriteToStream(root); root.Commit(true /* convert to raid now */); - + // Record the fact we've done this mDirsAdded.insert(MissingDirectoryID); - + // Remove the entry from the map, so this doesn't happen again mDirsWhichContainLostDirs.erase(missing); @@ -328,7 +337,7 @@ BackupStoreDirectoryFixer::BackupStoreDirectoryFixer(std::string storeRoot, // Generate filename StoreStructure::MakeObjectFilename(ID, mStoreRoot, mDiscSetNumber, mFilename, false /* don't make sure the dir exists */); - + // Read it in std::auto_ptr<RaidFileRead> file( RaidFileRead::Open(mDiscSetNumber, mFilename)); @@ -347,7 +356,7 @@ void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, { // Directory -- simply generate a name for it. char name[32]; - ::sprintf(name, "dir%08x", lostDirNameSerial); + ::snprintf(name, sizeof(name), "dir%08x", lostDirNameSerial); objectStoreFilename.SetAsClearFilename(name); } else @@ -370,7 +379,7 @@ void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif +#endif )) { // This should never happen, everything has been @@ -393,7 +402,7 @@ BackupStoreDirectoryFixer::~BackupStoreDirectoryFixer() { // Fix any flags which have been broken, which there's a good chance of doing mDirectory.CheckAndFix(); - + // Write it out RaidFileWrite root(mDiscSetNumber, mFilename); root.Open(true /* allow overwriting */); @@ -438,7 +447,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() while(true) { char name[32]; - ::sprintf(name, "lost+found%d", n++); + ::snprintf(name, sizeof(name), "lost+found%d", n++); lostAndFound.SetAsClearFilename(name); if(!dir.NameInUse(lostAndFound)) { @@ -453,7 +462,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() // Create a blank directory CreateBlankDirectory(id, BACKUPSTORE_ROOT_DIRECTORY_ID); - + // Add an entry for it dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0); @@ -462,7 +471,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() root.Open(true /* allow overwriting */); dir.WriteToStream(root); root.Commit(true /* convert to raid now */); - + // Store mLostAndFoundDirectoryID = id; @@ -494,7 +503,7 @@ void BackupStoreCheck::FixDirsWithWrongContainerID() int32_t index = 0; IDBlock *pblock = LookupID(*i, index); if(pblock == 0) continue; - + // Load in BackupStoreDirectory dir; std::string filename; @@ -506,7 +515,7 @@ void BackupStoreCheck::FixDirsWithWrongContainerID() // Adjust container ID dir.SetContainerID(pblock->mContainer[index]); - + // Write it out RaidFileWrite root(mDiscSetNumber, filename); root.Open(true /* allow overwriting */); @@ -539,7 +548,7 @@ void BackupStoreCheck::FixDirsWithLostDirs() int32_t index = 0; IDBlock *pblock = LookupID(i->second, index); if(pblock == 0) continue; - + // Load in BackupStoreDirectory dir; std::string filename; @@ -551,10 +560,10 @@ void BackupStoreCheck::FixDirsWithLostDirs() // Delete the dodgy entry dir.DeleteEntry(i->first); - + // Fix it up dir.CheckAndFix(); - + // Write it out RaidFileWrite root(mDiscSetNumber, filename); root.Open(true /* allow overwriting */); @@ -587,49 +596,42 @@ void BackupStoreCheck::WriteNewStoreInfo() ++mNumberErrorsFound; } - BOX_NOTICE("Total files: " << mNumFiles << " (of which " + BOX_INFO("Current files: " << mNumCurrentFiles << ", " "old files: " << mNumOldFiles << ", " - "deleted files: " << mNumDeletedFiles << "), " + "deleted files: " << mNumDeletedFiles << ", " "directories: " << mNumDirectories); - // Minimum soft and hard limits + // Minimum soft and hard limits to ensure that nothing gets deleted + // by housekeeping. int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024; int64_t minHard = ((minSoft * 11) / 10) + 1024; - // Need to do anything? - if(pOldInfo.get() != 0 && - mNumberErrorsFound == 0 && - pOldInfo->GetAccountID() == mAccountID) - { - // Leave the store info as it is, no need to alter it because nothing really changed, - // and the only essential thing was that the account ID was correct, which is was. - return; - } - - // NOTE: We will always build a new store info, so the client store marker gets changed. + int64_t softLimit = pOldInfo.get() ? pOldInfo->GetBlocksSoftLimit() : minSoft; + int64_t hardLimit = pOldInfo.get() ? pOldInfo->GetBlocksHardLimit() : minHard; - // Work out the new limits - int64_t softLimit = minSoft; - int64_t hardLimit = minHard; - if(pOldInfo.get() != 0 && pOldInfo->GetBlocksSoftLimit() > minSoft) + if(mNumberErrorsFound && pOldInfo.get()) { - softLimit = pOldInfo->GetBlocksSoftLimit(); - } - else - { - BOX_WARNING("Soft limit for account changed to ensure " - "housekeeping doesn't delete files on next run."); - } - if(pOldInfo.get() != 0 && pOldInfo->GetBlocksHardLimit() > minHard) - { - hardLimit = pOldInfo->GetBlocksHardLimit(); - } - else - { - BOX_WARNING("Hard limit for account changed to ensure " - "housekeeping doesn't delete files on next run."); + if(pOldInfo->GetBlocksSoftLimit() > minSoft) + { + softLimit = pOldInfo->GetBlocksSoftLimit(); + } + else + { + BOX_WARNING("Soft limit for account changed to ensure " + "housekeeping doesn't delete files on next run."); + } + + if(pOldInfo->GetBlocksHardLimit() > minHard) + { + hardLimit = pOldInfo->GetBlocksHardLimit(); + } + else + { + BOX_WARNING("Hard limit for account changed to ensure " + "housekeeping doesn't delete files on next run."); + } } - + // Object ID int64_t lastObjID = mLastIDInInfo; if(mLostAndFoundDirectoryID != 0) @@ -662,11 +664,24 @@ void BackupStoreCheck::WriteNewStoreInfo() hardLimit, (pOldInfo.get() ? pOldInfo->IsAccountEnabled() : true), *extra_data)); - info->AdjustNumFiles(mNumFiles); + info->AdjustNumCurrentFiles(mNumCurrentFiles); info->AdjustNumOldFiles(mNumOldFiles); info->AdjustNumDeletedFiles(mNumDeletedFiles); info->AdjustNumDirectories(mNumDirectories); + // If there are any errors (apart from wrong block counts), then we + // should reset the ClientStoreMarker to zero, which + // CreateForRegeneration does. But if there are no major errors, then + // we should maintain the old ClientStoreMarker, to avoid invalidating + // the client's directory cache. + if (pOldInfo.get() && !mNumberErrorsFound) + { + BOX_INFO("No major errors found, preserving old " + "ClientStoreMarker: " << + pOldInfo->GetClientStoreMarker()); + info->SetClientStoreMarker(pOldInfo->GetClientStoreMarker()); + } + if(pOldInfo.get()) { mNumberErrorsFound += info->ReportChangesTo(*pOldInfo); @@ -676,7 +691,7 @@ void BackupStoreCheck::WriteNewStoreInfo() if(mFixErrors) { info->Save(); - BOX_NOTICE("New store info file written successfully."); + BOX_INFO("New store info file written successfully."); } } @@ -695,7 +710,7 @@ void BackupStoreCheck::WriteNewStoreInfo() bool BackupStoreDirectory::CheckAndFix() { bool changed = false; - + // Check that if a file depends on a new version, that version is in this directory bool restart; @@ -718,11 +733,11 @@ bool BackupStoreDirectory::CheckAndFix() "on newer version " << FMT_OID(dependsNewer) << " which doesn't exist"); - + // Remove delete *i; mEntries.erase(i); - + // Mark as changed changed = true; @@ -751,7 +766,7 @@ bool BackupStoreDirectory::CheckAndFix() } } while(restart); - + // Check that if a file has a dependency marked, it exists, and remove it if it doesn't { std::vector<Entry*>::iterator i(mEntries.begin()); @@ -768,7 +783,7 @@ bool BackupStoreDirectory::CheckAndFix() "info cleared"); (*i)->SetDependsOlder(0); - + // Mark as changed changed = true; } @@ -780,7 +795,7 @@ bool BackupStoreDirectory::CheckAndFix() { // Reset change marker ch = false; - + // Search backwards -- so see newer versions first std::vector<Entry*>::iterator i(mEntries.end()); if(i == mEntries.begin()) @@ -806,10 +821,8 @@ bool BackupStoreDirectory::CheckAndFix() } else { - bool isDir = (((*i)->GetFlags() & Entry::Flags_Dir) == Entry::Flags_Dir); - // Check mutually exclusive flags - if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File)) + if((*i)->IsDir() && (*i)->IsFile()) { // Bad! Unset the file flag BOX_TRACE("Entry " << FMT_i << @@ -863,29 +876,29 @@ bool BackupStoreDirectory::CheckAndFix() } } } - + if(removeEntry) { // Mark something as changed, in loop ch = true; - + // Mark something as globally changed changed = true; - + // erase the thing from the list Entry *pentry = (*i); mEntries.erase(i); // And delete the entry object delete pentry; - + // Stop going around this loop, as the iterator is now invalid break; } } while(i != mEntries.begin()); } while(ch != false); - + return changed; } diff --git a/lib/backupstore/BackupStoreContext.cpp b/lib/backupstore/BackupStoreContext.cpp index 2c98b1d7..1a782df4 100644 --- a/lib/backupstore/BackupStoreContext.cpp +++ b/lib/backupstore/BackupStoreContext.cpp @@ -30,13 +30,14 @@ #include "MemLeakFindOn.h" -// Maximum number of directories to keep in the cache -// When the cache is bigger than this, everything gets -// deleted. +// Maximum number of directories to keep in the cache When the cache is bigger +// than this, everything gets deleted. In tests, we set the cache size to zero +// to ensure that it's always flushed, which is very inefficient but helps to +// catch programming errors (use of freed data). #ifdef BOX_RELEASE_BUILD #define MAX_CACHE_SIZE 32 #else - #define MAX_CACHE_SIZE 2 + #define MAX_CACHE_SIZE 0 #endif // Allow the housekeeping process 4 seconds to release an account @@ -54,19 +55,22 @@ // // -------------------------------------------------------------------------- BackupStoreContext::BackupStoreContext(int32_t ClientID, - HousekeepingInterface &rDaemon, const std::string& rConnectionDetails) - : mConnectionDetails(rConnectionDetails), - mClientID(ClientID), - mrDaemon(rDaemon), - mProtocolPhase(Phase_START), - mClientHasAccount(false), - mStoreDiscSet(-1), - mReadOnly(true), - mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), - mpTestHook(NULL) + HousekeepingInterface* pHousekeeping, const std::string& rConnectionDetails) +: mConnectionDetails(rConnectionDetails), + mClientID(ClientID), + mpHousekeeping(pHousekeeping), + mProtocolPhase(Phase_START), + mClientHasAccount(false), + mStoreDiscSet(-1), + mReadOnly(true), + mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), + mpTestHook(NULL) +// If you change the initialisers, be sure to update +// BackupStoreContext::ReceivedFinishCommand as well! { } + // -------------------------------------------------------------------------- // // Function @@ -77,11 +81,19 @@ BackupStoreContext::BackupStoreContext(int32_t ClientID, // -------------------------------------------------------------------------- BackupStoreContext::~BackupStoreContext() { + ClearDirectoryCache(); +} + + +void BackupStoreContext::ClearDirectoryCache() +{ // Delete the objects in the cache - for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); + i != mDirectoryCache.end(); ++i) { delete (i->second); } + mDirectoryCache.clear(); } @@ -103,6 +115,7 @@ void BackupStoreContext::CleanUp() } } + // -------------------------------------------------------------------------- // // Function @@ -118,6 +131,20 @@ void BackupStoreContext::ReceivedFinishCommand() // Save the store info, not delayed SaveStoreInfo(false); } + + // Just in case someone wants to reuse a local protocol object, + // put the context back to its initial state. + mProtocolPhase = BackupStoreContext::Phase_Version; + + // Avoid the need to check version again, by not resetting + // mClientHasAccount, mAccountRootDir or mStoreDiscSet + + mReadOnly = true; + mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; + mpTestHook = NULL; + mapStoreInfo.reset(); + mapRefCount.reset(); + ClearDirectoryCache(); } @@ -133,19 +160,19 @@ bool BackupStoreContext::AttemptToGetWriteLock() { // Make the filename of the write lock file std::string writeLockFile; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); + StoreStructure::MakeWriteLockFilename(mAccountRootDir, mStoreDiscSet, writeLockFile); // Request the lock bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); - - if(!gotLock) + + if(!gotLock && mpHousekeeping) { // The housekeeping process might have the thing open -- ask it to stop char msg[256]; - int msgLen = sprintf(msg, "r%x\n", mClientID); + int msgLen = snprintf(msg, sizeof(msg), "r%x\n", mClientID); // Send message - mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen); - + mpHousekeeping->SendMessageToHousekeepingProcess(msg, msgLen); + // Then try again a few times int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; do @@ -153,16 +180,16 @@ bool BackupStoreContext::AttemptToGetWriteLock() ::sleep(1 /* second */); --tries; gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); - + } while(!gotLock && tries > 0); } - + if(gotLock) { // Got the lock, mark as not read only mReadOnly = false; } - + return gotLock; } @@ -181,16 +208,16 @@ void BackupStoreContext::LoadStoreInfo() { THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded) } - + // Load it up! - std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly)); - + std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mAccountRootDir, mStoreDiscSet, mReadOnly)); + // Check it if(i->GetAccountID() != mClientID) { THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount) } - + // Keep the pointer to it mapStoreInfo = i; @@ -203,12 +230,11 @@ void BackupStoreContext::LoadStoreInfo() } catch(BoxException &e) { - BOX_WARNING("Reference count database is missing or corrupted, " - "creating a new one, expect housekeeping to find and " - "fix problems with reference counts later."); - - BackupStoreRefCountDatabase::CreateForRegeneration(account); - mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CorruptReferenceCountDatabase, "Reference count " + "database is missing or corrupted, cannot safely open " + "account. Housekeeping will fix this automatically " + "when it next runs."); } } @@ -227,6 +253,7 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -242,7 +269,7 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) } } - // Want to save now + // Want to save now mapStoreInfo->Save(); // Set count for next delay @@ -263,83 +290,111 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) void BackupStoreContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists) { // Delegate to utility function - StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists); + StoreStructure::MakeObjectFilename(ObjectID, mAccountRootDir, mStoreDiscSet, rOutput, EnsureDirectoryExists); } // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::GetDirectoryInternal(int64_t) -// Purpose: Return a reference to a directory. Valid only until the -// next time a function which affects directories is called. -// Mainly this funciton, and creation of files. -// Private version of this, which returns non-const directories. +// Name: BackupStoreContext::GetDirectoryInternal(int64_t, +// bool) +// Purpose: Return a reference to a directory. Valid only until +// the next time a function which affects directories +// is called. Mainly this function, and creation of +// files. Private version of this, which returns +// non-const directories. Unless called with +// AllowFlushCache == false, the cache may be flushed, +// invalidating all directory references that you may +// be holding, so beware. // Created: 2003/09/02 // // -------------------------------------------------------------------------- -BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) +BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID, + bool AllowFlushCache) { // Get the filename std::string filename; MakeObjectFilename(ObjectID, filename); - + int64_t oldRevID = 0, newRevID = 0; + // Already in cache? std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID)); - if(item != mDirectoryCache.end()) - { - // Check the revision ID of the file -- does it need refreshing? - int64_t revID = 0; - if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID)) - { - THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) - } - - if(revID == item->second->GetRevisionID()) + if(item != mDirectoryCache.end()) { +#ifndef BOX_RELEASE_BUILD // it might be in the cache, but invalidated + // in which case, delete it instead of returning it. + if(!item->second->IsInvalidated()) +#else + if(true) +#endif { - // Looks good... return the cached object - BOX_TRACE("Returning object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " from cache, modtime = " << revID); - return *(item->second); + oldRevID = item->second->GetRevisionID(); + + // Check the revision ID of the file -- does it need refreshing? + if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &newRevID)) + { + THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) + } + + if(newRevID == oldRevID) + { + // Looks good... return the cached object + BOX_TRACE("Returning object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " from cache, modtime = " << newRevID) + return *(item->second); + } } - - BOX_TRACE("Refreshing object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " in cache, modtime changed from " << - item->second->GetRevisionID() << " to " << revID); // Delete this cached object delete item->second; mDirectoryCache.erase(item); } - + // Need to load it up - + // First check to see if the cache is too big - if(mDirectoryCache.size() > MAX_CACHE_SIZE) + if(mDirectoryCache.size() > MAX_CACHE_SIZE && AllowFlushCache) { - // Very simple. Just delete everything! - for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + // Very simple. Just delete everything! But in debug builds, + // leave the entries in the cache and invalidate them instead, + // so that any attempt to access them will cause an assertion + // failure that helps to track down the error. +#ifdef BOX_RELEASE_BUILD + ClearDirectoryCache(); +#else + for(std::map<int64_t, BackupStoreDirectory*>::iterator + i = mDirectoryCache.begin(); + i != mDirectoryCache.end(); i++) { - delete (i->second); + i->second->Invalidate(); } - mDirectoryCache.clear(); +#endif } // Get a RaidFileRead to read it - int64_t revID = 0; - std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID)); - ASSERT(revID != 0); - - // New directory object - std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory); - + std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, + filename, &newRevID)); + + ASSERT(newRevID != 0); + + if (oldRevID == 0) + { + BOX_TRACE("Loading object " << BOX_FORMAT_OBJECTID(ObjectID) << + " with modtime " << newRevID); + } + else + { + BOX_TRACE("Refreshing object " << BOX_FORMAT_OBJECTID(ObjectID) << + " in cache, modtime changed from " << oldRevID << + " to " << newRevID); + } + // Read it from the stream, then set it's revision ID BufferedStream buf(*objectFile); - dir->ReadFromStream(buf, IOStream::TimeOutInfinite); - dir->SetRevisionID(revID); - + std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory(buf)); + dir->SetRevisionID(newRevID); + // Make sure the size of the directory is available for writing the dir back int64_t dirSize = objectFile->GetDiscUsageInBlocks(); ASSERT(dirSize > 0); @@ -348,7 +403,7 @@ BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) // Store in cache BackupStoreDirectory *pdir = dir.release(); try - { + { mDirectoryCache[ObjectID] = pdir; } catch(...) @@ -356,11 +411,12 @@ BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) delete pdir; throw; } - + // Return it return *pdir; } + // -------------------------------------------------------------------------- // // Function @@ -381,12 +437,12 @@ int64_t BackupStoreContext::AllocateObjectID() // to try for finding an unused ID. // (Sizes used in the store info are fixed by the housekeeping process) int retryLimit = (STORE_INFO_SAVE_DELAY * 2); - + while(retryLimit > 0) { // Attempt to allocate an ID from the store int64_t id = mapStoreInfo->AllocateObjectID(); - + // Generate filename std::string filename; MakeObjectFilename(id, filename); @@ -396,17 +452,17 @@ int64_t BackupStoreContext::AllocateObjectID() // Success! return id; } - + // Decrement retry count, and try again --retryLimit; - + // Mark that the store info should be saved as soon as possible mSaveStoreInfoDelay = 0; - + BOX_WARNING("When allocating object ID, found that " << BOX_FORMAT_OBJECTID(id) << " is already in use"); } - + THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation) } @@ -431,11 +487,12 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) } - + // This is going to be a bit complex to make sure it copes OK // with things going wrong. // The only thing which isn't safe is incrementing the object ID @@ -444,13 +501,13 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // be corrected the next time the account has a housekeeping run, // and the object ID allocation code is tolerant of missed IDs. // (the info is written lazily, so these are necessary) - + // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - + // Allocate the next ID int64_t id = AllocateObjectID(); - + // Stream the file to disc std::string fn; MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); @@ -458,12 +515,13 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, RaidFileWrite *ppreviousVerStoreFile = 0; bool reversedDiffIsCompletelyDifferent = false; int64_t oldVersionNewBlocksUsed = 0; + BackupStoreInfo::Adjustment adjustment = {}; + try { RaidFileWrite storeFile(mStoreDiscSet, fn); storeFile.Open(false /* no overwriting */); - // size adjustment from use of patch in old file int64_t spaceSavedByConversionToPatch = 0; // Diff or full file? @@ -482,12 +540,12 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, { THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory) } - + // Diff file, needs to be recreated. // Choose a temporary filename. std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp", 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */)); - + try { // Open it twice @@ -506,13 +564,13 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, THROW_EXCEPTION(CommonException, OSFileError); } #endif - + // Stream the incoming diff to this temporary file if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) { THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) } - + // Verify the diff diff.Seek(0, IOStream::SeekType_Absolute); if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) @@ -526,7 +584,7 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Filename of the old version std::string oldVersionFilename; MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */); - + // Reassemble that diff -- open previous file, and combine the patch and file std::auto_ptr<RaidFileRead> from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); BackupStoreFile::CombineFile(diff, diff2, *from, storeFile); @@ -539,15 +597,26 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, diff.Seek(0, IOStream::SeekType_Absolute); BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile, DiffFromFileID, &reversedDiffIsCompletelyDifferent); - + // Store disc space used oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks(); - + // And make a space adjustment for the size calculation spaceSavedByConversionToPatch = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed; + adjustment.mBlocksUsed -= spaceSavedByConversionToPatch; + // The code below will change the patch from a + // Current file to an Old file, so we need to + // account for it as a Current file here. + adjustment.mBlocksInCurrentFiles -= + spaceSavedByConversionToPatch; + + // Don't adjust anything else here. We'll do it + // when we update the directory just below, + // which also accounts for non-diff replacements. + // Everything cleans up here... } catch(...) @@ -557,14 +626,17 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, throw; } } - + // Get the blocks used newObjectBlocksUsed = storeFile.GetDiscUsageInBlocks(); - + adjustment.mBlocksUsed += newObjectBlocksUsed; + adjustment.mBlocksInCurrentFiles += newObjectBlocksUsed; + adjustment.mNumCurrentFiles++; + // Exceeds the hard limit? - int64_t newBlocksUsed = mapStoreInfo->GetBlocksUsed() + - newObjectBlocksUsed - spaceSavedByConversionToPatch; - if(newBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) + int64_t newTotalBlocksUsed = mapStoreInfo->GetBlocksUsed() + + adjustment.mBlocksUsed; + if(newTotalBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) { THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) // The store file will be deleted automatically by the RaidFile object @@ -581,7 +653,7 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, delete ppreviousVerStoreFile; ppreviousVerStoreFile = 0; } - + throw; } @@ -596,21 +668,34 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Error! Delete the file RaidFileWrite del(mStoreDiscSet, fn); del.Delete(); - + // Exception THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) } - } - + } + // Modify the directory -- first make all files with the same name // marked as an old version - int64_t blocksInOldFiles = 0; try { + // Adjust the entry for the object that we replaced with a + // patch, above. + BackupStoreDirectory::Entry *poldEntry = NULL; + + if(DiffFromFileID != 0) + { + // Get old version entry + poldEntry = dir.FindEntryByID(DiffFromFileID); + ASSERT(poldEntry != 0); + + // Adjust size of old entry + int64_t oldSize = poldEntry->GetSizeInBlocks(); + poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); + } + if(MarkFileWithSameNameAsOldVersions) { BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *e = 0; while((e = i.Next()) != 0) { @@ -626,43 +711,30 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion); // Can safely do this, because we know we won't be here if it's already // an old version - blocksInOldFiles += e->GetSizeInBlocks(); + adjustment.mBlocksInOldFiles += e->GetSizeInBlocks(); + adjustment.mBlocksInCurrentFiles -= e->GetSizeInBlocks(); + adjustment.mNumOldFiles++; + adjustment.mNumCurrentFiles--; } } } } - + // Then the new entry BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename, - ModificationTime, id, newObjectBlocksUsed, - BackupStoreDirectory::Entry::Flags_File, - AttributesHash); + ModificationTime, id, newObjectBlocksUsed, + BackupStoreDirectory::Entry::Flags_File, + AttributesHash); - // Adjust for the patch back stuff? - if(DiffFromFileID != 0) + // Adjust dependency info of file? + if(DiffFromFileID && poldEntry && !reversedDiffIsCompletelyDifferent) { - // Get old version entry - BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID); - ASSERT(poldEntry != 0); - - // Adjust dependency info of file? - if(!reversedDiffIsCompletelyDifferent) - { - poldEntry->SetDependsNewer(id); - pnewEntry->SetDependsOlder(DiffFromFileID); - } - - // Adjust size of old entry - int64_t oldSize = poldEntry->GetSizeInBlocks(); - poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); - - // And adjust blocks used count, for later adjustment - newObjectBlocksUsed += (oldVersionNewBlocksUsed - oldSize); - blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize); + poldEntry->SetDependsNewer(id); + pnewEntry->SetDependsOlder(DiffFromFileID); } // Write the directory back to disc - SaveDirectory(dir, InDirectory); + SaveDirectory(dir); // Commit the old version's new patched version, now that the directory safely reflects // the state of the files on disc. @@ -678,47 +750,42 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Back out on adding that file RaidFileWrite del(mStoreDiscSet, fn); del.Delete(); - + // Remove this entry from the cache RemoveDirectoryFromCache(InDirectory); - + // Delete any previous version store file if(ppreviousVerStoreFile != 0) { delete ppreviousVerStoreFile; ppreviousVerStoreFile = 0; } - + // Don't worry about the incremented number in the store info throw; } - + // Check logic ASSERT(ppreviousVerStoreFile == 0); - + // Modify the store info + mapStoreInfo->AdjustNumCurrentFiles(adjustment.mNumCurrentFiles); + mapStoreInfo->AdjustNumOldFiles(adjustment.mNumOldFiles); + mapStoreInfo->AdjustNumDeletedFiles(adjustment.mNumDeletedFiles); + mapStoreInfo->AdjustNumDirectories(adjustment.mNumDirectories); + mapStoreInfo->ChangeBlocksUsed(adjustment.mBlocksUsed); + mapStoreInfo->ChangeBlocksInCurrentFiles(adjustment.mBlocksInCurrentFiles); + mapStoreInfo->ChangeBlocksInOldFiles(adjustment.mBlocksInOldFiles); + mapStoreInfo->ChangeBlocksInDeletedFiles(adjustment.mBlocksInDeletedFiles); + mapStoreInfo->ChangeBlocksInDirectories(adjustment.mBlocksInDirectories); - if(DiffFromFileID == 0) - { - mapStoreInfo->AdjustNumFiles(1); - } - else - { - mapStoreInfo->AdjustNumOldFiles(1); - } - - mapStoreInfo->ChangeBlocksUsed(newObjectBlocksUsed); - mapStoreInfo->ChangeBlocksInCurrentFiles(newObjectBlocksUsed - - blocksInOldFiles); - mapStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles); - // Increment reference count on the new directory to one mapRefCount->AddReference(id); - + // Save the store info -- can cope if this exceptions because infomation // will be rebuilt by housekeeping, and ID allocation can recover. SaveStoreInfo(false); - + // Return the ID to the caller return id; } @@ -728,8 +795,11 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) -// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. +// Name: BackupStoreContext::DeleteFile( +// const BackupStoreFilename &, int64_t, int64_t &) +// Purpose: Deletes a file by name, returning true if the file +// existed. Object ID returned too, set to zero if not +// found. // Created: 2003/10/21 // // -------------------------------------------------------------------------- @@ -754,9 +824,6 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ bool madeChanges = false; rObjectIDOut = 0; // not found - // Count of deleted blocks - int64_t blocksDel = 0; - try { // Iterate through directory, only looking at files which haven't been deleted @@ -769,14 +836,28 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ if(e->GetName() == rFilename) { // Check that it's definately not already deleted - ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0); + ASSERT(!e->IsDeleted()); // Set deleted flag e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); // Mark as made a change madeChanges = true; - // Can safely do this, because we know we won't be here if it's already - // an old version - blocksDel += e->GetSizeInBlocks(); + + int64_t blocks = e->GetSizeInBlocks(); + mapStoreInfo->AdjustNumDeletedFiles(1); + mapStoreInfo->ChangeBlocksInDeletedFiles(blocks); + + // We're marking all old versions as deleted. + // This is how a file can be old and deleted + // at the same time. So we don't subtract from + // number or size of old files. But if it was + // a current file, then it's not any more, so + // we do need to adjust the current counts. + if(!e->IsOld()) + { + mapStoreInfo->AdjustNumCurrentFiles(-1); + mapStoreInfo->ChangeBlocksInCurrentFiles(-blocks); + } + // Is this the last version? if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) { @@ -786,19 +867,12 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ } } } - + // Save changes? if(madeChanges) { // Save the directory back - SaveDirectory(dir, InDirectory); - - // Modify the store info, and write - // It definitely wasn't an old or deleted version - mapStoreInfo->AdjustNumFiles(-1); - mapStoreInfo->AdjustNumDeletedFiles(1); - mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); - + SaveDirectory(dir); SaveStoreInfo(false); } } @@ -871,16 +945,16 @@ bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory) } } } - + // Save changes? if(madeChanges) { // Save the directory back - SaveDirectory(dir, InDirectory); - + SaveDirectory(dir); + // Modify the store info, and write mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); - + // Maybe postponed save of store info SaveStoreInfo(); } @@ -919,27 +993,27 @@ void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID) // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &, int64_t) +// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &) // Purpose: Save directory back to disc, update time in cache // Created: 2003/09/04 // // -------------------------------------------------------------------------- -void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID) +void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir) { if(mapStoreInfo.get() == 0) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - if(rDir.GetObjectID() != ObjectID) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } + + int64_t ObjectID = rDir.GetObjectID(); try { // Write to disc, adjust size in store info std::string dirfn; MakeObjectFilename(ObjectID, dirfn); + int64_t old_dir_size = rDir.GetUserInfo1_SizeInBlocks(); + { RaidFileWrite writeDir(mStoreDiscSet, dirfn); writeDir.Open(true /* allow overwriting */); @@ -953,7 +1027,7 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - + // Make sure the size of the directory is available for writing the dir back ASSERT(dirSize > 0); int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks(); @@ -962,6 +1036,7 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec // Update size stored in directory rDir.SetUserInfo1_SizeInBlocks(dirSize); } + // Refresh revision ID in cache { int64_t revid = 0; @@ -969,8 +1044,41 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec { THROW_EXCEPTION(BackupStoreException, Internal) } + + BOX_TRACE("Saved directory " << + BOX_FORMAT_OBJECTID(ObjectID) << + ", modtime = " << revid); + rDir.SetRevisionID(revid); } + + // Update the directory entry in the grandparent, to ensure + // that it reflects the current size of the parent directory. + int64_t new_dir_size = rDir.GetUserInfo1_SizeInBlocks(); + if(new_dir_size != old_dir_size && + ObjectID != BACKUPSTORE_ROOT_DIRECTORY_ID) + { + int64_t ContainerID = rDir.GetContainerID(); + BackupStoreDirectory& parent( + GetDirectoryInternal(ContainerID)); + // rDir is now invalid + BackupStoreDirectory::Entry* en = + parent.FindEntryByID(ObjectID); + if(!en) + { + BOX_ERROR("Missing entry for directory " << + BOX_FORMAT_OBJECTID(ObjectID) << + " in directory " << + BOX_FORMAT_OBJECTID(ContainerID) << + " while trying to update dir size in parent"); + } + else + { + ASSERT(en->GetSizeInBlocks() == old_dir_size); + en->SetSizeInBlocks(new_dir_size); + SaveDirectory(parent); + } + } } catch(...) { @@ -991,20 +1099,26 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec // Created: 2003/09/04 // // -------------------------------------------------------------------------- -int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists) +int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, + const BackupStoreFilename &rFilename, + const StreamableMemBlock &Attributes, + int64_t AttributesModTime, + int64_t ModificationTime, + bool &rAlreadyExists) { if(mapStoreInfo.get() == 0) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) } - + // Flags as not already existing rAlreadyExists = false; - + // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); @@ -1030,19 +1144,31 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF // Create an empty directory with the given attributes on disc std::string fn; MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); + int64_t dirSize; + { BackupStoreDirectory emptyDir(id, InDirectory); // add the atttribues emptyDir.SetAttributes(Attributes, AttributesModTime); - + // Write... RaidFileWrite dirFile(mStoreDiscSet, fn); dirFile.Open(false /* no overwriting */); emptyDir.WriteToStream(dirFile); // Get disc usage, before it's commited - int64_t dirSize = dirFile.GetDiscUsageInBlocks(); + dirSize = dirFile.GetDiscUsageInBlocks(); + + // Exceeds the hard limit? + int64_t newTotalBlocksUsed = mapStoreInfo->GetBlocksUsed() + + dirSize; + if(newTotalBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) + { + THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) + // The file will be deleted automatically by the RaidFile object + } + // Commit the file - dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); // Make sure the size of the directory is added to the usage counts in the info ASSERT(dirSize > 0); @@ -1050,12 +1176,14 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF mapStoreInfo->ChangeBlocksInDirectories(dirSize); // Not added to cache, so don't set the size in the directory } - + // Then add it into the parent directory try { - dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */); - SaveDirectory(dir, InDirectory); + dir.AddEntry(rFilename, ModificationTime, id, dirSize, + BackupStoreDirectory::Entry::Flags_Dir, + 0 /* attributes hash */); + SaveDirectory(dir); // Increment reference count on the new directory to one mapRefCount->AddReference(id); @@ -1065,12 +1193,12 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF // Back out on adding that directory RaidFileWrite del(mStoreDiscSet, fn); del.Delete(); - + // Remove this entry from the cache RemoveDirectoryFromCache(InDirectory); - + // Don't worry about the incremented number in the store info - throw; + throw; } // Save the store info (may not be postponed) @@ -1081,6 +1209,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF return id; } + // -------------------------------------------------------------------------- // // Function @@ -1096,6 +1225,7 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -1103,9 +1233,6 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) // Containing directory int64_t InDirectory = 0; - - // Count of blocks deleted - int64_t blocksDeleted = 0; try { @@ -1113,18 +1240,18 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) { // In block, because dir may not be valid after the delete directory call BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - + // Store the directory it's in for later InDirectory = dir.GetContainerID(); // Depth first delete of contents - DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete); + DeleteDirectoryRecurse(ObjectID, Undelete); } - + // Remove the entry from the directory it's in ASSERT(InDirectory != 0); BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory)); - + BackupStoreDirectory::Iterator i(parentDir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), @@ -1141,18 +1268,16 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) { en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); } - + // Save it - SaveDirectory(parentDir, InDirectory); - + SaveDirectory(parentDir); + // Done break; } } - + // Update blocks deleted count - mapStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted)); - mapStoreInfo->AdjustNumDirectories(-1); SaveStoreInfo(false); } catch(...) @@ -1162,6 +1287,7 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) } } + // -------------------------------------------------------------------------- // // Function @@ -1170,18 +1296,18 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) // Created: 2003/10/21 // // -------------------------------------------------------------------------- -void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete) +void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete) { try { // Does things carefully to avoid using a directory in the cache after recursive call // because it may have been deleted. - + // Do sub directories { // Get the directory... BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - + // Then scan it for directories std::vector<int64_t> subDirs; BackupStoreDirectory::Iterator i(dir); @@ -1204,30 +1330,46 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBloc subDirs.push_back(en->GetObjectID()); } } - + // Done with the directory for now. Recurse to sub directories for(std::vector<int64_t>::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i) { - DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete); + DeleteDirectoryRecurse(*i, Undelete); } } - + // Then, delete the files. Will need to load the directory again because it might have // been removed from the cache. { // Get the directory... BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - + // Changes made? bool changesMade = false; - - // Run through files + + // Run through files BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) { + // Keep count of the deleted blocks + if(en->IsFile()) + { + int64_t size = en->GetSizeInBlocks(); + ASSERT(en->IsDeleted() == Undelete); + // Don't adjust counters for old files, + // because it can be both old and deleted. + if(!en->IsOld()) + { + mapStoreInfo->ChangeBlocksInCurrentFiles(Undelete ? size : -size); + mapStoreInfo->AdjustNumCurrentFiles(Undelete ? 1 : -1); + } + mapStoreInfo->ChangeBlocksInDeletedFiles(Undelete ? -size : size); + mapStoreInfo->AdjustNumDeletedFiles(Undelete ? -1 : 1); + } + // Add/remove the deleted flags if(Undelete) { @@ -1237,21 +1379,15 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBloc { en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); } - - // Keep count of the deleted blocks - if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) - { - rBlocksDeletedOut += en->GetSizeInBlocks(); - } - + // Did something changesMade = true; } - + // Save the directory if(changesMade) { - SaveDirectory(dir, ObjectID); + SaveDirectory(dir); } } } @@ -1263,7 +1399,6 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBloc } - // -------------------------------------------------------------------------- // // Function @@ -1284,15 +1419,15 @@ void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const Streamable } try - { + { // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(Directory)); - + // Set attributes dir.SetAttributes(Attributes, AttributesModTime); - + // Save back - SaveDirectory(dir, Directory); + SaveDirectory(dir); } catch(...) { @@ -1301,6 +1436,7 @@ void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const Streamable } } + // -------------------------------------------------------------------------- // // Function @@ -1324,7 +1460,7 @@ bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilena { // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - + // Find the file entry BackupStoreDirectory::Entry *en = 0; // Iterate through current versions of files, only @@ -1338,10 +1474,10 @@ bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilena { // Set attributes en->SetAttributes(Attributes, AttributesHash); - + // Tell caller the object ID rObjectIDOut = en->GetObjectID(); - + // Done break; } @@ -1351,16 +1487,16 @@ bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilena // Didn't find it return false; } - + // Save back - SaveDirectory(dir, InDirectory); + SaveDirectory(dir); } catch(...) { RemoveDirectoryFromCache(InDirectory); throw; } - + // Changed, everything OK return true; } @@ -1380,7 +1516,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + // Note that we need to allow object IDs a little bit greater than the last one in the store info, // because the store info may not have got saved in an error condition. Max greater ID is // STORE_INFO_SAVE_DELAY in this case, *2 to be safe. @@ -1389,7 +1525,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) // Obviously bad object ID return false; } - + // Test to see if it exists on the disc std::string filename; MakeObjectFilename(ObjectID, filename); @@ -1398,7 +1534,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) // RaidFile reports no file there return false; } - + // Do we need to be more specific? if(MustBe != ObjectExists_Anything) { @@ -1406,7 +1542,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename)); // Read the first integer - u_int32_t magic; + uint32_t magic; if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */)) { // Failed to get any bytes, must have failed @@ -1422,17 +1558,17 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) #endif // Right one? - u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; - + uint32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; + // Check if(ntohl(magic) != requiredMagic) { return false; } - + // File is implicitly closed } - + return true; } @@ -1451,7 +1587,7 @@ std::auto_ptr<IOStream> BackupStoreContext::OpenObject(int64_t ObjectID) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + // Attempt to open the file std::string fn; MakeObjectFilename(ObjectID, fn); @@ -1473,7 +1609,7 @@ int64_t BackupStoreContext::GetClientStoreMarker() { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + return mapStoreInfo->GetClientStoreMarker(); } @@ -1536,7 +1672,7 @@ void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) } - + mapStoreInfo->SetClientStoreMarker(ClientStoreMarker); SaveStoreInfo(false /* don't delay saving this */); } @@ -1550,7 +1686,9 @@ void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker) // Created: 12/11/03 // // -------------------------------------------------------------------------- -void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) +void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, + int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, + bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) { if(mReadOnly) { @@ -1561,7 +1699,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject) ?(BackupStoreDirectory::Entry::Flags_Deleted) :(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); - + // Special case if the directories are the same... if(MoveFromDirectory == MoveToDirectory) { @@ -1569,16 +1707,16 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, { // Get the first directory BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory)); - + // Find the file entry BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); - + // Error if not found if(en == 0) { THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) } - + // Check the new name doens't already exist (optionally ignoring deleted files) { BackupStoreDirectory::Iterator i(dir); @@ -1591,7 +1729,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, } } } - + // Need to get all the entries with the same name? if(MoveAllWithSameName) { @@ -1612,45 +1750,46 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, // Just copy this one en->SetName(rNewFilename); } - + // Save the directory back - SaveDirectory(dir, MoveFromDirectory); + SaveDirectory(dir); } catch(...) { RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same throw; } - + return; } - // Got to be careful how this is written, as we can't guarentte that if we have two - // directories open, the first won't be deleted as the second is opened. (cache) + // Got to be careful how this is written, as we can't guarantee that + // if we have two directories open, the first won't be deleted as the + // second is opened. (cache) // List of entries to move std::vector<BackupStoreDirectory::Entry *> moving; - + // list of directory IDs which need to have containing dir id changed std::vector<int64_t> dirsToChangeContainingID; try { // First of all, get copies of the entries to move to the to directory. - + { // Get the first directory BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); - + // Find the file entry BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID); - + // Error if not found if(en == 0) { THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) } - + // Need to get all the entries with the same name? if(MoveAllWithSameName) { @@ -1663,7 +1802,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, { // Copy moving.push_back(new BackupStoreDirectory::Entry(*c)); - + // Check for containing directory correction if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID()); } @@ -1679,13 +1818,13 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID()); } } - + // Secondly, insert them into the to directory, and save it - + { // To directory BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); - + // Check the new name doens't already exist { BackupStoreDirectory::Iterator i(to); @@ -1698,7 +1837,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, } } } - + // Copy the entries into it, changing the name as we go for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) { @@ -1706,9 +1845,9 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, en->SetName(rNewFilename); to.AddEntry(*en); // adds copy } - + // Save back - SaveDirectory(to, MoveToDirectory); + SaveDirectory(to); } // Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory @@ -1716,57 +1855,57 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, { // Get directory BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); - + // Delete each one for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) { from.DeleteEntry((*i)->GetObjectID()); } - + // Save back - SaveDirectory(from, MoveFromDirectory); + SaveDirectory(from); } catch(...) { // UNDO modification to To directory - + // Get directory BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); - + // Delete each one for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) { to.DeleteEntry((*i)->GetObjectID()); } - + // Save back - SaveDirectory(to, MoveToDirectory); + SaveDirectory(to); // Throw the error throw; } - + // Finally... for all the directories we moved, modify their containing directory ID for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) { // Load the directory BackupStoreDirectory &change(GetDirectoryInternal(*i)); - + // Modify containing dir ID change.SetContainerID(MoveToDirectory); - + // Save it back - SaveDirectory(change, *i); + SaveDirectory(change); } } catch(...) { - // Make sure directories aren't in the cache, as they may have been modified + // Make sure directories aren't in the cache, as they may have been modified RemoveDirectoryFromCache(MoveToDirectory); RemoveDirectoryFromCache(MoveFromDirectory); for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) { - RemoveDirectoryFromCache(*i); + RemoveDirectoryFromCache(*i); } while(!moving.empty()) @@ -1775,7 +1914,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, moving.pop_back(); } throw; - } + } // Clean up while(!moving.empty()) @@ -1786,7 +1925,6 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, } - // -------------------------------------------------------------------------- // // Function @@ -1801,8 +1939,7 @@ const BackupStoreInfo &BackupStoreContext::GetBackupStoreInfo() const { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + return *(mapStoreInfo.get()); } - diff --git a/lib/backupstore/BackupStoreContext.h b/lib/backupstore/BackupStoreContext.h index c33e7d50..48448360 100644 --- a/lib/backupstore/BackupStoreContext.h +++ b/lib/backupstore/BackupStoreContext.h @@ -45,7 +45,8 @@ class HousekeepingInterface class BackupStoreContext { public: - BackupStoreContext(int32_t ClientID, HousekeepingInterface &rDaemon, + BackupStoreContext(int32_t ClientID, + HousekeepingInterface* mpHousekeeping, const std::string& rConnectionDetails); ~BackupStoreContext(); private: @@ -64,7 +65,7 @@ public: Phase_Login = 1, Phase_Commands = 2 }; - + int GetPhase() const {return mProtocolPhase;} std::string GetPhaseName() const { @@ -80,14 +81,23 @@ public: } } void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;} - + // Read only locking bool SessionIsReadOnly() {return mReadOnly;} bool AttemptToGetWriteLock(); - void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;} + // Not really an API, but useful for BackupProtocolLocal2. + void ReleaseWriteLock() + { + if(mWriteLock.GotLock()) + { + mWriteLock.ReleaseLock(); + } + } + + void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mAccountRootDir = rStoreRoot; mStoreDiscSet = StoreDiscSet;} bool GetClientHasAccount() const {return mClientHasAccount;} - const std::string &GetStoreRoot() const {return mStoreRoot;} + const std::string &GetAccountRoot() const {return mAccountRootDir;} int GetStoreDiscSet() const {return mStoreDiscSet;} // Store info @@ -106,7 +116,7 @@ public: // Client marker int64_t GetClientStoreMarker(); void SetClientStoreMarker(int64_t ClientStoreMarker); - + // Usage information void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit); bool HardLimitExceeded(); @@ -125,13 +135,24 @@ public: const BackupStoreDirectory &GetDirectory(int64_t ObjectID) { // External callers aren't allowed to change it -- this function - // merely turns the the returned directory const. + // merely turns the returned directory const. return GetDirectoryInternal(ObjectID); } - + // Manipulating files/directories - int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions); - int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists); + int64_t AddFile(IOStream &rFile, + int64_t InDirectory, + int64_t ModificationTime, + int64_t AttributesHash, + int64_t DiffFromFileID, + const BackupStoreFilename &rFilename, + bool MarkFileWithSameNameAsOldVersions); + int64_t AddDirectory(int64_t InDirectory, + const BackupStoreFilename &rFilename, + const StreamableMemBlock &Attributes, + int64_t AttributesModTime, + int64_t ModificationTime, + bool &rAlreadyExists); void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime); bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut); bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut); @@ -155,29 +176,32 @@ public: private: void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false); - BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID); - void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID); + BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID, + bool AllowFlushCache = true); + void SaveDirectory(BackupStoreDirectory &rDir); void RemoveDirectoryFromCache(int64_t ObjectID); - void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete); + void ClearDirectoryCache(); + void DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete); int64_t AllocateObjectID(); std::string mConnectionDetails; int32_t mClientID; - HousekeepingInterface &mrDaemon; + HousekeepingInterface *mpHousekeeping; int mProtocolPhase; bool mClientHasAccount; - std::string mStoreRoot; // has final directory separator + std::string mAccountRootDir; // has final directory separator int mStoreDiscSet; + bool mReadOnly; NamedLock mWriteLock; int mSaveStoreInfoDelay; // how many times to delay saving the store info - + // Store info std::auto_ptr<BackupStoreInfo> mapStoreInfo; // Refcount database std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount; - + // Directory cache std::map<int64_t, BackupStoreDirectory*> mDirectoryCache; diff --git a/lib/backupstore/BackupStoreDirectory.cpp b/lib/backupstore/BackupStoreDirectory.cpp index 81126ede..6946f06e 100644 --- a/lib/backupstore/BackupStoreDirectory.cpp +++ b/lib/backupstore/BackupStoreDirectory.cpp @@ -1,7 +1,7 @@ // -------------------------------------------------------------------------- // // File -// Name: BackupStoreDirectory.h +// Name: BackupStoreDirectory.cpp // Purpose: Representation of a backup directory // Created: 2003/08/26 // @@ -36,11 +36,6 @@ typedef struct // Then a StreamableMemBlock for attributes } dir_StreamFormat; -typedef enum -{ - Option_DependencyInfoPresent = 1 -} dir_StreamFormatOptions; - typedef struct { uint64_t mModificationTime; @@ -75,9 +70,17 @@ END_STRUCTURE_PACKING_FOR_WIRE // // -------------------------------------------------------------------------- BackupStoreDirectory::BackupStoreDirectory() - : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mRevisionID(0), + mObjectID(0), + mContainerID(0), + mAttributesModTime(0), + mUserInfo1(0) { - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); + ASSERT(sizeof(uint64_t) == sizeof(box_time_t)); } @@ -90,7 +93,15 @@ BackupStoreDirectory::BackupStoreDirectory() // // -------------------------------------------------------------------------- BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID) - : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mRevisionID(0), + mObjectID(ObjectID), + mContainerID(ContainerID), + mAttributesModTime(0), + mUserInfo1(0) { } @@ -122,6 +133,7 @@ BackupStoreDirectory::~BackupStoreDirectory() // -------------------------------------------------------------------------- void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) { + ASSERT(!mInvalidated); // Compiled out of release builds // Get the header dir_StreamFormat hdr; if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) @@ -133,35 +145,35 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue)) { THROW_EXCEPTION_MESSAGE(BackupStoreException, BadDirectoryFormat, - "Wrong magic number in directory object " << - BOX_FORMAT_OBJECTID(mObjectID) << ": expected " << + "Wrong magic number for directory: expected " << BOX_FORMAT_HEX32(OBJECTMAGIC_DIR_MAGIC_VALUE) << " but found " << - BOX_FORMAT_HEX32(ntohl(hdr.mMagicValue))); + BOX_FORMAT_HEX32(ntohl(hdr.mMagicValue)) << " in " << + rStream.ToString()); } - + // Get data mObjectID = box_ntoh64(hdr.mObjectID); mContainerID = box_ntoh64(hdr.mContainerID); mAttributesModTime = box_ntoh64(hdr.mAttributesModTime); - + // Options int32_t options = ntohl(hdr.mOptionsPresent); - + // Get attributes mAttributes.ReadFromStream(rStream, Timeout); - + // Decode count int count = ntohl(hdr.mNumEntries); - + // Clear existing list - for(std::vector<Entry*>::iterator i = mEntries.begin(); + for(std::vector<Entry*>::iterator i = mEntries.begin(); i != mEntries.end(); i++) { delete (*i); } mEntries.clear(); - + // Read them in! for(int c = 0; c < count; ++c) { @@ -170,7 +182,7 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) { // Read from stream pen->ReadFromStream(rStream, Timeout); - + // Add to list mEntries.push_back(pen); } @@ -180,7 +192,7 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) throw; } } - + // Read in dependency info? if(options & Option_DependencyInfoPresent) { @@ -202,6 +214,7 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) // -------------------------------------------------------------------------- void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const { + ASSERT(!mInvalidated); // Compiled out of release builds // Get count of entries int32_t count = mEntries.size(); if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING) @@ -214,11 +227,11 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS count++; } } - + // Check that sensible IDs have been set ASSERT(mObjectID != 0); ASSERT(mContainerID != 0); - + // Need dependency info? bool dependencyInfoRequired = false; if(StreamDependencyInfo) @@ -231,9 +244,9 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS { dependencyInfoRequired = true; } - } + } } - + // Options int32_t options = 0; if(dependencyInfoRequired) options |= Option_DependencyInfoPresent; @@ -246,10 +259,10 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS hdr.mContainerID = box_hton64(mContainerID); hdr.mAttributesModTime = box_hton64(mAttributesModTime); hdr.mOptionsPresent = htonl(options); - + // Write header rStream.Write(&hdr, sizeof(hdr)); - + // Write the attributes? if(StreamAttributes) { @@ -268,7 +281,7 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS { pen->WriteToStream(rStream); } - + // Write dependency info? if(dependencyInfoRequired) { @@ -277,7 +290,7 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) { pen->WriteToStreamDependencyInfo(rStream); - } + } } } @@ -291,6 +304,7 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS // -------------------------------------------------------------------------- BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy) { + ASSERT(!mInvalidated); // Compiled out of release builds Entry *pnew = new Entry(rEntryToCopy); try { @@ -301,7 +315,7 @@ BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryT delete pnew; throw; } - + return pnew; } @@ -318,6 +332,7 @@ BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) { + ASSERT(!mInvalidated); // Compiled out of release builds Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesHash); try @@ -329,7 +344,7 @@ BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, delete pnew; throw; } - + return pnew; } @@ -343,6 +358,7 @@ BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, // -------------------------------------------------------------------------- void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) { + ASSERT(!mInvalidated); // Compiled out of release builds for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) { @@ -356,9 +372,11 @@ void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) return; } } - + // Not found - THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) + THROW_EXCEPTION_MESSAGE(BackupStoreException, CouldNotFindEntryInDirectory, + "Failed to find entry " << BOX_FORMAT_OBJECTID(ObjectID) << + " in directory " << BOX_FORMAT_OBJECTID(mObjectID)); } @@ -372,6 +390,7 @@ void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) // -------------------------------------------------------------------------- BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const { + ASSERT(!mInvalidated); // Compiled out of release builds for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i) { @@ -396,15 +415,19 @@ BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectI // // -------------------------------------------------------------------------- BackupStoreDirectory::Entry::Entry() - : mModificationTime(0), - mObjectID(0), - mSizeInBlocks(0), - mFlags(0), - mAttributesHash(0), - mMinMarkNumber(0), - mMarkNumber(0), - mDependsNewer(0), - mDependsOlder(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mModificationTime(0), + mObjectID(0), + mSizeInBlocks(0), + mFlags(0), + mAttributesHash(0), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) { } @@ -429,17 +452,21 @@ BackupStoreDirectory::Entry::~Entry() // // -------------------------------------------------------------------------- BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) - : mName(rToCopy.mName), - mModificationTime(rToCopy.mModificationTime), - mObjectID(rToCopy.mObjectID), - mSizeInBlocks(rToCopy.mSizeInBlocks), - mFlags(rToCopy.mFlags), - mAttributesHash(rToCopy.mAttributesHash), - mAttributes(rToCopy.mAttributes), - mMinMarkNumber(rToCopy.mMinMarkNumber), - mMarkNumber(rToCopy.mMarkNumber), - mDependsNewer(rToCopy.mDependsNewer), - mDependsOlder(rToCopy.mDependsOlder) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mName(rToCopy.mName), + mModificationTime(rToCopy.mModificationTime), + mObjectID(rToCopy.mObjectID), + mSizeInBlocks(rToCopy.mSizeInBlocks), + mFlags(rToCopy.mFlags), + mAttributesHash(rToCopy.mAttributesHash), + mAttributes(rToCopy.mAttributes), + mMinMarkNumber(rToCopy.mMinMarkNumber), + mMarkNumber(rToCopy.mMarkNumber), + mDependsNewer(rToCopy.mDependsNewer), + mDependsOlder(rToCopy.mDependsOlder) { } @@ -453,16 +480,20 @@ BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) // // -------------------------------------------------------------------------- BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) - : mName(rName), - mModificationTime(ModificationTime), - mObjectID(ObjectID), - mSizeInBlocks(SizeInBlocks), - mFlags(Flags), - mAttributesHash(AttributesHash), - mMinMarkNumber(0), - mMarkNumber(0), - mDependsNewer(0), - mDependsOlder(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mName(rName), + mModificationTime(ModificationTime), + mObjectID(ObjectID), + mSizeInBlocks(SizeInBlocks), + mFlags(Flags), + mAttributesHash(AttributesHash), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) { } @@ -478,19 +509,21 @@ BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) { + ASSERT(!mInvalidated); // Compiled out of release builds // Grab the raw bytes from the stream which compose the header en_StreamFormat entry; - if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + if(!rStream.ReadFullBuffer(&entry, sizeof(entry), + 0 /* not interested in bytes read if this fails */, Timeout)) { THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) } // Do reading first before modifying the variables, to be more exception safe - + // Get the filename BackupStoreFilename name; name.ReadFromStream(rStream, Timeout); - + // Get the attributes mAttributes.ReadFromStream(rStream, Timeout); @@ -514,6 +547,7 @@ void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const { + ASSERT(!mInvalidated); // Compiled out of release builds // Build a structure en_StreamFormat entry; entry.mModificationTime = box_hton64(mModificationTime); @@ -521,13 +555,13 @@ void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const entry.mSizeInBlocks = box_hton64(mSizeInBlocks); entry.mAttributesHash = box_hton64(mAttributesHash); entry.mFlags = htons(mFlags); - + // Write it rStream.Write(&entry, sizeof(entry)); - + // Write the filename mName.WriteToStream(rStream); - + // Write any attributes mAttributes.WriteToStream(rStream); } @@ -543,6 +577,7 @@ void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout) { + ASSERT(!mInvalidated); // Compiled out of release builds // Grab the raw bytes from the stream which compose the header en_StreamFormatDepends depends; if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout)) @@ -566,13 +601,11 @@ void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const { + ASSERT(!mInvalidated); // Compiled out of release builds // Build structure - en_StreamFormatDepends depends; + en_StreamFormatDepends depends; depends.mDependsNewer = box_hton64(mDependsNewer); depends.mDependsOlder = box_hton64(mDependsOlder); // Write rStream.Write(&depends, sizeof(depends)); } - - - diff --git a/lib/backupstore/BackupStoreDirectory.h b/lib/backupstore/BackupStoreDirectory.h index 1348f4e6..788a3ad0 100644 --- a/lib/backupstore/BackupStoreDirectory.h +++ b/lib/backupstore/BackupStoreDirectory.h @@ -29,9 +29,48 @@ class IOStream; // -------------------------------------------------------------------------- class BackupStoreDirectory { +private: +#ifndef BOX_RELEASE_BUILD + bool mInvalidated; +#endif + public: +#ifndef BOX_RELEASE_BUILD + void Invalidate() + { + mInvalidated = true; + for (std::vector<Entry*>::iterator i = mEntries.begin(); + i != mEntries.end(); i++) + { + (*i)->Invalidate(); + } + } +#endif + + typedef enum + { + Option_DependencyInfoPresent = 1 + } dir_StreamFormatOptions; + BackupStoreDirectory(); BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID); + // Convenience constructor from a stream + BackupStoreDirectory(IOStream& rStream, + int Timeout = IOStream::TimeOutInfinite) +#ifndef BOX_RELEASE_BUILD + : mInvalidated(false) +#endif + { + ReadFromStream(rStream, Timeout); + } + BackupStoreDirectory(std::auto_ptr<IOStream> apStream, + int Timeout = IOStream::TimeOutInfinite) +#ifndef BOX_RELEASE_BUILD + : mInvalidated(false) +#endif + { + ReadFromStream(*apStream, Timeout); + } private: // Copying not allowed BackupStoreDirectory(const BackupStoreDirectory &rToCopy); @@ -40,40 +79,117 @@ public: class Entry { + private: +#ifndef BOX_RELEASE_BUILD + bool mInvalidated; +#endif + public: +#ifndef BOX_RELEASE_BUILD + void Invalidate() { mInvalidated = true; } +#endif + friend class BackupStoreDirectory; Entry(); ~Entry(); Entry(const Entry &rToCopy); Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash); - + void ReadFromStream(IOStream &rStream, int Timeout); void WriteToStream(IOStream &rStream) const; - - const BackupStoreFilename &GetName() const {return mName;} - box_time_t GetModificationTime() const {return mModificationTime;} - int64_t GetObjectID() const {return mObjectID;} - int64_t GetSizeInBlocks() const {return mSizeInBlocks;} - int16_t GetFlags() const {return mFlags;} - void AddFlags(int16_t Flags) {mFlags |= Flags;} - void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;} + + const BackupStoreFilename &GetName() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mName; + } + box_time_t GetModificationTime() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mModificationTime; + } + int64_t GetObjectID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mObjectID; + } + // SetObjectID is dangerous! It should only be used when + // creating a snapshot. + void SetObjectID(int64_t NewObjectID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mObjectID = NewObjectID; + } + int64_t GetSizeInBlocks() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mSizeInBlocks; + } + int16_t GetFlags() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mFlags; + } + void AddFlags(int16_t Flags) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mFlags |= Flags; + } + void RemoveFlags(int16_t Flags) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mFlags &= ~Flags; + } // Some things can be changed - void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;} - void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;} + void SetName(const BackupStoreFilename &rNewName) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mName = rNewName; + } + void SetSizeInBlocks(int64_t SizeInBlocks) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mSizeInBlocks = SizeInBlocks; + } // Attributes - bool HasAttributes() const {return !mAttributes.IsEmpty();} - void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;} - const StreamableMemBlock &GetAttributes() const {return mAttributes;} - uint64_t GetAttributesHash() const {return mAttributesHash;} - + bool HasAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return !mAttributes.IsEmpty(); + } + void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mAttributes.Set(rAttr); + mAttributesHash = AttributesHash; + } + const StreamableMemBlock &GetAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributes; + } + uint64_t GetAttributesHash() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributesHash; + } + // Marks // The lowest mark number a version of a file of this name has ever had - uint32_t GetMinMarkNumber() const {return mMinMarkNumber;} + uint32_t GetMinMarkNumber() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mMinMarkNumber; + } // The mark number on this file - uint32_t GetMarkNumber() const {return mMarkNumber;} + uint32_t GetMarkNumber() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mMarkNumber; + } // Make sure these flags are synced with those in backupprocotol.txt // ListDirectory command @@ -94,41 +210,66 @@ public: // convenience methods bool inline IsDir() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_Dir; } bool inline IsFile() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_File; } bool inline IsOld() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_OldVersion; } bool inline IsDeleted() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_Deleted; } bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { + ASSERT(!mInvalidated); // Compiled out of release builds return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet)) && ((mFlags & FlagsNotToBeSet) == 0); }; // Get dependency info // new version this depends on - int64_t GetDependsNewer() const {return mDependsNewer;} - void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;} + int64_t GetDependsNewer() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mDependsNewer; + } + void SetDependsNewer(int64_t ObjectID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mDependsNewer = ObjectID; + } // older version which depends on this - int64_t GetDependsOlder() const {return mDependsOlder;} - void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;} + int64_t GetDependsOlder() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mDependsOlder; + } + void SetDependsOlder(int64_t ObjectID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mDependsOlder = ObjectID; + } // Dependency info saving - bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;} + bool HasDependencies() + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mDependsNewer != 0 || mDependsOlder != 0; + } void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout); void WriteToStreamDependencyInfo(IOStream &rStream) const; private: - BackupStoreFilename mName; + BackupStoreFilename mName; box_time_t mModificationTime; int64_t mObjectID; int64_t mSizeInBlocks; @@ -137,11 +278,18 @@ public: StreamableMemBlock mAttributes; uint32_t mMinMarkNumber; uint32_t mMarkNumber; - + uint64_t mDependsNewer; // new version this depends on uint64_t mDependsOlder; // older version which depends on this }; - + +#ifndef BOX_RELEASE_BUILD + bool IsInvalidated() + { + return mInvalidated; + } +#endif // !BOX_RELEASE_BUILD + void ReadFromStream(IOStream &rStream, int Timeout); void WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, @@ -155,28 +303,78 @@ public: uint64_t AttributesHash); void DeleteEntry(int64_t ObjectID); Entry *FindEntryByID(int64_t ObjectID) const; - - int64_t GetObjectID() const {return mObjectID;} - int64_t GetContainerID() const {return mContainerID;} - + + int64_t GetObjectID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mObjectID; + } + + int64_t GetContainerID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mContainerID; + } // Need to be able to update the container ID when moving objects - void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;} + void SetContainerID(int64_t ContainerID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mContainerID = ContainerID; + } + + // Purely for use of server -- not serialised into streams + int64_t GetRevisionID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mRevisionID; + } + void SetRevisionID(int64_t RevisionID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mRevisionID = RevisionID; + } - // Purely for use of server -- not serialised into streams - int64_t GetRevisionID() const {return mRevisionID;} - void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;} - - unsigned int GetNumberOfEntries() const {return mEntries.size();} + unsigned int GetNumberOfEntries() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mEntries.size(); + } // User info -- not serialised into streams - int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;} - void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;} + int64_t GetUserInfo1_SizeInBlocks() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mUserInfo1; + } + void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mUserInfo1 = UserInfo1; + } // Attributes - bool HasAttributes() const {return !mAttributes.IsEmpty();} - void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;} - const StreamableMemBlock &GetAttributes() const {return mAttributes;} - box_time_t GetAttributesModTime() const {return mAttributesModTime;} + bool HasAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return !mAttributes.IsEmpty(); + } + void SetAttributes(const StreamableMemBlock &rAttr, + box_time_t AttributesModTime) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mAttributes.Set(rAttr); + mAttributesModTime = AttributesModTime; + } + const StreamableMemBlock &GetAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributes; + } + box_time_t GetAttributesModTime() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributesModTime; + } class Iterator { @@ -184,10 +382,12 @@ public: Iterator(const BackupStoreDirectory &rDir) : mrDir(rDir), i(rDir.mEntries.begin()) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds } - + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds // Skip over things which don't match the required flags while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) { @@ -207,6 +407,7 @@ public: // In a looping situation, cache the decrypted filenames in another memory structure. BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds // Skip over things which don't match the required flags or filename while( (i != mrDir.mEntries.end()) && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) @@ -227,7 +428,7 @@ public: const BackupStoreDirectory &mrDir; std::vector<Entry*>::const_iterator i; }; - + friend class Iterator; class ReverseIterator @@ -236,10 +437,12 @@ public: ReverseIterator(const BackupStoreDirectory &rDir) : mrDir(rDir), i(rDir.mEntries.rbegin()) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds } - + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds // Skip over things which don't match the required flags while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) { @@ -253,12 +456,12 @@ public: // Return entry, and increment return (*(i++)); } - + private: const BackupStoreDirectory &mrDir; std::vector<Entry*>::const_reverse_iterator i; }; - + friend class ReverseIterator; // For recovery of the store @@ -286,4 +489,3 @@ private: }; #endif // BACKUPSTOREDIRECTORY__H - diff --git a/lib/backupstore/BackupStoreException.txt b/lib/backupstore/BackupStoreException.txt index ece772c0..efdbcf68 100644 --- a/lib/backupstore/BackupStoreException.txt +++ b/lib/backupstore/BackupStoreException.txt @@ -70,3 +70,7 @@ DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file m PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. MultiplyReferencedObject 69 Attempted to modify an object with multiple references, should be uncloned first +CorruptReferenceCountDatabase 70 The account's refcount database is corrupt and must be rebuilt by housekeeping. +CancelledByBackgroundTask 71 The current task was cancelled on request by the background task. +ObjectDoesNotExist 72 The specified object ID does not exist in the store. +AccountAlreadyExists 73 Tried to create an account that already exists. diff --git a/lib/backupstore/BackupStoreFile.cpp b/lib/backupstore/BackupStoreFile.cpp index 519305ff..99562685 100644 --- a/lib/backupstore/BackupStoreFile.cpp +++ b/lib/backupstore/BackupStoreFile.cpp @@ -22,29 +22,30 @@ #include <stdio.h> #endif +#include "BackupClientFileAttributes.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" #include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFileWire.h" #include "BackupStoreFilename.h" -#include "BackupStoreException.h" -#include "IOStream.h" -#include "Guards.h" -#include "FileModificationTime.h" -#include "FileStream.h" -#include "BackupClientFileAttributes.h" +#include "BackupStoreInfo.h" #include "BackupStoreObjectMagic.h" -#include "Compress.h" -#include "CipherContext.h" -#include "CipherBlowfish.h" #include "CipherAES.h" -#include "BackupStoreConstants.h" +#include "CipherBlowfish.h" +#include "CipherContext.h" #include "CollectInBufferStream.h" -#include "RollingChecksum.h" +#include "Compress.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "Guards.h" +#include "IOStream.h" +#include "Logging.h" #include "MD5Digest.h" -#include "ReadGatherStream.h" #include "Random.h" -#include "BackupStoreFileEncodeStream.h" -#include "Logging.h" +#include "ReadGatherStream.h" +#include "RollingChecksum.h" #include "MemLeakFindOn.h" @@ -78,7 +79,8 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) + RunStatusProvider* pRunStatusProvider, + BackgroundTask* pBackgroundTask) { // Create the stream std::auto_ptr<BackupStoreFileEncodeStream> stream( @@ -86,8 +88,9 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( // Do the initial setup stream->Setup(Filename, 0 /* no recipe, just encode */, ContainerID, - rStoreFilename, pModificationTime, pLogger, pRunStatusProvider); - + rStoreFilename, pModificationTime, pLogger, pRunStatusProvider, + pBackgroundTask); + // Return the stream for the caller return stream; } @@ -100,7 +103,15 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( // requirements. Doesn't verify that the data is intact // and can be decoded. Optionally returns the ID of the // file which it is diffed from, and the (original) -// container ID. +// container ID. This is more efficient than +// BackupStoreFile::VerifyStream() when the file data +// already exists on disk and we can Seek() around in +// it, but less efficient if we are reading the stream +// from the network and not intending to Write() it to +// a file first, so we need both unfortunately. +// TODO FIXME: use a modified VerifyStream() which +// repositions the file pointer and Close()s early to +// deduplicate this code. // Created: 2003/08/28 // // -------------------------------------------------------------------------- @@ -120,7 +131,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read header return false; } - + // Check magic number if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -130,7 +141,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro { return false; } - + // Get a filename, see if it loads OK try { @@ -142,7 +153,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // an error occured while reading it, so that's not good return false; } - + // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not try { @@ -163,10 +174,10 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Get current position in file -- the end of the header int64_t headerEnd = rFile.GetPosition(); - + // Get number of blocks int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - + // Calculate where the block index will be, check it's reasonable int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); if(blockIndexLoc < headerEnd) @@ -183,7 +194,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read block index header -- assume bad file return false; } - + // Check header if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -195,10 +206,10 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Bad header -- either magic value or number of blocks is wrong return false; } - + // Flag for recording whether a block is referenced from another file bool blockFromOtherFileReferenced = false; - + // Read the index, checking that the length values all make sense int64_t currentBlockStart = headerEnd; for(int64_t b = 0; b < numBlocks; ++b) @@ -210,7 +221,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read block index entry -- assume bad file return false; } - + // Check size and location int64_t blkSize = box_ntoh64(blk.mEncodedSize); if(blkSize <= 0) @@ -226,19 +237,20 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Encoded size makes the block run over the index return false; } - - // Move the current block start ot the end of this block + + // Move the current block start to the end of this block currentBlockStart += blkSize; } } - + // Check that there's no empty space if(currentBlockStart != blockIndexLoc) { return false; } - - // Check that if another block is references, then the ID is there, and if one isn't there is no ID. + + // Check that if another file is referenced, then the ID is there, and if one + // isn't then there is no ID. int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); if((otherID != 0 && blockFromOtherFileReferenced == false) || (otherID == 0 && blockFromOtherFileReferenced == true)) @@ -246,13 +258,13 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Doesn't look good! return false; } - + // Does the caller want the other ID? if(pDiffFromObjectIDOut) { *pDiffFromObjectIDOut = otherID; } - + // Does the caller want the container ID? if(pContainerIDOut) { @@ -263,6 +275,346 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro return true; } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyStream::Write() +// Purpose: Handles writes to the verifying stream. If the write +// completes the current unit, then verify it, copy it +// to mpCopyToStream if not NULL, and move on to the +// next unit, otherwise throw an exception. +// Created: 2015/08/07 +// +// -------------------------------------------------------------------------- + +void BackupStoreFile::VerifyStream::Write(const void *pBuffer, int NBytes, int Timeout) +{ + // Check that we haven't already written too much to the current unit + size_t BytesToAdd; + if(mState == State_Blocks) + { + // We don't know how many bytes to expect + ASSERT(mCurrentUnitSize == 0); + BytesToAdd = NBytes; + } + else + { + ASSERT(mCurrentUnitData.GetSize() < mCurrentUnitSize); + size_t BytesLeftInCurrentUnit = mCurrentUnitSize - + mCurrentUnitData.GetSize(); + // Add however many bytes are needed/available to the current unit's buffer. + BytesToAdd = std::min(BytesLeftInCurrentUnit, (size_t)NBytes); + } + + // We must make progress here, or we could have infinite recursion. + ASSERT(BytesToAdd > 0); + + CollectInBufferStream* pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Write(pBuffer, BytesToAdd, Timeout); + if(mpCopyToStream) + { + mpCopyToStream->Write(pBuffer, BytesToAdd, Timeout); + } + + pBuffer = (uint8_t *)pBuffer + BytesToAdd; + NBytes -= BytesToAdd; + mCurrentPosition += BytesToAdd; + + if(mState == State_Blocks) + { + // The block index follows the blocks themselves, but without seeing the + // index we don't know how big the blocks are. So we just have to keep + // reading, holding the last mBlockIndexSize bytes in two buffers, until + // we reach the end of the file (when Close() is called) when we can look + // back over those buffers and extract the block index from them. + if(pCurrentBuffer->GetSize() >= mBlockIndexSize) + { + // Time to swap buffers, and clear the one we're about to + // overwrite. + mCurrentBufferIsAlternate = !mCurrentBufferIsAlternate; + pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Reset(); + } + + // We don't want to move data into the finished buffer while we're in this + // state, and we don't need to call ourselves recursively, so just return. + return; + } + + ASSERT(mState != State_Blocks); + + // If the current unit is not complete, just return now. + if(mCurrentUnitData.GetSize() < mCurrentUnitSize) + { + return; + } + + ASSERT(mCurrentUnitData.GetSize() == mCurrentUnitSize); + mCurrentUnitData.SetForReading(); + CollectInBufferStream finished(mCurrentUnitData); + + // This should leave mCurrentUnitData empty in write phase + ASSERT(!mCurrentUnitData.StreamClosed()); + ASSERT(mCurrentUnitData.GetSize() == 0); + + // Advance automatically to next state (to reduce code duplication) if the current + // state is anything but State_Blocks. We remain in that state for more than one + // read (processing one block at a time), and exit it manually. + int oldState = mState; + if(mState != State_Blocks) + { + mState++; + } + + // Process and complete the current unit, depending what it is. + if(oldState == State_Header) + { + // Get the header... + file_StreamFormat hdr; + ASSERT(finished.GetSize() == sizeof(hdr)); + memcpy(&hdr, finished.GetBuffer(), sizeof(hdr)); + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid header magic in stream: expected " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_MAGIC_VALUE_V1) << + " or " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_MAGIC_VALUE_V0) << + " but found " << + BOX_FORMAT_HEX32(ntohl(hdr.mMagicValue))); + } + + mNumBlocks = box_ntoh64(hdr.mNumBlocks); + mBlockIndexSize = (mNumBlocks * sizeof(file_BlockIndexEntry)) + + sizeof(file_BlockIndexHeader); + mContainerID = box_ntoh64(hdr.mContainerID); + + ASSERT(mState == State_FilenameHeader); + mCurrentUnitSize = 2; + } + else if(oldState == State_FilenameHeader) + { + // Check that the encoding is an accepted value. + unsigned int encoding = + BACKUPSTOREFILENAME_GET_ENCODING( + (uint8_t *)finished.GetBuffer()); + if(encoding < BackupStoreFilename::Encoding_Min || + encoding > BackupStoreFilename::Encoding_Max) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid encoding in filename: " << encoding); + } + + ASSERT(mState == State_Filename); + mCurrentUnitSize = BACKUPSTOREFILENAME_GET_SIZE( + (uint8_t *)finished.GetBuffer()); + // Copy the first two bytes back into the new buffer, to be used by the + // completed filename. + finished.CopyStreamTo(mCurrentUnitData); + } + else if(oldState == State_Filename) + { + BackupStoreFilename fn; + fn.ReadFromStream(finished, IOStream::TimeOutInfinite); + ASSERT(mState == State_AttributesSize); + mCurrentUnitSize = sizeof(int32_t); + } + else if(oldState == State_AttributesSize) + { + ASSERT(mState == State_Attributes); + mCurrentUnitSize = ntohl(*(int32_t *)finished.GetBuffer()); + } + else if(oldState == State_Attributes) + { + // Skip the attributes -- because they're encrypted, the server can't tell + // whether they're OK or not. + ASSERT(mState == State_Blocks); + mBlockDataPosition = mCurrentPosition; + mCurrentUnitSize = 0; + } + else if(oldState == State_Blocks) + { + // The block index follows the blocks themselves, but without seeing the + // index we don't know how big the blocks are. So we just have to keep + // reading, holding the last mBlockIndexSize bytes in two buffers, until + // we reach the end of the file (when Close() is called) when we can look + // back over those buffers and extract the block index from them. + ASSERT(mState == State_Blocks); + if(pCurrentBuffer->GetSize() >= mBlockIndexSize) + { + // Time to swap buffers, and clear the one we're about to + // overwrite. + mCurrentBufferIsAlternate = !mCurrentBufferIsAlternate; + pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Reset(); + } + } + + if(NBytes > 0) + { + // Still some data to process, so call recursively to deal with it. + Write(pBuffer, NBytes, Timeout); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyStream::Write() +// Purpose: Handles closing the verifying stream, which tells us +// that there is no more incoming data, at which point +// we can extract the block index from the data most +// recently read, and verify it. +// Created: 2015/08/07 +// +// -------------------------------------------------------------------------- + + +void BackupStoreFile::VerifyStream::Close(bool CloseCopyStream) +{ + if(mpCopyToStream && CloseCopyStream) + { + mpCopyToStream->Close(); + } + + if(mState != State_Blocks) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Stream closed too early to be a valid BackupStoreFile: was in " + "state " << mState << " instead of " << State_Blocks); + } + + // Find the last mBlockIndexSize bytes. + CollectInBufferStream finished; + + CollectInBufferStream* pCurrentBuffer = mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData; + CollectInBufferStream* pPreviousBuffer = mCurrentBufferIsAlternate ? + &mCurrentUnitData : &mAlternateData; + + int64_t BytesFromCurrentBuffer = std::min(mBlockIndexSize, + (int64_t)pCurrentBuffer->GetSize()); + int64_t BytesFromPreviousBuffer = mBlockIndexSize - BytesFromCurrentBuffer; + + if(pPreviousBuffer->GetSize() < BytesFromPreviousBuffer) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Not enough bytes for block index: expected " << + mBlockIndexSize << " but had collected " << + (pPreviousBuffer->GetSize() + pCurrentBuffer->GetSize()) << + " at " << mCurrentPosition << " bytes into file"); + } + + size_t PreviousBufferOffset = pPreviousBuffer->GetSize() - + BytesFromPreviousBuffer; + size_t CurrentBufferOffset = pCurrentBuffer->GetSize() - + BytesFromCurrentBuffer; + + file_BlockIndexHeader blkhdr; + finished.Write((uint8_t *)pPreviousBuffer->GetBuffer() + PreviousBufferOffset, + BytesFromPreviousBuffer); + finished.Write((uint8_t *)pCurrentBuffer->GetBuffer() + CurrentBufferOffset, + BytesFromCurrentBuffer); + ASSERT(finished.GetSize() == mBlockIndexSize); + + // Load the block index header + memcpy(&blkhdr, finished.GetBuffer(), sizeof(blkhdr)); + + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index magic in stream: expected " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) << + " or " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0) << + " but found " << + BOX_FORMAT_HEX32(ntohl(blkhdr.mMagicValue))); + } + + if((int64_t)box_ntoh64(blkhdr.mNumBlocks) != mNumBlocks) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index size in stream: expected " << + BOX_FORMAT_OBJECTID(mNumBlocks) << + " but found " << + BOX_FORMAT_OBJECTID(box_ntoh64(blkhdr.mNumBlocks))); + } + + // Flag for recording whether a block is referenced from another file + mBlockFromOtherFileReferenced = false; + int64_t blockIndexLoc = mCurrentPosition - mBlockIndexSize; + + // Read the index, checking that the length values all make sense + int64_t currentBlockStart = mBlockDataPosition; + file_BlockIndexEntry* pBlk = (file_BlockIndexEntry *) + ((uint8_t *)finished.GetBuffer() + sizeof(blkhdr)); + for(int64_t b = 0; b < mNumBlocks; ++b) + { + // Check size and location + int64_t blkSize = box_ntoh64(pBlk[b].mEncodedSize); + if(blkSize <= 0) + { + // Mark that this file references another file + mBlockFromOtherFileReferenced = true; + } + else + { + // This block is actually in this file + if((currentBlockStart + blkSize) > blockIndexLoc) + { + // Encoded size makes the block run over the index + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index: entry " << b << " makes " + "total size of block data exceed the available " + "space"); + } + + // Move the current block start to the end of this block + currentBlockStart += blkSize; + } + } + + // Check that there's no empty space + if(currentBlockStart != blockIndexLoc) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index: total size of blocks does not match the " + "actual amount of block data in the stream: expected " << + (blockIndexLoc - mBlockDataPosition) << " but found " << + (currentBlockStart - mBlockDataPosition)); + } + + // Check that if another file is referenced, then the ID is there, and if one + // isn't then there is no ID. + int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); + if((otherID != 0 && mBlockFromOtherFileReferenced == false) + || (otherID == 0 && mBlockFromOtherFileReferenced == true)) + { + // Doesn't look good! + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index header: actual dependency status does not " + "match the file header: expected to depend on file object " << + BOX_FORMAT_OBJECTID(otherID)); + } + + mDiffFromObjectID = otherID; +} + + // -------------------------------------------------------------------------- // // Function @@ -279,7 +631,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile { THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) } - + // Try, delete output file if error try { @@ -288,7 +640,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile // Get the decoding stream std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr)); - + // Is it a symlink? if(!stream->IsSymLink()) { @@ -311,7 +663,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile // of the block index. I hope that reading an extra byte // doesn't hurt! // ASSERT(drained == 0); - + // Write the attributes try { @@ -348,10 +700,10 @@ std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream( { // Create stream std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout)); - + // Get it ready stream->Setup(pAlterativeAttr); - + // Return to caller return stream; } @@ -431,7 +783,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - bool inFileOrder = true; + bool inFileOrder = true; switch(ntohl(magic)) { #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -455,7 +807,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl default: THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // If not in file order, then the index list must be read now if(!inFileOrder) { @@ -484,7 +836,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - } + } // Check magic number if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 @@ -498,7 +850,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Get the filename mFilename.ReadFromStream(mrEncodedFile, mTimeout); - + // Get the attributes (either from stream, or supplied attributes) if(pAlterativeAttr != 0) { @@ -514,7 +866,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Read the attributes from the stream mAttributes.ReadFromStream(mrEncodedFile, mTimeout); } - + // If it is in file order, go and read the file attributes // Requires that the stream can seek if(inFileOrder) @@ -524,30 +876,30 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl { THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) } - + // Store current location (beginning of encoded blocks) int64_t endOfHeaderPos = mrEncodedFile.GetPosition(); - + // Work out where the index is int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - + // Seek to that position mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute); - + // Read the block index - ReadBlockIndex(false /* magic number still to be read */); - + ReadBlockIndex(false /* magic number still to be read */); + // Seek back to the end of header position, ready for reading the chunks mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute); } - + // Check view of blocks from block header and file header match if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks)) { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Need to allocate some memory for the two blocks for reading encoded data, and clear data if(mNumBlocks > 0) { @@ -560,11 +912,11 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Get the clear and encoded size int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize); ASSERT(encodedSize > 0); - + // Larger? if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize; } - + // Allocate those blocks! mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32); @@ -589,7 +941,7 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) { // Header file_BlockIndexHeader blkhdr; - + // Read it in -- way depends on how whether the magic number has already been read if(MagicAlreadyRead) { @@ -609,7 +961,7 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - + // Check magic value if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -620,26 +972,26 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } } - + // Get the number of blocks out of the header mNumBlocks = box_ntoh64(blkhdr.mNumBlocks); - + // Read the IV base mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase); - + // Load the block entries in? if(mNumBlocks > 0) { // How big is the index? int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks; - + // Allocate some memory mpBlockIndex = ::malloc(indexSize); if(mpBlockIndex == 0) { throw std::bad_alloc(); } - + // Read it in if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout)) { @@ -676,7 +1028,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) int bytesToRead = NBytes; uint8_t *output = (uint8_t*)pBuffer; - + while(bytesToRead > 0 && mCurrentBlock < mNumBlocks) { // Anything left in the current block? @@ -685,16 +1037,16 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // Copy data out of this buffer int s = mCurrentBlockClearSize - mPositionInCurrentBlock; if(s > bytesToRead) s = bytesToRead; // limit to requested data - + // Copy ::memcpy(output, mpClearData + mPositionInCurrentBlock, s); - + // Update positions output += s; mPositionInCurrentBlock += s; bytesToRead -= s; } - + // Need to get some more data? if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize) { @@ -705,7 +1057,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // Stop now! break; } - + // Get the size from the block index const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize); @@ -716,14 +1068,14 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // It needs to be combined with the previous version first. THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining) } - + // Load in next block if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout)) { // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - + // Decode the data mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize); @@ -734,7 +1086,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // platforms with different endiannesses. iv = box_hton64(iv); sBlowfishDecryptBlockEntry.SetIV(&iv); - + // Decrypt the encrypted section file_BlockIndexEntryEnc entryEnc; int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), @@ -779,7 +1131,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) #endif } - + // Check the digest MD5Digest md5; md5.Add(mpClearData, mCurrentBlockClearSize); @@ -788,12 +1140,12 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck) } - + // Set vars to say what's happening mPositionInCurrentBlock = 0; } } - + ASSERT(bytesToRead >= 0); ASSERT(bytesToRead <= NBytes); @@ -816,14 +1168,14 @@ bool BackupStoreFile::DecodedStream::IsSymLink() { return false; } - + // So the attributes think it is a symlink. // Consistency check... if(mNumBlocks != 0) { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + return true; } @@ -836,7 +1188,8 @@ bool BackupStoreFile::DecodedStream::IsSymLink() // Created: 9/12/03 // // -------------------------------------------------------------------------- -void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes) +void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes, + int Timeout) { THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream) } @@ -916,7 +1269,7 @@ void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength) sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); sAESDecrypt.Reset(); sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); - + // Set encryption to use this key, instead of the "default" blowfish key spEncrypt = &sAESEncrypt; sEncryptCipherType = HEADER_AES_ENCODING; @@ -961,9 +1314,9 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi { rOutput.Reallocate(256); } - + // Check alignment of the block - ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + ASSERT((((uint64_t)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); // Want to compress it? bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE); @@ -981,10 +1334,10 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi const void *iv = spEncrypt->SetRandomIV(ivLen); ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen); outOffset += ivLen; - + // Start encryption process spEncrypt->Begin(); - + #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \ { \ if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \ @@ -992,13 +1345,13 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \ } \ } - + // Encode the chunk if(compressChunk) { // buffer to compress into uint8_t buffer[2048]; - + // Set compressor with all the chunk as an input Compress<true> compress; compress.Input(Chunk, ChunkSize); @@ -1011,7 +1364,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi if(s > 0) { ENCODECHUNK_CHECK_SPACE(s) - outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); } else { @@ -1031,7 +1384,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi ENCODECHUNK_CHECK_SPACE(16) outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); } - + ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check return outOffset; @@ -1051,7 +1404,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize) { // Check alignment of the encoded block - ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + ASSERT((((uint64_t)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); // First check if(EncodedSize < 1) @@ -1060,7 +1413,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } const uint8_t *input = (uint8_t*)Encoded; - + // Get header, make checks, etc uint8_t header = input[0]; bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED; @@ -1069,7 +1422,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out { THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding) } - + #ifndef HAVE_OLD_SSL // Choose cipher CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt); @@ -1081,7 +1434,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } CipherContext &cipher(sBlowfishDecrypt); #endif - + // Check enough space for header, an IV and one byte of input int ivLen = cipher.GetIVLength(); if(EncodedSize < (1 + ivLen + 1)) @@ -1092,7 +1445,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out // Set IV in decrypt context, and start cipher.SetIV(input + 1); cipher.Begin(); - + // Setup vars for code int inOffset = 1 + ivLen; uint8_t *output = (uint8_t*)Output; @@ -1104,10 +1457,10 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out // Do things in chunks uint8_t buffer[2048]; int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer)); - + // Decompressor Compress<false> decompress; - + while(inOffset < EncodedSize) { // Decrypt a block @@ -1115,7 +1468,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl); inOffset += bl; - + // Decompress the decrypted data if(s > 0) { @@ -1126,7 +1479,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out os = decompress.Output(output + outOffset, OutputSize - outOffset); outOffset += os; } while(os > 0); - + // Check that there's space left in the output buffer -- there always should be if(outOffset >= OutputSize) { @@ -1134,7 +1487,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } } } - + // Get any compressed data remaining in the cipher context and compression int s = cipher.Final(buffer, sizeof(buffer)); decompress.Input(buffer, s); @@ -1157,7 +1510,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset); outOffset += cipher.Final(output + outOffset, OutputSize - outOffset); } - + return outOffset; } @@ -1209,10 +1562,10 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Get number of blocks int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - + // Calculate where the block index will be, check it's reasonable int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); int64_t blockIndexLoc = fileSize - blockIndexSize; @@ -1221,10 +1574,10 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr // Doesn't look good! THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Build a reordered stream std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership)); - + // Set it up... ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get())); int component = rreordered.AddComponent(pStream); @@ -1232,7 +1585,7 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc); // And then the rest of the file rreordered.AddBlock(component, blockIndexLoc, true, 0); - + return reordered; } @@ -1287,7 +1640,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, { in.reset(new FileStream(Filename)); } - + // Read header file_BlockIndexHeader hdr; if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) @@ -1313,17 +1666,17 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, // Get basic information int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase); - + //TODO: Verify that these sizes look reasonable - + // setup void *data = 0; int32_t dataSize = -1; bool matches = true; int64_t totalSizeInBlockIndex = 0; - + try - { + { for(int64_t b = 0; b < numBlocks; ++b) { // Read an entry from the stream @@ -1332,8 +1685,8 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, { // Couldn't read entry THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - + } + // Calculate IV for this entry uint64_t iv = entryIVBase; iv += b; @@ -1345,8 +1698,8 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, iv = box_swap64(iv); } #endif - sBlowfishDecryptBlockEntry.SetIV(&iv); - + sBlowfishDecryptBlockEntry.SetIV(&iv); + // Decrypt the encrypted section file_BlockIndexEntryEnc entryEnc; int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), @@ -1381,7 +1734,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } dataSize = blockClearSize + 128; } - + // Load in the block from the file, if it's not a symlink if(!sourceIsSymlink) { @@ -1403,7 +1756,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } } } - + // Keep on going regardless, to make sure the entire block index stream is read // -- must always be consistent about what happens with the stream. } @@ -1418,14 +1771,14 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } throw; } - + // free block if(data != 0) { ::free(data); data = 0; } - + // Check for data left over if it's not a symlink if(!sourceIsSymlink) { @@ -1436,13 +1789,13 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, matches = false; } } - + // Symlinks must have zero size on server if(sourceIsSymlink) { matches = (totalSizeInBlockIndex == 0); } - + return matches; } @@ -1522,10 +1875,10 @@ void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) } // Copy data ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize); - + // Free old BackupStoreFile::CodingChunkFree(mpBuffer); - + // Store new buffer mpBuffer = buffer; mBufferSize = NewSize; @@ -1554,5 +1907,44 @@ DiffTimer::DiffTimer() // // -------------------------------------------------------------------------- DiffTimer::~DiffTimer() -{ +{ +} + +// Shortcut interface +int64_t BackupStoreFile::QueryStoreFileDiff(BackupProtocolCallable& protocol, + const std::string& LocalFilename, int64_t DirectoryObjectID, + int64_t DiffFromFileID, int64_t AttributesHash, + const BackupStoreFilenameClear& StoreFilename, int Timeout, + DiffTimer *pDiffTimer, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) +{ + int64_t ModificationTime; + std::auto_ptr<BackupStoreFileEncodeStream> pStream; + + if(DiffFromFileID) + { + // Fetch the block index for this one + std::auto_ptr<BackupProtocolSuccess> getblockindex = + protocol.QueryGetBlockIndexByName(DirectoryObjectID, + StoreFilename); + ASSERT(getblockindex->GetObjectID() == DiffFromFileID); + std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream()); + + pStream = EncodeFileDiff(LocalFilename, + DirectoryObjectID, StoreFilename, DiffFromFileID, + *(blockIndexStream.get()), Timeout, pDiffTimer, + &ModificationTime, NULL // pIsCompletelyDifferent + ); + } + else + { + pStream = BackupStoreFile::EncodeFile(LocalFilename, + DirectoryObjectID, StoreFilename, &ModificationTime); + } + + std::auto_ptr<IOStream> upload(pStream.release()); + return protocol.QueryStoreFile(DirectoryObjectID, + ModificationTime, AttributesHash, DiffFromFileID, + StoreFilename, upload)->GetObjectID(); } + diff --git a/lib/backupstore/BackupStoreFile.h b/lib/backupstore/BackupStoreFile.h index 7c72e010..fe69caeb 100644 --- a/lib/backupstore/BackupStoreFile.h +++ b/lib/backupstore/BackupStoreFile.h @@ -14,8 +14,11 @@ #include <memory> #include <cstdlib> +#include "autogen_BackupProtocol.h" #include "BackupClientFileAttributes.h" +#include "BackupStoreFileWire.h" #include "BackupStoreFilename.h" +#include "CollectInBufferStream.h" #include "IOStream.h" #include "ReadLoggingStream.h" @@ -26,6 +29,7 @@ typedef struct int64_t mTotalFileStreamSize; } BackupStoreFileStats; +class BackgroundTask; class RunStatusProvider; // Uncomment to disable backwards compatibility @@ -40,6 +44,8 @@ class RunStatusProvider; // Have some memory allocation commands, note closing "Off" at end of file. #include "MemLeakFindOn.h" +class BackupStoreFileEncodeStream; + // -------------------------------------------------------------------------- // // Class @@ -60,8 +66,6 @@ public: virtual bool IsManaged() = 0; }; -class BackupStoreFileEncodeStream; - // -------------------------------------------------------------------------- // // Class @@ -83,9 +87,10 @@ public: public: ~DecodedStream(); - // Stream functions + // Stream functions virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); @@ -119,6 +124,64 @@ public: #endif }; + class VerifyStream : public IOStream + { + private: + enum + { + State_Header = 0, + State_FilenameHeader, + State_Filename, + State_AttributesSize, + State_Attributes, + State_Blocks, + }; + + int mState; + IOStream* mpCopyToStream; + CollectInBufferStream mCurrentUnitData; + size_t mCurrentUnitSize; + int64_t mNumBlocks; + int64_t mBlockIndexSize; + int64_t mCurrentPosition; + int64_t mBlockDataPosition; + bool mCurrentBufferIsAlternate; + CollectInBufferStream mAlternateData; + bool mBlockFromOtherFileReferenced; + int64_t mContainerID; + int64_t mDiffFromObjectID; + + public: + VerifyStream(IOStream* pCopyToStream = NULL) + : mState(State_Header), + mpCopyToStream(pCopyToStream), + mCurrentUnitSize(sizeof(file_StreamFormat)), + mNumBlocks(0), + mBlockIndexSize(0), + mCurrentPosition(0), + mBlockDataPosition(0), + mCurrentBufferIsAlternate(false), + mBlockFromOtherFileReferenced(false), + mContainerID(0), + mDiffFromObjectID(0) + { } + virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); + virtual void Close(bool CloseCopyStream = true); + virtual bool StreamDataLeft() + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual bool StreamClosed() + { + THROW_EXCEPTION(CommonException, NotSupported); + } + }; // Main interface static std::auto_ptr<BackupStoreFileEncodeStream> EncodeFile @@ -127,7 +190,8 @@ public: int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0, ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL + RunStatusProvider* pRunStatusProvider = NULL, + BackgroundTask* pBackgroundTask = NULL ); static std::auto_ptr<BackupStoreFileEncodeStream> EncodeFileDiff ( @@ -137,8 +201,19 @@ public: int Timeout, DiffTimer *pDiffTimer, int64_t *pModificationTime = 0, - bool *pIsCompletelyDifferent = 0 + bool *pIsCompletelyDifferent = 0, + BackgroundTask* pBackgroundTask = NULL ); + // Shortcut interface + static int64_t QueryStoreFileDiff(BackupProtocolCallable& protocol, + const std::string& LocalFilename, int64_t DirectoryObjectID, + int64_t DiffFromFileID, int64_t AttributesHash, + const BackupStoreFilenameClear& StoreFilename, + int Timeout = IOStream::TimeOutInfinite, + DiffTimer *pDiffTimer = NULL, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL); + static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); diff --git a/lib/backupstore/BackupStoreFileCmbIdx.cpp b/lib/backupstore/BackupStoreFileCmbIdx.cpp index c8bcc3b9..0eec3872 100644 --- a/lib/backupstore/BackupStoreFileCmbIdx.cpp +++ b/lib/backupstore/BackupStoreFileCmbIdx.cpp @@ -32,7 +32,8 @@ public: ~BSFCombinedIndexStream(); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); virtual void Initialise(IOStream &rFrom); @@ -289,7 +290,8 @@ int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout) // Created: 8/7/04 // // -------------------------------------------------------------------------- -void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes) +void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes, + int Timeout) { THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) } diff --git a/lib/backupstore/BackupStoreFileDiff.cpp b/lib/backupstore/BackupStoreFileDiff.cpp index fa8cb892..e6df11a6 100644 --- a/lib/backupstore/BackupStoreFileDiff.cpp +++ b/lib/backupstore/BackupStoreFileDiff.cpp @@ -16,7 +16,7 @@ #ifdef HAVE_TIME_H #include <time.h> -#elif HAVE_SYS_TIME_H +#elif defined HAVE_SYS_TIME_H #include <sys/time.h> #endif @@ -114,21 +114,21 @@ void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream) // Function // Name: BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *) // Purpose: Similar to EncodeFile, but takes the object ID of the file it's -// diffing from, and the index of the blocks in a stream. It'll then -// calculate which blocks can be reused from that old file. -// The timeout is the timeout value for reading the diff block index. -// If pIsCompletelyDifferent != 0, it will be set to true if the -// the two files are completely different (do not share any block), false otherwise. -// +// diffing from, and the index of the blocks in a stream. It'll then +// calculate which blocks can be reused from that old file. +// The timeout is the timeout value for reading the diff block index. +// If pIsCompletelyDifferent != 0, it will be set to true if the +// the two files are completely different (do not share any block), false otherwise. // Created: 12/1/04 // // -------------------------------------------------------------------------- std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff ( const std::string& Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, - IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, - int64_t *pModificationTime, bool *pIsCompletelyDifferent) + const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, + IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, + int64_t *pModificationTime, bool *pIsCompletelyDifferent, + BackgroundTask* pBackgroundTask) { // Is it a symlink? { @@ -144,7 +144,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff { *pIsCompletelyDifferent = true; } - return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + return EncodeFile(Filename, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); // BackgroundTask } } @@ -162,7 +166,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff { *pIsCompletelyDifferent = true; } - return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + return EncodeFile(Filename, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); // BackgroundTask } // Pointer to recipe we're going to create @@ -210,7 +218,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff new BackupStoreFileEncodeStream); // Do the initial setup - stream->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime); + stream->Setup(Filename, precipe, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); precipe = 0; // Stream has taken ownership of this // Tell user about completely different status? @@ -515,7 +527,7 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> // Search for each block size in turn // NOTE: Do the smallest size first, so that the scheme for adding - // entries in the found list works as expected and replaces smallers block + // entries in the found list works as expected and replaces smaller blocks // with larger blocks when it finds matches at the same offset in the file. for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s) { @@ -629,7 +641,7 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> { if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks)) { - BOX_TRACE("Found block match for " << hash << " of " << Sizes[s] << " bytes at offset " << fileOffset); + BOX_TRACE("Found block match of " << Sizes[s] << " bytes with hash " << hash << " at offset " << fileOffset); goodnessOfFit[fileOffset] = Sizes[s]; // Block matched, roll the checksum forward to the next block without doing @@ -657,7 +669,8 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> } else { - BOX_TRACE("False alarm match for " << hash << " of " << Sizes[s] << " bytes at offset " << fileOffset); + // Too many to log + // BOX_TRACE("False alarm match of " << Sizes[s] << " bytes with hash " << hash << " at offset " << fileOffset); } int64_t NumBlocksFound = static_cast<int64_t>( @@ -1029,9 +1042,11 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA { char b[64]; #ifdef WIN32 - sprintf(b, "%8I64d", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); + snprintf(b, sizeof(b), "%8I64d", (int64_t) + (rRecipe[e].mpStartBlock - pIndex)); #else - sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); + snprintf(b, sizeof(b), "%8lld", (int64_t) + (rRecipe[e].mpStartBlock - pIndex)); #endif BOX_TRACE(std::setw(8) << rRecipe[e].mSpaceBefore << diff --git a/lib/backupstore/BackupStoreFileEncodeStream.cpp b/lib/backupstore/BackupStoreFileEncodeStream.cpp index b53c4c26..83333a5b 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.cpp +++ b/lib/backupstore/BackupStoreFileEncodeStream.cpp @@ -11,6 +11,7 @@ #include <string.h> +#include "BackgroundTask.h" #include "BackupClientFileAttributes.h" #include "BackupStoreConstants.h" #include "BackupStoreException.h" @@ -40,25 +41,28 @@ using namespace BackupStoreFileCryptVar; // // -------------------------------------------------------------------------- BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() - : mpRecipe(0), - mpFile(0), - mpLogging(0), - mpRunStatusProvider(NULL), - mStatus(Status_Header), - mSendData(true), - mTotalBlocks(0), - mAbsoluteBlockNumber(-1), - mInstructionNumber(-1), - mNumBlocks(0), - mCurrentBlock(-1), - mCurrentBlockEncodedSize(0), - mPositionInCurrentBlock(0), - mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), - mLastBlockSize(0), - mTotalBytesSent(0), - mpRawBuffer(0), - mAllocatedBufferSize(0), - mEntryIVBase(0) +: mpRecipe(0), + mpFile(0), + mpLogging(0), + mpRunStatusProvider(NULL), + mpBackgroundTask(NULL), + mStatus(Status_Header), + mSendData(true), + mTotalBlocks(0), + mBytesToUpload(0), + mBytesUploaded(0), + mAbsoluteBlockNumber(-1), + mInstructionNumber(-1), + mNumBlocks(0), + mCurrentBlock(-1), + mCurrentBlockEncodedSize(0), + mPositionInCurrentBlock(0), + mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), + mLastBlockSize(0), + mTotalBytesSent(0), + mpRawBuffer(0), + mAllocatedBufferSize(0), + mEntryIVBase(0) { } @@ -78,21 +82,21 @@ BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() ::free(mpRawBuffer); mpRawBuffer = 0; } - + // Close the file, which we might have open if(mpFile) { delete mpFile; mpFile = 0; } - + // Clear up logging stream if(mpLogging) { delete mpLogging; mpLogging = 0; } - + // Free the recipe if(mpRecipe != 0) { @@ -115,7 +119,8 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, BackupStoreFileEncodeStream::Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) + RunStatusProvider* pRunStatusProvider, + BackgroundTask* pBackgroundTask) { // Pointer to a blank recipe which we might create BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; @@ -128,27 +133,27 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, BackupClientFileAttributes attr; attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime, 0 /* not interested in attr mod time */, &fileSize); - + // Might need to create a blank recipe... if(pRecipe == 0) { pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); - + BackupStoreFileEncodeStream::RecipeInstruction instruction; instruction.mSpaceBefore = fileSize; // whole file instruction.mBlocks = 0; // no blocks instruction.mpStartBlock = 0; // no block - pblankRecipe->push_back(instruction); + pblankRecipe->push_back(instruction); pRecipe = pblankRecipe; } - + // Tell caller? if(pModificationTime != 0) { *pModificationTime = modTime; } - + // Go through each instruction in the recipe and work out how many blocks // it will add, and the max clear size of these blocks int maxBlockClearSize = 0; @@ -162,14 +167,15 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize); // Add to accumlated total mTotalBlocks += numBlocks; + mBytesToUpload += (*pRecipe)[inst].mSpaceBefore; // Update maximum clear size if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize; if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize; } - + // Add number of blocks copied from the previous file mTotalBlocks += (*pRecipe)[inst].mBlocks; - + // Check for bad things if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0)) { @@ -182,7 +188,7 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize; } } - + // Send data? (symlinks don't have any data in them) mSendData = !attr.IsSymLink(); @@ -191,7 +197,7 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, { maxBlockClearSize = 0; } - + // Header file_StreamFormat hdr; hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1); @@ -201,16 +207,16 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128); hdr.mOptions = 0; // no options defined yet - + // Write header to stream mData.Write(&hdr, sizeof(hdr)); - + // Write filename to stream rStoreFilename.WriteToStream(mData); - + // Write attributes to stream attr.WriteToStream(mData); - + // Allocate some buffers for writing data if(mSendData) { @@ -229,10 +235,10 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, mpLogging = mpFile; mpFile = NULL; } - + // Work out the largest possible block required for the encoded data mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); - + // Then allocate two blocks of this size mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize); if(mpRawBuffer == 0) @@ -256,13 +262,13 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, blkhdr.mNumBlocks = box_hton64(0); mData.Write(&blkhdr, sizeof(blkhdr)); } - + // Ready for reading mData.SetForReading(); - + // Update stats BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize; - + // Finally, store the pointer to the recipe, when we know exceptions won't occur mpRecipe = pRecipe; } @@ -276,8 +282,9 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, } throw; } - + mpRunStatusProvider = pRunStatusProvider; + mpBackgroundTask = pBackgroundTask; } @@ -296,14 +303,14 @@ void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t do { rBlockSizeOut *= 2; - + rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut; - + } while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); - + // Last block size rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut); - + // Avoid small blocks? if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN && rNumBlocksOut > 1) @@ -312,7 +319,7 @@ void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t --rNumBlocksOut; rLastBlockSizeOut += rBlockSizeOut; } - + // checks! ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize); //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut); @@ -335,26 +342,39 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { return 0; } - + if(mpRunStatusProvider && mpRunStatusProvider->StopRun()) { THROW_EXCEPTION(BackupStoreException, SignalReceived); } + if(mpBackgroundTask) + { + BackgroundTask::State state = (mpRecipe->at(0).mBlocks == 0) + ? BackgroundTask::Uploading_Full + : BackgroundTask::Uploading_Patch; + if(!mpBackgroundTask->RunBackgroundTask(state, mBytesUploaded, + mBytesToUpload)) + { + THROW_EXCEPTION(BackupStoreException, + CancelledByBackgroundTask); + } + } + int bytesToRead = NBytes; uint8_t *buffer = (uint8_t*)pBuffer; - + while(bytesToRead > 0 && mStatus != Status_Finished) { if(mStatus == Status_Header || mStatus == Status_BlockListing) { // Header or block listing phase -- send from the buffered stream - + // Send bytes from the data buffer int b = mData.Read(buffer, bytesToRead, Timeout); bytesToRead -= b; buffer += b; - + // Check to see if all the data has been used from this stream if(!mData.StreamDataLeft()) { @@ -367,7 +387,7 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { // Reset the buffer so it can be used for the next phase mData.Reset(); - + // Get buffer ready for index? if(mStatus == Status_Header) { @@ -377,14 +397,14 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) ASSERT(mpRecipe != 0); blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID()); blkhdr.mNumBlocks = box_hton64(mTotalBlocks); - + // Generate the IV base Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase)); blkhdr.mEntryIVBase = box_hton64(mEntryIVBase); - + mData.Write(&blkhdr, sizeof(blkhdr)); } - + ++mStatus; } } @@ -392,7 +412,7 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) else if(mStatus == Status_Blocks) { // Block sending phase - + if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize) { // Next block! @@ -405,10 +425,10 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { SkipPreviousBlocksInInstruction(); } - + // Is there another instruction to go? ++mInstructionNumber; - + // Skip instructions which don't contain any data while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size()) && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0) @@ -416,12 +436,12 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) SkipPreviousBlocksInInstruction(); ++mInstructionNumber; } - + if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size())) { // End of blocks, go to next phase ++mStatus; - + // Set the data to reading so the index can be written mData.SetForReading(); } @@ -438,17 +458,17 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) EncodeCurrentBlock(); } } - + // Send data from the current block (if there's data to send) if(mPositionInCurrentBlock < mCurrentBlockEncodedSize) { // How much data to put in the buffer? int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock; if(s > bytesToRead) s = bytesToRead; - + // Copy it in ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s); - + // Update variables bytesToRead -= s; buffer += s; @@ -461,11 +481,11 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) ASSERT(false); } } - + // Add encoded size to stats BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead); mTotalBytesSent += (NBytes - bytesToRead); - + // Return size of data to caller return NBytes - bytesToRead; } @@ -490,27 +510,27 @@ void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction() // Index of the first block in old file (being diffed from) int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock); - + int64_t sizeToSkip = 0; for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b) { // Update stats BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; - + // Store the entry StoreBlockIndexEntry(0 - (firstIndex + b), (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize, (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum, - (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); // Increment the absolute block number -- kept encryption IV in sync ++mAbsoluteBlockNumber; - + // Add the size of this block to the size to skip sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; } - + // Move forward in the stream mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative); } @@ -528,7 +548,7 @@ void BackupStoreFileEncodeStream::SetForInstruction() { // Calculate block sizes CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize); - + // Set variables mCurrentBlock = 0; mCurrentBlockEncodedSize = 0; @@ -561,7 +581,7 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() // File should be open, but isn't. So logical error. THROW_EXCEPTION(BackupStoreException, Internal) } - + // Read the data in if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */)) @@ -571,13 +591,15 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer) } - + // Encode it mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer); - + + mBytesUploaded += blockRawSize; + //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); - + // Create block listing data -- generate checksums RollingChecksum weakChecksum(mpRawBuffer, blockRawSize); MD5Digest strongChecksum; @@ -587,7 +609,7 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() // Add entry to the index StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); - + // Set vars to reading this block mPositionInCurrentBlock = 0; } @@ -611,7 +633,7 @@ void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex // Then the clear section file_BlockIndexEntry entry; entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex)); - + // Then encrypt the encryted section // Generate the IV from the block number if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase)) @@ -645,7 +667,8 @@ void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex // Created: 8/12/03 // // -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes) +void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes, + int Timeout) { THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream) } @@ -686,11 +709,12 @@ bool BackupStoreFileEncodeStream::StreamClosed() // Created: 15/1/04 // // -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, - int64_t NumBlocksInIndex, int64_t OtherFileID) - : mpBlockIndex(pBlockIndex), - mNumBlocksInIndex(NumBlocksInIndex), - mOtherFileID(OtherFileID) +BackupStoreFileEncodeStream::Recipe::Recipe( + BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, + int64_t NumBlocksInIndex, int64_t OtherFileID) +: mpBlockIndex(pBlockIndex), + mNumBlocksInIndex(NumBlocksInIndex), + mOtherFileID(OtherFileID) { ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) } diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h index a169e036..5b9b4a61 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.h +++ b/lib/backupstore/BackupStoreFileEncodeStream.h @@ -79,14 +79,20 @@ public: const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL); + RunStatusProvider* pRunStatusProvider = NULL, + BackgroundTask* pBackgroundTask = NULL); virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); + int64_t GetBytesToUpload() { return mBytesToUpload; } int64_t GetTotalBytesSent() { return mTotalBytesSent; } + static void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, + int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); + private: enum { @@ -95,28 +101,29 @@ private: Status_BlockListing = 2, Status_Finished = 3 }; - -private: + void EncodeCurrentBlock(); - void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); void SkipPreviousBlocksInInstruction(); void SetForInstruction(); void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum); -private: Recipe *mpRecipe; IOStream *mpFile; // source file CollectInBufferStream mData; // buffer for header and index entries IOStream *mpLogging; RunStatusProvider* mpRunStatusProvider; + BackgroundTask* mpBackgroundTask; int mStatus; bool mSendData; // true if there's file data to send (ie not a symlink) int64_t mTotalBlocks; // Total number of blocks in the file + int64_t mBytesToUpload; // Total number of clear bytes to encode and upload + int64_t mBytesUploaded; // Total number of clear bytes already encoded + // excluding reused blocks already on the server. int64_t mAbsoluteBlockNumber; // The absolute block number currently being output // Instruction number int64_t mInstructionNumber; // All the below are within the current instruction - int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases + int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases int64_t mCurrentBlock; int32_t mCurrentBlockEncodedSize; int32_t mPositionInCurrentBlock; // for reading out diff --git a/lib/backupstore/BackupStoreFilenameClear.h b/lib/backupstore/BackupStoreFilenameClear.h index 595d1158..b7cf555f 100644 --- a/lib/backupstore/BackupStoreFilenameClear.h +++ b/lib/backupstore/BackupStoreFilenameClear.h @@ -42,7 +42,7 @@ public: #endif void SetClearFilename(const std::string &rToEncode); - // Setup for encryption of filenames + // Setup for encryption of filenames static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength); static void SetEncodingMethod(int Method); diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp index b6714709..efe3f7bb 100644 --- a/lib/backupstore/BackupStoreInfo.cpp +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -34,22 +34,24 @@ // // -------------------------------------------------------------------------- BackupStoreInfo::BackupStoreInfo() - : mAccountID(-1), - mDiscSet(-1), - mReadOnly(true), - mIsModified(false), - mClientStoreMarker(0), - mLastObjectIDUsed(-1), - mBlocksUsed(0), - mBlocksInCurrentFiles(0), - mBlocksInOldFiles(0), - mBlocksInDeletedFiles(0), - mBlocksInDirectories(0), - mNumFiles(0), - mNumOldFiles(0), - mNumDeletedFiles(0), - mNumDirectories(0), - mAccountEnabled(true) +: mAccountID(-1), + mDiscSet(-1), + mReadOnly(true), + mIsModified(false), + mClientStoreMarker(0), + mLastObjectIDUsed(-1), + mBlocksUsed(0), + mBlocksInCurrentFiles(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0), + mBlocksSoftLimit(0), + mBlocksHardLimit(0), + mNumCurrentFiles(0), + mNumOldFiles(0), + mNumDeletedFiles(0), + mNumDirectories(0), + mAccountEnabled(true) { } @@ -92,6 +94,31 @@ void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, info.Save(false); } +BackupStoreInfo::BackupStoreInfo(int32_t AccountID, const std::string &FileName, + int64_t BlockSoftLimit, int64_t BlockHardLimit) +: mAccountID(AccountID), + mDiscSet(-1), + mFilename(FileName), + mReadOnly(false), + mIsModified(false), + mClientStoreMarker(0), + mLastObjectIDUsed(0), + mBlocksUsed(0), + mBlocksInCurrentFiles(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0), + mBlocksSoftLimit(BlockSoftLimit), + mBlocksHardLimit(BlockHardLimit), + mNumCurrentFiles(0), + mNumOldFiles(0), + mNumDeletedFiles(0), + mNumDirectories(0), + mAccountEnabled(true) +{ + mExtraData.SetForReading(); // extra data is empty in this case +} + // -------------------------------------------------------------------------- // // Function @@ -108,16 +135,31 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { // Generate the filename std::string fn(rRootDir + INFO_FILENAME); - + // Open the file for reading (passing on optional request for revision ID) std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID)); + std::auto_ptr<BackupStoreInfo> info = Load(*rf, fn, ReadOnly); + + // Check it + if(info->GetAccountID() != AccountID) + { + THROW_FILE_ERROR("Found wrong account ID in store info", + fn, BackupStoreException, BadStoreInfoOnLoad); + } + + info->mDiscSet = DiscSet; + return info; +} +std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(IOStream& rStream, + const std::string FileName, bool ReadOnly) +{ // Read in format and version int32_t magic; - if(!rf->ReadFullBuffer(&magic, sizeof(magic), 0)) + if(!rStream.ReadFullBuffer(&magic, sizeof(magic), 0)) { THROW_FILE_ERROR("Failed to read store info file: " - "short read of magic number", fn, + "short read of magic number", FileName, BackupStoreException, CouldNotLoadStoreInfo); } @@ -135,16 +177,14 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { THROW_FILE_ERROR("Failed to read store info file: " "unknown magic " << BOX_FORMAT_HEX32(ntohl(magic)), - fn, BackupStoreException, BadStoreInfoOnLoad); + FileName, BackupStoreException, BadStoreInfoOnLoad); } // Make new object std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); - + // Put in basic location info - info->mAccountID = AccountID; - info->mDiscSet = DiscSet; - info->mFilename = fn; + info->mFilename = FileName; info->mReadOnly = ReadOnly; int64_t numDelObj = 0; @@ -152,23 +192,17 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { // Read in a header info_StreamFormat_1 hdr; - rf->Seek(0, IOStream::SeekType_Absolute); + rStream.Seek(0, IOStream::SeekType_Absolute); - if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), + if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) { THROW_FILE_ERROR("Failed to read store info header", - fn, BackupStoreException, CouldNotLoadStoreInfo); - } - - // Check it - if((int32_t)ntohl(hdr.mAccountID) != AccountID) - { - THROW_FILE_ERROR("Found wrong account ID in store info", - fn, BackupStoreException, BadStoreInfoOnLoad); + FileName, BackupStoreException, CouldNotLoadStoreInfo); } - + // Insert info from file + info->mAccountID = ntohl(hdr.mAccountID); info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker); info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed); info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); @@ -177,24 +211,16 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories); info->mBlocksSoftLimit = box_ntoh64(hdr.mBlocksSoftLimit); info->mBlocksHardLimit = box_ntoh64(hdr.mBlocksHardLimit); - + // Load up array of deleted objects numDelObj = box_ntoh64(hdr.mNumberDeletedDirectories); } else if(v2) { - Archive archive(*rf, IOStream::TimeOutInfinite); + Archive archive(rStream, IOStream::TimeOutInfinite); // Check it - int32_t FileAccountID; - archive.Read(FileAccountID); - if (FileAccountID != AccountID) - { - THROW_FILE_ERROR("Found wrong account ID in store " - "info: " << BOX_FORMAT_HEX32(FileAccountID), - fn, BackupStoreException, BadStoreInfoOnLoad); - } - + archive.Read(info->mAccountID); archive.Read(info->mAccountName); archive.Read(info->mClientStoreMarker); archive.Read(info->mLastObjectIDUsed); @@ -205,35 +231,35 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, archive.Read(info->mBlocksInDirectories); archive.Read(info->mBlocksSoftLimit); archive.Read(info->mBlocksHardLimit); - archive.Read(info->mNumFiles); - archive.Read(info->mNumOldFiles); - archive.Read(info->mNumDeletedFiles); - archive.Read(info->mNumDirectories); - archive.Read(numDelObj); + archive.Read(info->mNumCurrentFiles); + archive.Read(info->mNumOldFiles); + archive.Read(info->mNumDeletedFiles); + archive.Read(info->mNumDirectories); + archive.Read(numDelObj); } // Then load the list of deleted directories if(numDelObj > 0) { int64_t objs[NUM_DELETED_DIRS_BLOCK]; - + int64_t toload = numDelObj; while(toload > 0) { // How many in this one? int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload)); - - if(!rf->ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */)) + + if(!rStream.ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */)) { THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) } - + // Add them for(int t = 0; t < b; ++t) { info->mDeletedDirectories.push_back(box_ntoh64(objs[t])); } - + // Number loaded toload -= b; } @@ -247,24 +273,24 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, if(v2) { - Archive archive(*rf, IOStream::TimeOutInfinite); + Archive archive(rStream, IOStream::TimeOutInfinite); archive.ReadIfPresent(info->mAccountEnabled, true); } else { info->mAccountEnabled = true; } - + // If there's any data left in the info file, from future additions to // the file format, then we need to load it so that it won't be lost when // we resave the file. - IOStream::pos_type bytesLeft = rf->BytesLeftToRead(); + IOStream::pos_type bytesLeft = rStream.BytesLeftToRead(); if (bytesLeft > 0) { - rf->CopyStreamTo(info->mExtraData); + rStream.CopyStreamTo(info->mExtraData); } info->mExtraData.SetForReading(); - + // return it to caller return info; } @@ -289,10 +315,10 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( { // Generate the filename std::string fn(rRootDir + INFO_FILENAME); - + // Make new object std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); - + // Put in basic info info->mAccountID = AccountID; info->mAccountName = rAccountName; @@ -311,7 +337,7 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( info->mBlocksSoftLimit = BlockSoftLimit; info->mBlocksHardLimit = BlockHardLimit; info->mAccountEnabled = AccountEnabled; - + ExtraData.CopyStreamTo(info->mExtraData); info->mExtraData.SetForReading(); @@ -341,15 +367,22 @@ void BackupStoreInfo::Save(bool allowOverwrite) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + // Then... open a write file RaidFileWrite rf(mDiscSet, mFilename); rf.Open(allowOverwrite); - + Save(rf); + + // Commit it to disc, converting it to RAID now + rf.Commit(true); +} + +void BackupStoreInfo::Save(IOStream& rOutStream) +{ // Make header int32_t magic = htonl(INFO_MAGIC_VALUE_2); - rf.Write(&magic, sizeof(magic)); - Archive archive(rf, IOStream::TimeOutInfinite); + rOutStream.Write(&magic, sizeof(magic)); + Archive archive(rOutStream, IOStream::TimeOutInfinite); archive.Write(mAccountID); archive.Write(mAccountName); @@ -362,7 +395,7 @@ void BackupStoreInfo::Save(bool allowOverwrite) archive.Write(mBlocksInDirectories); archive.Write(mBlocksSoftLimit); archive.Write(mBlocksHardLimit); - archive.Write(mNumFiles); + archive.Write(mNumCurrentFiles); archive.Write(mNumOldFiles); archive.Write(mNumDeletedFiles); archive.Write(mNumDirectories); @@ -374,14 +407,14 @@ void BackupStoreInfo::Save(bool allowOverwrite) if(mDeletedDirectories.size() > 0) { int64_t objs[NUM_DELETED_DIRS_BLOCK]; - + int tosave = mDeletedDirectories.size(); std::vector<int64_t>::iterator i(mDeletedDirectories.begin()); while(tosave > 0) { // How many in this one? int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); - + // Add them for(int t = 0; t < b; ++t) { @@ -390,23 +423,20 @@ void BackupStoreInfo::Save(bool allowOverwrite) i++; } - // Write - rf.Write(objs, b * sizeof(int64_t)); - + // Write + rOutStream.Write(objs, b * sizeof(int64_t)); + // Number saved tosave -= b; } } - + archive.Write(mAccountEnabled); - + mExtraData.Seek(0, IOStream::SeekType_Absolute); - mExtraData.CopyStreamTo(rf); + mExtraData.CopyStreamTo(rOutStream); mExtraData.Seek(0, IOStream::SeekType_Absolute); - // Commit it to disc, converting it to RAID now - rf.Commit(true); - // Mark is as not modified mIsModified = false; } @@ -426,7 +456,6 @@ int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) COMPARE(AccountID); COMPARE(AccountName); - COMPARE(LastObjectIDUsed); COMPARE(BlocksUsed); COMPARE(BlocksInCurrentFiles); COMPARE(BlocksInOldFiles); @@ -434,32 +463,47 @@ int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) COMPARE(BlocksInDirectories); COMPARE(BlocksSoftLimit); COMPARE(BlocksHardLimit); - COMPARE(NumFiles); + COMPARE(NumCurrentFiles); COMPARE(NumOldFiles); COMPARE(NumDeletedFiles); COMPARE(NumDirectories); #undef COMPARE + if (rOldInfo.GetLastObjectIDUsed() != GetLastObjectIDUsed()) + { + BOX_NOTICE("LastObjectIDUsed changed from " << + rOldInfo.GetLastObjectIDUsed() << " to " << + GetLastObjectIDUsed()); + // Not important enough to be an error + // numChanges++; + } + return numChanges; } -#define APPLY_DELTA(field, delta) \ - if(mReadOnly) \ - { \ - THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) \ - } \ - \ - if((field + delta) < 0) \ - { \ - THROW_EXCEPTION_MESSAGE(BackupStoreException, \ - StoreInfoBlockDeltaMakesValueNegative, \ - "Failed to reduce " << #field << " from " << \ - field << " by " << delta); \ - } \ - \ - field += delta; \ +void BackupStoreInfo::ApplyDelta(int64_t& field, const std::string& field_name, + const int64_t delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly); + } + + if((field + delta) < 0) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + StoreInfoBlockDeltaMakesValueNegative, + "Failed to reduce " << field_name << " from " << + field << " by " << delta); + } + + field += delta; mIsModified = true; +} + +#define APPLY_DELTA(field, delta) \ + ApplyDelta(field, #field, delta) // -------------------------------------------------------------------------- // @@ -527,9 +571,9 @@ void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta) APPLY_DELTA(mBlocksInDirectories, Delta); } -void BackupStoreInfo::AdjustNumFiles(int64_t increase) +void BackupStoreInfo::AdjustNumCurrentFiles(int64_t increase) { - APPLY_DELTA(mNumFiles, increase); + APPLY_DELTA(mNumCurrentFiles, increase); } void BackupStoreInfo::AdjustNumOldFiles(int64_t increase) @@ -563,13 +607,13 @@ void BackupStoreInfo::CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + // Set the values mBlocksUsed = Used; mBlocksInOldFiles = InOldFiles; mBlocksInDeletedFiles = InDeletedFiles; mBlocksInDirectories = InDirectories; - + mIsModified = true; } @@ -588,9 +632,8 @@ void BackupStoreInfo::AddDeletedDirectory(int64_t DirID) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + mDeletedDirectories.push_back(DirID); - mIsModified = true; } @@ -608,14 +651,14 @@ void BackupStoreInfo::RemovedDeletedDirectory(int64_t DirID) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + std::vector<int64_t>::iterator i(std::find(mDeletedDirectories.begin(), mDeletedDirectories.end(), DirID)); if(i == mDeletedDirectories.end()) { THROW_EXCEPTION(BackupStoreException, StoreInfoDirNotInList) } + mDeletedDirectories.erase(i); - mIsModified = true; } @@ -636,7 +679,7 @@ void BackupStoreInfo::ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimi mBlocksSoftLimit = BlockSoftLimit; mBlocksHardLimit = BlockHardLimit; - + mIsModified = true; } @@ -655,15 +698,16 @@ int64_t BackupStoreInfo::AllocateObjectID() { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } + if(mLastObjectIDUsed < 0) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotInitialised) } - + + mIsModified = true; + // Return the next object ID return ++mLastObjectIDUsed; - - mIsModified = true; } @@ -682,9 +726,8 @@ void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + mClientStoreMarker = ClientStoreMarker; - mIsModified = true; } @@ -703,9 +746,8 @@ void BackupStoreInfo::SetAccountName(const std::string& rName) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + mAccountName = rName; - mIsModified = true; } diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h index 752cc44a..ec6cd2cb 100644 --- a/lib/backupstore/BackupStoreInfo.h +++ b/lib/backupstore/BackupStoreInfo.h @@ -77,20 +77,28 @@ private: BackupStoreInfo(); // No copying allowed BackupStoreInfo(const BackupStoreInfo &); - + public: // Create a New account, saving a blank info object to the disc - static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit); - + static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, + int64_t BlockSoftLimit, int64_t BlockHardLimit); + BackupStoreInfo(int32_t AccountID, const std::string &FileName, + int64_t BlockSoftLimit, int64_t BlockHardLimit); + // Load it from the store static std::auto_ptr<BackupStoreInfo> Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0); - + + // Load it from a stream (file or RaidFile) + static std::auto_ptr<BackupStoreInfo> Load(IOStream& rStream, + const std::string FileName, bool ReadOnly); + // Has info been modified? bool IsModified() const {return mIsModified;} - + // Save modified infomation back to store void Save(bool allowOverwrite = true); - + void Save(IOStream& rOutStream); + // Data access functions int32_t GetAccountID() const {return mAccountID;} int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;} @@ -102,7 +110,7 @@ public: const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;} int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;} int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;} - int64_t GetNumFiles() const {return mNumFiles;} + int64_t GetNumCurrentFiles() const {return mNumCurrentFiles;} int64_t GetNumOldFiles() const {return mNumOldFiles;} int64_t GetNumDeletedFiles() const {return mNumDeletedFiles;} int64_t GetNumDirectories() const {return mNumDirectories;} @@ -111,7 +119,7 @@ public: int GetDiscSetNumber() const {return mDiscSet;} int ReportChangesTo(BackupStoreInfo& rOldInfo); - + // Data modification functions void ChangeBlocksUsed(int64_t Delta); void ChangeBlocksInCurrentFiles(int64_t Delta); @@ -122,14 +130,14 @@ public: void AddDeletedDirectory(int64_t DirID); void RemovedDeletedDirectory(int64_t DirID); void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit); - void AdjustNumFiles(int64_t increase); + void AdjustNumCurrentFiles(int64_t increase); void AdjustNumOldFiles(int64_t increase); void AdjustNumDeletedFiles(int64_t increase); void AdjustNumDirectories(int64_t increase); - + // Object IDs int64_t AllocateObjectID(); - + // Client marker set and get int64_t GetClientStoreMarker() const {return mClientStoreMarker;} void SetClientStoreMarker(int64_t ClientStoreMarker); @@ -152,6 +160,20 @@ public: int64_t BlockSoftLimit, int64_t BlockHardLimit, bool AccountEnabled, IOStream& ExtraData); + typedef struct + { + int64_t mLastObjectIDUsed; + int64_t mBlocksUsed; + int64_t mBlocksInCurrentFiles; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; + int64_t mNumCurrentFiles; + int64_t mNumOldFiles; + int64_t mNumDeletedFiles; + int64_t mNumDirectories; + } Adjustment; + private: // Location information // Be VERY careful about changing types of these values, as @@ -162,10 +184,10 @@ private: std::string mFilename; bool mReadOnly; bool mIsModified; - + // Client infomation int64_t mClientStoreMarker; - + // Account information int64_t mLastObjectIDUsed; int64_t mBlocksUsed; @@ -175,13 +197,16 @@ private: int64_t mBlocksInDirectories; int64_t mBlocksSoftLimit; int64_t mBlocksHardLimit; - int64_t mNumFiles; + int64_t mNumCurrentFiles; int64_t mNumOldFiles; int64_t mNumDeletedFiles; int64_t mNumDirectories; std::vector<int64_t> mDeletedDirectories; bool mAccountEnabled; CollectInBufferStream mExtraData; + + void ApplyDelta(int64_t& field, const std::string& field_name, + const int64_t delta); }; #endif // BACKUPSTOREINFO__H diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp index 26f9acca..b2ea1abd 100644 --- a/lib/backupstore/BackupStoreRefCountDatabase.cpp +++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp @@ -9,6 +9,8 @@ #include "Box.h" +#include <stdio.h> + #include <algorithm> #include "BackupStoreRefCountDatabase.h" @@ -34,12 +36,85 @@ // // -------------------------------------------------------------------------- BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const - BackupStoreAccountDatabase::Entry& rAccount) + BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly, + bool Temporary, std::auto_ptr<FileStream> apDatabaseFile) : mAccount(rAccount), - mFilename(GetFilename(rAccount)), - mReadOnly(true), - mIsModified(false) + mFilename(GetFilename(rAccount, Temporary)), + mReadOnly(ReadOnly), + mIsModified(false), + mIsTemporaryFile(Temporary), + mapDatabaseFile(apDatabaseFile) +{ + ASSERT(!(ReadOnly && Temporary)); // being both doesn't make sense +} + +void BackupStoreRefCountDatabase::Commit() { + if (!mIsTemporaryFile) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Cannot commit a permanent reference count database"); + } + + if (!mapDatabaseFile.get()) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Reference count database is already closed"); + } + + mapDatabaseFile->Close(); + mapDatabaseFile.reset(); + + std::string Final_Filename = GetFilename(mAccount, false); + + #ifdef WIN32 + if(FileExists(Final_Filename) && unlink(Final_Filename.c_str()) != 0) + { + THROW_EMU_FILE_ERROR("Failed to delete old permanent refcount " + "database file", mFilename, CommonException, + OSFileError); + } + #endif + + if(rename(mFilename.c_str(), Final_Filename.c_str()) != 0) + { + THROW_EMU_ERROR("Failed to rename temporary refcount database " + "file from " << mFilename << " to " << + Final_Filename, CommonException, OSFileError); + } + + mFilename = Final_Filename; + mIsModified = false; + mIsTemporaryFile = false; +} + +void BackupStoreRefCountDatabase::Discard() +{ + if (!mIsTemporaryFile) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Cannot discard a permanent reference count database"); + } + + // Under normal conditions, we should know whether the file is still + // open or not, and not Discard it unless it's open. However if the + // final rename() fails during Commit(), the file will already be + // closed, and we don't want to blow up here in that case. + if (mapDatabaseFile.get()) + { + mapDatabaseFile->Close(); + mapDatabaseFile.reset(); + } + + if(unlink(mFilename.c_str()) != 0) + { + THROW_EMU_FILE_ERROR("Failed to delete temporary refcount " + "database file", mFilename, CommonException, + OSFileError); + } + + mIsModified = false; + mIsTemporaryFile = false; } // -------------------------------------------------------------------------- @@ -52,16 +127,35 @@ BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const // -------------------------------------------------------------------------- BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase() { + if (mIsTemporaryFile) + { + // Don't throw exceptions in a destructor. + BOX_ERROR("BackupStoreRefCountDatabase destroyed without " + "explicit commit or discard"); + try + { + Discard(); + } + catch(BoxException &e) + { + BOX_LOG_SYS_ERROR("Failed to discard BackupStoreRefCountDatabase " + "in destructor: " << e.what()); + } + } } std::string BackupStoreRefCountDatabase::GetFilename(const - BackupStoreAccountDatabase::Entry& rAccount) + BackupStoreAccountDatabase::Entry& rAccount, bool Temporary) { std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount); ASSERT(RootDir[RootDir.size() - 1] == '/' || RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); - std::string fn(RootDir + REFCOUNT_FILENAME ".db"); + std::string fn(RootDir + REFCOUNT_FILENAME ".rdb"); + if(Temporary) + { + fn += "X"; + } RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet())); return RaidFileUtil::MakeWriteFileName(rdiscSet, fn); @@ -72,40 +166,53 @@ std::string BackupStoreRefCountDatabase::GetFilename(const // Function // Name: BackupStoreRefCountDatabase::Create(int32_t, // const std::string &, int, bool) -// Purpose: Create a new database, overwriting an existing -// one only if AllowOverwrite is true. +// Purpose: Create a blank database, using a temporary file that +// you must Discard() or Commit() to make permanent. // Created: 2003/08/28 // // -------------------------------------------------------------------------- -void BackupStoreRefCountDatabase::Create(const - BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite) +std::auto_ptr<BackupStoreRefCountDatabase> + BackupStoreRefCountDatabase::Create + (const BackupStoreAccountDatabase::Entry& rAccount) { // Initial header refcount_StreamFormat hdr; hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE); hdr.mAccountID = htonl(rAccount.GetID()); - // Generate the filename - std::string Filename = GetFilename(rAccount); + std::string Filename = GetFilename(rAccount, true); // temporary // Open the file for writing - if (FileExists(Filename) && !AllowOverwrite) - { - THROW_FILE_ERROR("Failed to overwrite refcount database: " - "not allowed here", Filename, RaidFileException, - CannotOverwriteExistingFile); - } - - int flags = O_CREAT | O_BINARY | O_RDWR; - if (!AllowOverwrite) + if (FileExists(Filename)) { - flags |= O_EXCL; + BOX_WARNING(BOX_FILE_MESSAGE(Filename, "Overwriting existing " + "temporary reference count database")); + if (unlink(Filename.c_str()) != 0) + { + THROW_SYS_FILE_ERROR("Failed to delete old temporary " + "reference count database file", Filename, + CommonException, OSFileError); + } } + int flags = O_CREAT | O_BINARY | O_RDWR | O_EXCL; std::auto_ptr<FileStream> DatabaseFile(new FileStream(Filename, flags)); // Write header DatabaseFile->Write(&hdr, sizeof(hdr)); + + // Make new object + std::auto_ptr<BackupStoreRefCountDatabase> refcount( + new BackupStoreRefCountDatabase(rAccount, false, true, + DatabaseFile)); + + // The root directory must always have one reference for a database + // to be valid, so set that now on the new database. This will leave + // mIsModified set to true. + refcount->SetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); + + // return it to caller + return refcount; } // -------------------------------------------------------------------------- @@ -122,12 +229,13 @@ void BackupStoreRefCountDatabase::Create(const std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly) { - // Generate the filename - std::string filename = GetFilename(rAccount); + // Generate the filename. Cannot open a temporary database, so it must + // be a permanent one. + std::string Filename = GetFilename(rAccount, false); int flags = ReadOnly ? O_RDONLY : O_RDWR; // Open the file for read/write - std::auto_ptr<FileStream> dbfile(new FileStream(filename, + std::auto_ptr<FileStream> dbfile(new FileStream(Filename, flags | O_BINARY)); // Read in a header @@ -135,7 +243,7 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) { THROW_FILE_ERROR("Failed to read refcount database: " - "short read", filename, BackupStoreException, + "short read", Filename, BackupStoreException, CouldNotLoadStoreInfo); } @@ -144,16 +252,14 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID()) { THROW_FILE_ERROR("Failed to read refcount database: " - "bad magic number", filename, BackupStoreException, + "bad magic number", Filename, BackupStoreException, BadStoreInfoOnLoad); } // Make new object - std::auto_ptr<BackupStoreRefCountDatabase> refcount(new BackupStoreRefCountDatabase(rAccount)); - - // Put in basic location info - refcount->mReadOnly = ReadOnly; - refcount->mapDatabaseFile = dbfile; + std::auto_ptr<BackupStoreRefCountDatabase> refcount( + new BackupStoreRefCountDatabase(rAccount, ReadOnly, false, + dbfile)); // return it to caller return refcount; @@ -229,6 +335,7 @@ void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID, mapDatabaseFile->Seek(offset, SEEK_SET); refcount_t RefCountNetOrder = htonl(NewRefCount); mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder)); + mIsModified = true; } bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) @@ -240,3 +347,31 @@ bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) return (refcount > 0); } +int BackupStoreRefCountDatabase::ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs) +{ + int ErrorCount = 0; + int64_t MaxOldObjectId = rOldRefs.GetLastObjectIDUsed(); + int64_t MaxNewObjectId = GetLastObjectIDUsed(); + + for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID; + ObjectID < std::max(MaxOldObjectId, MaxNewObjectId); + ObjectID++) + { + typedef BackupStoreRefCountDatabase::refcount_t refcount_t; + refcount_t OldRefs = (ObjectID <= MaxOldObjectId) ? + rOldRefs.GetRefCount(ObjectID) : 0; + refcount_t NewRefs = (ObjectID <= MaxNewObjectId) ? + this->GetRefCount(ObjectID) : 0; + + if (OldRefs != NewRefs) + { + BOX_WARNING("Reference count of object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " changed from " << OldRefs << + " to " << NewRefs); + ErrorCount++; + } + } + + return ErrorCount; +} diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h index 93c79afb..915653a4 100644 --- a/lib/backupstore/BackupStoreRefCountDatabase.h +++ b/lib/backupstore/BackupStoreRefCountDatabase.h @@ -15,6 +15,7 @@ #include <vector> #include "BackupStoreAccountDatabase.h" +#include "BackupStoreConstants.h" #include "FileStream.h" class BackupStoreCheck; @@ -59,48 +60,39 @@ public: private: // Creation through static functions only BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry& - rAccount); + rAccount, bool ReadOnly, bool Temporary, + std::auto_ptr<FileStream> apDatabaseFile); // No copying allowed BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &); public: - // Create a new database for a new account. This method will refuse - // to overwrite any existing file. - static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount) - { - Create(rAccount, false); - } - + // Create a blank database, using a temporary file that you must + // Discard() or Commit() to make permanent. + static std::auto_ptr<BackupStoreRefCountDatabase> Create + (const BackupStoreAccountDatabase::Entry& rAccount); + void Commit(); + void Discard(); + // Load it from the store static std::auto_ptr<BackupStoreRefCountDatabase> Load(const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); - + typedef uint32_t refcount_t; // Data access functions refcount_t GetRefCount(int64_t ObjectID) const; int64_t GetLastObjectIDUsed() const; - + // Data modification functions void AddReference(int64_t ObjectID); // RemoveReference returns false if refcount drops to zero bool RemoveReference(int64_t ObjectID); + int ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs); private: - // Create a new database for an existing account. Used during - // account checking if opening the old database throws an exception. - // This method will overwrite any existing file. - static void CreateForRegeneration(const - BackupStoreAccountDatabase::Entry& rAccount) - { - Create(rAccount, true); - } - - static void Create(const BackupStoreAccountDatabase::Entry& rAccount, - bool AllowOverwrite); - static std::string GetFilename(const BackupStoreAccountDatabase::Entry& - rAccount); + rAccount, bool Temporary); + IOStream::pos_type GetSize() const { return mapDatabaseFile->GetPosition() + @@ -122,7 +114,13 @@ private: std::string mFilename; bool mReadOnly; bool mIsModified; + bool mIsTemporaryFile; std::auto_ptr<FileStream> mapDatabaseFile; + + bool NeedsCommitOrDiscard() + { + return mapDatabaseFile.get() && mIsModified && mIsTemporaryFile; + } }; #endif // BACKUPSTOREREFCOUNTDATABASE__H diff --git a/lib/backupstore/HousekeepStoreAccount.cpp b/lib/backupstore/HousekeepStoreAccount.cpp index 75feda7f..d5acf62c 100644 --- a/lib/backupstore/HousekeepStoreAccount.cpp +++ b/lib/backupstore/HousekeepStoreAccount.cpp @@ -2,7 +2,7 @@ // // File // Name: HousekeepStoreAccount.cpp -// Purpose: +// Purpose: // Created: 11/12/03 // // -------------------------------------------------------------------------- @@ -51,6 +51,7 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, mDeletionSizeTarget(0), mPotentialDeletionsTotalSize(0), mMaxSizeInPotentialDeletions(0), + mErrorCount(0), mBlocksUsed(0), mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), @@ -61,8 +62,6 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, mBlocksInDirectoriesDelta(0), mFilesDeleted(0), mEmptyDirectoriesDeleted(0), - mSuppressRefCountChangeWarnings(false), - mRefCountsAdjusted(0), mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY) { std::ostringstream tag; @@ -80,6 +79,20 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, // -------------------------------------------------------------------------- HousekeepStoreAccount::~HousekeepStoreAccount() { + if(mapNewRefs.get()) + { + // Discard() can throw exception, but destructors aren't supposed to do that, so + // just catch and log them. + try + { + mapNewRefs->Discard(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to destroy housekeeper: discarding the refcount " + "database threw an exception: " << e.what()); + } + } } // -------------------------------------------------------------------------- @@ -105,7 +118,7 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) { if(KeepTryingForever) { - BOX_WARNING("Failed to lock account for housekeeping, " + BOX_INFO("Failed to lock account for housekeeping, " "still trying..."); while(!writeLock.TryAndGetLock(writeLockFilename, 0600 /* restrictive file permissions */)) @@ -143,204 +156,108 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) mDeletionSizeTarget = 0; } - // initialise the refcount database - mNewRefCounts.clear(); - // try to pre-allocate as much memory as we need - mNewRefCounts.reserve(info->GetLastObjectIDUsed()); - // initialise the refcount of the root entry - mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0); - mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1; + BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); + mapNewRefs = BackupStoreRefCountDatabase::Create(account); // Scan the directory for potential things to delete // This will also remove eligible items marked with RemoveASAP - bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID); + bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, + *info); - // If scan directory stopped for some reason, probably parent - // instructed to terminate, stop now. if(!continueHousekeeping) { - // If any files were marked "delete now", then update - // the size of the store. - if(mBlocksUsedDelta != 0 || - mBlocksInOldFilesDelta != 0 || - mBlocksInDeletedFilesDelta != 0) - { - info->ChangeBlocksUsed(mBlocksUsedDelta); - info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); - info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); - - // Save the store info back - info->ReportChangesTo(*pOldInfo); - info->Save(); - } - - return false; + // The scan was incomplete, so the new block counts are + // incorrect, we can't rely on them. It's better to discard + // the new info and adjust the old one instead. + info = pOldInfo; + + // We're about to reset counters and exit, so report what + // happened now. + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " removed " << + (0 - mBlocksUsedDelta) << " blocks (" << + mFilesDeleted << " files, " << + mEmptyDirectoriesDeleted << " dirs) and the directory " + "scan was interrupted"); } - // Log any difference in opinion between the values recorded in - // the store info, and the values just calculated for space usage. - // BLOCK - { - int64_t used = info->GetBlocksUsed(); - int64_t usedOld = info->GetBlocksInOldFiles(); - int64_t usedDeleted = info->GetBlocksInDeletedFiles(); - int64_t usedDirectories = info->GetBlocksInDirectories(); - - // If the counts were wrong, taking into account RemoveASAP - // items deleted, log a message - if((used + mBlocksUsedDelta) != mBlocksUsed - || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles - || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles - || usedDirectories != mBlocksInDirectories) - { - // Log this - BOX_ERROR("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " found " - "and fixed wrong block counts: " - "used (" << - (used + mBlocksUsedDelta) << "," << - mBlocksUsed << "), old (" << - (usedOld + mBlocksInOldFilesDelta) << "," << - mBlocksInOldFiles << "), deleted (" << - (usedDeleted + mBlocksInDeletedFilesDelta) << - "," << mBlocksInDeletedFiles << "), dirs (" << - usedDirectories << "," << mBlocksInDirectories - << ")"); - } - - // If the current values don't match, store them - if(used != mBlocksUsed - || usedOld != mBlocksInOldFiles - || usedDeleted != mBlocksInDeletedFiles - || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta)) - { - // Set corrected values in store info - info->CorrectAllUsedValues(mBlocksUsed, - mBlocksInOldFiles, mBlocksInDeletedFiles, - mBlocksInDirectories + mBlocksInDirectoriesDelta); - - info->ReportChangesTo(*pOldInfo); - info->Save(); - } - } - - // Reset the delta counts for files, as they will include - // RemoveASAP flagged files deleted during the initial scan. + // If housekeeping made any changes, such as deleting RemoveASAP files, + // the differences in block counts will be recorded in the deltas. + info->ChangeBlocksUsed(mBlocksUsedDelta); + info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); + info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); - // keep for reporting + // Reset the delta counts for files, as they will include + // RemoveASAP flagged files deleted during the initial scan. + // keep removeASAPBlocksUsedDelta for reporting int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta; - mBlocksUsedDelta = 0; mBlocksInOldFilesDelta = 0; mBlocksInDeletedFilesDelta = 0; - - // Go and delete items from the accounts - bool deleteInterrupted = DeleteFiles(); - - // If that wasn't interrupted, remove any empty directories which - // are also marked as deleted in their containing directory - if(!deleteInterrupted) - { - deleteInterrupted = DeleteEmptyDirectories(); - } - - // Log deletion if anything was deleted - if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) - { - BOX_INFO("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " " - "removed " << - (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << - " blocks (" << mFilesDeleted << " files, " << - mEmptyDirectoriesDeleted << " dirs)" << - (deleteInterrupted?" and was interrupted":"")); - } + // If scan directory stopped for some reason, probably parent + // instructed to terminate, stop now. + // // We can only update the refcount database if we successfully // finished our scan of all directories, otherwise we don't actually // know which of the new counts are valid and which aren't - // (we might not have seen second references to some objects, etc.) + // (we might not have seen second references to some objects, etc.). - BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); - std::auto_ptr<BackupStoreRefCountDatabase> apReferences; + if(!continueHousekeeping) + { + mapNewRefs->Discard(); + info->Save(); + return false; + } + + // Report any UNexpected changes, and consider them to be errors. + // Do this before applying the expected changes below. + mErrorCount += info->ReportChangesTo(*pOldInfo); + info->Save(); + + // Try to load the old reference count database and check whether + // any counts have changed. We want to compare the mapNewRefs to + // apOldRefs before we delete any files, because that will also change + // the reference count in a way that's not an error. - // try to load the reference count database try { - apReferences = BackupStoreRefCountDatabase::Load(account, - false); + std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs = + BackupStoreRefCountDatabase::Load(account, false); + mErrorCount += mapNewRefs->ReportChangesTo(*apOldRefs); } catch(BoxException &e) { - BOX_WARNING("Reference count database is missing or corrupted " - "during housekeeping, creating a new one."); - mSuppressRefCountChangeWarnings = true; - BackupStoreRefCountDatabase::CreateForRegeneration(account); - apReferences = BackupStoreRefCountDatabase::Load(account, - false); + BOX_WARNING("Reference count database was missing or " + "corrupted during housekeeping, cannot check it for " + "errors."); + mErrorCount++; } - int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed(); + // Go and delete items from the accounts + bool deleteInterrupted = DeleteFiles(*info); - for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID; - ObjectID < mNewRefCounts.size(); ObjectID++) + // If that wasn't interrupted, remove any empty directories which + // are also marked as deleted in their containing directory + if(!deleteInterrupted) { - if (ObjectID > LastUsedObjectIdOnDisk) - { - if (!mSuppressRefCountChangeWarnings) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " not found in database, added" - " with " << mNewRefCounts[ObjectID] << - " references"); - } - apReferences->SetRefCount(ObjectID, - mNewRefCounts[ObjectID]); - mRefCountsAdjusted++; - LastUsedObjectIdOnDisk = ObjectID; - continue; - } - - BackupStoreRefCountDatabase::refcount_t OldRefCount = - apReferences->GetRefCount(ObjectID); - - if (OldRefCount != mNewRefCounts[ObjectID]) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " changed from " << OldRefCount << - " to " << mNewRefCounts[ObjectID]); - apReferences->SetRefCount(ObjectID, - mNewRefCounts[ObjectID]); - mRefCountsAdjusted++; - } + deleteInterrupted = DeleteEmptyDirectories(*info); } - // zero excess references in the database - for (int64_t ObjectID = mNewRefCounts.size(); - ObjectID <= LastUsedObjectIdOnDisk; ObjectID++) + // Log deletion if anything was deleted + if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) { - BackupStoreRefCountDatabase::refcount_t OldRefCount = - apReferences->GetRefCount(ObjectID); - BackupStoreRefCountDatabase::refcount_t NewRefCount = 0; - - if (OldRefCount != NewRefCount) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " changed from " << OldRefCount << - " to " << NewRefCount << " (not found)"); - apReferences->SetRefCount(ObjectID, NewRefCount); - mRefCountsAdjusted++; - } + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " " + "removed " << + (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << + " blocks (" << mFilesDeleted << " files, " << + mEmptyDirectoriesDeleted << " dirs)" << + (deleteInterrupted?" and was interrupted":"")); } - // force file to be saved and closed before releasing the lock below - apReferences.reset(); - - // Make sure the delta's won't cause problems if the counts are - // really wrong, and it wasn't fixed because the store was + // Make sure the delta's won't cause problems if the counts are + // really wrong, and it wasn't fixed because the store was // updated during the scan. if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) { @@ -348,28 +265,31 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) } if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles())) { - mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); + mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); } if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles())) { - mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); + mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); } if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories())) { mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories()); } - + // Update the usage counts in the store info->ChangeBlocksUsed(mBlocksUsedDelta); info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta); - + // Save the store info back - info->ReportChangesTo(*pOldInfo); info->Save(); - - // Explicity release the lock (would happen automatically on + + // force file to be saved and closed before releasing the lock below + mapNewRefs->Commit(); + mapNewRefs.reset(); + + // Explicity release the lock (would happen automatically on // going out of scope, included for code clarity) writeLock.ReleaseLock(); @@ -405,7 +325,8 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF // Created: 11/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) +bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID, + BackupStoreInfo& rBackupStoreInfo) { #ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) @@ -430,18 +351,19 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Open it. std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, objectFilename)); - + // Add the size of the directory on disc to the size being calculated int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); mBlocksInDirectories += originalDirSizeInBlocks; mBlocksUsed += originalDirSizeInBlocks; - + // Read the directory in BackupStoreDirectory dir; BufferedStream buf(*dirStream); dir.ReadFromStream(buf, IOStream::TimeOutInfinite); + dir.SetUserInfo1_SizeInBlocks(originalDirSizeInBlocks); dirStream->Close(); - + // Is it empty? if(dir.GetNumberOfEntries() == 0) { @@ -459,11 +381,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) while((en = i.Next()) != 0) { // This directory references this object - if (mNewRefCounts.size() <= en->GetObjectID()) - { - mNewRefCounts.resize(en->GetObjectID() + 1, 0); - } - mNewRefCounts[en->GetObjectID()]++; + mapNewRefs->AddReference(en->GetObjectID()); } } @@ -482,14 +400,15 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { int16_t enFlags = en->GetFlags(); if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0 - && (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) + && (en->IsDeleted() || en->IsOld())) { // Delete this immediately. - DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks); - + DeleteFile(ObjectID, en->GetObjectID(), dir, + objectFilename, rBackupStoreInfo); + // flag as having done something deletedSomething = true; - + // Must start the loop from the beginning again, as iterator is now // probably invalid. break; @@ -497,7 +416,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } } while(deletedSomething); } - + // BLOCK { // Add files to the list of potential deletions @@ -517,9 +436,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) int16_t enFlags = en->GetFlags(); int64_t enSizeInBlocks = en->GetSizeInBlocks(); mBlocksUsed += enSizeInBlocks; - if(enFlags & BackupStoreDirectory::Entry::Flags_OldVersion) mBlocksInOldFiles += enSizeInBlocks; - if(enFlags & BackupStoreDirectory::Entry::Flags_Deleted) mBlocksInDeletedFiles += enSizeInBlocks; - + if(en->IsOld()) mBlocksInOldFiles += enSizeInBlocks; + if(en->IsDeleted()) mBlocksInDeletedFiles += enSizeInBlocks; + // Work out ages of this version from the last mark int32_t enVersionAge = 0; std::map<version_t, int32_t>::iterator enVersionAgeI( @@ -536,9 +455,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) markVersionAges[version_t(en->GetName().GetEncodedFilename(), en->GetMarkNumber())] = enVersionAge; } // enVersionAge is now the age of this version. - + // Potentially add it to the list if it's deleted, if it's an old version or deleted - if((enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) + if(en->IsOld() || en->IsDeleted()) { // Is deleted / old version. DelEn d; @@ -547,17 +466,15 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) d.mSizeInBlocks = en->GetSizeInBlocks(); d.mMarkNumber = en->GetMarkNumber(); d.mVersionAgeWithinMark = enVersionAge; - d.mIsFlagDeleted = (enFlags & - BackupStoreDirectory::Entry::Flags_Deleted) - ? true : false; - + d.mIsFlagDeleted = en->IsDeleted(); + // Add it to the list mPotentialDeletions.insert(d); - + // Update various counts mPotentialDeletionsTotalSize += d.mSizeInBlocks; if(d.mSizeInBlocks > mMaxSizeInPotentialDeletions) mMaxSizeInPotentialDeletions = d.mSizeInBlocks; - + // Too much in the list of potential deletions? // (check against the deletion target + the max size in deletions, so that we never delete things // and take the total size below the deletion size target) @@ -565,7 +482,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { int64_t sizeToRemove = mPotentialDeletionsTotalSize - (mDeletionSizeTarget + mMaxSizeInPotentialDeletions); bool recalcMaxSize = false; - + while(sizeToRemove > 0) { // Make iterator for the last element, while checking that there's something there in the first place. @@ -577,7 +494,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } // Make this into an iterator pointing to the last element in the set --i; - + // Delete this one? if(sizeToRemove > i->mSizeInBlocks) { @@ -595,7 +512,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) break; } } - + if(recalcMaxSize) { // Because an object which was the maximum size recorded was deleted from the set @@ -615,23 +532,22 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } } + // Recurse into subdirectories { - // Recurse into subdirectories BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0) { - // Next level - ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir); - - if(!ScanDirectory(en->GetObjectID())) + ASSERT(en->IsDir()); + + if(!ScanDirectory(en->GetObjectID(), rBackupStoreInfo)) { // Halting operation return false; } } } - + return true; } @@ -648,11 +564,11 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &x, const HousekeepStoreAccount::DelEn &y) { // STL spec says this: - // A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. + // A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. // The sort order here is intended to preserve the entries of most value, that is, the newest objects // which are on a mark boundary. - + // Reverse order age, so oldest goes first if(x.mVersionAgeWithinMark > y.mVersionAgeWithinMark) { @@ -687,7 +603,7 @@ bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount // Created: 15/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::DeleteFiles() +bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo) { // Only delete files if the deletion target is greater than zero // (otherwise we delete one file each time round, which gradually deletes the old versions) @@ -718,21 +634,33 @@ bool HousekeepStoreAccount::DeleteFiles() // Get the filename std::string dirFilename; BackupStoreDirectory dir; - int64_t dirSizeInBlocksOrig = 0; { MakeObjectFilename(i->mInDirectory, dirFilename); std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename)); - dirSizeInBlocksOrig = dirStream->GetDiscUsageInBlocks(); dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + dir.SetUserInfo1_SizeInBlocks(dirStream->GetDiscUsageInBlocks()); } - + // Delete the file - DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig); - BOX_INFO("Housekeeping removed " << - (i->mIsFlagDeleted ? "deleted" : "old") << - " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << - " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); - + BackupStoreRefCountDatabase::refcount_t refs = + DeleteFile(i->mInDirectory, i->mObjectID, dir, + dirFilename, rBackupStoreInfo); + if(refs == 0) + { + BOX_INFO("Housekeeping removed " << + (i->mIsFlagDeleted ? "deleted" : "old") << + " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << + " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); + } + else + { + BOX_TRACE("Housekeeping preserved " << + (i->mIsFlagDeleted ? "deleted" : "old") << + " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << + " in dir " << BOX_FORMAT_OBJECTID(i->mInDirectory) << + " with " << refs << " references"); + } + // Stop if the deletion target has been matched or exceeded // (checking here rather than at the beginning will tend to reduce the // space to slightly less than the soft limit, which will allow the backup @@ -754,11 +682,17 @@ bool HousekeepStoreAccount::DeleteFiles() // BackupStoreDirectory &, const std::string &, int64_t) // Purpose: Delete a file. Takes the directory already loaded // in and the filename, for efficiency in both the -// usage scenarios. +// usage scenarios. Returns the number of references +// remaining. If it's zero, the file was removed from +// disk as unused. // Created: 15/7/04 // // -------------------------------------------------------------------------- -void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks) + +BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( + int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, + const std::string &rDirectoryFilename, + BackupStoreInfo& rBackupStoreInfo) { // Find the entry inside the directory bool wasDeleted = false; @@ -768,6 +702,9 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba std::auto_ptr<RaidFileWrite> padjustedEntry; // BLOCK { + BackupStoreRefCountDatabase::refcount_t refs = + mapNewRefs->GetRefCount(ObjectID); + BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID); if(pentry == 0) { @@ -775,26 +712,47 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba BOX_FORMAT_ACCOUNT(mAccountID) << " " "found error: object " << BOX_FORMAT_OBJECTID(ObjectID) << " " - "not found in dir " << + "not found in dir " << BOX_FORMAT_OBJECTID(InDirectory) << ", " "indicates logic error/corruption? Run " "bbstoreaccounts check <accid> fix"); - return; + mErrorCount++; + return refs; } - + // Record the flags it's got set - wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0); - wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0); + wasDeleted = pentry->IsDeleted(); + wasOldVersion = pentry->IsOld(); // Check this should be deleted if(!wasDeleted && !wasOldVersion) { - // Things changed size we were last around - return; + // Things changed since we were last around + return refs; } - + // Record size deletedFileSizeInBlocks = pentry->GetSizeInBlocks(); - + + if(refs > 1) + { + // Not safe to merge patches if someone else has a + // reference to this object, so just remove the + // directory entry and return. + rDirectory.DeleteEntry(ObjectID); + if(wasDeleted) + { + rBackupStoreInfo.AdjustNumDeletedFiles(-1); + } + + if(wasOldVersion) + { + rBackupStoreInfo.AdjustNumOldFiles(-1); + } + + mapNewRefs->RemoveReference(ObjectID); + return refs - 1; + } + // If the entry is involved in a chain of patches, it needs to be handled // a bit more carefully. if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0) @@ -826,7 +784,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba else { // This entry is in the middle of a chain, and two patches need combining. - + // First, adjust the directory entries BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer()); if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID @@ -838,7 +796,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba pnewer->SetDependsOlder(pentry->GetDependsOlder()); polder->SetDependsNewer(pentry->GetDependsNewer()); } - + // COMMON CODE to both cases // Generate the filename of the older version @@ -853,7 +811,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename)); // And open a write file to overwrite the other directory entry padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, - objFilenameOlder, mNewRefCounts[ObjectID])); + objFilenameOlder, mapNewRefs->GetRefCount(ObjectID))); padjustedEntry->Open(true /* allow overwriting */); if(pentry->GetDependsNewer() == 0) @@ -867,84 +825,86 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry); } // The file will be committed later when the directory is safely commited. - + // Work out the adjusted size int64_t newSize = padjustedEntry->GetDiscUsageInBlocks(); int64_t sizeDelta = newSize - polder->GetSizeInBlocks(); mBlocksUsedDelta += sizeDelta; - if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0) + if(polder->IsDeleted()) { mBlocksInDeletedFilesDelta += sizeDelta; } - if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0) + if(polder->IsOld()) { mBlocksInOldFilesDelta += sizeDelta; } polder->SetSizeInBlocks(newSize); } - + // pentry no longer valid } - + // Delete it from the directory rDirectory.DeleteEntry(ObjectID); - + // Save directory back to disc // BLOCK - int64_t dirRevisedSize = 0; { RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename, - mNewRefCounts[InDirectory]); + mapNewRefs->GetRefCount(InDirectory)); writeDir.Open(true /* allow overwriting */); rDirectory.WriteToStream(writeDir); - // get the disc usage (must do this before commiting it) - dirRevisedSize = writeDir.GetDiscUsageInBlocks(); + // Get the disc usage (must do this before commiting it) + int64_t new_size = writeDir.GetDiscUsageInBlocks(); // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - // adjust usage counts for this directory - if(dirRevisedSize > 0) - { - int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks; - mBlocksUsedDelta += adjust; - mBlocksInDirectoriesDelta += adjust; - } + // Adjust block counts if the directory itself changed in size + int64_t original_size = rDirectory.GetUserInfo1_SizeInBlocks(); + int64_t adjust = new_size - original_size; + mBlocksUsedDelta += adjust; + mBlocksInDirectoriesDelta += adjust; + + UpdateDirectorySize(rDirectory, new_size); } // Commit any new adjusted entry if(padjustedEntry.get() != 0) { padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - padjustedEntry.reset(); // delete it now + padjustedEntry.reset(); // delete it now } - // Drop reference count by one. If it reaches zero, delete the file. - if(--mNewRefCounts[ObjectID] == 0) + // Drop reference count by one. Must now be zero, to delete the file. + bool remaining_refs = mapNewRefs->RemoveReference(ObjectID); + ASSERT(!remaining_refs); + + // Delete from disc + BOX_TRACE("Removing unreferenced object " << + BOX_FORMAT_OBJECTID(ObjectID)); + std::string objFilename; + MakeObjectFilename(ObjectID, objFilename); + RaidFileWrite del(mStoreDiscSet, objFilename, mapNewRefs->GetRefCount(ObjectID)); + del.Delete(); + + // Adjust counts for the file + ++mFilesDeleted; + mBlocksUsedDelta -= deletedFileSizeInBlocks; + + if(wasDeleted) { - // Delete from disc - BOX_TRACE("Removing unreferenced object " << - BOX_FORMAT_OBJECTID(ObjectID)); - std::string objFilename; - MakeObjectFilename(ObjectID, objFilename); - RaidFileWrite del(mStoreDiscSet, objFilename, - mNewRefCounts[ObjectID]); - del.Delete(); + mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks; + rBackupStoreInfo.AdjustNumDeletedFiles(-1); } - else + + if(wasOldVersion) { - BOX_TRACE("Preserving object " << - BOX_FORMAT_OBJECTID(ObjectID) << " with " << - mNewRefCounts[ObjectID] << " references"); + mBlocksInOldFilesDelta -= deletedFileSizeInBlocks; + rBackupStoreInfo.AdjustNumOldFiles(-1); } - // Adjust counts for the file - ++mFilesDeleted; - mBlocksUsedDelta -= deletedFileSizeInBlocks; - if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks; - if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks; - // Delete the directory? // Do this if... dir has zero entries, and is marked as deleted in it's containing directory if(rDirectory.GetNumberOfEntries() == 0) @@ -952,8 +912,81 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba // Candidate for deletion mEmptyDirectories.push_back(InDirectory); } + + return 0; } +// -------------------------------------------------------------------------- +// +// Function +// Name: HousekeepStoreAccount::UpdateDirectorySize( +// BackupStoreDirectory& rDirectory, +// IOStream::pos_type new_size_in_blocks) +// Purpose: Update the directory size, modifying the parent +// directory's entry for this directory if necessary. +// Created: 05/03/14 +// +// -------------------------------------------------------------------------- + +void HousekeepStoreAccount::UpdateDirectorySize( + BackupStoreDirectory& rDirectory, + IOStream::pos_type new_size_in_blocks) +{ +#ifndef BOX_RELEASE_BUILD + { + std::string dirFilename; + MakeObjectFilename(rDirectory.GetObjectID(), dirFilename); + std::auto_ptr<RaidFileRead> dirStream( + RaidFileRead::Open(mStoreDiscSet, dirFilename)); + ASSERT(new_size_in_blocks == dirStream->GetDiscUsageInBlocks()); + } +#endif + + IOStream::pos_type old_size_in_blocks = + rDirectory.GetUserInfo1_SizeInBlocks(); + + if(new_size_in_blocks == old_size_in_blocks) + { + return; + } + + rDirectory.SetUserInfo1_SizeInBlocks(new_size_in_blocks); + + if (rDirectory.GetObjectID() == BACKUPSTORE_ROOT_DIRECTORY_ID) + { + return; + } + + std::string parentFilename; + MakeObjectFilename(rDirectory.GetContainerID(), parentFilename); + std::auto_ptr<RaidFileRead> parentStream( + RaidFileRead::Open(mStoreDiscSet, parentFilename)); + BackupStoreDirectory parent(*parentStream); + parentStream.reset(); + + BackupStoreDirectory::Entry* en = + parent.FindEntryByID(rDirectory.GetObjectID()); + ASSERT(en); + + if (en->GetSizeInBlocks() != old_size_in_blocks) + { + BOX_WARNING("Directory " << + BOX_FORMAT_OBJECTID(rDirectory.GetObjectID()) << + " entry in directory " << + BOX_FORMAT_OBJECTID(rDirectory.GetContainerID()) << + " had incorrect size " << en->GetSizeInBlocks() << + ", should have been " << old_size_in_blocks); + mErrorCount++; + } + + en->SetSizeInBlocks(new_size_in_blocks); + + RaidFileWrite writeDir(mStoreDiscSet, parentFilename, + mapNewRefs->GetRefCount(rDirectory.GetContainerID())); + writeDir.Open(true /* allow overwriting */); + parent.WriteToStream(writeDir); + writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); +} // -------------------------------------------------------------------------- // @@ -964,7 +997,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba // Created: 15/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::DeleteEmptyDirectories() +bool HousekeepStoreAccount::DeleteEmptyDirectories(BackupStoreInfo& rBackupStoreInfo) { while(mEmptyDirectories.size() > 0) { @@ -992,7 +1025,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() continue; } - DeleteEmptyDirectory(*i, toExamine); + DeleteEmptyDirectory(*i, toExamine, rBackupStoreInfo); } // Remove contents of empty directories @@ -1000,13 +1033,13 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() // Swap in new, so it's examined next time round mEmptyDirectories.swap(toExamine); } - + // Not interrupted return false; } void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, - std::vector<int64_t>& rToExamine) + std::vector<int64_t>& rToExamine, BackupStoreInfo& rBackupStoreInfo) { // Load up the directory to potentially delete std::string dirFilename; @@ -1046,16 +1079,18 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, std::auto_ptr<RaidFileRead> containingDirStream( RaidFileRead::Open(mStoreDiscSet, containingDirFilename)); - containingDirSizeInBlocksOrig = + containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks(); containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite); + containingDir.SetUserInfo1_SizeInBlocks(containingDirSizeInBlocksOrig); } // Find the entry - BackupStoreDirectory::Entry *pdirentry = + BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID()); - if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)) + // TODO FIXME invert test and reduce indentation + if((pdirentry != 0) && pdirentry->IsDeleted()) { // Should be deleted containingDir.DeleteEntry(dir.GetObjectID()); @@ -1068,7 +1103,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Write revised parent directory RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename, - mNewRefCounts[containingDir.GetObjectID()]); + mapNewRefs->GetRefCount(containingDir.GetObjectID())); writeDir.Open(true /* allow overwriting */); containingDir.WriteToStream(writeDir); @@ -1077,6 +1112,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + UpdateDirectorySize(containingDir, dirSize); // adjust usage counts for this directory if(dirSize > 0) @@ -1086,17 +1122,22 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, mBlocksInDirectoriesDelta += adjust; } - - if (--mNewRefCounts[dir.GetObjectID()] == 0) + if (mapNewRefs->RemoveReference(dir.GetObjectID())) { - // Delete the directory itself - RaidFileWrite del(mStoreDiscSet, dirFilename, - mNewRefCounts[dir.GetObjectID()]); - del.Delete(); + // Still referenced + BOX_TRACE("Housekeeping spared empty deleted dir " << + BOX_FORMAT_OBJECTID(dirId) << " due to " << + mapNewRefs->GetRefCount(dir.GetObjectID()) << + "remaining references"); + return; } - BOX_INFO("Housekeeping removed empty deleted dir " << + // Delete the directory itself + BOX_INFO("Housekeeping removing empty deleted dir " << BOX_FORMAT_OBJECTID(dirId)); + RaidFileWrite del(mStoreDiscSet, dirFilename, + mapNewRefs->GetRefCount(dir.GetObjectID())); + del.Delete(); // And adjust usage counts for the directory that's // just been deleted @@ -1105,6 +1146,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Update count ++mEmptyDirectoriesDeleted; + rBackupStoreInfo.AdjustNumDirectories(-1); } } diff --git a/lib/backupstore/HousekeepStoreAccount.h b/lib/backupstore/HousekeepStoreAccount.h index cfa05a2e..ff9e9ffe 100644 --- a/lib/backupstore/HousekeepStoreAccount.h +++ b/lib/backupstore/HousekeepStoreAccount.h @@ -14,6 +14,8 @@ #include <set> #include <vector> +#include "BackupStoreRefCountDatabase.h" + class BackupStoreDirectory; class HousekeepingCallback @@ -39,20 +41,25 @@ public: ~HousekeepStoreAccount(); bool DoHousekeeping(bool KeepTryingForever = false); - int GetRefCountsAdjusted() { return mRefCountsAdjusted; } + int GetErrorCount() { return mErrorCount; } private: // utility functions void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut); - bool ScanDirectory(int64_t ObjectID); - bool DeleteFiles(); - bool DeleteEmptyDirectories(); - void DeleteEmptyDirectory(int64_t dirId, - std::vector<int64_t>& rToExamine); - void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks); + bool ScanDirectory(int64_t ObjectID, BackupStoreInfo& rBackupStoreInfo); + bool DeleteFiles(BackupStoreInfo& rBackupStoreInfo); + bool DeleteEmptyDirectories(BackupStoreInfo& rBackupStoreInfo); + void DeleteEmptyDirectory(int64_t dirId, std::vector<int64_t>& rToExamine, + BackupStoreInfo& rBackupStoreInfo); + BackupStoreRefCountDatabase::refcount_t DeleteFile(int64_t InDirectory, + int64_t ObjectID, + BackupStoreDirectory &rDirectory, + const std::string &rDirectoryFilename, + BackupStoreInfo& rBackupStoreInfo); + void UpdateDirectorySize(BackupStoreDirectory &rDirectory, + IOStream::pos_type new_size_in_blocks); -private: typedef struct { int64_t mObjectID; @@ -81,6 +88,9 @@ private: // List of directories which are empty, and might be good for deleting std::vector<int64_t> mEmptyDirectories; + + // Count of errors found and fixed + int64_t mErrorCount; // The re-calculated blocks used stats int64_t mBlocksUsed; @@ -99,9 +109,7 @@ private: int64_t mEmptyDirectoriesDeleted; // New reference count list - std::vector<uint32_t> mNewRefCounts; - bool mSuppressRefCountChangeWarnings; - int mRefCountsAdjusted; + std::auto_ptr<BackupStoreRefCountDatabase> mapNewRefs; // Poll frequency int mCountUntilNextInterprocessMsgCheck; diff --git a/lib/backupstore/Makefile.extra b/lib/backupstore/Makefile.extra index c55fd549..6f181abd 100644 --- a/lib/backupstore/Makefile.extra +++ b/lib/backupstore/Makefile.extra @@ -1,9 +1,9 @@ MAKEPROTOCOL = ../../lib/server/makeprotocol.pl -GEN_CMD = $(MAKEPROTOCOL) backupprotocol.txt +GEN_CMD = $(MAKEPROTOCOL) BackupProtocol.txt # AUTOGEN SEEDING -autogen_BackupProtocol.cpp autogen_BackupProtocol.h: $(MAKEPROTOCOL) backupprotocol.txt +autogen_BackupProtocol.cpp autogen_BackupProtocol.h: $(MAKEPROTOCOL) BackupProtocol.txt $(_PERL) $(GEN_CMD) diff --git a/lib/backupstore/StoreTestUtils.cpp b/lib/backupstore/StoreTestUtils.cpp new file mode 100644 index 00000000..2b773cb1 --- /dev/null +++ b/lib/backupstore/StoreTestUtils.cpp @@ -0,0 +1,300 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreTestUtils.cpp +// Purpose: Utilities for housekeeping and checking a test store +// Created: 18/02/14 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <cstdio> +#include <vector> + +#include "autogen_BackupProtocol.h" +#include "BoxPortsAndFiles.h" +#include "BackupStoreAccounts.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreConfigVerify.h" +#include "BackupStoreConstants.h" +#include "BackupStoreInfo.h" +#include "HousekeepStoreAccount.h" +#include "Logging.h" +#include "ServerControl.h" +#include "SocketStreamTLS.h" +#include "StoreTestUtils.h" +#include "TLSContext.h" +#include "Test.h" + +bool create_account(int soft, int hard) +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + + Logger::LevelGuard guard(Logging::GetConsole(), Log::WARNING); + int result = control.CreateAccount(0x01234567, 0, soft, hard); + TEST_EQUAL(0, result); + return (result == 0); +} + +bool delete_account() +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + Logger::LevelGuard guard(Logging::GetConsole(), Log::WARNING); + TEST_THAT_THROWONFAIL(control.DeleteAccount(0x01234567, false) == 0); + return true; +} + +std::vector<uint32_t> ExpectedRefCounts; +int bbstored_pid = 0, bbackupd_pid = 0; + +void set_refcount(int64_t ObjectID, uint32_t RefCount) +{ + if ((int64_t)ExpectedRefCounts.size() <= ObjectID) + { + ExpectedRefCounts.resize(ObjectID + 1, 0); + } + ExpectedRefCounts[ObjectID] = RefCount; + for (size_t i = ExpectedRefCounts.size() - 1; i >= 1; i--) + { + if (ExpectedRefCounts[i] == 0) + { + // BackupStoreCheck and housekeeping will both + // regenerate the refcount DB without any missing + // items at the end, so we need to prune ourselves + // of all items with no references to match. + ExpectedRefCounts.resize(i); + } + } +} + +void init_context(TLSContext& rContext) +{ + rContext.Initialise(false /* client */, + "testfiles/clientCerts.pem", + "testfiles/clientPrivKey.pem", + "testfiles/clientTrustedCAs.pem"); +} + +std::auto_ptr<SocketStream> open_conn(const char *hostname, + TLSContext& rContext) +{ + init_context(rContext); + std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS); + conn->Open(rContext, Socket::TypeINET, hostname, + BOX_PORT_BBSTORED_TEST); + return static_cast<std::auto_ptr<SocketStream> >(conn); +} + +std::auto_ptr<BackupProtocolCallable> connect_to_bbstored(TLSContext& rContext) +{ + // Make a protocol + std::auto_ptr<BackupProtocolCallable> protocol(new + BackupProtocolClient(open_conn("localhost", rContext))); + + // Check the version + std::auto_ptr<BackupProtocolVersion> serverVersion( + protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION)); + TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + + return protocol; +} + +std::auto_ptr<BackupProtocolCallable> connect_and_login(TLSContext& rContext, + int flags) +{ + // Make a protocol + std::auto_ptr<BackupProtocolCallable> protocol = + connect_to_bbstored(rContext); + + // Login + protocol->QueryLogin(0x01234567, flags); + + return protocol; +} + +bool check_num_files(int files, int old, int deleted, int dirs) +{ + std::auto_ptr<BackupStoreInfo> apInfo = + BackupStoreInfo::Load(0x1234567, + "backup/01234567/", 0, true); + TEST_EQUAL_LINE(files, apInfo->GetNumCurrentFiles(), + "current files"); + TEST_EQUAL_LINE(old, apInfo->GetNumOldFiles(), + "old files"); + TEST_EQUAL_LINE(deleted, apInfo->GetNumDeletedFiles(), + "deleted files"); + TEST_EQUAL_LINE(dirs, apInfo->GetNumDirectories(), + "directories"); + + return (files == apInfo->GetNumCurrentFiles() && + old == apInfo->GetNumOldFiles() && + deleted == apInfo->GetNumDeletedFiles() && + dirs == apInfo->GetNumDirectories()); +} + +bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, + int Deleted, int Dirs, int Total) +{ + std::auto_ptr<BackupProtocolAccountUsage2> usage = + Client.QueryGetAccountUsage2(); + TEST_EQUAL_LINE(Total, usage->GetBlocksUsed(), "wrong BlocksUsed"); + TEST_EQUAL_LINE(Current, usage->GetBlocksInCurrentFiles(), + "wrong BlocksInCurrentFiles"); + TEST_EQUAL_LINE(Old, usage->GetBlocksInOldFiles(), + "wrong BlocksInOldFiles"); + TEST_EQUAL_LINE(Deleted, usage->GetBlocksInDeletedFiles(), + "wrong BlocksInDeletedFiles"); + TEST_EQUAL_LINE(Dirs, usage->GetBlocksInDirectories(), + "wrong BlocksInDirectories"); + return (Total == usage->GetBlocksUsed() && + Current == usage->GetBlocksInCurrentFiles() && + Old == usage->GetBlocksInOldFiles() && + Deleted == usage->GetBlocksInDeletedFiles() && + Dirs == usage->GetBlocksInDirectories()); +} + +bool change_account_limits(const char* soft, const char* hard) +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + int result = control.SetLimit(0x01234567, soft, hard); + TEST_EQUAL(0, result); + return (result == 0); +} + +int check_account_for_errors(Log::Level log_level) +{ + Logger::LevelGuard guard(Logging::GetConsole(), log_level); + Logging::Tagger tag("check fix", true); + Logging::ShowTagOnConsole show; + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify("testfiles/bbstored.conf", + &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + int errors_fixed = control.CheckAccount(0x01234567, + true, // FixErrors + false, // Quiet + true); // ReturnNumErrorsFound + return errors_fixed; +} + +bool check_account(Log::Level log_level) +{ + int errors_fixed = check_account_for_errors(log_level); + TEST_EQUAL(0, errors_fixed); + return (errors_fixed == 0); +} + +int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount) +{ + std::string rootDir = BackupStoreAccounts::GetAccountRoot(rAccount); + int discSet = rAccount.GetDiscSet(); + + // Do housekeeping on this account + HousekeepStoreAccount housekeeping(rAccount.GetID(), rootDir, + discSet, NULL); + TEST_THAT(housekeeping.DoHousekeeping(true /* keep trying forever */)); + return housekeeping.GetErrorCount(); +} + +// Run housekeeping (for which we need to disconnect ourselves) and check +// that it doesn't change the numbers of files. +// +// Also check that bbstoreaccounts doesn't change anything + +bool run_housekeeping_and_check_account() +{ + int error_count; + + { + Logging::Tagger tag("", true); + Logging::ShowTagOnConsole show; + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + BackupStoreAccountDatabase::Entry account = + apAccounts->GetEntry(0x1234567); + error_count = run_housekeeping(account); + } + + TEST_EQUAL_LINE(0, error_count, "housekeeping errors"); + + bool check_account_is_ok = check_account(); + TEST_THAT(check_account_is_ok); + + return error_count == 0 && check_account_is_ok; +} + +bool check_reference_counts() +{ + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + BackupStoreAccountDatabase::Entry account = + apAccounts->GetEntry(0x1234567); + + std::auto_ptr<BackupStoreRefCountDatabase> apReferences( + BackupStoreRefCountDatabase::Load(account, true)); + TEST_EQUAL(ExpectedRefCounts.size(), + apReferences->GetLastObjectIDUsed() + 1); + + bool counts_ok = true; + + for (unsigned int i = BackupProtocolListDirectory::RootDirectory; + i < ExpectedRefCounts.size(); i++) + { + TEST_EQUAL_LINE(ExpectedRefCounts[i], + apReferences->GetRefCount(i), + "object " << BOX_FORMAT_OBJECTID(i)); + if (ExpectedRefCounts[i] != apReferences->GetRefCount(i)) + { + counts_ok = false; + } + } + + return counts_ok; +} + +bool StartServer() +{ + bbstored_pid = StartDaemon(bbstored_pid, + BBSTORED " " + bbstored_args + " testfiles/bbstored.conf", + "testfiles/bbstored.pid"); + return bbstored_pid != 0; +} + +bool StopServer(bool wait_for_process) +{ + bool result = StopDaemon(bbstored_pid, "testfiles/bbstored.pid", + "bbstored.memleaks", wait_for_process); + bbstored_pid = 0; + return result; +} + +bool StartClient(const std::string& bbackupd_conf_file) +{ + bbackupd_pid = StartDaemon(bbackupd_pid, + BBACKUPD " " + bbackupd_args + " " + bbackupd_conf_file, + "testfiles/bbackupd.pid"); + return bbackupd_pid != 0; +} + +bool StopClient(bool wait_for_process) +{ + bool result = StopDaemon(bbackupd_pid, "testfiles/bbackupd.pid", + "bbackupd.memleaks", wait_for_process); + bbackupd_pid = 0; + return result; +} + diff --git a/lib/backupstore/StoreTestUtils.h b/lib/backupstore/StoreTestUtils.h new file mode 100644 index 00000000..b3faebb5 --- /dev/null +++ b/lib/backupstore/StoreTestUtils.h @@ -0,0 +1,118 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreTestUtils.h +// Purpose: Utilities for housekeeping and checking a test store +// Created: 18/02/14 +// +// -------------------------------------------------------------------------- + +#ifndef STORETESTUTILS__H +#define STORETESTUTILS__H + +#include "Test.h" + +class BackupProtocolCallable; +class BackupProtocolClient; +class SocketStreamTLS; +class TLSContext; + +//! Holds the expected reference counts of each object. +extern std::vector<uint32_t> ExpectedRefCounts; + +//! Holds the PID of the currently running bbstored test server. +extern int bbstored_pid, bbackupd_pid; + +//! Sets the expected refcount of an object, resizing vector if necessary. +void set_refcount(int64_t ObjectID, uint32_t RefCount = 1); + +//! Initialises a TLSContext object using the standard certficate filenames. +void init_context(TLSContext& rContext); + +//! Opens a connection to the server (bbstored). +std::auto_ptr<SocketStream> open_conn(const char *hostname, + TLSContext& rContext); + +//! Opens a connection to the server (bbstored) without logging in. +std::auto_ptr<BackupProtocolCallable> connect_to_bbstored(TLSContext& rContext); + +//! Opens a connection to the server (bbstored) and logs in. +std::auto_ptr<BackupProtocolCallable> connect_and_login(TLSContext& rContext, + int flags = 0); + +//! Checks the number of files of each type in the store against expectations. +bool check_num_files(int files, int old, int deleted, int dirs); + +//! Checks the number of blocks in files of each type against expectations. +bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, + int Deleted, int Dirs, int Total); + +//! Change the soft and hard limits on the test account. +bool change_account_limits(const char* soft, const char* hard); + +//! Checks an account for errors, returning the number of errors found and fixed. +int check_account_for_errors(Log::Level log_level = Log::WARNING); + +//! Checks an account for errors, returning true if it's OK, for use in assertions. +bool check_account(Log::Level log_level = Log::WARNING); + +//! Runs housekeeping on an account, to remove old and deleted files if necessary. +int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount); + +//! Runs housekeeping and checks the account, returning true if it's OK. +bool run_housekeeping_and_check_account(); + +//! Tests that all object reference counts have the expected values. +bool check_reference_counts(); + +//! Starts the bbstored test server running, which must not already be running. +bool StartServer(); + +//! Stops the currently running bbstored test server. +bool StopServer(bool wait_for_process = false); + +//! Starts the bbackupd client running, which must not already be running. +bool StartClient(const std::string& bbackupd_conf_file = "testfiles/bbackupd.conf"); + +//! Stops the currently running bbackupd client. +bool StopClient(bool wait_for_process = false); + +//! Creates the standard test account, for example after delete_account(). +bool create_account(int soft, int hard); + +//! Deletes the standard test account, for testing behaviour with no account. +bool delete_account(); + +#define TEST_PROTOCOL_ERROR_OR(protocol, error, or_statements) \ + { \ + int type, subtype; \ + (protocol).GetLastError(type, subtype); \ + if (type == BackupProtocolError::ErrorType) \ + { \ + TEST_EQUAL_LINE(BackupProtocolError::error, subtype, \ + "command returned error: " << \ + BackupProtocolError::GetMessage(subtype)); \ + if (subtype != BackupProtocolError::error) \ + { \ + or_statements; \ + } \ + } \ + else \ + { \ + TEST_FAIL_WITH_MESSAGE("command did not return an error, but a " \ + "response of type " << type << ", subtype " << subtype << \ + " instead"); \ + or_statements; \ + } \ + } + +#define TEST_COMMAND_RETURNS_ERROR_OR(protocol, command, error, or_statements) \ + TEST_CHECK_THROWS_AND_OR((protocol) . command, ConnectionException, \ + Protocol_UnexpectedReply, /* and_command */, or_statements); \ + TEST_PROTOCOL_ERROR_OR(protocol, error, or_statements) + +#define TEST_COMMAND_RETURNS_ERROR(protocol, command, error) \ + TEST_COMMAND_RETURNS_ERROR_OR(protocol, command, error,) + +#endif // STORETESTUTILS__H + |