diff options
Diffstat (limited to 'lib')
181 files changed, 21606 insertions, 4022 deletions
diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp index db72c4bd..d3300604 100644 --- a/lib/backupclient/BackupClientRestore.cpp +++ b/lib/backupclient/BackupClientRestore.cpp @@ -204,13 +204,13 @@ typedef struct // -------------------------------------------------------------------------- // // Function -// Name: BackupClientRestoreDir(BackupProtocolClient &, +// Name: BackupClientRestoreDir(BackupProtocolCallable &, // int64_t, const char *, bool) // Purpose: Restore a directory // Created: 23/11/03 // // -------------------------------------------------------------------------- -static int BackupClientRestoreDir(BackupProtocolClient &rConnection, +static int BackupClientRestoreDir(BackupProtocolCallable &rConnection, int64_t DirectoryID, const std::string &rRemoteDirectoryName, const std::string &rLocalDirectoryName, RestoreParams &Params, RestoreResumeInfo &rLevel) @@ -224,7 +224,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, DIRECTORY_SEPARATOR_ASCHAR + rLevel.mNextLevelLocalName); BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, - rRemoteDirectoryName + '/' + + rRemoteDirectoryName + '/' + rLevel.mNextLevelLocalName, localDirname, Params, *rLevel.mpNextLevel); @@ -232,7 +232,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, rLevel.mRestoredObjects.insert(rLevel.mNextLevelID); // Remove the level for the recursed directory - rLevel.RemoveLevel(); + rLevel.RemoveLevel(); } // Create the local directory, if not already done. @@ -299,7 +299,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, } std::string parentDirectoryName(rLocalDirectoryName); - if(parentDirectoryName[parentDirectoryName.size() - 1] == + if(parentDirectoryName[parentDirectoryName.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR) { parentDirectoryName.resize(parentDirectoryName.size() - 1); @@ -309,7 +309,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, if(lastSlash == std::string::npos) { - // might be a forward slash separator, + // might be a forward slash separator, // especially in the unit tests! lastSlash = parentDirectoryName.rfind('/'); } @@ -813,7 +813,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, // -------------------------------------------------------------------------- // // Function -// Name: BackupClientRestore(BackupProtocolClient &, int64_t, +// Name: BackupClientRestore(BackupProtocolCallable &, int64_t, // const char *, bool, bool, bool, bool, bool) // Purpose: Restore a directory on the server to a local // directory on the disc. The local directory must not @@ -840,7 +840,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, // Created: 23/11/03 // // -------------------------------------------------------------------------- -int BackupClientRestore(BackupProtocolClient &rConnection, +int BackupClientRestore(BackupProtocolCallable &rConnection, int64_t DirectoryID, const std::string& RemoteDirectoryName, const std::string& LocalDirectoryName, bool PrintDots, bool RestoreDeleted, bool UndeleteAfterRestoreDeleted, bool Resume, @@ -889,7 +889,7 @@ int BackupClientRestore(BackupProtocolClient &rConnection, } // Restore the directory - int result = BackupClientRestoreDir(rConnection, DirectoryID, + int result = BackupClientRestoreDir(rConnection, DirectoryID, RemoteDirectoryName, LocalDirectoryName, params, params.mResumeInfo); if (result != Restore_Complete) diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h index 77f09c2e..cdbedea7 100644 --- a/lib/backupclient/BackupClientRestore.h +++ b/lib/backupclient/BackupClientRestore.h @@ -7,10 +7,10 @@ // // -------------------------------------------------------------------------- -#ifndef BACKUPSCLIENTRESTORE_H -#define BACKUPSCLIENTRESTORE__H +#ifndef BACKUPCLIENTRESTORE_H +#define BACKUPCLIENTRESTORE_H -class BackupProtocolClient; +class BackupProtocolCallable; enum { @@ -22,7 +22,7 @@ enum Restore_CompleteWithErrors, }; -int BackupClientRestore(BackupProtocolClient &rConnection, +int BackupClientRestore(BackupProtocolCallable &rConnection, int64_t DirectoryID, const std::string& RemoteDirectoryName, const std::string& LocalDirectoryName, @@ -32,5 +32,5 @@ int BackupClientRestore(BackupProtocolClient &rConnection, bool Resume, bool ContinueAfterErrors); -#endif // BACKUPSCLIENTRESTORE__H +#endif // BACKUPCLIENTRESTORE_H diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index a3b95335..865ee413 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -11,11 +11,12 @@ #include "BackupDaemonConfigVerify.h" #include "Daemon.h" #include "BoxPortsAndFiles.h" +#include "BackupConstants.h" #include "MemLeakFindOn.h" -static const ConfigurationVerifyKey backuplocationkeys[] = +static const ConfigurationVerifyKey backuplocationkeys[] = { ConfigurationVerifyKey("ExcludeFile", ConfigTest_MultiValueAllowed), ConfigurationVerifyKey("ExcludeFilesRegex", ConfigTest_MultiValueAllowed), @@ -28,7 +29,7 @@ static const ConfigurationVerifyKey backuplocationkeys[] = ConfigurationVerifyKey("Path", ConfigTest_Exists | ConfigTest_LastEntry) }; -static const ConfigurationVerify backuplocations[] = +static const ConfigurationVerify backuplocations[] = { { "*", @@ -39,12 +40,22 @@ static const ConfigurationVerify backuplocations[] = } }; -static const ConfigurationVerifyKey verifyserverkeys[] = +static const ConfigurationVerifyKey verifyserverkeys[] = { DAEMON_VERIFY_SERVER_KEYS }; -static const ConfigurationVerify verifyserver[] = +static const ConfigurationVerifyKey verifys3keys[] = +{ + // These values are only required for Amazon S3-compatible stores + ConfigurationVerifyKey("HostName", ConfigTest_Exists), + ConfigurationVerifyKey("Port", ConfigTest_Exists | ConfigTest_IsInt, 80), + ConfigurationVerifyKey("BasePath", ConfigTest_Exists), + ConfigurationVerifyKey("AccessKey", ConfigTest_Exists), + ConfigurationVerifyKey("SecretKey", ConfigTest_Exists | ConfigTest_LastEntry) +}; + +static const ConfigurationVerify verifyserver[] = { { "Server", @@ -54,6 +65,13 @@ static const ConfigurationVerify verifyserver[] = 0 }, { + "S3Store", + 0, + verifys3keys, + 0, + 0 + }, + { "BackupLocations", backuplocations, 0, @@ -62,12 +80,12 @@ static const ConfigurationVerify verifyserver[] = } }; -static const ConfigurationVerifyKey verifyrootkeys[] = +static const ConfigurationVerifyKey verifyrootkeys[] = { - ConfigurationVerifyKey("AccountNumber", - ConfigTest_Exists | ConfigTest_IsUint32), ConfigurationVerifyKey("UpdateStoreInterval", ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("BackupErrorDelay", + ConfigTest_IsInt, BACKUP_ERROR_RETRY_SECONDS), ConfigurationVerifyKey("MinimumFileAge", ConfigTest_Exists | ConfigTest_IsInt), ConfigurationVerifyKey("MaxUploadWait", @@ -89,9 +107,6 @@ static const ConfigurationVerifyKey verifyrootkeys[] = ConfigTest_Exists | ConfigTest_IsInt), ConfigurationVerifyKey("DiffingUploadSizeThreshold", ConfigTest_Exists | ConfigTest_IsInt), - ConfigurationVerifyKey("StoreHostname", ConfigTest_Exists), - ConfigurationVerifyKey("StorePort", ConfigTest_IsInt, - BOX_PORT_BBSTORED), ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), // extended log to syslog ConfigurationVerifyKey("ExtendedLogFile", 0), @@ -102,6 +117,8 @@ static const ConfigurationVerifyKey verifyrootkeys[] = // enable logging to a file ConfigurationVerifyKey("LogFileLevel", 0), // set the level of verbosity of file logging + ConfigurationVerifyKey("LogFileOverwrite", ConfigTest_IsBool, false), + // overwrite the log file on each backup ConfigurationVerifyKey("CommandSocket", 0), // not compulsory to have this ConfigurationVerifyKey("KeepAliveTime", ConfigTest_IsInt), @@ -120,13 +137,18 @@ static const ConfigurationVerifyKey verifyrootkeys[] = ConfigurationVerifyKey("TcpNice", ConfigTest_IsBool, false), // optional enable of tcp nice/background mode - ConfigurationVerifyKey("CertificateFile", ConfigTest_Exists), - ConfigurationVerifyKey("PrivateKeyFile", ConfigTest_Exists), - ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_Exists), ConfigurationVerifyKey("KeysFile", ConfigTest_Exists), - ConfigurationVerifyKey("DataDirectory", - ConfigTest_Exists | ConfigTest_LastEntry), + ConfigurationVerifyKey("DataDirectory", ConfigTest_Exists), + // These values are only required for bbstored stores: + ConfigurationVerifyKey("StoreHostname", 0), + ConfigurationVerifyKey("StorePort", ConfigTest_IsInt, + BOX_PORT_BBSTORED), + ConfigurationVerifyKey("AccountNumber", + ConfigTest_IsUint32), + ConfigurationVerifyKey("CertificateFile", 0), + ConfigurationVerifyKey("PrivateKeyFile", 0), + ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_LastEntry), }; const ConfigurationVerify BackupDaemonConfigVerify = diff --git a/lib/backupclient/ClientException.txt b/lib/backupclient/ClientException.txt new file mode 100644 index 00000000..04f88620 --- /dev/null +++ b/lib/backupclient/ClientException.txt @@ -0,0 +1,11 @@ + +# NOTE: Exception descriptions are for public distributions of Box Backup only -- do not rely for other applications. + + +EXCEPTION Client 13 + +Internal 0 +AssertFailed 1 +ClockWentBackwards 2 Invalid (negative) sync period: perhaps your clock is going backwards? +FailedToDeleteStoreObjectInfoFile 3 Failed to delete the StoreObjectInfoFile, backup cannot continue safely. +CorruptStoreObjectInfoFile 4 The store object info file contained an invalid value and is probably corrupt. Try deleting it. diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra new file mode 100644 index 00000000..25ceb1e7 --- /dev/null +++ b/lib/backupclient/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_ClientException.h autogen_ClientException.cpp: $(MAKEEXCEPTION) ClientException.txt + $(_PERL) $(MAKEEXCEPTION) ClientException.txt + diff --git a/lib/backupstore/BackgroundTask.h b/lib/backupstore/BackgroundTask.h new file mode 100644 index 00000000..bae9162f --- /dev/null +++ b/lib/backupstore/BackgroundTask.h @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackgroundTask.h +// Purpose: Declares the BackgroundTask interface. +// Created: 2014/04/07 +// +// -------------------------------------------------------------------------- + +#ifndef BACKGROUNDTASK__H +#define BACKGROUNDTASK__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackgroundTask +// Purpose: Provides a RunBackgroundTask() method which allows +// background tasks such as polling the command socket +// to happen while a file is being uploaded. If it +// returns false, the current task should be aborted. +// Created: 2014/04/07 +// +// -------------------------------------------------------------------------- +class BackgroundTask +{ + public: + enum State { + Unknown = 0, + Scanning_Dirs, + Searching_Blocks, + Uploading_Full, + Uploading_Patch, + }; + virtual ~BackgroundTask() { } + virtual bool RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum) = 0; +}; + +#endif // BACKGROUNDTASK__H diff --git a/lib/backupstore/BackupAccountControl.cpp b/lib/backupstore/BackupAccountControl.cpp new file mode 100644 index 00000000..331ef841 --- /dev/null +++ b/lib/backupstore/BackupAccountControl.cpp @@ -0,0 +1,267 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupAccountControl.cpp +// Purpose: Client-side account management for Amazon S3 stores +// Created: 2015/06/27 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <climits> +#include <iostream> + +#include "autogen_CommonException.h" +#include "autogen_BackupStoreException.h" +#include "BackupAccountControl.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreInfo.h" +#include "Configuration.h" +#include "HTTPResponse.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +void BackupAccountControl::CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit) +{ + if(SoftLimit > HardLimit) + { + BOX_FATAL("Soft limit must be less than the hard limit."); + exit(1); + } + if(SoftLimit > ((HardLimit * MAX_SOFT_LIMIT_SIZE) / 100)) + { + BOX_WARNING("We recommend setting the soft limit below " << + MAX_SOFT_LIMIT_SIZE << "% of the hard limit, or " << + HumanReadableSize((HardLimit * MAX_SOFT_LIMIT_SIZE) + / 100) << " in this case."); + } +} + +int64_t BackupAccountControl::SizeStringToBlocks(const char *string, int blockSize) +{ + // Get number + char *endptr = (char*)string; + int64_t number = strtol(string, &endptr, 0); + if(endptr == string || number == LONG_MIN || number == LONG_MAX) + { + BOX_FATAL("'" << string << "' is not a valid number."); + exit(1); + } + + // Check units + switch(*endptr) + { + case 'M': + case 'm': + // Units: Mb + return (number * 1024*1024) / blockSize; + break; + + case 'G': + case 'g': + // Units: Gb + return (number * 1024*1024*1024) / blockSize; + break; + + case 'B': + case 'b': + // Units: Blocks + // Easy! Just return the number specified. + return number; + break; + + default: + BOX_FATAL(string << " has an invalid units specifier " + "(use B for blocks, M for MB, G for GB, eg 2GB)"); + exit(1); + break; + } +} + +std::string BackupAccountControl::BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int BlockSize) +{ + return FormatUsageBar(Blocks, Blocks * BlockSize, MaxBlocks * BlockSize, + mMachineReadableOutput); +} + +int BackupAccountControl::PrintAccountInfo(const BackupStoreInfo& info, + int BlockSize) +{ + // Then print out lots of info + std::cout << FormatUsageLineStart("Account ID", mMachineReadableOutput) << + BOX_FORMAT_ACCOUNT(info.GetAccountID()) << std::endl; + std::cout << FormatUsageLineStart("Account Name", mMachineReadableOutput) << + info.GetAccountName() << std::endl; + std::cout << FormatUsageLineStart("Last object ID", mMachineReadableOutput) << + BOX_FORMAT_OBJECTID(info.GetLastObjectIDUsed()) << std::endl; + std::cout << FormatUsageLineStart("Used", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksUsed(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Current files", + mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInCurrentFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Old files", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInOldFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Deleted files", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInDeletedFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Directories", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInDirectories(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Soft limit", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksSoftLimit(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Hard limit", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksHardLimit(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Client store marker", mMachineReadableOutput) << + info.GetClientStoreMarker() << std::endl; + std::cout << FormatUsageLineStart("Current Files", mMachineReadableOutput) << + info.GetNumCurrentFiles() << std::endl; + std::cout << FormatUsageLineStart("Old Files", mMachineReadableOutput) << + info.GetNumOldFiles() << std::endl; + std::cout << FormatUsageLineStart("Deleted Files", mMachineReadableOutput) << + info.GetNumDeletedFiles() << std::endl; + std::cout << FormatUsageLineStart("Directories", mMachineReadableOutput) << + info.GetNumDirectories() << std::endl; + std::cout << FormatUsageLineStart("Enabled", mMachineReadableOutput) << + (info.IsAccountEnabled() ? "yes" : "no") << std::endl; + + return 0; +} + +S3BackupAccountControl::S3BackupAccountControl(const Configuration& config, + bool machineReadableOutput) +: BackupAccountControl(config, machineReadableOutput) +{ + if(!mConfig.SubConfigurationExists("S3Store")) + { + THROW_EXCEPTION_MESSAGE(CommonException, + InvalidConfiguration, + "The S3Store configuration subsection is required " + "when S3Store mode is enabled"); + } + const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); + + mBasePath = s3config.GetKeyValue("BasePath"); + if(mBasePath.size() == 0) + { + mBasePath = "/"; + } + else + { + if(mBasePath[0] != '/' || mBasePath[mBasePath.size() - 1] != '/') + { + THROW_EXCEPTION_MESSAGE(CommonException, + InvalidConfiguration, + "If S3Store.BasePath is not empty then it must start and " + "end with a slash, e.g. '/subdir/', but it currently does not."); + } + } + + mapS3Client.reset(new S3Client( + s3config.GetKeyValue("HostName"), + s3config.GetKeyValueInt("Port"), + s3config.GetKeyValue("AccessKey"), + s3config.GetKeyValue("SecretKey"))); + + mapFileSystem.reset(new S3BackupFileSystem(mConfig, mBasePath, *mapS3Client)); +} + +std::string S3BackupAccountControl::GetFullURL(const std::string ObjectPath) const +{ + const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); + return std::string("http://") + s3config.GetKeyValue("HostName") + ":" + + s3config.GetKeyValue("Port") + GetFullPath(ObjectPath); +} + +int S3BackupAccountControl::CreateAccount(const std::string& name, int32_t SoftLimit, + int32_t HardLimit) +{ + // Try getting the info file. If we get a 200 response then it already + // exists, and we should bail out. If we get a 404 then it's safe to + // continue. Otherwise something else is wrong and we should bail out. + std::string info_url = GetFullURL(S3_INFO_FILE_NAME); + + HTTPResponse response = GetObject(S3_INFO_FILE_NAME); + if(response.GetResponseCode() == HTTPResponse::Code_OK) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, AccountAlreadyExists, + "The BackupStoreInfo file already exists at this URL: " << + info_url); + } + + if(response.GetResponseCode() != HTTPResponse::Code_NotFound) + { + mapS3Client->CheckResponse(response, std::string("Failed to check for an " + "existing BackupStoreInfo file at this URL: ") + info_url); + } + + BackupStoreInfo info(0, // fake AccountID for S3 stores + info_url, // FileName, + SoftLimit, HardLimit); + info.SetAccountName(name); + + // And an empty directory + BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); + int64_t rootDirSize = mapFileSystem->PutDirectory(rootDir); + + // Update the store info to reflect the size of the root directory + info.ChangeBlocksUsed(rootDirSize); + info.ChangeBlocksInDirectories(rootDirSize); + info.AdjustNumDirectories(1); + int64_t id = info.AllocateObjectID(); + ASSERT(id == BACKUPSTORE_ROOT_DIRECTORY_ID); + + CollectInBufferStream out; + info.Save(out); + out.SetForReading(); + + response = PutObject(S3_INFO_FILE_NAME, out); + mapS3Client->CheckResponse(response, std::string("Failed to upload the new BackupStoreInfo " + "file to this URL: ") + info_url); + + // Now get the file again, to check that it really worked. + response = GetObject(S3_INFO_FILE_NAME); + mapS3Client->CheckResponse(response, std::string("Failed to download the new BackupStoreInfo " + "file that we just created: ") + info_url); + + return 0; +} + +std::string S3BackupFileSystem::GetDirectoryURI(int64_t ObjectID) +{ + std::ostringstream out; + out << mBasePath << "dirs/" << BOX_FORMAT_OBJECTID(ObjectID) << ".dir"; + return out.str(); +} + +std::auto_ptr<HTTPResponse> S3BackupFileSystem::GetDirectory(BackupStoreDirectory& rDir) +{ + std::string uri = GetDirectoryURI(rDir.GetObjectID()); + HTTPResponse response = mrClient.GetObject(uri); + mrClient.CheckResponse(response, + std::string("Failed to download directory: ") + uri); + return std::auto_ptr<HTTPResponse>(new HTTPResponse(response)); +} + +int S3BackupFileSystem::PutDirectory(BackupStoreDirectory& rDir) +{ + CollectInBufferStream out; + rDir.WriteToStream(out); + out.SetForReading(); + + std::string uri = GetDirectoryURI(rDir.GetObjectID()); + HTTPResponse response = mrClient.PutObject(uri, out); + mrClient.CheckResponse(response, + std::string("Failed to upload directory: ") + uri); + + int blocks = (out.GetSize() + S3_NOTIONAL_BLOCK_SIZE - 1) / S3_NOTIONAL_BLOCK_SIZE; + return blocks; +} + diff --git a/lib/backupstore/BackupAccountControl.h b/lib/backupstore/BackupAccountControl.h new file mode 100644 index 00000000..00118ec2 --- /dev/null +++ b/lib/backupstore/BackupAccountControl.h @@ -0,0 +1,91 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupAccountControl.h +// Purpose: Client-side account management for Amazon S3 stores +// Created: 2015/06/27 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPACCOUNTCONTROL__H +#define BACKUPACCOUNTCONTROL__H + +#include <string> + +#include "BackupStoreAccountDatabase.h" +#include "HTTPResponse.h" +#include "S3Client.h" + +class BackupStoreDirectory; +class BackupStoreInfo; +class Configuration; + +class BackupAccountControl +{ +protected: + const Configuration& mConfig; + bool mMachineReadableOutput; + +public: + BackupAccountControl(const Configuration& config, + bool machineReadableOutput = false) + : mConfig(config), + mMachineReadableOutput(machineReadableOutput) + { } + void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit); + int64_t SizeStringToBlocks(const char *string, int BlockSize); + std::string BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int BlockSize); + int PrintAccountInfo(const BackupStoreInfo& info, int BlockSize); +}; + +class S3BackupFileSystem +{ +private: + std::string mBasePath; + S3Client& mrClient; +public: + S3BackupFileSystem(const Configuration& config, const std::string& BasePath, + S3Client& rClient) + : mBasePath(BasePath), + mrClient(rClient) + { } + std::string GetDirectoryURI(int64_t ObjectID); + std::auto_ptr<HTTPResponse> GetDirectory(BackupStoreDirectory& rDir); + int PutDirectory(BackupStoreDirectory& rDir); +}; + +class S3BackupAccountControl : public BackupAccountControl +{ +private: + std::string mBasePath; + std::auto_ptr<S3Client> mapS3Client; + std::auto_ptr<S3BackupFileSystem> mapFileSystem; +public: + S3BackupAccountControl(const Configuration& config, + bool machineReadableOutput = false); + std::string GetFullPath(const std::string ObjectPath) const + { + return mBasePath + ObjectPath; + } + std::string GetFullURL(const std::string ObjectPath) const; + int CreateAccount(const std::string& name, int32_t SoftLimit, int32_t HardLimit); + int GetBlockSize() { return 4096; } + HTTPResponse GetObject(const std::string& name) + { + return mapS3Client->GetObject(GetFullPath(name)); + } + HTTPResponse PutObject(const std::string& name, IOStream& rStreamToSend, + const char* pContentType = NULL) + { + return mapS3Client->PutObject(GetFullPath(name), rStreamToSend, + pContentType); + } +}; + +// max size of soft limit as percent of hard limit +#define MAX_SOFT_LIMIT_SIZE 97 +#define S3_INFO_FILE_NAME "boxbackup.info" +#define S3_NOTIONAL_BLOCK_SIZE 1048576 + +#endif // BACKUPACCOUNTCONTROL__H + diff --git a/lib/backupstore/BackupClientFileAttributes.cpp b/lib/backupstore/BackupClientFileAttributes.cpp index d76432ba..37140301 100644 --- a/lib/backupstore/BackupClientFileAttributes.cpp +++ b/lib/backupstore/BackupClientFileAttributes.cpp @@ -55,20 +55,20 @@ BEGIN_STRUCTURE_PACKING_FOR_WIRE typedef struct { int32_t AttributeType; - u_int32_t UID; - u_int32_t GID; - u_int64_t ModificationTime; - u_int64_t AttrModificationTime; - u_int32_t UserDefinedFlags; - u_int32_t FileGenerationNumber; - u_int16_t Mode; + uint32_t UID; + uint32_t GID; + uint64_t ModificationTime; + uint64_t AttrModificationTime; + uint32_t UserDefinedFlags; + uint32_t FileGenerationNumber; + uint16_t Mode; // Symbolic link filename may follow // Extended attribute (xattr) information may follow, format is: - // u_int32_t Size of extended attribute block (excluding this word) + // uint32_t Size of extended attribute block (excluding this word) // For each of NumberOfAttributes (sorted by AttributeName): - // u_int16_t AttributeNameLength + // uint16_t AttributeNameLength // char AttributeName[AttributeNameLength] - // u_int32_t AttributeValueLength + // uint32_t AttributeValueLength // unsigned char AttributeValue[AttributeValueLength] // AttributeName is 0 terminated, AttributeValue is not (and may be binary data) } attr_StreamFormat; @@ -117,7 +117,7 @@ namespace BackupClientFileAttributes::BackupClientFileAttributes() : mpClearAttributes(0) { - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); + ASSERT(sizeof(uint64_t) == sizeof(box_time_t)); } // -------------------------------------------------------------------------- @@ -131,7 +131,7 @@ BackupClientFileAttributes::BackupClientFileAttributes() BackupClientFileAttributes::BackupClientFileAttributes(const EMU_STRUCT_STAT &st) : mpClearAttributes(0) { - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); + ASSERT(sizeof(uint64_t) == sizeof(box_time_t)); StreamableMemBlock *pnewAttr = new StreamableMemBlock; FillAttributes(*pnewAttr, (const char *)NULL, st, true); @@ -411,7 +411,7 @@ void BackupClientFileAttributes::ReadAttributes(const std::string& Filename, // __time64_t winTime = BoxTimeToSeconds( // pnewAttr->ModificationTime); - u_int64_t modTime = box_ntoh64(pattr->ModificationTime); + uint64_t modTime = box_ntoh64(pattr->ModificationTime); box_time_t modSecs = BoxTimeToSeconds(modTime); __time64_t winTime = modSecs; @@ -545,7 +545,7 @@ void BackupClientFileAttributes::FillAttributesLink( void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const std::string& Filename) { -#ifdef HAVE_SYS_XATTR_H +#if defined HAVE_LLISTXATTR && defined HAVE_LGETXATTR int listBufferSize = 10000; char* list = new char[listBufferSize]; @@ -582,30 +582,30 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc // Leave space for attr block size later int xattrBlockSizeOffset = xattrSize; - xattrSize += sizeof(u_int32_t); + xattrSize += sizeof(uint32_t); // Loop for each attribute for(std::vector<std::string>::const_iterator attrKeyI = attrKeys.begin(); attrKeyI!=attrKeys.end(); ++attrKeyI) { std::string attrKey(*attrKeyI); - if(xattrSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t)>static_cast<unsigned int>(xattrBufferSize)) + if(xattrSize+sizeof(uint16_t)+attrKey.size()+1+sizeof(uint32_t)>static_cast<unsigned int>(xattrBufferSize)) { - xattrBufferSize = (xattrBufferSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t))*2; + xattrBufferSize = (xattrBufferSize+sizeof(uint16_t)+attrKey.size()+1+sizeof(uint32_t))*2; outputBlock.ResizeBlock(xattrBufferSize); buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); } // Store length and text for attibute name - u_int16_t keyLength = htons(attrKey.size()+1); - std::memcpy(buffer+xattrSize, &keyLength, sizeof(u_int16_t)); - xattrSize += sizeof(u_int16_t); + uint16_t keyLength = htons(attrKey.size()+1); + std::memcpy(buffer+xattrSize, &keyLength, sizeof(uint16_t)); + xattrSize += sizeof(uint16_t); std::memcpy(buffer+xattrSize, attrKey.c_str(), attrKey.size()+1); xattrSize += attrKey.size()+1; // Leave space for value size int valueSizeOffset = xattrSize; - xattrSize += sizeof(u_int32_t); + xattrSize += sizeof(uint32_t); // Find size of attribute (must call with buffer and length 0 on some platforms, // as -1 is returned if the data doesn't fit.) @@ -642,21 +642,30 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc xattrSize += valueSize; // Fill in value size - u_int32_t valueLength = htonl(valueSize); - std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(u_int32_t)); + uint32_t valueLength = htonl(valueSize); + std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(uint32_t)); } // Fill in attribute block size - u_int32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(u_int32_t)); - std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(u_int32_t)); + uint32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(uint32_t)); + std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(uint32_t)); outputBlock.ResizeBlock(xattrSize); } else if(listSize<0) { - if(errno == EOPNOTSUPP || errno == EACCES) + if(errno == EOPNOTSUPP || errno == EACCES +#if HAVE_DECL_ENOTSUP + // NetBSD uses ENOTSUP instead + // https://mail-index.netbsd.org/tech-kern/2011/12/13/msg012185.html + || errno == ENOTSUP +#endif + ) { - // fail silently + // Not supported by OS, or not on this filesystem + BOX_TRACE(BOX_SYS_ERRNO_MESSAGE(errno, + BOX_FILE_MESSAGE(Filename, "Failed to " + "list extended attributes"))); } else if(errno == ERANGE) { @@ -664,12 +673,17 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc "attributes of '" << Filename << "': " "buffer too small, not backed up"); } + else if(errno == ENOENT) + { + BOX_ERROR("Failed to list extended " + "attributes of '" << Filename << "': " + "file no longer exists"); + } else { - BOX_LOG_SYS_ERROR("Failed to list extended " - "attributes of '" << Filename << "', " - "not backed up"); - THROW_EXCEPTION(CommonException, OSFileError); + THROW_SYS_FILE_ERROR("Failed to list extended " + "attributes for unknown reason", Filename, + CommonException, OSFileError); } } } @@ -679,7 +693,7 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc throw; } delete[] list; -#endif +#endif // defined HAVE_LLISTXATTR && defined HAVE_LGETXATTR } // -------------------------------------------------------------------------- @@ -839,7 +853,7 @@ void BackupClientFileAttributes::WriteAttributes(const std::string& Filename, #endif } - if(static_cast<int>(xattrOffset+sizeof(u_int32_t))<=mpClearAttributes->GetSize()) + if(static_cast<int>(xattrOffset+sizeof(uint32_t))<=mpClearAttributes->GetSize()) { WriteExtendedAttr(Filename, xattrOffset); } @@ -978,13 +992,13 @@ void BackupClientFileAttributes::EnsureClearAvailable() const // -------------------------------------------------------------------------- void BackupClientFileAttributes::WriteExtendedAttr(const std::string& Filename, int xattrOffset) const { -#ifdef HAVE_SYS_XATTR_H +#if defined HAVE_LSETXATTR const char* buffer = static_cast<char*>(mpClearAttributes->GetBuffer()); - u_int32_t xattrBlockLength = 0; - std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(u_int32_t)); + uint32_t xattrBlockLength = 0; + std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(uint32_t)); int xattrBlockSize = ntohl(xattrBlockLength); - xattrOffset += sizeof(u_int32_t); + xattrOffset += sizeof(uint32_t); int xattrEnd = xattrOffset+xattrBlockSize; if(xattrEnd>mpClearAttributes->GetSize()) @@ -995,18 +1009,18 @@ void BackupClientFileAttributes::WriteExtendedAttr(const std::string& Filename, while(xattrOffset<xattrEnd) { - u_int16_t keyLength = 0; - std::memcpy(&keyLength, buffer+xattrOffset, sizeof(u_int16_t)); + uint16_t keyLength = 0; + std::memcpy(&keyLength, buffer+xattrOffset, sizeof(uint16_t)); int keySize = ntohs(keyLength); - xattrOffset += sizeof(u_int16_t); + xattrOffset += sizeof(uint16_t); const char* key = buffer+xattrOffset; xattrOffset += keySize; - u_int32_t valueLength = 0; - std::memcpy(&valueLength, buffer+xattrOffset, sizeof(u_int32_t)); + uint32_t valueLength = 0; + std::memcpy(&valueLength, buffer+xattrOffset, sizeof(uint32_t)); int valueSize = ntohl(valueLength); - xattrOffset += sizeof(u_int32_t); + xattrOffset += sizeof(uint32_t); // FIXME: Warn on EOPNOTSUPP if(::lsetxattr(Filename.c_str(), key, buffer+xattrOffset, diff --git a/lib/backupstore/BackupCommands.cpp b/lib/backupstore/BackupCommands.cpp index 318ce55a..22ef0215 100644 --- a/lib/backupstore/BackupCommands.cpp +++ b/lib/backupstore/BackupCommands.cpp @@ -52,11 +52,66 @@ return PROTOCOL_ERROR(Err_SessionReadOnly); \ } + // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolVersion::DoCommand(Protocol &, BackupStoreContext &) -// Purpose: Return the current version, or an error if the requested version isn't allowed +// Name: BackupProtocolMessage::HandleException(BoxException& e) +// Purpose: Return an error message appropriate to the passed-in +// exception, or rethrow it. +// Created: 2014/09/14 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupProtocolMessage> BackupProtocolReplyable::HandleException(BoxException& e) const +{ + if(e.GetType() == RaidFileException::ExceptionType && + e.GetSubType() == RaidFileException::RaidFileDoesntExist) + { + return PROTOCOL_ERROR(Err_DoesNotExist); + } + else if (e.GetType() == BackupStoreException::ExceptionType) + { + if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify) + { + return PROTOCOL_ERROR(Err_FileDoesNotVerify); + } + else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit) + { + return PROTOCOL_ERROR(Err_StorageLimitExceeded); + } + else if(e.GetSubType() == BackupStoreException::MultiplyReferencedObject) + { + return PROTOCOL_ERROR(Err_MultiplyReferencedObject); + } + else if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory) + { + return PROTOCOL_ERROR(Err_DoesNotExistInDirectory); + } + else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory) + { + return PROTOCOL_ERROR(Err_TargetNameExists); + } + else if(e.GetSubType() == BackupStoreException::ObjectDoesNotExist) + { + return PROTOCOL_ERROR(Err_DoesNotExist); + } + else if(e.GetSubType() == BackupStoreException::PatchChainInfoBadInDirectory) + { + return PROTOCOL_ERROR(Err_PatchConsistencyError); + } + } + + throw; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolVersion::DoCommand(Protocol &, +// BackupStoreContext &) +// Purpose: Return the current version, or an error if the +// requested version isn't allowed // Created: 2003/08/20 // // -------------------------------------------------------------------------- @@ -93,7 +148,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc // and that the client actually has an account on this machine if(mClientID != rContext.GetClientID()) { - BOX_WARNING("Failed login from client ID " << + BOX_WARNING("Failed login from client ID " << BOX_FORMAT_ACCOUNT(mClientID) << ": " "wrong certificate for this account"); return PROTOCOL_ERROR(Err_BadLogin); @@ -101,7 +156,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc if(!rContext.GetClientHasAccount()) { - BOX_WARNING("Failed login from client ID " << + BOX_WARNING("Failed login from client ID " << BOX_FORMAT_ACCOUNT(mClientID) << ": " "no such account on this server"); return PROTOCOL_ERROR(Err_BadLogin); @@ -117,17 +172,17 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc BOX_FORMAT_ACCOUNT(mClientID)); return PROTOCOL_ERROR(Err_CannotLockStoreForWriting); } - + // Debug: check we got the lock ASSERT(!rContext.SessionIsReadOnly()); } - + // Load the store info rContext.LoadStoreInfo(); if(!rContext.GetBackupStoreInfo().IsAccountEnabled()) { - BOX_WARNING("Refused login from disabled client ID " << + BOX_WARNING("Refused login from disabled client ID " << BOX_FORMAT_ACCOUNT(mClientID)); return PROTOCOL_ERROR(Err_DisabledAccount); } @@ -137,9 +192,9 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc // Mark the next phase rContext.SetPhase(BackupStoreContext::Phase_Commands); - + // Log login - BOX_NOTICE("Login from Client ID " << + BOX_NOTICE("Login from Client ID " << BOX_FORMAT_ACCOUNT(mClientID) << " " "(name=" << rContext.GetAccountName() << "): " << (((mFlags & Flags_ReadOnly) != Flags_ReadOnly) @@ -164,14 +219,15 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc // -------------------------------------------------------------------------- std::auto_ptr<BackupProtocolMessage> BackupProtocolFinished::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const { - BOX_NOTICE("Session finished for Client ID " << + // can be called in any phase + + BOX_NOTICE("Session finished for Client ID " << BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << " " "(name=" << rContext.GetAccountName() << ")"); // Let the context know about it rContext.ReceivedFinishCommand(); - // can be called in any phase return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolFinished); } @@ -191,26 +247,15 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolListDirectory::DoCommand(Back // Store the listing to a stream std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream); - try - { - // Ask the context for a directory - const BackupStoreDirectory &rdir( - rContext.GetDirectory(mObjectID)); - rdir.WriteToStream(*stream, mFlagsMustBeSet, - mFlagsNotToBeSet, mSendAttributes, - false /* never send dependency info to the client */); - } - catch (RaidFileException &e) - { - if (e.GetSubType() == RaidFileException::RaidFileDoesntExist) - { - return PROTOCOL_ERROR(Err_DoesNotExist); - } - throw; - } + // Ask the context for a directory + const BackupStoreDirectory &rdir( + rContext.GetDirectory(mObjectID)); + rdir.WriteToStream(*stream, mFlagsMustBeSet, + mFlagsNotToBeSet, mSendAttributes, + false /* never send dependency info to the client */); stream->SetForReading(); - + // Get the protocol to send the stream rProtocol.SendStreamAfterCommand(static_cast< std::auto_ptr<IOStream> > (stream)); @@ -226,7 +271,9 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolListDirectory::DoCommand(Back // Created: 2003/09/02 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -237,7 +284,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand(BackupPr { return hookResult; } - + // Check that the diff from file actually exists, if it's specified if(mDiffFromFileID != 0) { @@ -247,35 +294,13 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand(BackupPr return PROTOCOL_ERROR(Err_DiffFromFileDoesNotExist); } } - - // A stream follows, which contains the file - std::auto_ptr<IOStream> filestream(rProtocol.ReceiveStream()); - + // Ask the context to store it - int64_t id = 0; - try - { - id = rContext.AddFile(*filestream, mDirectoryObjectID, - mModificationTime, mAttributesHash, mDiffFromFileID, - mFilename, - true /* mark files with same name as old versions */); - } - catch(BackupStoreException &e) - { - if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify) - { - return PROTOCOL_ERROR(Err_FileDoesNotVerify); - } - else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit) - { - return PROTOCOL_ERROR(Err_StorageLimitExceeded); - } - else - { - throw; - } - } - + int64_t id = rContext.AddFile(rDataStream, mDirectoryObjectID, + mModificationTime, mAttributesHash, mDiffFromFileID, + mFilename, + true /* mark files with same name as old versions */); + // Tell the caller what the file ID was return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(id)); } @@ -298,7 +323,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObject::DoCommand(BackupPr // Check the object exists if(!rContext.ObjectExists(mObjectID)) { - return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(NoObject)); + return PROTOCOL_ERROR(Err_DoesNotExist); } // Open the object @@ -315,7 +340,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObject::DoCommand(BackupPr // // Function // Name: BackupProtocolGetFile::DoCommand(Protocol &, BackupStoreContext &) -// Purpose: Command to get an file object from the server -- may have to do a bit of +// Purpose: Command to get an file object from the server -- may have to do a bit of // work to get the object. // Created: 2003/09/03 // @@ -357,13 +382,13 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt en = rdir.FindEntryByID(id); if(en == 0) { - BOX_ERROR("Object " << + BOX_ERROR("Object " << BOX_FORMAT_OBJECTID(mObjectID) << - " in dir " << + " in dir " << BOX_FORMAT_OBJECTID(mInDirectory) << " for account " << BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << - " references object " << + " references object " << BOX_FORMAT_OBJECTID(id) << " which does not exist in dir"); return PROTOCOL_ERROR(Err_PatchConsistencyError); @@ -371,89 +396,73 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt id = en->GetDependsNewer(); } while(en != 0 && id != 0); - + // OK! The last entry in the chain is the full file, the others are patches back from it. // Open the last one, which is the current from file std::auto_ptr<IOStream> from(rContext.OpenObject(patchChain[patchChain.size() - 1])); - + // Then, for each patch in the chain, do a combine for(int p = ((int)patchChain.size()) - 2; p >= 0; --p) { // ID of patch int64_t patchID = patchChain[p]; - + // Open it a couple of times std::auto_ptr<IOStream> diff(rContext.OpenObject(patchID)); std::auto_ptr<IOStream> diff2(rContext.OpenObject(patchID)); - + // Choose a temporary filename for the result of the combination std::ostringstream fs; - fs << rContext.GetStoreRoot() << ".recombinetemp." << p; - std::string tempFn = + fs << rContext.GetAccountRoot() << ".recombinetemp." << p; + std::string tempFn = RaidFileController::DiscSetPathToFileSystemPath( rContext.GetStoreDiscSet(), fs.str(), p + 16); - + // Open the temporary file - std::auto_ptr<IOStream> combined; - try - { - { - // Write nastily to allow this to work with gcc 2.x - std::auto_ptr<IOStream> t( - new InvisibleTempFileStream( - tempFn.c_str(), - O_RDWR | O_CREAT | - O_EXCL | O_BINARY | - O_TRUNC)); - combined = t; - } - } - catch(...) - { - // Make sure it goes - ::unlink(tempFn.c_str()); - throw; - } - + std::auto_ptr<IOStream> combined( + new InvisibleTempFileStream( + tempFn, O_RDWR | O_CREAT | O_EXCL | + O_BINARY | O_TRUNC)); + // Do the combining BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined); - + // Move to the beginning of the combined file combined->Seek(0, IOStream::SeekType_Absolute); - + // Then shuffle round for the next go if (from.get()) from->Close(); from = combined; } - + // Now, from contains a nice file to send to the client. Reorder it { // Write nastily to allow this to work with gcc 2.x std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */)); stream = t; } - + // Release from file to avoid double deletion from.release(); } else { // Simple case: file already exists on disc ready to go - + // Open the object std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID)); BufferedStream buf(*object); - + // Verify it if(!BackupStoreFile::VerifyEncodedFileFormat(buf)) { return PROTOCOL_ERROR(Err_FileDoesNotVerify); } - + // Reset stream -- seek to beginning object->Seek(0, IOStream::SeekType_Absolute); - + // Reorder the stream/file into stream order { // Write nastily to allow this to work with gcc 2.x @@ -461,15 +470,15 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt stream = t; } - // Object will be deleted when the stream is deleted, - // so can release the object auto_ptr here to avoid + // Object will be deleted when the stream is deleted, + // so can release the object auto_ptr here to avoid // premature deletion object.release(); } // Stream the reordered stream to the peer rProtocol.SendStreamAfterCommand(stream); - + // Tell the caller what the file was return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(mObjectID)); } @@ -483,18 +492,38 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt // Created: 2003/09/04 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const +{ + return BackupProtocolCreateDirectory2(mContainingDirectoryID, + mAttributesModTime, 0 /* ModificationTime */, + mDirectoryName).DoCommand(rProtocol, rContext, rDataStream); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolCreateDirectory2::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Create directory command, with a specific +// modification time. +// Created: 2014/02/11 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory2::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - - // Get the stream containing the attributes - std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); - // Collect the attributes -- do this now so no matter what the outcome, + + // Collect the attributes -- do this now so no matter what the outcome, // the data has been absorbed. StreamableMemBlock attr; - attr.Set(*attrstream, rProtocol.GetTimeout()); - + attr.Set(rDataStream, rProtocol.GetTimeout()); + // Check to see if the hard limit has been exceeded if(rContext.HardLimitExceeded()) { @@ -503,8 +532,10 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand(Ba } bool alreadyExists = false; - int64_t id = rContext.AddDirectory(mContainingDirectoryID, mDirectoryName, attr, mAttributesModTime, alreadyExists); - + int64_t id = rContext.AddDirectory(mContainingDirectoryID, + mDirectoryName, attr, mAttributesModTime, mModificationTime, + alreadyExists); + if(alreadyExists) { return PROTOCOL_ERROR(Err_DirectoryAlreadyExists); @@ -524,17 +555,17 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand(Ba // Created: 2003/09/06 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolChangeDirAttributes::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> BackupProtocolChangeDirAttributes::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - // Get the stream containing the attributes - std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); - // Collect the attributes -- do this now so no matter what the outcome, + // Collect the attributes -- do this now so no matter what the outcome, // the data has been absorbed. StreamableMemBlock attr; - attr.Set(*attrstream, rProtocol.GetTimeout()); + attr.Set(rDataStream, rProtocol.GetTimeout()); // Get the context to do it's magic rContext.ChangeDirAttributes(mObjectID, attr, mAttributesModTime); @@ -552,17 +583,18 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolChangeDirAttributes::DoComman // Created: 2003/09/06 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolSetReplacementFileAttributes::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> +BackupProtocolSetReplacementFileAttributes::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - // Get the stream containing the attributes - std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); - // Collect the attributes -- do this now so no matter what the outcome, + // Collect the attributes -- do this now so no matter what the outcome, // the data has been absorbed. StreamableMemBlock attr; - attr.Set(*attrstream, rProtocol.GetTimeout()); + attr.Set(rDataStream, rProtocol.GetTimeout()); // Get the context to do it's magic int64_t objectID = 0; @@ -577,7 +609,6 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolSetReplacementFileAttributes: } - // -------------------------------------------------------------------------- // // Function @@ -644,19 +675,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolDeleteDirectory::DoCommand(Ba } // Context handles this - try - { - rContext.DeleteDirectory(mObjectID); - } - catch (BackupStoreException &e) - { - if(e.GetSubType() == BackupStoreException::MultiplyReferencedObject) - { - return PROTOCOL_ERROR(Err_MultiplyReferencedObject); - } - - throw; - } + rContext.DeleteDirectory(mObjectID); // return the object ID return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(mObjectID)); @@ -722,29 +741,11 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolMoveObject::DoCommand(BackupP { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - + // Let context do this, but modify error reporting on exceptions... - try - { - rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory, - mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName, - (mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject); - } - catch(BackupStoreException &e) - { - if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory) - { - return PROTOCOL_ERROR(Err_DoesNotExist); - } - else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory) - { - return PROTOCOL_ERROR(Err_TargetNameExists); - } - else - { - throw; - } - } + rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory, + mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName, + (mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject); // Return the object ID return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(mObjectID)); @@ -762,21 +763,21 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolMoveObject::DoCommand(BackupP std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const { CHECK_PHASE(Phase_Commands) - + // Create a stream for the list of filenames std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream); // Object and directory IDs int64_t objectID = mObjectID; int64_t dirID = mContainingDirectoryID; - + // Data to return in the reply int32_t numNameElements = 0; int16_t objectFlags = 0; int64_t modTime = 0; uint64_t attrModHash = 0; bool haveModTimes = false; - + do { // Check the directory really exists @@ -786,7 +787,28 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back } // Load up the directory - const BackupStoreDirectory &rdir(rContext.GetDirectory(dirID)); + const BackupStoreDirectory *pDir; + + try + { + pDir = &rContext.GetDirectory(dirID); + } + catch(BackupStoreException &e) + { + if(e.GetSubType() == BackupStoreException::ObjectDoesNotExist) + { + // If this can't be found, then there is a problem... + // tell the caller it can't be found. + return std::auto_ptr<BackupProtocolMessage>( + new BackupProtocolObjectName( + BackupProtocolObjectName::NumNameElements_ObjectDoesntExist, + 0, 0, 0)); + } + + throw; + } + + const BackupStoreDirectory& rdir(*pDir); // Find the element in this directory and store it's name if(objectID != ObjectID_DirectoryOnly) @@ -799,13 +821,13 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back // Abort! return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolObjectName(BackupProtocolObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0)); } - + // Store flags? if(objectFlags == 0) { objectFlags = en->GetFlags(); } - + // Store modification times? if(!haveModTimes) { @@ -813,14 +835,14 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back attrModHash = en->GetAttributesHash(); haveModTimes = true; } - + // Store the name in the stream en->GetName().WriteToStream(*stream); - + // Count of name elements ++numNameElements; } - + // Setup for next time round objectID = dirID; dirID = rdir.GetContainerID(); @@ -831,7 +853,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back if(numNameElements > 0) { // Get the stream ready to go - stream->SetForReading(); + stream->SetForReading(); // Tell the protocol to send the stream rProtocol.SendStreamAfterCommand(static_cast< std::auto_ptr<IOStream> >(stream)); } @@ -856,10 +878,10 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByID::DoCommand( // Open the file std::auto_ptr<IOStream> stream(rContext.OpenObject(mObjectID)); - + // Move the file pointer to the block index BackupStoreFile::MoveStreamPositionToBlockIndex(*stream); - + // Return the stream to the client rProtocol.SendStreamAfterCommand(stream); @@ -882,7 +904,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByName::DoComman // Get the directory const BackupStoreDirectory &dir(rContext.GetDirectory(mInDirectory)); - + // Find the latest object ID within it which has the same name int64_t objectID = 0; BackupStoreDirectory::Iterator i(dir); @@ -898,7 +920,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByName::DoComman } } } - + // Found anything? if(objectID == 0) { @@ -908,10 +930,10 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByName::DoComman // Open the file std::auto_ptr<IOStream> stream(rContext.OpenObject(objectID)); - + // Move the file pointer to the block index BackupStoreFile::MoveStreamPositionToBlockIndex(*stream); - + // Return the stream to the client rProtocol.SendStreamAfterCommand(stream); @@ -934,11 +956,11 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetAccountUsage::DoCommand(Ba // Get store info from context const BackupStoreInfo &rinfo(rContext.GetBackupStoreInfo()); - + // Find block size RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(rinfo.GetDiscSetNumber())); - + // Return info return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolAccountUsage( rinfo.GetBlocksUsed(), @@ -968,3 +990,48 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetIsAlive::DoCommand(BackupP // return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolIsAlive()); } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolGetAccountUsage2::DoCommand(BackupProtocolReplyable &, BackupStoreContext &) +// Purpose: Return the amount of disc space used +// Created: 26/12/13 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupProtocolMessage> BackupProtocolGetAccountUsage2::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +{ + CHECK_PHASE(Phase_Commands) + + // Get store info from context + const BackupStoreInfo &info(rContext.GetBackupStoreInfo()); + + // Find block size + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(info.GetDiscSetNumber())); + + // Return info + BackupProtocolAccountUsage2* usage = new BackupProtocolAccountUsage2(); + std::auto_ptr<BackupProtocolMessage> reply(usage); + #define COPY(name) usage->Set ## name (info.Get ## name ()) + COPY(AccountName); + usage->SetAccountEnabled(info.IsAccountEnabled()); + COPY(ClientStoreMarker); + usage->SetBlockSize(rdiscSet.GetBlockSize()); + COPY(LastObjectIDUsed); + COPY(BlocksUsed); + COPY(BlocksInCurrentFiles); + COPY(BlocksInOldFiles); + COPY(BlocksInDeletedFiles); + COPY(BlocksInDirectories); + COPY(BlocksSoftLimit); + COPY(BlocksHardLimit); + COPY(NumCurrentFiles); + COPY(NumOldFiles); + COPY(NumDeletedFiles); + COPY(NumDirectories); + #undef COPY + + return reply; +} diff --git a/lib/backupstore/BackupConstants.h b/lib/backupstore/BackupConstants.h index 19d06a15..195dc621 100644 --- a/lib/backupstore/BackupConstants.h +++ b/lib/backupstore/BackupConstants.h @@ -13,6 +13,9 @@ // 15 minutes to timeout (milliseconds) #define BACKUP_STORE_TIMEOUT (15*60*1000) +// Time to wait for retry after a backup error +#define BACKUP_ERROR_RETRY_SECONDS 100 + // Should the store daemon convert files to Raid immediately? #define BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY true diff --git a/lib/backupstore/BackupProtocol.h b/lib/backupstore/BackupProtocol.h new file mode 100644 index 00000000..d9070c73 --- /dev/null +++ b/lib/backupstore/BackupProtocol.h @@ -0,0 +1,71 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupProtocol.h +// Purpose: A thin wrapper around autogen_BackupProtocol.h +// Created: 2014/01/05 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPPROTOCOL__H +#define BACKUPPROTOCOL__H + +#include <autogen_BackupProtocol.h> +#include <BackupStoreConstants.h> +#include <BackupStoreContext.h> + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupProtocolLocal2 +// Purpose: BackupProtocolLocal with a few more IQ points +// Created: 2014/09/20 +// +// -------------------------------------------------------------------------- +class BackupProtocolLocal2 : public BackupProtocolLocal +{ +private: + BackupStoreContext mContext; + int32_t mAccountNumber; + bool mReadOnly; + +protected: + BackupStoreContext& GetContext() { return mContext; } + +public: + BackupProtocolLocal2(int32_t AccountNumber, + const std::string& ConnectionDetails, + const std::string& AccountRootDir, int DiscSetNumber, + bool ReadOnly) + // This is rather ugly: the BackupProtocolLocal constructor must not + // touch the Context, because it's not initialised yet! + : BackupProtocolLocal(mContext), + mContext(AccountNumber, (HousekeepingInterface *)NULL, + ConnectionDetails), + mAccountNumber(AccountNumber), + mReadOnly(ReadOnly) + { + mContext.SetClientHasAccount(AccountRootDir, DiscSetNumber); + QueryVersion(BACKUP_STORE_SERVER_VERSION); + QueryLogin(AccountNumber, + ReadOnly ? BackupProtocolLogin::Flags_ReadOnly : 0); + } + virtual ~BackupProtocolLocal2() { } + + std::auto_ptr<BackupProtocolFinished> Query(const BackupProtocolFinished &rQuery) + { + std::auto_ptr<BackupProtocolFinished> finished = + BackupProtocolLocal::Query(rQuery); + mContext.ReleaseWriteLock(); + return finished; + } + + void Reopen() + { + QueryVersion(BACKUP_STORE_SERVER_VERSION); + QueryLogin(mAccountNumber, + mReadOnly ? BackupProtocolLogin::Flags_ReadOnly : 0); + } +}; + +#endif // BACKUPPROTOCOL__H diff --git a/lib/backupstore/backupprotocol.txt b/lib/backupstore/BackupProtocol.txt index aa987e70..5921d009 100644 --- a/lib/backupstore/backupprotocol.txt +++ b/lib/backupstore/BackupProtocol.txt @@ -7,6 +7,7 @@ IdentString Box-Backup:v=C ServerContextClass BackupStoreContext BackupStoreContext.h AddType Filename BackupStoreFilenameClear BackupStoreFilenameClear.h +AddType String std::string ImplementLog Server syslog ImplementLog Client syslog @@ -76,8 +77,7 @@ SetClientStoreMarker 6 Command(Success) GetObject 10 Command(Success) int64 ObjectID - CONSTANT NoObject 0 - # reply has stream following, if ObjectID != NoObject + # reply has stream following (if successful) MoveObject 11 Command(Success) @@ -123,6 +123,14 @@ CreateDirectory 20 Command(Success) StreamWithCommand # stream following containing attributes +CreateDirectory2 46 Command(Success) StreamWithCommand + int64 ContainingDirectoryID + int64 AttributesModTime + int64 ModificationTime + Filename DirectoryName + # stream following containing attributes + + ListDirectory 21 Command(Success) int64 ObjectID int16 FlagsMustBeSet @@ -151,6 +159,7 @@ ChangeDirAttributes 22 Command(Success) StreamWithCommand DeleteDirectory 23 Command(Success) int64 ObjectID + UndeleteDirectory 24 Command(Success) int64 ObjectID # may not have exactly the desired effect if files within in have been deleted before the directory was deleted. @@ -233,3 +242,25 @@ GetIsAlive 42 Command(IsAlive) IsAlive 43 Reply # no data members +GetAccountUsage2 44 Command(AccountUsage2) + # no data members + +AccountUsage2 45 Reply + String AccountName + bool AccountEnabled + int64 ClientStoreMarker + int32 BlockSize + int64 LastObjectIDUsed + int64 BlocksUsed + int64 BlocksInCurrentFiles + int64 BlocksInOldFiles + int64 BlocksInDeletedFiles + int64 BlocksInDirectories + int64 BlocksSoftLimit + int64 BlocksHardLimit + int64 NumCurrentFiles + int64 NumOldFiles + int64 NumDeletedFiles + int64 NumDirectories + +# 46 is CreateDirectory2 diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp index 201491a3..c5f012fc 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.cpp +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -247,7 +247,8 @@ void BackupStoreAccountDatabase::Write() { // Write out the entry char line[256]; // more than enough for a couple of integers in string form - int s = ::sprintf(line, "%x:%d\n", i->second.GetID(), i->second.GetDiscSet()); + int s = ::snprintf(line, sizeof(line), "%x:%d\n", + i->second.GetID(), i->second.GetDiscSet()); if(::write(file, line, s) != s) { THROW_EXCEPTION(CommonException, OSFileError) diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp index 18500fc1..7955b3c4 100644 --- a/lib/backupstore/BackupStoreAccounts.cpp +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -9,19 +9,29 @@ #include "Box.h" -#include <stdio.h> +#include <algorithm> +#include <climits> +#include <cstdio> +#include <cstring> +#include <iostream> #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" +#include "BackupStoreCheck.h" +#include "BackupStoreConfigVerify.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BackupStoreInfo.h" #include "BackupStoreRefCountDatabase.h" #include "BoxPortsAndFiles.h" +#include "HousekeepStoreAccount.h" +#include "NamedLock.h" +#include "RaidFileController.h" #include "RaidFileWrite.h" #include "StoreStructure.h" #include "UnixUser.h" +#include "Utils.h" #include "MemLeakFindOn.h" @@ -110,10 +120,7 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, info->Save(); // Create the refcount database - BackupStoreRefCountDatabase::CreateNew(Entry); - std::auto_ptr<BackupStoreRefCountDatabase> refcount( - BackupStoreRefCountDatabase::Load(Entry, false)); - refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID); + BackupStoreRefCountDatabase::Create(Entry)->Commit(); } // As the original user... @@ -151,9 +158,9 @@ void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, i std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) { char accid[64]; // big enough! - ::sprintf(accid, "%08x" DIRECTORY_SEPARATOR, ID); - return std::string(std::string(BOX_RAIDFILE_ROOT_BBSTORED - DIRECTORY_SEPARATOR) + accid); + ::snprintf(accid, sizeof(accid) - 1, "%08x" DIRECTORY_SEPARATOR, ID); + return std::string(BOX_RAIDFILE_ROOT_BBSTORED DIRECTORY_SEPARATOR) + + accid; } @@ -198,6 +205,391 @@ void BackupStoreAccounts::LockAccount(int32_t ID, NamedLock& rNamedLock) { THROW_EXCEPTION_MESSAGE(BackupStoreException, CouldNotLockStoreAccount, "Failed to get exclusive " - "lock on account " << ID); + "lock on account " << BOX_FORMAT_ACCOUNT(ID)); } } + +int BackupStoreAccountsControl::BlockSizeOfDiscSet(int discSetNum) +{ + // Get controller, check disc set number + RaidFileController &controller(RaidFileController::GetController()); + if(discSetNum < 0 || discSetNum >= controller.GetNumDiscSets()) + { + BOX_FATAL("Disc set " << discSetNum << " does not exist."); + exit(1); + } + + // Return block size + return controller.GetDiscSet(discSetNum).GetBlockSize(); +} + +int BackupStoreAccountsControl::SetLimit(int32_t ID, const char *SoftLimitStr, + const char *HardLimitStr) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change limits."); + return 1; + } + + // Load the info + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, + discSetNum, false /* Read/Write */)); + + // Change the limits + int blocksize = BlockSizeOfDiscSet(discSetNum); + int64_t softlimit = SizeStringToBlocks(SoftLimitStr, blocksize); + int64_t hardlimit = SizeStringToBlocks(HardLimitStr, blocksize); + CheckSoftHardLimits(softlimit, hardlimit); + info->ChangeLimits(softlimit, hardlimit); + + // Save + info->Save(); + + BOX_NOTICE("Limits on account " << BOX_FORMAT_ACCOUNT(ID) << + " changed to " << softlimit << " soft, " << + hardlimit << " hard."); + + return 0; +} + +int BackupStoreAccountsControl::SetAccountName(int32_t ID, const std::string& rNewAccountName) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change name."); + return 1; + } + + // Load the info + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, false /* Read/Write */)); + + info->SetAccountName(rNewAccountName); + + // Save + info->Save(); + + BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << + " name changed to " << rNewAccountName); + + return 0; +} + +int BackupStoreAccountsControl::PrintAccountInfo(int32_t ID) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + + if(!OpenAccount(ID, rootDir, discSetNum, user, + NULL /* no write lock needed for this read-only operation */)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to display info."); + return 1; + } + + // Load it in + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, true /* ReadOnly */)); + + return BackupAccountControl::PrintAccountInfo(*info, + BlockSizeOfDiscSet(discSetNum)); +} + +int BackupStoreAccountsControl::SetAccountEnabled(int32_t ID, bool enabled) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change enabled flag."); + return 1; + } + + // Load it in + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, false /* ReadOnly */)); + info->SetAccountEnabled(enabled); + info->Save(); + return 0; +} + +int BackupStoreAccountsControl::DeleteAccount(int32_t ID, bool AskForConfirmation) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + // Obtain a write lock, as the daemon user + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for deletion."); + return 1; + } + + // Check user really wants to do this + if(AskForConfirmation) + { + BOX_WARNING("Really delete account " << + BOX_FORMAT_ACCOUNT(ID) << "? (type 'yes' to confirm)"); + char response[256]; + if(::fgets(response, sizeof(response), stdin) == 0 || ::strcmp(response, "yes\n") != 0) + { + BOX_NOTICE("Deletion cancelled."); + return 0; + } + } + + // Back to original user, but write lock is maintained + user.reset(); + + std::auto_ptr<BackupStoreAccountDatabase> db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Delete from account database + db->DeleteEntry(ID); + + // Write back to disc + db->Write(); + + // Remove the store files... + + // First, become the user specified in the config file + std::string username; + { + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Become the right user + if(!username.empty()) + { + // Username specified, change... + user.reset(new UnixUser(username)); + user->ChangeProcessUser(true /* temporary */); + // Change will be undone when user goes out of scope + } + + // Secondly, work out which directories need wiping + std::vector<std::string> toDelete; + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet discSet(rcontroller.GetDiscSet(discSetNum)); + for(RaidFileDiscSet::const_iterator i(discSet.begin()); i != discSet.end(); ++i) + { + if(std::find(toDelete.begin(), toDelete.end(), *i) == toDelete.end()) + { + toDelete.push_back((*i) + DIRECTORY_SEPARATOR + rootDir); + } + } + + // NamedLock will throw an exception if it can't delete the lockfile, + // which it can't if it doesn't exist. Now that we've deleted the account, + // nobody can open it anyway, so it's safe to unlock. + writeLock.ReleaseLock(); + + int retcode = 0; + + // Thirdly, delete the directories... + for(std::vector<std::string>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) + { + BOX_NOTICE("Deleting store directory " << (*d) << "..."); + // Just use the rm command to delete the files +#ifdef WIN32 + std::string cmd("rmdir /s/q "); + std::string dir = *d; + + // rmdir doesn't understand forward slashes, so replace them all. + for(std::string::iterator i = dir.begin(); i != dir.end(); i++) + { + if(*i == '/') + { + *i = '\\'; + } + } + cmd += dir; +#else + std::string cmd("rm -rf "); + cmd += *d; +#endif + // Run command + if(::system(cmd.c_str()) != 0) + { + BOX_ERROR("Failed to delete files in " << (*d) << + ", delete them manually."); + retcode = 1; + } + } + + // Success! + return retcode; +} + +bool BackupStoreAccountsControl::OpenAccount(int32_t ID, std::string &rRootDirOut, + int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock) +{ + // Load in the account database + std::auto_ptr<BackupStoreAccountDatabase> db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Exists? + if(!db->EntryExists(ID)) + { + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " does not exist."); + return false; + } + + // Get info from the database + BackupStoreAccounts acc(*db); + acc.GetAccountRoot(ID, rRootDirOut, rDiscSetOut); + + // Get the user under which the daemon runs + std::string username; + { + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Become the right user + if(!username.empty()) + { + // Username specified, change... + apUser.reset(new UnixUser(username)); + apUser->ChangeProcessUser(true /* temporary */); + // Change will be undone when apUser goes out of scope + // in the caller. + } + + if(pLock) + { + acc.LockAccount(ID, *pLock); + } + + return true; +} + +int BackupStoreAccountsControl::CheckAccount(int32_t ID, bool FixErrors, bool Quiet, + bool ReturnNumErrorsFound) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, + FixErrors ? &writeLock : NULL)) // don't need a write lock if not making changes + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for checking."); + return 1; + } + + // Check it + BackupStoreCheck check(rootDir, discSetNum, ID, FixErrors, Quiet); + check.Check(); + + if(ReturnNumErrorsFound) + { + return check.GetNumErrorsFound(); + } + else + { + return check.ErrorsFound() ? 1 : 0; + } +} + +int BackupStoreAccountsControl::CreateAccount(int32_t ID, int32_t DiscNumber, + int32_t SoftLimit, int32_t HardLimit) +{ + // Load in the account database + std::auto_ptr<BackupStoreAccountDatabase> db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Already exists? + if(db->EntryExists(ID)) + { + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " already exists."); + return 1; + } + + // Get the user under which the daemon runs + std::string username; + { + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Create it. + BackupStoreAccounts acc(*db); + acc.Create(ID, DiscNumber, SoftLimit, HardLimit, username); + + BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << " created."); + + return 0; +} + +int BackupStoreAccountsControl::HousekeepAccountNow(int32_t ID) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + + if(!OpenAccount(ID, rootDir, discSetNum, user, + NULL /* housekeeping locks the account itself */)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for housekeeping."); + return 1; + } + + HousekeepStoreAccount housekeeping(ID, rootDir, discSetNum, NULL); + bool success = housekeeping.DoHousekeeping(); + + if(!success) + { + BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(ID) + << " for housekeeping: perhaps a client is " + "still connected?"); + return 1; + } + else + { + BOX_TRACE("Finished housekeeping on account " << + BOX_FORMAT_ACCOUNT(ID)); + return 0; + } +} + diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h index 3163f15c..bcc3cf1c 100644 --- a/lib/backupstore/BackupStoreAccounts.h +++ b/lib/backupstore/BackupStoreAccounts.h @@ -13,6 +13,7 @@ #include <string> #include "BackupStoreAccountDatabase.h" +#include "BackupAccountControl.h" #include "NamedLock.h" // -------------------------------------------------------------------------- @@ -51,5 +52,34 @@ private: BackupStoreAccountDatabase &mrDatabase; }; +class Configuration; +class UnixUser; + +class BackupStoreAccountsControl : public BackupAccountControl +{ +public: + BackupStoreAccountsControl(const Configuration& config, + bool machineReadableOutput = false) + : BackupAccountControl(config, machineReadableOutput) + { } + int BlockSizeOfDiscSet(int discSetNum); + bool OpenAccount(int32_t ID, std::string &rRootDirOut, + int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock); + int SetLimit(int32_t ID, const char *SoftLimitStr, + const char *HardLimitStr); + int SetAccountName(int32_t ID, const std::string& rNewAccountName); + int PrintAccountInfo(int32_t ID); + int SetAccountEnabled(int32_t ID, bool enabled); + int DeleteAccount(int32_t ID, bool AskForConfirmation); + int CheckAccount(int32_t ID, bool FixErrors, bool Quiet, + bool ReturnNumErrorsFound = false); + int CreateAccount(int32_t ID, int32_t DiscNumber, int32_t SoftLimit, + int32_t HardLimit); + int HousekeepAccountNow(int32_t ID); +}; + +// max size of soft limit as percent of hard limit +#define MAX_SOFT_LIMIT_SIZE 97 + #endif // BACKUPSTOREACCOUNTS__H diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index f2302337..b53ebf6d 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -17,11 +17,13 @@ #endif #include "autogen_BackupStoreException.h" +#include "BackupStoreAccountDatabase.h" #include "BackupStoreCheck.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreFile.h" #include "BackupStoreObjectMagic.h" +#include "BackupStoreRefCountDatabase.h" #include "RaidFileController.h" #include "RaidFileException.h" #include "RaidFileRead.h" @@ -58,7 +60,7 @@ BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNum mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), mBlocksInDirectories(0), - mNumFiles(0), + mNumCurrentFiles(0), mNumOldFiles(0), mNumDeletedFiles(0), mNumDirectories(0) @@ -78,6 +80,24 @@ BackupStoreCheck::~BackupStoreCheck() { // Clean up FreeInfo(); + + // Avoid an exception if we forget to discard mapNewRefs + if (mapNewRefs.get()) + { + // Discard() can throw exception, but destructors aren't supposed to do that, so + // just catch and log them. + try + { + mapNewRefs->Discard(); + } + catch(BoxException &e) + { + BOX_ERROR("Error while destroying BackupStoreCheck: discarding " + "the refcount database threw an exception: " << e.what()); + } + + mapNewRefs.reset(); + } } @@ -92,15 +112,21 @@ BackupStoreCheck::~BackupStoreCheck() // -------------------------------------------------------------------------- void BackupStoreCheck::Check() { - std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); - ASSERT(FileExists(writeLockFilename)); + if(mFixErrors) + { + std::string writeLockFilename; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); + ASSERT(FileExists(writeLockFilename)); + } if(!mQuiet && mFixErrors) { - BOX_NOTICE("Will fix errors encountered during checking."); + BOX_INFO("Will fix errors encountered during checking."); } + BackupStoreAccountDatabase::Entry account(mAccountID, mDiscSetNumber); + mapNewRefs = BackupStoreRefCountDatabase::Create(account); + // Phase 1, check objects if(!mQuiet) { @@ -109,14 +135,14 @@ void BackupStoreCheck::Check() BOX_INFO("Phase 1, check objects..."); } CheckObjects(); - + // Phase 2, check directories if(!mQuiet) { BOX_INFO("Phase 2, check directories..."); } CheckDirectories(); - + // Phase 3, check root if(!mQuiet) { @@ -138,16 +164,38 @@ void BackupStoreCheck::Check() } FixDirsWithWrongContainerID(); FixDirsWithLostDirs(); - + // Phase 6, regenerate store info if(!mQuiet) { BOX_INFO("Phase 6, regenerate store info..."); } WriteNewStoreInfo(); - -// DUMP_OBJECT_INFO - + + try + { + std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs = + BackupStoreRefCountDatabase::Load(account, false); + mNumberErrorsFound += mapNewRefs->ReportChangesTo(*apOldRefs); + } + catch(BoxException &e) + { + BOX_WARNING("Reference count database was missing or " + "corrupted, cannot check it for errors."); + mNumberErrorsFound++; + } + + // force file to be saved and closed before releasing the lock below + if(mFixErrors) + { + mapNewRefs->Commit(); + } + else + { + mapNewRefs->Discard(); + } + mapNewRefs.reset(); + if(mNumberErrorsFound > 0) { BOX_WARNING("Finished checking store account ID " << @@ -250,16 +298,18 @@ void BackupStoreCheck::CheckObjects() { // Make sure the starting root dir doesn't end with '/'. std::string start(mStoreRoot); - if(start.size() > 0 && start[start.size() - 1] == '/') + if(start.size() > 0 && ( + start[start.size() - 1] == '/' || + start[start.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR)) { start.resize(start.size() - 1); } - - maxDir = CheckObjectsScanDir(0, 1, mStoreRoot); + + maxDir = CheckObjectsScanDir(0, 1, start); BOX_TRACE("Max dir starting ID is " << BOX_FORMAT_OBJECTID(maxDir)); } - + // Then go through and scan all the objects within those directories for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH)) { @@ -287,24 +337,25 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const // If any of the directories is missing, create it. RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); - + if(!rdiscSet.IsNonRaidSet()) { unsigned int numDiscs = rdiscSet.size(); - + for(unsigned int l = 0; l < numDiscs; ++l) { // build name std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName); - struct stat st; + EMU_STRUCT_STAT st; - if(stat(dn.c_str(), &st) != 0 && errno == ENOENT) + if(EMU_STAT(dn.c_str(), &st) != 0 && + errno == ENOENT) { if(mkdir(dn.c_str(), 0755) != 0) { THROW_SYS_FILE_ERROR("Failed to " "create missing RaidFile " - "directory", dn, + "directory", dn, RaidFileException, OSError); } } @@ -334,7 +385,7 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const else { BOX_ERROR("Spurious or invalid directory " << - rDirName << DIRECTORY_SEPARATOR << + rDirName << DIRECTORY_SEPARATOR << (*i) << " found, " << (mFixErrors?"deleting":"delete manually")); ++mNumberErrorsFound; @@ -361,11 +412,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) std::string dirName; StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */); // Check expectations - ASSERT(dirName.size() > 4 && + ASSERT(dirName.size() > 4 && dirName[dirName.size() - 4] == DIRECTORY_SEPARATOR_ASCHAR); // Remove the filename from it dirName.resize(dirName.size() - 4); // four chars for "/o00" - + // Check directory exists if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName)) { @@ -377,14 +428,14 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) std::vector<std::string> files; RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName, RaidFileRead::DirReadType_FilesOnly, files); - + // Array of things present bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)]; for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l) { idsPresent[l] = false; } - + // Parse each entry, building up a list of object IDs which are present in the dir. // This is done so that whatever order is retured from the directory, objects are scanned // in order. @@ -405,7 +456,8 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) fileOK = false; } // info and refcount databases are OK in the root directory - else if(*i == "info" || *i == "refcount.db") + else if(*i == "info" || *i == "refcount.db" || + *i == "refcount.rdb" || *i == "refcount.rdbX") { fileOK = true; } @@ -413,11 +465,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) { fileOK = false; } - + if(!fileOK) { // Unexpected or bad file, delete it - BOX_ERROR("Spurious file " << dirName << + BOX_ERROR("Spurious file " << dirName << DIRECTORY_SEPARATOR << (*i) << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; @@ -428,7 +480,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) } } } - + // Check all the objects found in this directory for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i) { @@ -436,7 +488,8 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) { // Check the object is OK, and add entry char leaf[8]; - ::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i); + ::snprintf(leaf, sizeof(leaf), + DIRECTORY_SEPARATOR "o%02x", i); if(!CheckAndAddObject(StartID | i, dirName + leaf)) { // File was bad, delete it @@ -480,7 +533,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, std::auto_ptr<RaidFileRead> file( RaidFileRead::Open(mDiscSetNumber, rFilename)); size = file->GetDiscUsageInBlocks(); - + // Read in first four bytes -- don't have to worry about // retrying if not all bytes read as is RaidFile uint32_t signature; @@ -491,7 +544,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, } // Seek back to beginning file->Seek(0, IOStream::SeekType_Absolute); - + // Then... check depending on the type switch(ntohl(signature)) { @@ -519,7 +572,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, // Error caught, not a good file then, let it be deleted return false; } - + // Got a container ID? (ie check was successful) if(containerID == -1) { @@ -538,13 +591,13 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, // If it looks like a good object, and it's non-RAID, and // this is a RAID set, then convert it to RAID. - + RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); if(!rdiscSet.IsNonRaidSet()) { // See if the file exists - RaidFileUtil::ExistType existance = + RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, rFilename); if(existance == RaidFileUtil::NonRaid) { @@ -565,7 +618,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, if(mFixErrors) { std::auto_ptr<RaidFileRead> read( - RaidFileRead::Open(mDiscSetNumber, + RaidFileRead::Open(mDiscSetNumber, rFilename)); RaidFileWrite write(mDiscSetNumber, rFilename); write.Open(true /* overwrite */); @@ -575,7 +628,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, } } } - + // Report success return true; } @@ -636,7 +689,7 @@ int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream) // Wrong object ID return -1; } - + // Return container ID return dir.GetContainerID(); } @@ -669,7 +722,7 @@ void BackupStoreCheck::CheckDirectories() { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; - + for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); @@ -703,9 +756,9 @@ void BackupStoreCheck::CheckDirectories() BOX_FORMAT_OBJECTID(pblock->mID[e]) << " was OK after fixing"); } - + if(isModified && mFixErrors) - { + { BOX_WARNING("Writing modified directory to disk: " << BOX_FORMAT_OBJECTID(pblock->mID[e])); RaidFileWrite fixed(mDiscSetNumber, filename); @@ -714,64 +767,7 @@ void BackupStoreCheck::CheckDirectories() fixed.Commit(true /* convert to raid representation now */); } - // Count valid entries - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next()) != 0) - { - int32_t iIndex; - IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); - - ASSERT(piBlock != 0 || - mDirsWhichContainLostDirs.find(en->GetObjectID()) - != mDirsWhichContainLostDirs.end()); - if (piBlock) - { - // Normally it would exist and this - // check would not be necessary, but - // we might have missing directories - // that we will recreate later. - // cf mDirsWhichContainLostDirs. - uint8_t iflags = GetFlags(piBlock, iIndex); - SetFlags(piBlock, iIndex, iflags | Flags_IsContained); - } - - if(en->IsDir()) - { - mNumDirectories++; - } - else if(!en->IsFile()) - { - BOX_TRACE("Not counting object " << - BOX_FORMAT_OBJECTID(en->GetObjectID()) << - " with flags " << en->GetFlags()); - } - else // it's a good file, add to sizes - if(en->IsOld() && en->IsDeleted()) - { - BOX_WARNING("File " << - BOX_FORMAT_OBJECTID(en->GetObjectID()) << - " is both old and deleted, " - "this should not happen!"); - } - else if(en->IsOld()) - { - mNumFiles++; - mNumOldFiles++; - mBlocksInOldFiles += en->GetSizeInBlocks(); - } - else if(en->IsDeleted()) - { - mNumFiles++; - mNumDeletedFiles++; - mBlocksInDeletedFiles += en->GetSizeInBlocks(); - } - else - { - mNumFiles++; - mBlocksInCurrentFiles += en->GetSizeInBlocks(); - } - } + CountDirectoryEntries(dir); } } } @@ -798,8 +794,8 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) ++mNumberErrorsFound; isModified = true; } - - // Go through, and check that everything in that directory exists and is valid + + // Go through, and check that every entry exists and is valid BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) @@ -824,14 +820,14 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) { // Just remove the entry badEntry = true; - BOX_ERROR("Directory ID " << + BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(dir.GetObjectID()) << - " references object " << + " references object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " which does not exist."); ++mNumberErrorsFound; } - + // Is this entry worth keeping? if(badEntry) { @@ -846,16 +842,16 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) { BOX_ERROR("Removing directory entry " << BOX_FORMAT_OBJECTID(*d) << " from " - "directory " << + "directory " << BOX_FORMAT_OBJECTID(dir.GetObjectID())); ++mNumberErrorsFound; dir.DeleteEntry(*d); } - + // Mark as modified restart = true; isModified = true; - + // Errors found } } @@ -863,6 +859,84 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) return isModified; } +// Count valid remaining entries and the number of blocks in them. +void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir) +{ + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + int32_t iIndex; + IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); + bool badEntry = false; + bool wasAlreadyContained = false; + + ASSERT(piBlock != 0 || + mDirsWhichContainLostDirs.find(en->GetObjectID()) + != mDirsWhichContainLostDirs.end()); + + if (piBlock) + { + // Normally it would exist and this + // check would not be necessary, but + // we might have missing directories + // that we will recreate later. + // cf mDirsWhichContainLostDirs. + uint8_t iflags = GetFlags(piBlock, iIndex); + wasAlreadyContained = (iflags & Flags_IsContained); + SetFlags(piBlock, iIndex, iflags | Flags_IsContained); + } + + if(wasAlreadyContained) + { + // don't double-count objects that are + // contained by another directory as well. + } + else if(en->IsDir()) + { + mNumDirectories++; + } + else if(!en->IsFile()) + { + BOX_TRACE("Not counting object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " with flags " << en->GetFlags()); + } + else // it's a file + { + // Add to sizes? + // If piBlock was zero, then wasAlreadyContained + // might be uninitialized; but we only process + // files here, and if a file's piBlock was zero + // then badEntry would be set above, so we + // wouldn't be here. + ASSERT(!badEntry) + + // It can be both old and deleted. + // If neither, then it's current. + if(en->IsDeleted()) + { + mNumDeletedFiles++; + mBlocksInDeletedFiles += en->GetSizeInBlocks(); + } + + if(en->IsOld()) + { + mNumOldFiles++; + mBlocksInOldFiles += en->GetSizeInBlocks(); + } + + if(!en->IsDeleted() && !en->IsOld()) + { + mNumCurrentFiles++; + mBlocksInCurrentFiles += en->GetSizeInBlocks(); + } + } + + mapNewRefs->AddReference(en->GetObjectID()); + } +} + bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified) { @@ -871,8 +945,7 @@ bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, ASSERT(piBlock != 0); uint8_t iflags = GetFlags(piBlock, IndexInDirBlock); - bool badEntry = false; - + // Is the type the same? if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir()) { @@ -882,73 +955,67 @@ bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, " references object " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " which has a different type than expected."); - badEntry = true; ++mNumberErrorsFound; + return false; // remove this entry } + // Check that the entry is not already contained. - else if(iflags & Flags_IsContained) + if(iflags & Flags_IsContained) { BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(DirectoryID) << " references object " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " which is already contained."); - badEntry = true; ++mNumberErrorsFound; + return false; // remove this entry } - else + + // Not already contained by another directory. + // Don't set the flag until later, after we finish repairing + // the directory and removing all bad entries. + + // Check that the container ID of the object is correct + if(piBlock->mContainer[IndexInDirBlock] != DirectoryID) { - // Not already contained by another directory. - // Don't set the flag until later, after we finish repairing - // the directory and removing all bad entries. - - // Check that the container ID of the object is correct - if(piBlock->mContainer[IndexInDirBlock] != DirectoryID) + // Needs fixing... + if(iflags & Flags_IsDir) { - // Needs fixing... - if(iflags & Flags_IsDir) - { - // Add to will fix later list - BOX_ERROR("Directory ID " << - BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) - << " has wrong container ID."); - mDirsWithWrongContainerID.push_back(rEntry.GetObjectID()); - ++mNumberErrorsFound; - } - else - { - // This is OK for files, they might move - BOX_NOTICE("File ID " << - BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) - << " has different container ID, " - "probably moved"); - } - - // Fix entry for now - piBlock->mContainer[IndexInDirBlock] = DirectoryID; + // Add to will fix later list + BOX_ERROR("Directory ID " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) + << " has wrong container ID."); + mDirsWithWrongContainerID.push_back(rEntry.GetObjectID()); + ++mNumberErrorsFound; + } + else + { + // This is OK for files, they might move + BOX_INFO("File ID " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) + << " has different container ID, " + "probably moved"); } + + // Fix entry for now + piBlock->mContainer[IndexInDirBlock] = DirectoryID; } - - // Check the object size, if it's OK and a file - if(!badEntry && !rEntry.IsDir()) + + // Check the object size + if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock]) { - if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock]) - { - // Wrong size, correct it. - rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]); + // Wrong size, correct it. + BOX_ERROR("Directory " << BOX_FORMAT_OBJECTID(DirectoryID) << + " entry for " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << + " has wrong size " << rEntry.GetSizeInBlocks() << + ", should be " << piBlock->mObjectSizeInBlocks[IndexInDirBlock]); - // Mark as changed - rIsModified = true; - ++mNumberErrorsFound; + rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]); - // Tell user - BOX_ERROR("Directory ID " << - BOX_FORMAT_OBJECTID(DirectoryID) << - " has wrong size for object " << - BOX_FORMAT_OBJECTID(rEntry.GetObjectID())); - } + // Mark as changed + rIsModified = true; + ++mNumberErrorsFound; } - return !badEntry; + return true; // don't delete this entry } - diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h index 178a873a..5353c968 100644 --- a/lib/backupstore/BackupStoreCheck.h +++ b/lib/backupstore/BackupStoreCheck.h @@ -20,6 +20,7 @@ class IOStream; class BackupStoreFilename; +class BackupStoreRefCountDatabase; /* @@ -130,8 +131,9 @@ private: bool CheckDirectory(BackupStoreDirectory& dir); bool CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified); + void CountDirectoryEntries(BackupStoreDirectory& dir); int64_t CheckFile(int64_t ObjectID, IOStream &rStream); - int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); + int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); // Fixing functions bool TryToRecreateDirectory(int64_t MissingDirectoryID); @@ -195,6 +197,9 @@ private: // Set of extra directories added std::set<BackupStoreCheck_ID_t> mDirsAdded; + + // The refcount database, being reconstructed as the check/fix progresses + std::auto_ptr<BackupStoreRefCountDatabase> mapNewRefs; // Misc stuff int32_t mLostDirNameSerial; @@ -206,7 +211,7 @@ private: int64_t mBlocksInOldFiles; int64_t mBlocksInDeletedFiles; int64_t mBlocksInDirectories; - int64_t mNumFiles; + int64_t mNumCurrentFiles; int64_t mNumOldFiles; int64_t mNumDeletedFiles; int64_t mNumDirectories; diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp index 90e21e7f..13831a09 100644 --- a/lib/backupstore/BackupStoreCheck2.cpp +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -20,6 +20,7 @@ #include "BackupStoreFileWire.h" #include "BackupStoreInfo.h" #include "BackupStoreObjectMagic.h" +#include "BackupStoreRefCountDatabase.h" #include "MemBlockStream.h" #include "RaidFileRead.h" #include "RaidFileWrite.h" @@ -40,7 +41,7 @@ void BackupStoreCheck::CheckRoot() { int32_t index = 0; IDBlock *pblock = LookupID(BACKUPSTORE_ROOT_DIRECTORY_ID, index); - + if(pblock != 0) { // Found it. Which is lucky. Mark it as contained. @@ -49,9 +50,9 @@ void BackupStoreCheck::CheckRoot() else { BOX_WARNING("Root directory doesn't exist"); - + ++mNumberErrorsFound; - + if(mFixErrors) { // Create a new root directory @@ -78,7 +79,7 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain } BackupStoreDirectory dir(DirectoryID, ContainingDirID); - + // Serialise to disc std::string filename; StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); @@ -87,10 +88,10 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain dir.WriteToStream(obj); int64_t size = obj.GetDiscUsageInBlocks(); obj.Commit(true /* convert to raid now */); - + // Record the fact we've done this mDirsAdded.insert(DirectoryID); - + // Add to sizes mBlocksUsed += size; mBlocksInDirectories += size; @@ -131,15 +132,16 @@ void BackupStoreCheck::CheckUnattachedObjects() { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; - + for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); if((flags & Flags_IsContained) == 0) { // Unattached object... + int64_t ObjectID = pblock->mID[e]; BOX_ERROR("Object " << - BOX_FORMAT_OBJECTID(pblock->mID[e]) << + BOX_FORMAT_OBJECTID(ObjectID) << " is unattached."); ++mNumberErrorsFound; @@ -149,6 +151,8 @@ void BackupStoreCheck::CheckUnattachedObjects() if((flags & Flags_IsDir) == Flags_IsDir) { // Directory. Just put into lost and found. + // (It doesn't contain its filename, so we + // can't recreate the entry in the parent) putIntoDirectoryID = GetLostAndFoundDirID(); } else @@ -157,7 +161,9 @@ void BackupStoreCheck::CheckUnattachedObjects() { int64_t diffFromObjectID = 0; std::string filename; - StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */); + StoreStructure::MakeObjectFilename(ObjectID, + mStoreRoot, mDiscSetNumber, filename, + false /* don't attempt to make sure the dir exists */); // The easiest way to do this is to verify it again. Not such a bad penalty, because // this really shouldn't be done very often. @@ -170,20 +176,22 @@ void BackupStoreCheck::CheckUnattachedObjects() // Just delete it to be safe. if(diffFromObjectID != 0) { - BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached, and is a patch. Deleting, cannot reliably recover."); - + BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(ObjectID) << " is unattached, and is a patch. Deleting, cannot reliably recover."); + // Delete this object instead if(mFixErrors) { RaidFileWrite del(mDiscSetNumber, filename); del.Delete(); } - + + mBlocksUsed -= pblock->mObjectSizeInBlocks[e]; + // Move on to next item continue; } } - + // Files contain their original filename, so perhaps the orginal directory still exists, // or we can infer the existance of a directory? // Look for a matching entry in the mDirsWhichContainLostDirs map. @@ -249,9 +257,10 @@ void BackupStoreCheck::CheckUnattachedObjects() } // Add it to the directory - pFixer->InsertObject(pblock->mID[e], + pFixer->InsertObject(ObjectID, ((flags & Flags_IsDir) == Flags_IsDir), lostDirNameSerial); + mapNewRefs->AddReference(ObjectID); } } } @@ -284,7 +293,7 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) // Not a missing directory, can't recreate. return false; } - + // Can recreate this! Wooo! if(!mFixErrors) { @@ -297,12 +306,12 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) BOX_WARNING("Recreating missing directory " << BOX_FORMAT_OBJECTID(MissingDirectoryID)); - + // Create a blank directory BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */); // Note that this directory already contains a directory entry pointing to // this dir, so it doesn't have to be added. - + // Serialise to disc std::string filename; StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); @@ -310,10 +319,10 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) root.Open(false /* don't allow overwriting */); dir.WriteToStream(root); root.Commit(true /* convert to raid now */); - + // Record the fact we've done this mDirsAdded.insert(MissingDirectoryID); - + // Remove the entry from the map, so this doesn't happen again mDirsWhichContainLostDirs.erase(missing); @@ -328,7 +337,7 @@ BackupStoreDirectoryFixer::BackupStoreDirectoryFixer(std::string storeRoot, // Generate filename StoreStructure::MakeObjectFilename(ID, mStoreRoot, mDiscSetNumber, mFilename, false /* don't make sure the dir exists */); - + // Read it in std::auto_ptr<RaidFileRead> file( RaidFileRead::Open(mDiscSetNumber, mFilename)); @@ -347,7 +356,7 @@ void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, { // Directory -- simply generate a name for it. char name[32]; - ::sprintf(name, "dir%08x", lostDirNameSerial); + ::snprintf(name, sizeof(name), "dir%08x", lostDirNameSerial); objectStoreFilename.SetAsClearFilename(name); } else @@ -370,7 +379,7 @@ void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif +#endif )) { // This should never happen, everything has been @@ -393,7 +402,7 @@ BackupStoreDirectoryFixer::~BackupStoreDirectoryFixer() { // Fix any flags which have been broken, which there's a good chance of doing mDirectory.CheckAndFix(); - + // Write it out RaidFileWrite root(mDiscSetNumber, mFilename); root.Open(true /* allow overwriting */); @@ -438,7 +447,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() while(true) { char name[32]; - ::sprintf(name, "lost+found%d", n++); + ::snprintf(name, sizeof(name), "lost+found%d", n++); lostAndFound.SetAsClearFilename(name); if(!dir.NameInUse(lostAndFound)) { @@ -453,7 +462,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() // Create a blank directory CreateBlankDirectory(id, BACKUPSTORE_ROOT_DIRECTORY_ID); - + // Add an entry for it dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0); @@ -462,7 +471,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() root.Open(true /* allow overwriting */); dir.WriteToStream(root); root.Commit(true /* convert to raid now */); - + // Store mLostAndFoundDirectoryID = id; @@ -494,7 +503,7 @@ void BackupStoreCheck::FixDirsWithWrongContainerID() int32_t index = 0; IDBlock *pblock = LookupID(*i, index); if(pblock == 0) continue; - + // Load in BackupStoreDirectory dir; std::string filename; @@ -506,7 +515,7 @@ void BackupStoreCheck::FixDirsWithWrongContainerID() // Adjust container ID dir.SetContainerID(pblock->mContainer[index]); - + // Write it out RaidFileWrite root(mDiscSetNumber, filename); root.Open(true /* allow overwriting */); @@ -539,7 +548,7 @@ void BackupStoreCheck::FixDirsWithLostDirs() int32_t index = 0; IDBlock *pblock = LookupID(i->second, index); if(pblock == 0) continue; - + // Load in BackupStoreDirectory dir; std::string filename; @@ -551,10 +560,10 @@ void BackupStoreCheck::FixDirsWithLostDirs() // Delete the dodgy entry dir.DeleteEntry(i->first); - + // Fix it up dir.CheckAndFix(); - + // Write it out RaidFileWrite root(mDiscSetNumber, filename); root.Open(true /* allow overwriting */); @@ -587,49 +596,42 @@ void BackupStoreCheck::WriteNewStoreInfo() ++mNumberErrorsFound; } - BOX_NOTICE("Total files: " << mNumFiles << " (of which " + BOX_INFO("Current files: " << mNumCurrentFiles << ", " "old files: " << mNumOldFiles << ", " - "deleted files: " << mNumDeletedFiles << "), " + "deleted files: " << mNumDeletedFiles << ", " "directories: " << mNumDirectories); - // Minimum soft and hard limits + // Minimum soft and hard limits to ensure that nothing gets deleted + // by housekeeping. int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024; int64_t minHard = ((minSoft * 11) / 10) + 1024; - // Need to do anything? - if(pOldInfo.get() != 0 && - mNumberErrorsFound == 0 && - pOldInfo->GetAccountID() == mAccountID) - { - // Leave the store info as it is, no need to alter it because nothing really changed, - // and the only essential thing was that the account ID was correct, which is was. - return; - } - - // NOTE: We will always build a new store info, so the client store marker gets changed. + int64_t softLimit = pOldInfo.get() ? pOldInfo->GetBlocksSoftLimit() : minSoft; + int64_t hardLimit = pOldInfo.get() ? pOldInfo->GetBlocksHardLimit() : minHard; - // Work out the new limits - int64_t softLimit = minSoft; - int64_t hardLimit = minHard; - if(pOldInfo.get() != 0 && pOldInfo->GetBlocksSoftLimit() > minSoft) + if(mNumberErrorsFound && pOldInfo.get()) { - softLimit = pOldInfo->GetBlocksSoftLimit(); - } - else - { - BOX_WARNING("Soft limit for account changed to ensure " - "housekeeping doesn't delete files on next run."); - } - if(pOldInfo.get() != 0 && pOldInfo->GetBlocksHardLimit() > minHard) - { - hardLimit = pOldInfo->GetBlocksHardLimit(); - } - else - { - BOX_WARNING("Hard limit for account changed to ensure " - "housekeeping doesn't delete files on next run."); + if(pOldInfo->GetBlocksSoftLimit() > minSoft) + { + softLimit = pOldInfo->GetBlocksSoftLimit(); + } + else + { + BOX_WARNING("Soft limit for account changed to ensure " + "housekeeping doesn't delete files on next run."); + } + + if(pOldInfo->GetBlocksHardLimit() > minHard) + { + hardLimit = pOldInfo->GetBlocksHardLimit(); + } + else + { + BOX_WARNING("Hard limit for account changed to ensure " + "housekeeping doesn't delete files on next run."); + } } - + // Object ID int64_t lastObjID = mLastIDInInfo; if(mLostAndFoundDirectoryID != 0) @@ -662,11 +664,24 @@ void BackupStoreCheck::WriteNewStoreInfo() hardLimit, (pOldInfo.get() ? pOldInfo->IsAccountEnabled() : true), *extra_data)); - info->AdjustNumFiles(mNumFiles); + info->AdjustNumCurrentFiles(mNumCurrentFiles); info->AdjustNumOldFiles(mNumOldFiles); info->AdjustNumDeletedFiles(mNumDeletedFiles); info->AdjustNumDirectories(mNumDirectories); + // If there are any errors (apart from wrong block counts), then we + // should reset the ClientStoreMarker to zero, which + // CreateForRegeneration does. But if there are no major errors, then + // we should maintain the old ClientStoreMarker, to avoid invalidating + // the client's directory cache. + if (pOldInfo.get() && !mNumberErrorsFound) + { + BOX_INFO("No major errors found, preserving old " + "ClientStoreMarker: " << + pOldInfo->GetClientStoreMarker()); + info->SetClientStoreMarker(pOldInfo->GetClientStoreMarker()); + } + if(pOldInfo.get()) { mNumberErrorsFound += info->ReportChangesTo(*pOldInfo); @@ -676,7 +691,7 @@ void BackupStoreCheck::WriteNewStoreInfo() if(mFixErrors) { info->Save(); - BOX_NOTICE("New store info file written successfully."); + BOX_INFO("New store info file written successfully."); } } @@ -695,7 +710,7 @@ void BackupStoreCheck::WriteNewStoreInfo() bool BackupStoreDirectory::CheckAndFix() { bool changed = false; - + // Check that if a file depends on a new version, that version is in this directory bool restart; @@ -718,11 +733,11 @@ bool BackupStoreDirectory::CheckAndFix() "on newer version " << FMT_OID(dependsNewer) << " which doesn't exist"); - + // Remove delete *i; mEntries.erase(i); - + // Mark as changed changed = true; @@ -751,7 +766,7 @@ bool BackupStoreDirectory::CheckAndFix() } } while(restart); - + // Check that if a file has a dependency marked, it exists, and remove it if it doesn't { std::vector<Entry*>::iterator i(mEntries.begin()); @@ -768,7 +783,7 @@ bool BackupStoreDirectory::CheckAndFix() "info cleared"); (*i)->SetDependsOlder(0); - + // Mark as changed changed = true; } @@ -780,7 +795,7 @@ bool BackupStoreDirectory::CheckAndFix() { // Reset change marker ch = false; - + // Search backwards -- so see newer versions first std::vector<Entry*>::iterator i(mEntries.end()); if(i == mEntries.begin()) @@ -806,10 +821,8 @@ bool BackupStoreDirectory::CheckAndFix() } else { - bool isDir = (((*i)->GetFlags() & Entry::Flags_Dir) == Entry::Flags_Dir); - // Check mutually exclusive flags - if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File)) + if((*i)->IsDir() && (*i)->IsFile()) { // Bad! Unset the file flag BOX_TRACE("Entry " << FMT_i << @@ -863,29 +876,29 @@ bool BackupStoreDirectory::CheckAndFix() } } } - + if(removeEntry) { // Mark something as changed, in loop ch = true; - + // Mark something as globally changed changed = true; - + // erase the thing from the list Entry *pentry = (*i); mEntries.erase(i); // And delete the entry object delete pentry; - + // Stop going around this loop, as the iterator is now invalid break; } } while(i != mEntries.begin()); } while(ch != false); - + return changed; } diff --git a/lib/backupstore/BackupStoreContext.cpp b/lib/backupstore/BackupStoreContext.cpp index 2c98b1d7..1a782df4 100644 --- a/lib/backupstore/BackupStoreContext.cpp +++ b/lib/backupstore/BackupStoreContext.cpp @@ -30,13 +30,14 @@ #include "MemLeakFindOn.h" -// Maximum number of directories to keep in the cache -// When the cache is bigger than this, everything gets -// deleted. +// Maximum number of directories to keep in the cache When the cache is bigger +// than this, everything gets deleted. In tests, we set the cache size to zero +// to ensure that it's always flushed, which is very inefficient but helps to +// catch programming errors (use of freed data). #ifdef BOX_RELEASE_BUILD #define MAX_CACHE_SIZE 32 #else - #define MAX_CACHE_SIZE 2 + #define MAX_CACHE_SIZE 0 #endif // Allow the housekeeping process 4 seconds to release an account @@ -54,19 +55,22 @@ // // -------------------------------------------------------------------------- BackupStoreContext::BackupStoreContext(int32_t ClientID, - HousekeepingInterface &rDaemon, const std::string& rConnectionDetails) - : mConnectionDetails(rConnectionDetails), - mClientID(ClientID), - mrDaemon(rDaemon), - mProtocolPhase(Phase_START), - mClientHasAccount(false), - mStoreDiscSet(-1), - mReadOnly(true), - mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), - mpTestHook(NULL) + HousekeepingInterface* pHousekeeping, const std::string& rConnectionDetails) +: mConnectionDetails(rConnectionDetails), + mClientID(ClientID), + mpHousekeeping(pHousekeeping), + mProtocolPhase(Phase_START), + mClientHasAccount(false), + mStoreDiscSet(-1), + mReadOnly(true), + mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), + mpTestHook(NULL) +// If you change the initialisers, be sure to update +// BackupStoreContext::ReceivedFinishCommand as well! { } + // -------------------------------------------------------------------------- // // Function @@ -77,11 +81,19 @@ BackupStoreContext::BackupStoreContext(int32_t ClientID, // -------------------------------------------------------------------------- BackupStoreContext::~BackupStoreContext() { + ClearDirectoryCache(); +} + + +void BackupStoreContext::ClearDirectoryCache() +{ // Delete the objects in the cache - for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); + i != mDirectoryCache.end(); ++i) { delete (i->second); } + mDirectoryCache.clear(); } @@ -103,6 +115,7 @@ void BackupStoreContext::CleanUp() } } + // -------------------------------------------------------------------------- // // Function @@ -118,6 +131,20 @@ void BackupStoreContext::ReceivedFinishCommand() // Save the store info, not delayed SaveStoreInfo(false); } + + // Just in case someone wants to reuse a local protocol object, + // put the context back to its initial state. + mProtocolPhase = BackupStoreContext::Phase_Version; + + // Avoid the need to check version again, by not resetting + // mClientHasAccount, mAccountRootDir or mStoreDiscSet + + mReadOnly = true; + mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; + mpTestHook = NULL; + mapStoreInfo.reset(); + mapRefCount.reset(); + ClearDirectoryCache(); } @@ -133,19 +160,19 @@ bool BackupStoreContext::AttemptToGetWriteLock() { // Make the filename of the write lock file std::string writeLockFile; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); + StoreStructure::MakeWriteLockFilename(mAccountRootDir, mStoreDiscSet, writeLockFile); // Request the lock bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); - - if(!gotLock) + + if(!gotLock && mpHousekeeping) { // The housekeeping process might have the thing open -- ask it to stop char msg[256]; - int msgLen = sprintf(msg, "r%x\n", mClientID); + int msgLen = snprintf(msg, sizeof(msg), "r%x\n", mClientID); // Send message - mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen); - + mpHousekeeping->SendMessageToHousekeepingProcess(msg, msgLen); + // Then try again a few times int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; do @@ -153,16 +180,16 @@ bool BackupStoreContext::AttemptToGetWriteLock() ::sleep(1 /* second */); --tries; gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); - + } while(!gotLock && tries > 0); } - + if(gotLock) { // Got the lock, mark as not read only mReadOnly = false; } - + return gotLock; } @@ -181,16 +208,16 @@ void BackupStoreContext::LoadStoreInfo() { THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded) } - + // Load it up! - std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly)); - + std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mAccountRootDir, mStoreDiscSet, mReadOnly)); + // Check it if(i->GetAccountID() != mClientID) { THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount) } - + // Keep the pointer to it mapStoreInfo = i; @@ -203,12 +230,11 @@ void BackupStoreContext::LoadStoreInfo() } catch(BoxException &e) { - BOX_WARNING("Reference count database is missing or corrupted, " - "creating a new one, expect housekeeping to find and " - "fix problems with reference counts later."); - - BackupStoreRefCountDatabase::CreateForRegeneration(account); - mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CorruptReferenceCountDatabase, "Reference count " + "database is missing or corrupted, cannot safely open " + "account. Housekeeping will fix this automatically " + "when it next runs."); } } @@ -227,6 +253,7 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -242,7 +269,7 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) } } - // Want to save now + // Want to save now mapStoreInfo->Save(); // Set count for next delay @@ -263,83 +290,111 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) void BackupStoreContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists) { // Delegate to utility function - StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists); + StoreStructure::MakeObjectFilename(ObjectID, mAccountRootDir, mStoreDiscSet, rOutput, EnsureDirectoryExists); } // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::GetDirectoryInternal(int64_t) -// Purpose: Return a reference to a directory. Valid only until the -// next time a function which affects directories is called. -// Mainly this funciton, and creation of files. -// Private version of this, which returns non-const directories. +// Name: BackupStoreContext::GetDirectoryInternal(int64_t, +// bool) +// Purpose: Return a reference to a directory. Valid only until +// the next time a function which affects directories +// is called. Mainly this function, and creation of +// files. Private version of this, which returns +// non-const directories. Unless called with +// AllowFlushCache == false, the cache may be flushed, +// invalidating all directory references that you may +// be holding, so beware. // Created: 2003/09/02 // // -------------------------------------------------------------------------- -BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) +BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID, + bool AllowFlushCache) { // Get the filename std::string filename; MakeObjectFilename(ObjectID, filename); - + int64_t oldRevID = 0, newRevID = 0; + // Already in cache? std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID)); - if(item != mDirectoryCache.end()) - { - // Check the revision ID of the file -- does it need refreshing? - int64_t revID = 0; - if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID)) - { - THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) - } - - if(revID == item->second->GetRevisionID()) + if(item != mDirectoryCache.end()) { +#ifndef BOX_RELEASE_BUILD // it might be in the cache, but invalidated + // in which case, delete it instead of returning it. + if(!item->second->IsInvalidated()) +#else + if(true) +#endif { - // Looks good... return the cached object - BOX_TRACE("Returning object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " from cache, modtime = " << revID); - return *(item->second); + oldRevID = item->second->GetRevisionID(); + + // Check the revision ID of the file -- does it need refreshing? + if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &newRevID)) + { + THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) + } + + if(newRevID == oldRevID) + { + // Looks good... return the cached object + BOX_TRACE("Returning object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " from cache, modtime = " << newRevID) + return *(item->second); + } } - - BOX_TRACE("Refreshing object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " in cache, modtime changed from " << - item->second->GetRevisionID() << " to " << revID); // Delete this cached object delete item->second; mDirectoryCache.erase(item); } - + // Need to load it up - + // First check to see if the cache is too big - if(mDirectoryCache.size() > MAX_CACHE_SIZE) + if(mDirectoryCache.size() > MAX_CACHE_SIZE && AllowFlushCache) { - // Very simple. Just delete everything! - for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + // Very simple. Just delete everything! But in debug builds, + // leave the entries in the cache and invalidate them instead, + // so that any attempt to access them will cause an assertion + // failure that helps to track down the error. +#ifdef BOX_RELEASE_BUILD + ClearDirectoryCache(); +#else + for(std::map<int64_t, BackupStoreDirectory*>::iterator + i = mDirectoryCache.begin(); + i != mDirectoryCache.end(); i++) { - delete (i->second); + i->second->Invalidate(); } - mDirectoryCache.clear(); +#endif } // Get a RaidFileRead to read it - int64_t revID = 0; - std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID)); - ASSERT(revID != 0); - - // New directory object - std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory); - + std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, + filename, &newRevID)); + + ASSERT(newRevID != 0); + + if (oldRevID == 0) + { + BOX_TRACE("Loading object " << BOX_FORMAT_OBJECTID(ObjectID) << + " with modtime " << newRevID); + } + else + { + BOX_TRACE("Refreshing object " << BOX_FORMAT_OBJECTID(ObjectID) << + " in cache, modtime changed from " << oldRevID << + " to " << newRevID); + } + // Read it from the stream, then set it's revision ID BufferedStream buf(*objectFile); - dir->ReadFromStream(buf, IOStream::TimeOutInfinite); - dir->SetRevisionID(revID); - + std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory(buf)); + dir->SetRevisionID(newRevID); + // Make sure the size of the directory is available for writing the dir back int64_t dirSize = objectFile->GetDiscUsageInBlocks(); ASSERT(dirSize > 0); @@ -348,7 +403,7 @@ BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) // Store in cache BackupStoreDirectory *pdir = dir.release(); try - { + { mDirectoryCache[ObjectID] = pdir; } catch(...) @@ -356,11 +411,12 @@ BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) delete pdir; throw; } - + // Return it return *pdir; } + // -------------------------------------------------------------------------- // // Function @@ -381,12 +437,12 @@ int64_t BackupStoreContext::AllocateObjectID() // to try for finding an unused ID. // (Sizes used in the store info are fixed by the housekeeping process) int retryLimit = (STORE_INFO_SAVE_DELAY * 2); - + while(retryLimit > 0) { // Attempt to allocate an ID from the store int64_t id = mapStoreInfo->AllocateObjectID(); - + // Generate filename std::string filename; MakeObjectFilename(id, filename); @@ -396,17 +452,17 @@ int64_t BackupStoreContext::AllocateObjectID() // Success! return id; } - + // Decrement retry count, and try again --retryLimit; - + // Mark that the store info should be saved as soon as possible mSaveStoreInfoDelay = 0; - + BOX_WARNING("When allocating object ID, found that " << BOX_FORMAT_OBJECTID(id) << " is already in use"); } - + THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation) } @@ -431,11 +487,12 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) } - + // This is going to be a bit complex to make sure it copes OK // with things going wrong. // The only thing which isn't safe is incrementing the object ID @@ -444,13 +501,13 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // be corrected the next time the account has a housekeeping run, // and the object ID allocation code is tolerant of missed IDs. // (the info is written lazily, so these are necessary) - + // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - + // Allocate the next ID int64_t id = AllocateObjectID(); - + // Stream the file to disc std::string fn; MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); @@ -458,12 +515,13 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, RaidFileWrite *ppreviousVerStoreFile = 0; bool reversedDiffIsCompletelyDifferent = false; int64_t oldVersionNewBlocksUsed = 0; + BackupStoreInfo::Adjustment adjustment = {}; + try { RaidFileWrite storeFile(mStoreDiscSet, fn); storeFile.Open(false /* no overwriting */); - // size adjustment from use of patch in old file int64_t spaceSavedByConversionToPatch = 0; // Diff or full file? @@ -482,12 +540,12 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, { THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory) } - + // Diff file, needs to be recreated. // Choose a temporary filename. std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp", 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */)); - + try { // Open it twice @@ -506,13 +564,13 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, THROW_EXCEPTION(CommonException, OSFileError); } #endif - + // Stream the incoming diff to this temporary file if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) { THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) } - + // Verify the diff diff.Seek(0, IOStream::SeekType_Absolute); if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) @@ -526,7 +584,7 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Filename of the old version std::string oldVersionFilename; MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */); - + // Reassemble that diff -- open previous file, and combine the patch and file std::auto_ptr<RaidFileRead> from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); BackupStoreFile::CombineFile(diff, diff2, *from, storeFile); @@ -539,15 +597,26 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, diff.Seek(0, IOStream::SeekType_Absolute); BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile, DiffFromFileID, &reversedDiffIsCompletelyDifferent); - + // Store disc space used oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks(); - + // And make a space adjustment for the size calculation spaceSavedByConversionToPatch = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed; + adjustment.mBlocksUsed -= spaceSavedByConversionToPatch; + // The code below will change the patch from a + // Current file to an Old file, so we need to + // account for it as a Current file here. + adjustment.mBlocksInCurrentFiles -= + spaceSavedByConversionToPatch; + + // Don't adjust anything else here. We'll do it + // when we update the directory just below, + // which also accounts for non-diff replacements. + // Everything cleans up here... } catch(...) @@ -557,14 +626,17 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, throw; } } - + // Get the blocks used newObjectBlocksUsed = storeFile.GetDiscUsageInBlocks(); - + adjustment.mBlocksUsed += newObjectBlocksUsed; + adjustment.mBlocksInCurrentFiles += newObjectBlocksUsed; + adjustment.mNumCurrentFiles++; + // Exceeds the hard limit? - int64_t newBlocksUsed = mapStoreInfo->GetBlocksUsed() + - newObjectBlocksUsed - spaceSavedByConversionToPatch; - if(newBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) + int64_t newTotalBlocksUsed = mapStoreInfo->GetBlocksUsed() + + adjustment.mBlocksUsed; + if(newTotalBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) { THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) // The store file will be deleted automatically by the RaidFile object @@ -581,7 +653,7 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, delete ppreviousVerStoreFile; ppreviousVerStoreFile = 0; } - + throw; } @@ -596,21 +668,34 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Error! Delete the file RaidFileWrite del(mStoreDiscSet, fn); del.Delete(); - + // Exception THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) } - } - + } + // Modify the directory -- first make all files with the same name // marked as an old version - int64_t blocksInOldFiles = 0; try { + // Adjust the entry for the object that we replaced with a + // patch, above. + BackupStoreDirectory::Entry *poldEntry = NULL; + + if(DiffFromFileID != 0) + { + // Get old version entry + poldEntry = dir.FindEntryByID(DiffFromFileID); + ASSERT(poldEntry != 0); + + // Adjust size of old entry + int64_t oldSize = poldEntry->GetSizeInBlocks(); + poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); + } + if(MarkFileWithSameNameAsOldVersions) { BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *e = 0; while((e = i.Next()) != 0) { @@ -626,43 +711,30 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion); // Can safely do this, because we know we won't be here if it's already // an old version - blocksInOldFiles += e->GetSizeInBlocks(); + adjustment.mBlocksInOldFiles += e->GetSizeInBlocks(); + adjustment.mBlocksInCurrentFiles -= e->GetSizeInBlocks(); + adjustment.mNumOldFiles++; + adjustment.mNumCurrentFiles--; } } } } - + // Then the new entry BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename, - ModificationTime, id, newObjectBlocksUsed, - BackupStoreDirectory::Entry::Flags_File, - AttributesHash); + ModificationTime, id, newObjectBlocksUsed, + BackupStoreDirectory::Entry::Flags_File, + AttributesHash); - // Adjust for the patch back stuff? - if(DiffFromFileID != 0) + // Adjust dependency info of file? + if(DiffFromFileID && poldEntry && !reversedDiffIsCompletelyDifferent) { - // Get old version entry - BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID); - ASSERT(poldEntry != 0); - - // Adjust dependency info of file? - if(!reversedDiffIsCompletelyDifferent) - { - poldEntry->SetDependsNewer(id); - pnewEntry->SetDependsOlder(DiffFromFileID); - } - - // Adjust size of old entry - int64_t oldSize = poldEntry->GetSizeInBlocks(); - poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); - - // And adjust blocks used count, for later adjustment - newObjectBlocksUsed += (oldVersionNewBlocksUsed - oldSize); - blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize); + poldEntry->SetDependsNewer(id); + pnewEntry->SetDependsOlder(DiffFromFileID); } // Write the directory back to disc - SaveDirectory(dir, InDirectory); + SaveDirectory(dir); // Commit the old version's new patched version, now that the directory safely reflects // the state of the files on disc. @@ -678,47 +750,42 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Back out on adding that file RaidFileWrite del(mStoreDiscSet, fn); del.Delete(); - + // Remove this entry from the cache RemoveDirectoryFromCache(InDirectory); - + // Delete any previous version store file if(ppreviousVerStoreFile != 0) { delete ppreviousVerStoreFile; ppreviousVerStoreFile = 0; } - + // Don't worry about the incremented number in the store info throw; } - + // Check logic ASSERT(ppreviousVerStoreFile == 0); - + // Modify the store info + mapStoreInfo->AdjustNumCurrentFiles(adjustment.mNumCurrentFiles); + mapStoreInfo->AdjustNumOldFiles(adjustment.mNumOldFiles); + mapStoreInfo->AdjustNumDeletedFiles(adjustment.mNumDeletedFiles); + mapStoreInfo->AdjustNumDirectories(adjustment.mNumDirectories); + mapStoreInfo->ChangeBlocksUsed(adjustment.mBlocksUsed); + mapStoreInfo->ChangeBlocksInCurrentFiles(adjustment.mBlocksInCurrentFiles); + mapStoreInfo->ChangeBlocksInOldFiles(adjustment.mBlocksInOldFiles); + mapStoreInfo->ChangeBlocksInDeletedFiles(adjustment.mBlocksInDeletedFiles); + mapStoreInfo->ChangeBlocksInDirectories(adjustment.mBlocksInDirectories); - if(DiffFromFileID == 0) - { - mapStoreInfo->AdjustNumFiles(1); - } - else - { - mapStoreInfo->AdjustNumOldFiles(1); - } - - mapStoreInfo->ChangeBlocksUsed(newObjectBlocksUsed); - mapStoreInfo->ChangeBlocksInCurrentFiles(newObjectBlocksUsed - - blocksInOldFiles); - mapStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles); - // Increment reference count on the new directory to one mapRefCount->AddReference(id); - + // Save the store info -- can cope if this exceptions because infomation // will be rebuilt by housekeeping, and ID allocation can recover. SaveStoreInfo(false); - + // Return the ID to the caller return id; } @@ -728,8 +795,11 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) -// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. +// Name: BackupStoreContext::DeleteFile( +// const BackupStoreFilename &, int64_t, int64_t &) +// Purpose: Deletes a file by name, returning true if the file +// existed. Object ID returned too, set to zero if not +// found. // Created: 2003/10/21 // // -------------------------------------------------------------------------- @@ -754,9 +824,6 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ bool madeChanges = false; rObjectIDOut = 0; // not found - // Count of deleted blocks - int64_t blocksDel = 0; - try { // Iterate through directory, only looking at files which haven't been deleted @@ -769,14 +836,28 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ if(e->GetName() == rFilename) { // Check that it's definately not already deleted - ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0); + ASSERT(!e->IsDeleted()); // Set deleted flag e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); // Mark as made a change madeChanges = true; - // Can safely do this, because we know we won't be here if it's already - // an old version - blocksDel += e->GetSizeInBlocks(); + + int64_t blocks = e->GetSizeInBlocks(); + mapStoreInfo->AdjustNumDeletedFiles(1); + mapStoreInfo->ChangeBlocksInDeletedFiles(blocks); + + // We're marking all old versions as deleted. + // This is how a file can be old and deleted + // at the same time. So we don't subtract from + // number or size of old files. But if it was + // a current file, then it's not any more, so + // we do need to adjust the current counts. + if(!e->IsOld()) + { + mapStoreInfo->AdjustNumCurrentFiles(-1); + mapStoreInfo->ChangeBlocksInCurrentFiles(-blocks); + } + // Is this the last version? if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) { @@ -786,19 +867,12 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ } } } - + // Save changes? if(madeChanges) { // Save the directory back - SaveDirectory(dir, InDirectory); - - // Modify the store info, and write - // It definitely wasn't an old or deleted version - mapStoreInfo->AdjustNumFiles(-1); - mapStoreInfo->AdjustNumDeletedFiles(1); - mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); - + SaveDirectory(dir); SaveStoreInfo(false); } } @@ -871,16 +945,16 @@ bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory) } } } - + // Save changes? if(madeChanges) { // Save the directory back - SaveDirectory(dir, InDirectory); - + SaveDirectory(dir); + // Modify the store info, and write mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); - + // Maybe postponed save of store info SaveStoreInfo(); } @@ -919,27 +993,27 @@ void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID) // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &, int64_t) +// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &) // Purpose: Save directory back to disc, update time in cache // Created: 2003/09/04 // // -------------------------------------------------------------------------- -void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID) +void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir) { if(mapStoreInfo.get() == 0) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - if(rDir.GetObjectID() != ObjectID) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } + + int64_t ObjectID = rDir.GetObjectID(); try { // Write to disc, adjust size in store info std::string dirfn; MakeObjectFilename(ObjectID, dirfn); + int64_t old_dir_size = rDir.GetUserInfo1_SizeInBlocks(); + { RaidFileWrite writeDir(mStoreDiscSet, dirfn); writeDir.Open(true /* allow overwriting */); @@ -953,7 +1027,7 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - + // Make sure the size of the directory is available for writing the dir back ASSERT(dirSize > 0); int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks(); @@ -962,6 +1036,7 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec // Update size stored in directory rDir.SetUserInfo1_SizeInBlocks(dirSize); } + // Refresh revision ID in cache { int64_t revid = 0; @@ -969,8 +1044,41 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec { THROW_EXCEPTION(BackupStoreException, Internal) } + + BOX_TRACE("Saved directory " << + BOX_FORMAT_OBJECTID(ObjectID) << + ", modtime = " << revid); + rDir.SetRevisionID(revid); } + + // Update the directory entry in the grandparent, to ensure + // that it reflects the current size of the parent directory. + int64_t new_dir_size = rDir.GetUserInfo1_SizeInBlocks(); + if(new_dir_size != old_dir_size && + ObjectID != BACKUPSTORE_ROOT_DIRECTORY_ID) + { + int64_t ContainerID = rDir.GetContainerID(); + BackupStoreDirectory& parent( + GetDirectoryInternal(ContainerID)); + // rDir is now invalid + BackupStoreDirectory::Entry* en = + parent.FindEntryByID(ObjectID); + if(!en) + { + BOX_ERROR("Missing entry for directory " << + BOX_FORMAT_OBJECTID(ObjectID) << + " in directory " << + BOX_FORMAT_OBJECTID(ContainerID) << + " while trying to update dir size in parent"); + } + else + { + ASSERT(en->GetSizeInBlocks() == old_dir_size); + en->SetSizeInBlocks(new_dir_size); + SaveDirectory(parent); + } + } } catch(...) { @@ -991,20 +1099,26 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec // Created: 2003/09/04 // // -------------------------------------------------------------------------- -int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists) +int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, + const BackupStoreFilename &rFilename, + const StreamableMemBlock &Attributes, + int64_t AttributesModTime, + int64_t ModificationTime, + bool &rAlreadyExists) { if(mapStoreInfo.get() == 0) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) } - + // Flags as not already existing rAlreadyExists = false; - + // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); @@ -1030,19 +1144,31 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF // Create an empty directory with the given attributes on disc std::string fn; MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); + int64_t dirSize; + { BackupStoreDirectory emptyDir(id, InDirectory); // add the atttribues emptyDir.SetAttributes(Attributes, AttributesModTime); - + // Write... RaidFileWrite dirFile(mStoreDiscSet, fn); dirFile.Open(false /* no overwriting */); emptyDir.WriteToStream(dirFile); // Get disc usage, before it's commited - int64_t dirSize = dirFile.GetDiscUsageInBlocks(); + dirSize = dirFile.GetDiscUsageInBlocks(); + + // Exceeds the hard limit? + int64_t newTotalBlocksUsed = mapStoreInfo->GetBlocksUsed() + + dirSize; + if(newTotalBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) + { + THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) + // The file will be deleted automatically by the RaidFile object + } + // Commit the file - dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); // Make sure the size of the directory is added to the usage counts in the info ASSERT(dirSize > 0); @@ -1050,12 +1176,14 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF mapStoreInfo->ChangeBlocksInDirectories(dirSize); // Not added to cache, so don't set the size in the directory } - + // Then add it into the parent directory try { - dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */); - SaveDirectory(dir, InDirectory); + dir.AddEntry(rFilename, ModificationTime, id, dirSize, + BackupStoreDirectory::Entry::Flags_Dir, + 0 /* attributes hash */); + SaveDirectory(dir); // Increment reference count on the new directory to one mapRefCount->AddReference(id); @@ -1065,12 +1193,12 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF // Back out on adding that directory RaidFileWrite del(mStoreDiscSet, fn); del.Delete(); - + // Remove this entry from the cache RemoveDirectoryFromCache(InDirectory); - + // Don't worry about the incremented number in the store info - throw; + throw; } // Save the store info (may not be postponed) @@ -1081,6 +1209,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF return id; } + // -------------------------------------------------------------------------- // // Function @@ -1096,6 +1225,7 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -1103,9 +1233,6 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) // Containing directory int64_t InDirectory = 0; - - // Count of blocks deleted - int64_t blocksDeleted = 0; try { @@ -1113,18 +1240,18 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) { // In block, because dir may not be valid after the delete directory call BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - + // Store the directory it's in for later InDirectory = dir.GetContainerID(); // Depth first delete of contents - DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete); + DeleteDirectoryRecurse(ObjectID, Undelete); } - + // Remove the entry from the directory it's in ASSERT(InDirectory != 0); BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory)); - + BackupStoreDirectory::Iterator i(parentDir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), @@ -1141,18 +1268,16 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) { en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); } - + // Save it - SaveDirectory(parentDir, InDirectory); - + SaveDirectory(parentDir); + // Done break; } } - + // Update blocks deleted count - mapStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted)); - mapStoreInfo->AdjustNumDirectories(-1); SaveStoreInfo(false); } catch(...) @@ -1162,6 +1287,7 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) } } + // -------------------------------------------------------------------------- // // Function @@ -1170,18 +1296,18 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) // Created: 2003/10/21 // // -------------------------------------------------------------------------- -void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete) +void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete) { try { // Does things carefully to avoid using a directory in the cache after recursive call // because it may have been deleted. - + // Do sub directories { // Get the directory... BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - + // Then scan it for directories std::vector<int64_t> subDirs; BackupStoreDirectory::Iterator i(dir); @@ -1204,30 +1330,46 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBloc subDirs.push_back(en->GetObjectID()); } } - + // Done with the directory for now. Recurse to sub directories for(std::vector<int64_t>::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i) { - DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete); + DeleteDirectoryRecurse(*i, Undelete); } } - + // Then, delete the files. Will need to load the directory again because it might have // been removed from the cache. { // Get the directory... BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - + // Changes made? bool changesMade = false; - - // Run through files + + // Run through files BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) { + // Keep count of the deleted blocks + if(en->IsFile()) + { + int64_t size = en->GetSizeInBlocks(); + ASSERT(en->IsDeleted() == Undelete); + // Don't adjust counters for old files, + // because it can be both old and deleted. + if(!en->IsOld()) + { + mapStoreInfo->ChangeBlocksInCurrentFiles(Undelete ? size : -size); + mapStoreInfo->AdjustNumCurrentFiles(Undelete ? 1 : -1); + } + mapStoreInfo->ChangeBlocksInDeletedFiles(Undelete ? -size : size); + mapStoreInfo->AdjustNumDeletedFiles(Undelete ? -1 : 1); + } + // Add/remove the deleted flags if(Undelete) { @@ -1237,21 +1379,15 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBloc { en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); } - - // Keep count of the deleted blocks - if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) - { - rBlocksDeletedOut += en->GetSizeInBlocks(); - } - + // Did something changesMade = true; } - + // Save the directory if(changesMade) { - SaveDirectory(dir, ObjectID); + SaveDirectory(dir); } } } @@ -1263,7 +1399,6 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBloc } - // -------------------------------------------------------------------------- // // Function @@ -1284,15 +1419,15 @@ void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const Streamable } try - { + { // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(Directory)); - + // Set attributes dir.SetAttributes(Attributes, AttributesModTime); - + // Save back - SaveDirectory(dir, Directory); + SaveDirectory(dir); } catch(...) { @@ -1301,6 +1436,7 @@ void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const Streamable } } + // -------------------------------------------------------------------------- // // Function @@ -1324,7 +1460,7 @@ bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilena { // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - + // Find the file entry BackupStoreDirectory::Entry *en = 0; // Iterate through current versions of files, only @@ -1338,10 +1474,10 @@ bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilena { // Set attributes en->SetAttributes(Attributes, AttributesHash); - + // Tell caller the object ID rObjectIDOut = en->GetObjectID(); - + // Done break; } @@ -1351,16 +1487,16 @@ bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilena // Didn't find it return false; } - + // Save back - SaveDirectory(dir, InDirectory); + SaveDirectory(dir); } catch(...) { RemoveDirectoryFromCache(InDirectory); throw; } - + // Changed, everything OK return true; } @@ -1380,7 +1516,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + // Note that we need to allow object IDs a little bit greater than the last one in the store info, // because the store info may not have got saved in an error condition. Max greater ID is // STORE_INFO_SAVE_DELAY in this case, *2 to be safe. @@ -1389,7 +1525,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) // Obviously bad object ID return false; } - + // Test to see if it exists on the disc std::string filename; MakeObjectFilename(ObjectID, filename); @@ -1398,7 +1534,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) // RaidFile reports no file there return false; } - + // Do we need to be more specific? if(MustBe != ObjectExists_Anything) { @@ -1406,7 +1542,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename)); // Read the first integer - u_int32_t magic; + uint32_t magic; if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */)) { // Failed to get any bytes, must have failed @@ -1422,17 +1558,17 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) #endif // Right one? - u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; - + uint32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; + // Check if(ntohl(magic) != requiredMagic) { return false; } - + // File is implicitly closed } - + return true; } @@ -1451,7 +1587,7 @@ std::auto_ptr<IOStream> BackupStoreContext::OpenObject(int64_t ObjectID) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + // Attempt to open the file std::string fn; MakeObjectFilename(ObjectID, fn); @@ -1473,7 +1609,7 @@ int64_t BackupStoreContext::GetClientStoreMarker() { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + return mapStoreInfo->GetClientStoreMarker(); } @@ -1536,7 +1672,7 @@ void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) } - + mapStoreInfo->SetClientStoreMarker(ClientStoreMarker); SaveStoreInfo(false /* don't delay saving this */); } @@ -1550,7 +1686,9 @@ void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker) // Created: 12/11/03 // // -------------------------------------------------------------------------- -void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) +void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, + int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, + bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) { if(mReadOnly) { @@ -1561,7 +1699,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject) ?(BackupStoreDirectory::Entry::Flags_Deleted) :(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); - + // Special case if the directories are the same... if(MoveFromDirectory == MoveToDirectory) { @@ -1569,16 +1707,16 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, { // Get the first directory BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory)); - + // Find the file entry BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); - + // Error if not found if(en == 0) { THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) } - + // Check the new name doens't already exist (optionally ignoring deleted files) { BackupStoreDirectory::Iterator i(dir); @@ -1591,7 +1729,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, } } } - + // Need to get all the entries with the same name? if(MoveAllWithSameName) { @@ -1612,45 +1750,46 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, // Just copy this one en->SetName(rNewFilename); } - + // Save the directory back - SaveDirectory(dir, MoveFromDirectory); + SaveDirectory(dir); } catch(...) { RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same throw; } - + return; } - // Got to be careful how this is written, as we can't guarentte that if we have two - // directories open, the first won't be deleted as the second is opened. (cache) + // Got to be careful how this is written, as we can't guarantee that + // if we have two directories open, the first won't be deleted as the + // second is opened. (cache) // List of entries to move std::vector<BackupStoreDirectory::Entry *> moving; - + // list of directory IDs which need to have containing dir id changed std::vector<int64_t> dirsToChangeContainingID; try { // First of all, get copies of the entries to move to the to directory. - + { // Get the first directory BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); - + // Find the file entry BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID); - + // Error if not found if(en == 0) { THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) } - + // Need to get all the entries with the same name? if(MoveAllWithSameName) { @@ -1663,7 +1802,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, { // Copy moving.push_back(new BackupStoreDirectory::Entry(*c)); - + // Check for containing directory correction if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID()); } @@ -1679,13 +1818,13 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID()); } } - + // Secondly, insert them into the to directory, and save it - + { // To directory BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); - + // Check the new name doens't already exist { BackupStoreDirectory::Iterator i(to); @@ -1698,7 +1837,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, } } } - + // Copy the entries into it, changing the name as we go for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) { @@ -1706,9 +1845,9 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, en->SetName(rNewFilename); to.AddEntry(*en); // adds copy } - + // Save back - SaveDirectory(to, MoveToDirectory); + SaveDirectory(to); } // Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory @@ -1716,57 +1855,57 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, { // Get directory BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); - + // Delete each one for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) { from.DeleteEntry((*i)->GetObjectID()); } - + // Save back - SaveDirectory(from, MoveFromDirectory); + SaveDirectory(from); } catch(...) { // UNDO modification to To directory - + // Get directory BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); - + // Delete each one for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) { to.DeleteEntry((*i)->GetObjectID()); } - + // Save back - SaveDirectory(to, MoveToDirectory); + SaveDirectory(to); // Throw the error throw; } - + // Finally... for all the directories we moved, modify their containing directory ID for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) { // Load the directory BackupStoreDirectory &change(GetDirectoryInternal(*i)); - + // Modify containing dir ID change.SetContainerID(MoveToDirectory); - + // Save it back - SaveDirectory(change, *i); + SaveDirectory(change); } } catch(...) { - // Make sure directories aren't in the cache, as they may have been modified + // Make sure directories aren't in the cache, as they may have been modified RemoveDirectoryFromCache(MoveToDirectory); RemoveDirectoryFromCache(MoveFromDirectory); for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) { - RemoveDirectoryFromCache(*i); + RemoveDirectoryFromCache(*i); } while(!moving.empty()) @@ -1775,7 +1914,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, moving.pop_back(); } throw; - } + } // Clean up while(!moving.empty()) @@ -1786,7 +1925,6 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, } - // -------------------------------------------------------------------------- // // Function @@ -1801,8 +1939,7 @@ const BackupStoreInfo &BackupStoreContext::GetBackupStoreInfo() const { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + return *(mapStoreInfo.get()); } - diff --git a/lib/backupstore/BackupStoreContext.h b/lib/backupstore/BackupStoreContext.h index c33e7d50..48448360 100644 --- a/lib/backupstore/BackupStoreContext.h +++ b/lib/backupstore/BackupStoreContext.h @@ -45,7 +45,8 @@ class HousekeepingInterface class BackupStoreContext { public: - BackupStoreContext(int32_t ClientID, HousekeepingInterface &rDaemon, + BackupStoreContext(int32_t ClientID, + HousekeepingInterface* mpHousekeeping, const std::string& rConnectionDetails); ~BackupStoreContext(); private: @@ -64,7 +65,7 @@ public: Phase_Login = 1, Phase_Commands = 2 }; - + int GetPhase() const {return mProtocolPhase;} std::string GetPhaseName() const { @@ -80,14 +81,23 @@ public: } } void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;} - + // Read only locking bool SessionIsReadOnly() {return mReadOnly;} bool AttemptToGetWriteLock(); - void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;} + // Not really an API, but useful for BackupProtocolLocal2. + void ReleaseWriteLock() + { + if(mWriteLock.GotLock()) + { + mWriteLock.ReleaseLock(); + } + } + + void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mAccountRootDir = rStoreRoot; mStoreDiscSet = StoreDiscSet;} bool GetClientHasAccount() const {return mClientHasAccount;} - const std::string &GetStoreRoot() const {return mStoreRoot;} + const std::string &GetAccountRoot() const {return mAccountRootDir;} int GetStoreDiscSet() const {return mStoreDiscSet;} // Store info @@ -106,7 +116,7 @@ public: // Client marker int64_t GetClientStoreMarker(); void SetClientStoreMarker(int64_t ClientStoreMarker); - + // Usage information void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit); bool HardLimitExceeded(); @@ -125,13 +135,24 @@ public: const BackupStoreDirectory &GetDirectory(int64_t ObjectID) { // External callers aren't allowed to change it -- this function - // merely turns the the returned directory const. + // merely turns the returned directory const. return GetDirectoryInternal(ObjectID); } - + // Manipulating files/directories - int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions); - int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists); + int64_t AddFile(IOStream &rFile, + int64_t InDirectory, + int64_t ModificationTime, + int64_t AttributesHash, + int64_t DiffFromFileID, + const BackupStoreFilename &rFilename, + bool MarkFileWithSameNameAsOldVersions); + int64_t AddDirectory(int64_t InDirectory, + const BackupStoreFilename &rFilename, + const StreamableMemBlock &Attributes, + int64_t AttributesModTime, + int64_t ModificationTime, + bool &rAlreadyExists); void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime); bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut); bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut); @@ -155,29 +176,32 @@ public: private: void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false); - BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID); - void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID); + BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID, + bool AllowFlushCache = true); + void SaveDirectory(BackupStoreDirectory &rDir); void RemoveDirectoryFromCache(int64_t ObjectID); - void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete); + void ClearDirectoryCache(); + void DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete); int64_t AllocateObjectID(); std::string mConnectionDetails; int32_t mClientID; - HousekeepingInterface &mrDaemon; + HousekeepingInterface *mpHousekeeping; int mProtocolPhase; bool mClientHasAccount; - std::string mStoreRoot; // has final directory separator + std::string mAccountRootDir; // has final directory separator int mStoreDiscSet; + bool mReadOnly; NamedLock mWriteLock; int mSaveStoreInfoDelay; // how many times to delay saving the store info - + // Store info std::auto_ptr<BackupStoreInfo> mapStoreInfo; // Refcount database std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount; - + // Directory cache std::map<int64_t, BackupStoreDirectory*> mDirectoryCache; diff --git a/lib/backupstore/BackupStoreDirectory.cpp b/lib/backupstore/BackupStoreDirectory.cpp index 81126ede..6946f06e 100644 --- a/lib/backupstore/BackupStoreDirectory.cpp +++ b/lib/backupstore/BackupStoreDirectory.cpp @@ -1,7 +1,7 @@ // -------------------------------------------------------------------------- // // File -// Name: BackupStoreDirectory.h +// Name: BackupStoreDirectory.cpp // Purpose: Representation of a backup directory // Created: 2003/08/26 // @@ -36,11 +36,6 @@ typedef struct // Then a StreamableMemBlock for attributes } dir_StreamFormat; -typedef enum -{ - Option_DependencyInfoPresent = 1 -} dir_StreamFormatOptions; - typedef struct { uint64_t mModificationTime; @@ -75,9 +70,17 @@ END_STRUCTURE_PACKING_FOR_WIRE // // -------------------------------------------------------------------------- BackupStoreDirectory::BackupStoreDirectory() - : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mRevisionID(0), + mObjectID(0), + mContainerID(0), + mAttributesModTime(0), + mUserInfo1(0) { - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); + ASSERT(sizeof(uint64_t) == sizeof(box_time_t)); } @@ -90,7 +93,15 @@ BackupStoreDirectory::BackupStoreDirectory() // // -------------------------------------------------------------------------- BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID) - : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mRevisionID(0), + mObjectID(ObjectID), + mContainerID(ContainerID), + mAttributesModTime(0), + mUserInfo1(0) { } @@ -122,6 +133,7 @@ BackupStoreDirectory::~BackupStoreDirectory() // -------------------------------------------------------------------------- void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) { + ASSERT(!mInvalidated); // Compiled out of release builds // Get the header dir_StreamFormat hdr; if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) @@ -133,35 +145,35 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue)) { THROW_EXCEPTION_MESSAGE(BackupStoreException, BadDirectoryFormat, - "Wrong magic number in directory object " << - BOX_FORMAT_OBJECTID(mObjectID) << ": expected " << + "Wrong magic number for directory: expected " << BOX_FORMAT_HEX32(OBJECTMAGIC_DIR_MAGIC_VALUE) << " but found " << - BOX_FORMAT_HEX32(ntohl(hdr.mMagicValue))); + BOX_FORMAT_HEX32(ntohl(hdr.mMagicValue)) << " in " << + rStream.ToString()); } - + // Get data mObjectID = box_ntoh64(hdr.mObjectID); mContainerID = box_ntoh64(hdr.mContainerID); mAttributesModTime = box_ntoh64(hdr.mAttributesModTime); - + // Options int32_t options = ntohl(hdr.mOptionsPresent); - + // Get attributes mAttributes.ReadFromStream(rStream, Timeout); - + // Decode count int count = ntohl(hdr.mNumEntries); - + // Clear existing list - for(std::vector<Entry*>::iterator i = mEntries.begin(); + for(std::vector<Entry*>::iterator i = mEntries.begin(); i != mEntries.end(); i++) { delete (*i); } mEntries.clear(); - + // Read them in! for(int c = 0; c < count; ++c) { @@ -170,7 +182,7 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) { // Read from stream pen->ReadFromStream(rStream, Timeout); - + // Add to list mEntries.push_back(pen); } @@ -180,7 +192,7 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) throw; } } - + // Read in dependency info? if(options & Option_DependencyInfoPresent) { @@ -202,6 +214,7 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) // -------------------------------------------------------------------------- void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const { + ASSERT(!mInvalidated); // Compiled out of release builds // Get count of entries int32_t count = mEntries.size(); if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING) @@ -214,11 +227,11 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS count++; } } - + // Check that sensible IDs have been set ASSERT(mObjectID != 0); ASSERT(mContainerID != 0); - + // Need dependency info? bool dependencyInfoRequired = false; if(StreamDependencyInfo) @@ -231,9 +244,9 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS { dependencyInfoRequired = true; } - } + } } - + // Options int32_t options = 0; if(dependencyInfoRequired) options |= Option_DependencyInfoPresent; @@ -246,10 +259,10 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS hdr.mContainerID = box_hton64(mContainerID); hdr.mAttributesModTime = box_hton64(mAttributesModTime); hdr.mOptionsPresent = htonl(options); - + // Write header rStream.Write(&hdr, sizeof(hdr)); - + // Write the attributes? if(StreamAttributes) { @@ -268,7 +281,7 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS { pen->WriteToStream(rStream); } - + // Write dependency info? if(dependencyInfoRequired) { @@ -277,7 +290,7 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) { pen->WriteToStreamDependencyInfo(rStream); - } + } } } @@ -291,6 +304,7 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS // -------------------------------------------------------------------------- BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy) { + ASSERT(!mInvalidated); // Compiled out of release builds Entry *pnew = new Entry(rEntryToCopy); try { @@ -301,7 +315,7 @@ BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryT delete pnew; throw; } - + return pnew; } @@ -318,6 +332,7 @@ BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) { + ASSERT(!mInvalidated); // Compiled out of release builds Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesHash); try @@ -329,7 +344,7 @@ BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, delete pnew; throw; } - + return pnew; } @@ -343,6 +358,7 @@ BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, // -------------------------------------------------------------------------- void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) { + ASSERT(!mInvalidated); // Compiled out of release builds for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) { @@ -356,9 +372,11 @@ void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) return; } } - + // Not found - THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) + THROW_EXCEPTION_MESSAGE(BackupStoreException, CouldNotFindEntryInDirectory, + "Failed to find entry " << BOX_FORMAT_OBJECTID(ObjectID) << + " in directory " << BOX_FORMAT_OBJECTID(mObjectID)); } @@ -372,6 +390,7 @@ void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) // -------------------------------------------------------------------------- BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const { + ASSERT(!mInvalidated); // Compiled out of release builds for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i) { @@ -396,15 +415,19 @@ BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectI // // -------------------------------------------------------------------------- BackupStoreDirectory::Entry::Entry() - : mModificationTime(0), - mObjectID(0), - mSizeInBlocks(0), - mFlags(0), - mAttributesHash(0), - mMinMarkNumber(0), - mMarkNumber(0), - mDependsNewer(0), - mDependsOlder(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mModificationTime(0), + mObjectID(0), + mSizeInBlocks(0), + mFlags(0), + mAttributesHash(0), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) { } @@ -429,17 +452,21 @@ BackupStoreDirectory::Entry::~Entry() // // -------------------------------------------------------------------------- BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) - : mName(rToCopy.mName), - mModificationTime(rToCopy.mModificationTime), - mObjectID(rToCopy.mObjectID), - mSizeInBlocks(rToCopy.mSizeInBlocks), - mFlags(rToCopy.mFlags), - mAttributesHash(rToCopy.mAttributesHash), - mAttributes(rToCopy.mAttributes), - mMinMarkNumber(rToCopy.mMinMarkNumber), - mMarkNumber(rToCopy.mMarkNumber), - mDependsNewer(rToCopy.mDependsNewer), - mDependsOlder(rToCopy.mDependsOlder) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mName(rToCopy.mName), + mModificationTime(rToCopy.mModificationTime), + mObjectID(rToCopy.mObjectID), + mSizeInBlocks(rToCopy.mSizeInBlocks), + mFlags(rToCopy.mFlags), + mAttributesHash(rToCopy.mAttributesHash), + mAttributes(rToCopy.mAttributes), + mMinMarkNumber(rToCopy.mMinMarkNumber), + mMarkNumber(rToCopy.mMarkNumber), + mDependsNewer(rToCopy.mDependsNewer), + mDependsOlder(rToCopy.mDependsOlder) { } @@ -453,16 +480,20 @@ BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) // // -------------------------------------------------------------------------- BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) - : mName(rName), - mModificationTime(ModificationTime), - mObjectID(ObjectID), - mSizeInBlocks(SizeInBlocks), - mFlags(Flags), - mAttributesHash(AttributesHash), - mMinMarkNumber(0), - mMarkNumber(0), - mDependsNewer(0), - mDependsOlder(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mName(rName), + mModificationTime(ModificationTime), + mObjectID(ObjectID), + mSizeInBlocks(SizeInBlocks), + mFlags(Flags), + mAttributesHash(AttributesHash), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) { } @@ -478,19 +509,21 @@ BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) { + ASSERT(!mInvalidated); // Compiled out of release builds // Grab the raw bytes from the stream which compose the header en_StreamFormat entry; - if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + if(!rStream.ReadFullBuffer(&entry, sizeof(entry), + 0 /* not interested in bytes read if this fails */, Timeout)) { THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) } // Do reading first before modifying the variables, to be more exception safe - + // Get the filename BackupStoreFilename name; name.ReadFromStream(rStream, Timeout); - + // Get the attributes mAttributes.ReadFromStream(rStream, Timeout); @@ -514,6 +547,7 @@ void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const { + ASSERT(!mInvalidated); // Compiled out of release builds // Build a structure en_StreamFormat entry; entry.mModificationTime = box_hton64(mModificationTime); @@ -521,13 +555,13 @@ void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const entry.mSizeInBlocks = box_hton64(mSizeInBlocks); entry.mAttributesHash = box_hton64(mAttributesHash); entry.mFlags = htons(mFlags); - + // Write it rStream.Write(&entry, sizeof(entry)); - + // Write the filename mName.WriteToStream(rStream); - + // Write any attributes mAttributes.WriteToStream(rStream); } @@ -543,6 +577,7 @@ void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout) { + ASSERT(!mInvalidated); // Compiled out of release builds // Grab the raw bytes from the stream which compose the header en_StreamFormatDepends depends; if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout)) @@ -566,13 +601,11 @@ void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const { + ASSERT(!mInvalidated); // Compiled out of release builds // Build structure - en_StreamFormatDepends depends; + en_StreamFormatDepends depends; depends.mDependsNewer = box_hton64(mDependsNewer); depends.mDependsOlder = box_hton64(mDependsOlder); // Write rStream.Write(&depends, sizeof(depends)); } - - - diff --git a/lib/backupstore/BackupStoreDirectory.h b/lib/backupstore/BackupStoreDirectory.h index 1348f4e6..788a3ad0 100644 --- a/lib/backupstore/BackupStoreDirectory.h +++ b/lib/backupstore/BackupStoreDirectory.h @@ -29,9 +29,48 @@ class IOStream; // -------------------------------------------------------------------------- class BackupStoreDirectory { +private: +#ifndef BOX_RELEASE_BUILD + bool mInvalidated; +#endif + public: +#ifndef BOX_RELEASE_BUILD + void Invalidate() + { + mInvalidated = true; + for (std::vector<Entry*>::iterator i = mEntries.begin(); + i != mEntries.end(); i++) + { + (*i)->Invalidate(); + } + } +#endif + + typedef enum + { + Option_DependencyInfoPresent = 1 + } dir_StreamFormatOptions; + BackupStoreDirectory(); BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID); + // Convenience constructor from a stream + BackupStoreDirectory(IOStream& rStream, + int Timeout = IOStream::TimeOutInfinite) +#ifndef BOX_RELEASE_BUILD + : mInvalidated(false) +#endif + { + ReadFromStream(rStream, Timeout); + } + BackupStoreDirectory(std::auto_ptr<IOStream> apStream, + int Timeout = IOStream::TimeOutInfinite) +#ifndef BOX_RELEASE_BUILD + : mInvalidated(false) +#endif + { + ReadFromStream(*apStream, Timeout); + } private: // Copying not allowed BackupStoreDirectory(const BackupStoreDirectory &rToCopy); @@ -40,40 +79,117 @@ public: class Entry { + private: +#ifndef BOX_RELEASE_BUILD + bool mInvalidated; +#endif + public: +#ifndef BOX_RELEASE_BUILD + void Invalidate() { mInvalidated = true; } +#endif + friend class BackupStoreDirectory; Entry(); ~Entry(); Entry(const Entry &rToCopy); Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash); - + void ReadFromStream(IOStream &rStream, int Timeout); void WriteToStream(IOStream &rStream) const; - - const BackupStoreFilename &GetName() const {return mName;} - box_time_t GetModificationTime() const {return mModificationTime;} - int64_t GetObjectID() const {return mObjectID;} - int64_t GetSizeInBlocks() const {return mSizeInBlocks;} - int16_t GetFlags() const {return mFlags;} - void AddFlags(int16_t Flags) {mFlags |= Flags;} - void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;} + + const BackupStoreFilename &GetName() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mName; + } + box_time_t GetModificationTime() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mModificationTime; + } + int64_t GetObjectID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mObjectID; + } + // SetObjectID is dangerous! It should only be used when + // creating a snapshot. + void SetObjectID(int64_t NewObjectID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mObjectID = NewObjectID; + } + int64_t GetSizeInBlocks() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mSizeInBlocks; + } + int16_t GetFlags() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mFlags; + } + void AddFlags(int16_t Flags) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mFlags |= Flags; + } + void RemoveFlags(int16_t Flags) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mFlags &= ~Flags; + } // Some things can be changed - void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;} - void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;} + void SetName(const BackupStoreFilename &rNewName) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mName = rNewName; + } + void SetSizeInBlocks(int64_t SizeInBlocks) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mSizeInBlocks = SizeInBlocks; + } // Attributes - bool HasAttributes() const {return !mAttributes.IsEmpty();} - void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;} - const StreamableMemBlock &GetAttributes() const {return mAttributes;} - uint64_t GetAttributesHash() const {return mAttributesHash;} - + bool HasAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return !mAttributes.IsEmpty(); + } + void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mAttributes.Set(rAttr); + mAttributesHash = AttributesHash; + } + const StreamableMemBlock &GetAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributes; + } + uint64_t GetAttributesHash() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributesHash; + } + // Marks // The lowest mark number a version of a file of this name has ever had - uint32_t GetMinMarkNumber() const {return mMinMarkNumber;} + uint32_t GetMinMarkNumber() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mMinMarkNumber; + } // The mark number on this file - uint32_t GetMarkNumber() const {return mMarkNumber;} + uint32_t GetMarkNumber() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mMarkNumber; + } // Make sure these flags are synced with those in backupprocotol.txt // ListDirectory command @@ -94,41 +210,66 @@ public: // convenience methods bool inline IsDir() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_Dir; } bool inline IsFile() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_File; } bool inline IsOld() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_OldVersion; } bool inline IsDeleted() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_Deleted; } bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { + ASSERT(!mInvalidated); // Compiled out of release builds return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet)) && ((mFlags & FlagsNotToBeSet) == 0); }; // Get dependency info // new version this depends on - int64_t GetDependsNewer() const {return mDependsNewer;} - void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;} + int64_t GetDependsNewer() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mDependsNewer; + } + void SetDependsNewer(int64_t ObjectID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mDependsNewer = ObjectID; + } // older version which depends on this - int64_t GetDependsOlder() const {return mDependsOlder;} - void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;} + int64_t GetDependsOlder() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mDependsOlder; + } + void SetDependsOlder(int64_t ObjectID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mDependsOlder = ObjectID; + } // Dependency info saving - bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;} + bool HasDependencies() + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mDependsNewer != 0 || mDependsOlder != 0; + } void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout); void WriteToStreamDependencyInfo(IOStream &rStream) const; private: - BackupStoreFilename mName; + BackupStoreFilename mName; box_time_t mModificationTime; int64_t mObjectID; int64_t mSizeInBlocks; @@ -137,11 +278,18 @@ public: StreamableMemBlock mAttributes; uint32_t mMinMarkNumber; uint32_t mMarkNumber; - + uint64_t mDependsNewer; // new version this depends on uint64_t mDependsOlder; // older version which depends on this }; - + +#ifndef BOX_RELEASE_BUILD + bool IsInvalidated() + { + return mInvalidated; + } +#endif // !BOX_RELEASE_BUILD + void ReadFromStream(IOStream &rStream, int Timeout); void WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, @@ -155,28 +303,78 @@ public: uint64_t AttributesHash); void DeleteEntry(int64_t ObjectID); Entry *FindEntryByID(int64_t ObjectID) const; - - int64_t GetObjectID() const {return mObjectID;} - int64_t GetContainerID() const {return mContainerID;} - + + int64_t GetObjectID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mObjectID; + } + + int64_t GetContainerID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mContainerID; + } // Need to be able to update the container ID when moving objects - void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;} + void SetContainerID(int64_t ContainerID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mContainerID = ContainerID; + } + + // Purely for use of server -- not serialised into streams + int64_t GetRevisionID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mRevisionID; + } + void SetRevisionID(int64_t RevisionID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mRevisionID = RevisionID; + } - // Purely for use of server -- not serialised into streams - int64_t GetRevisionID() const {return mRevisionID;} - void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;} - - unsigned int GetNumberOfEntries() const {return mEntries.size();} + unsigned int GetNumberOfEntries() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mEntries.size(); + } // User info -- not serialised into streams - int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;} - void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;} + int64_t GetUserInfo1_SizeInBlocks() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mUserInfo1; + } + void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mUserInfo1 = UserInfo1; + } // Attributes - bool HasAttributes() const {return !mAttributes.IsEmpty();} - void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;} - const StreamableMemBlock &GetAttributes() const {return mAttributes;} - box_time_t GetAttributesModTime() const {return mAttributesModTime;} + bool HasAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return !mAttributes.IsEmpty(); + } + void SetAttributes(const StreamableMemBlock &rAttr, + box_time_t AttributesModTime) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mAttributes.Set(rAttr); + mAttributesModTime = AttributesModTime; + } + const StreamableMemBlock &GetAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributes; + } + box_time_t GetAttributesModTime() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributesModTime; + } class Iterator { @@ -184,10 +382,12 @@ public: Iterator(const BackupStoreDirectory &rDir) : mrDir(rDir), i(rDir.mEntries.begin()) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds } - + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds // Skip over things which don't match the required flags while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) { @@ -207,6 +407,7 @@ public: // In a looping situation, cache the decrypted filenames in another memory structure. BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds // Skip over things which don't match the required flags or filename while( (i != mrDir.mEntries.end()) && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) @@ -227,7 +428,7 @@ public: const BackupStoreDirectory &mrDir; std::vector<Entry*>::const_iterator i; }; - + friend class Iterator; class ReverseIterator @@ -236,10 +437,12 @@ public: ReverseIterator(const BackupStoreDirectory &rDir) : mrDir(rDir), i(rDir.mEntries.rbegin()) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds } - + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds // Skip over things which don't match the required flags while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) { @@ -253,12 +456,12 @@ public: // Return entry, and increment return (*(i++)); } - + private: const BackupStoreDirectory &mrDir; std::vector<Entry*>::const_reverse_iterator i; }; - + friend class ReverseIterator; // For recovery of the store @@ -286,4 +489,3 @@ private: }; #endif // BACKUPSTOREDIRECTORY__H - diff --git a/lib/backupstore/BackupStoreException.txt b/lib/backupstore/BackupStoreException.txt index ece772c0..efdbcf68 100644 --- a/lib/backupstore/BackupStoreException.txt +++ b/lib/backupstore/BackupStoreException.txt @@ -70,3 +70,7 @@ DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file m PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. MultiplyReferencedObject 69 Attempted to modify an object with multiple references, should be uncloned first +CorruptReferenceCountDatabase 70 The account's refcount database is corrupt and must be rebuilt by housekeeping. +CancelledByBackgroundTask 71 The current task was cancelled on request by the background task. +ObjectDoesNotExist 72 The specified object ID does not exist in the store. +AccountAlreadyExists 73 Tried to create an account that already exists. diff --git a/lib/backupstore/BackupStoreFile.cpp b/lib/backupstore/BackupStoreFile.cpp index 519305ff..99562685 100644 --- a/lib/backupstore/BackupStoreFile.cpp +++ b/lib/backupstore/BackupStoreFile.cpp @@ -22,29 +22,30 @@ #include <stdio.h> #endif +#include "BackupClientFileAttributes.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" #include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFileWire.h" #include "BackupStoreFilename.h" -#include "BackupStoreException.h" -#include "IOStream.h" -#include "Guards.h" -#include "FileModificationTime.h" -#include "FileStream.h" -#include "BackupClientFileAttributes.h" +#include "BackupStoreInfo.h" #include "BackupStoreObjectMagic.h" -#include "Compress.h" -#include "CipherContext.h" -#include "CipherBlowfish.h" #include "CipherAES.h" -#include "BackupStoreConstants.h" +#include "CipherBlowfish.h" +#include "CipherContext.h" #include "CollectInBufferStream.h" -#include "RollingChecksum.h" +#include "Compress.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "Guards.h" +#include "IOStream.h" +#include "Logging.h" #include "MD5Digest.h" -#include "ReadGatherStream.h" #include "Random.h" -#include "BackupStoreFileEncodeStream.h" -#include "Logging.h" +#include "ReadGatherStream.h" +#include "RollingChecksum.h" #include "MemLeakFindOn.h" @@ -78,7 +79,8 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) + RunStatusProvider* pRunStatusProvider, + BackgroundTask* pBackgroundTask) { // Create the stream std::auto_ptr<BackupStoreFileEncodeStream> stream( @@ -86,8 +88,9 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( // Do the initial setup stream->Setup(Filename, 0 /* no recipe, just encode */, ContainerID, - rStoreFilename, pModificationTime, pLogger, pRunStatusProvider); - + rStoreFilename, pModificationTime, pLogger, pRunStatusProvider, + pBackgroundTask); + // Return the stream for the caller return stream; } @@ -100,7 +103,15 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( // requirements. Doesn't verify that the data is intact // and can be decoded. Optionally returns the ID of the // file which it is diffed from, and the (original) -// container ID. +// container ID. This is more efficient than +// BackupStoreFile::VerifyStream() when the file data +// already exists on disk and we can Seek() around in +// it, but less efficient if we are reading the stream +// from the network and not intending to Write() it to +// a file first, so we need both unfortunately. +// TODO FIXME: use a modified VerifyStream() which +// repositions the file pointer and Close()s early to +// deduplicate this code. // Created: 2003/08/28 // // -------------------------------------------------------------------------- @@ -120,7 +131,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read header return false; } - + // Check magic number if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -130,7 +141,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro { return false; } - + // Get a filename, see if it loads OK try { @@ -142,7 +153,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // an error occured while reading it, so that's not good return false; } - + // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not try { @@ -163,10 +174,10 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Get current position in file -- the end of the header int64_t headerEnd = rFile.GetPosition(); - + // Get number of blocks int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - + // Calculate where the block index will be, check it's reasonable int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); if(blockIndexLoc < headerEnd) @@ -183,7 +194,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read block index header -- assume bad file return false; } - + // Check header if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -195,10 +206,10 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Bad header -- either magic value or number of blocks is wrong return false; } - + // Flag for recording whether a block is referenced from another file bool blockFromOtherFileReferenced = false; - + // Read the index, checking that the length values all make sense int64_t currentBlockStart = headerEnd; for(int64_t b = 0; b < numBlocks; ++b) @@ -210,7 +221,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read block index entry -- assume bad file return false; } - + // Check size and location int64_t blkSize = box_ntoh64(blk.mEncodedSize); if(blkSize <= 0) @@ -226,19 +237,20 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Encoded size makes the block run over the index return false; } - - // Move the current block start ot the end of this block + + // Move the current block start to the end of this block currentBlockStart += blkSize; } } - + // Check that there's no empty space if(currentBlockStart != blockIndexLoc) { return false; } - - // Check that if another block is references, then the ID is there, and if one isn't there is no ID. + + // Check that if another file is referenced, then the ID is there, and if one + // isn't then there is no ID. int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); if((otherID != 0 && blockFromOtherFileReferenced == false) || (otherID == 0 && blockFromOtherFileReferenced == true)) @@ -246,13 +258,13 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Doesn't look good! return false; } - + // Does the caller want the other ID? if(pDiffFromObjectIDOut) { *pDiffFromObjectIDOut = otherID; } - + // Does the caller want the container ID? if(pContainerIDOut) { @@ -263,6 +275,346 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro return true; } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyStream::Write() +// Purpose: Handles writes to the verifying stream. If the write +// completes the current unit, then verify it, copy it +// to mpCopyToStream if not NULL, and move on to the +// next unit, otherwise throw an exception. +// Created: 2015/08/07 +// +// -------------------------------------------------------------------------- + +void BackupStoreFile::VerifyStream::Write(const void *pBuffer, int NBytes, int Timeout) +{ + // Check that we haven't already written too much to the current unit + size_t BytesToAdd; + if(mState == State_Blocks) + { + // We don't know how many bytes to expect + ASSERT(mCurrentUnitSize == 0); + BytesToAdd = NBytes; + } + else + { + ASSERT(mCurrentUnitData.GetSize() < mCurrentUnitSize); + size_t BytesLeftInCurrentUnit = mCurrentUnitSize - + mCurrentUnitData.GetSize(); + // Add however many bytes are needed/available to the current unit's buffer. + BytesToAdd = std::min(BytesLeftInCurrentUnit, (size_t)NBytes); + } + + // We must make progress here, or we could have infinite recursion. + ASSERT(BytesToAdd > 0); + + CollectInBufferStream* pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Write(pBuffer, BytesToAdd, Timeout); + if(mpCopyToStream) + { + mpCopyToStream->Write(pBuffer, BytesToAdd, Timeout); + } + + pBuffer = (uint8_t *)pBuffer + BytesToAdd; + NBytes -= BytesToAdd; + mCurrentPosition += BytesToAdd; + + if(mState == State_Blocks) + { + // The block index follows the blocks themselves, but without seeing the + // index we don't know how big the blocks are. So we just have to keep + // reading, holding the last mBlockIndexSize bytes in two buffers, until + // we reach the end of the file (when Close() is called) when we can look + // back over those buffers and extract the block index from them. + if(pCurrentBuffer->GetSize() >= mBlockIndexSize) + { + // Time to swap buffers, and clear the one we're about to + // overwrite. + mCurrentBufferIsAlternate = !mCurrentBufferIsAlternate; + pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Reset(); + } + + // We don't want to move data into the finished buffer while we're in this + // state, and we don't need to call ourselves recursively, so just return. + return; + } + + ASSERT(mState != State_Blocks); + + // If the current unit is not complete, just return now. + if(mCurrentUnitData.GetSize() < mCurrentUnitSize) + { + return; + } + + ASSERT(mCurrentUnitData.GetSize() == mCurrentUnitSize); + mCurrentUnitData.SetForReading(); + CollectInBufferStream finished(mCurrentUnitData); + + // This should leave mCurrentUnitData empty in write phase + ASSERT(!mCurrentUnitData.StreamClosed()); + ASSERT(mCurrentUnitData.GetSize() == 0); + + // Advance automatically to next state (to reduce code duplication) if the current + // state is anything but State_Blocks. We remain in that state for more than one + // read (processing one block at a time), and exit it manually. + int oldState = mState; + if(mState != State_Blocks) + { + mState++; + } + + // Process and complete the current unit, depending what it is. + if(oldState == State_Header) + { + // Get the header... + file_StreamFormat hdr; + ASSERT(finished.GetSize() == sizeof(hdr)); + memcpy(&hdr, finished.GetBuffer(), sizeof(hdr)); + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid header magic in stream: expected " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_MAGIC_VALUE_V1) << + " or " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_MAGIC_VALUE_V0) << + " but found " << + BOX_FORMAT_HEX32(ntohl(hdr.mMagicValue))); + } + + mNumBlocks = box_ntoh64(hdr.mNumBlocks); + mBlockIndexSize = (mNumBlocks * sizeof(file_BlockIndexEntry)) + + sizeof(file_BlockIndexHeader); + mContainerID = box_ntoh64(hdr.mContainerID); + + ASSERT(mState == State_FilenameHeader); + mCurrentUnitSize = 2; + } + else if(oldState == State_FilenameHeader) + { + // Check that the encoding is an accepted value. + unsigned int encoding = + BACKUPSTOREFILENAME_GET_ENCODING( + (uint8_t *)finished.GetBuffer()); + if(encoding < BackupStoreFilename::Encoding_Min || + encoding > BackupStoreFilename::Encoding_Max) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid encoding in filename: " << encoding); + } + + ASSERT(mState == State_Filename); + mCurrentUnitSize = BACKUPSTOREFILENAME_GET_SIZE( + (uint8_t *)finished.GetBuffer()); + // Copy the first two bytes back into the new buffer, to be used by the + // completed filename. + finished.CopyStreamTo(mCurrentUnitData); + } + else if(oldState == State_Filename) + { + BackupStoreFilename fn; + fn.ReadFromStream(finished, IOStream::TimeOutInfinite); + ASSERT(mState == State_AttributesSize); + mCurrentUnitSize = sizeof(int32_t); + } + else if(oldState == State_AttributesSize) + { + ASSERT(mState == State_Attributes); + mCurrentUnitSize = ntohl(*(int32_t *)finished.GetBuffer()); + } + else if(oldState == State_Attributes) + { + // Skip the attributes -- because they're encrypted, the server can't tell + // whether they're OK or not. + ASSERT(mState == State_Blocks); + mBlockDataPosition = mCurrentPosition; + mCurrentUnitSize = 0; + } + else if(oldState == State_Blocks) + { + // The block index follows the blocks themselves, but without seeing the + // index we don't know how big the blocks are. So we just have to keep + // reading, holding the last mBlockIndexSize bytes in two buffers, until + // we reach the end of the file (when Close() is called) when we can look + // back over those buffers and extract the block index from them. + ASSERT(mState == State_Blocks); + if(pCurrentBuffer->GetSize() >= mBlockIndexSize) + { + // Time to swap buffers, and clear the one we're about to + // overwrite. + mCurrentBufferIsAlternate = !mCurrentBufferIsAlternate; + pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Reset(); + } + } + + if(NBytes > 0) + { + // Still some data to process, so call recursively to deal with it. + Write(pBuffer, NBytes, Timeout); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyStream::Write() +// Purpose: Handles closing the verifying stream, which tells us +// that there is no more incoming data, at which point +// we can extract the block index from the data most +// recently read, and verify it. +// Created: 2015/08/07 +// +// -------------------------------------------------------------------------- + + +void BackupStoreFile::VerifyStream::Close(bool CloseCopyStream) +{ + if(mpCopyToStream && CloseCopyStream) + { + mpCopyToStream->Close(); + } + + if(mState != State_Blocks) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Stream closed too early to be a valid BackupStoreFile: was in " + "state " << mState << " instead of " << State_Blocks); + } + + // Find the last mBlockIndexSize bytes. + CollectInBufferStream finished; + + CollectInBufferStream* pCurrentBuffer = mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData; + CollectInBufferStream* pPreviousBuffer = mCurrentBufferIsAlternate ? + &mCurrentUnitData : &mAlternateData; + + int64_t BytesFromCurrentBuffer = std::min(mBlockIndexSize, + (int64_t)pCurrentBuffer->GetSize()); + int64_t BytesFromPreviousBuffer = mBlockIndexSize - BytesFromCurrentBuffer; + + if(pPreviousBuffer->GetSize() < BytesFromPreviousBuffer) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Not enough bytes for block index: expected " << + mBlockIndexSize << " but had collected " << + (pPreviousBuffer->GetSize() + pCurrentBuffer->GetSize()) << + " at " << mCurrentPosition << " bytes into file"); + } + + size_t PreviousBufferOffset = pPreviousBuffer->GetSize() - + BytesFromPreviousBuffer; + size_t CurrentBufferOffset = pCurrentBuffer->GetSize() - + BytesFromCurrentBuffer; + + file_BlockIndexHeader blkhdr; + finished.Write((uint8_t *)pPreviousBuffer->GetBuffer() + PreviousBufferOffset, + BytesFromPreviousBuffer); + finished.Write((uint8_t *)pCurrentBuffer->GetBuffer() + CurrentBufferOffset, + BytesFromCurrentBuffer); + ASSERT(finished.GetSize() == mBlockIndexSize); + + // Load the block index header + memcpy(&blkhdr, finished.GetBuffer(), sizeof(blkhdr)); + + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index magic in stream: expected " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) << + " or " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0) << + " but found " << + BOX_FORMAT_HEX32(ntohl(blkhdr.mMagicValue))); + } + + if((int64_t)box_ntoh64(blkhdr.mNumBlocks) != mNumBlocks) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index size in stream: expected " << + BOX_FORMAT_OBJECTID(mNumBlocks) << + " but found " << + BOX_FORMAT_OBJECTID(box_ntoh64(blkhdr.mNumBlocks))); + } + + // Flag for recording whether a block is referenced from another file + mBlockFromOtherFileReferenced = false; + int64_t blockIndexLoc = mCurrentPosition - mBlockIndexSize; + + // Read the index, checking that the length values all make sense + int64_t currentBlockStart = mBlockDataPosition; + file_BlockIndexEntry* pBlk = (file_BlockIndexEntry *) + ((uint8_t *)finished.GetBuffer() + sizeof(blkhdr)); + for(int64_t b = 0; b < mNumBlocks; ++b) + { + // Check size and location + int64_t blkSize = box_ntoh64(pBlk[b].mEncodedSize); + if(blkSize <= 0) + { + // Mark that this file references another file + mBlockFromOtherFileReferenced = true; + } + else + { + // This block is actually in this file + if((currentBlockStart + blkSize) > blockIndexLoc) + { + // Encoded size makes the block run over the index + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index: entry " << b << " makes " + "total size of block data exceed the available " + "space"); + } + + // Move the current block start to the end of this block + currentBlockStart += blkSize; + } + } + + // Check that there's no empty space + if(currentBlockStart != blockIndexLoc) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index: total size of blocks does not match the " + "actual amount of block data in the stream: expected " << + (blockIndexLoc - mBlockDataPosition) << " but found " << + (currentBlockStart - mBlockDataPosition)); + } + + // Check that if another file is referenced, then the ID is there, and if one + // isn't then there is no ID. + int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); + if((otherID != 0 && mBlockFromOtherFileReferenced == false) + || (otherID == 0 && mBlockFromOtherFileReferenced == true)) + { + // Doesn't look good! + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index header: actual dependency status does not " + "match the file header: expected to depend on file object " << + BOX_FORMAT_OBJECTID(otherID)); + } + + mDiffFromObjectID = otherID; +} + + // -------------------------------------------------------------------------- // // Function @@ -279,7 +631,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile { THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) } - + // Try, delete output file if error try { @@ -288,7 +640,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile // Get the decoding stream std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr)); - + // Is it a symlink? if(!stream->IsSymLink()) { @@ -311,7 +663,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile // of the block index. I hope that reading an extra byte // doesn't hurt! // ASSERT(drained == 0); - + // Write the attributes try { @@ -348,10 +700,10 @@ std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream( { // Create stream std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout)); - + // Get it ready stream->Setup(pAlterativeAttr); - + // Return to caller return stream; } @@ -431,7 +783,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - bool inFileOrder = true; + bool inFileOrder = true; switch(ntohl(magic)) { #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -455,7 +807,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl default: THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // If not in file order, then the index list must be read now if(!inFileOrder) { @@ -484,7 +836,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - } + } // Check magic number if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 @@ -498,7 +850,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Get the filename mFilename.ReadFromStream(mrEncodedFile, mTimeout); - + // Get the attributes (either from stream, or supplied attributes) if(pAlterativeAttr != 0) { @@ -514,7 +866,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Read the attributes from the stream mAttributes.ReadFromStream(mrEncodedFile, mTimeout); } - + // If it is in file order, go and read the file attributes // Requires that the stream can seek if(inFileOrder) @@ -524,30 +876,30 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl { THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) } - + // Store current location (beginning of encoded blocks) int64_t endOfHeaderPos = mrEncodedFile.GetPosition(); - + // Work out where the index is int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - + // Seek to that position mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute); - + // Read the block index - ReadBlockIndex(false /* magic number still to be read */); - + ReadBlockIndex(false /* magic number still to be read */); + // Seek back to the end of header position, ready for reading the chunks mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute); } - + // Check view of blocks from block header and file header match if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks)) { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Need to allocate some memory for the two blocks for reading encoded data, and clear data if(mNumBlocks > 0) { @@ -560,11 +912,11 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Get the clear and encoded size int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize); ASSERT(encodedSize > 0); - + // Larger? if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize; } - + // Allocate those blocks! mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32); @@ -589,7 +941,7 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) { // Header file_BlockIndexHeader blkhdr; - + // Read it in -- way depends on how whether the magic number has already been read if(MagicAlreadyRead) { @@ -609,7 +961,7 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - + // Check magic value if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -620,26 +972,26 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } } - + // Get the number of blocks out of the header mNumBlocks = box_ntoh64(blkhdr.mNumBlocks); - + // Read the IV base mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase); - + // Load the block entries in? if(mNumBlocks > 0) { // How big is the index? int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks; - + // Allocate some memory mpBlockIndex = ::malloc(indexSize); if(mpBlockIndex == 0) { throw std::bad_alloc(); } - + // Read it in if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout)) { @@ -676,7 +1028,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) int bytesToRead = NBytes; uint8_t *output = (uint8_t*)pBuffer; - + while(bytesToRead > 0 && mCurrentBlock < mNumBlocks) { // Anything left in the current block? @@ -685,16 +1037,16 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // Copy data out of this buffer int s = mCurrentBlockClearSize - mPositionInCurrentBlock; if(s > bytesToRead) s = bytesToRead; // limit to requested data - + // Copy ::memcpy(output, mpClearData + mPositionInCurrentBlock, s); - + // Update positions output += s; mPositionInCurrentBlock += s; bytesToRead -= s; } - + // Need to get some more data? if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize) { @@ -705,7 +1057,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // Stop now! break; } - + // Get the size from the block index const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize); @@ -716,14 +1068,14 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // It needs to be combined with the previous version first. THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining) } - + // Load in next block if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout)) { // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - + // Decode the data mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize); @@ -734,7 +1086,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // platforms with different endiannesses. iv = box_hton64(iv); sBlowfishDecryptBlockEntry.SetIV(&iv); - + // Decrypt the encrypted section file_BlockIndexEntryEnc entryEnc; int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), @@ -779,7 +1131,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) #endif } - + // Check the digest MD5Digest md5; md5.Add(mpClearData, mCurrentBlockClearSize); @@ -788,12 +1140,12 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck) } - + // Set vars to say what's happening mPositionInCurrentBlock = 0; } } - + ASSERT(bytesToRead >= 0); ASSERT(bytesToRead <= NBytes); @@ -816,14 +1168,14 @@ bool BackupStoreFile::DecodedStream::IsSymLink() { return false; } - + // So the attributes think it is a symlink. // Consistency check... if(mNumBlocks != 0) { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + return true; } @@ -836,7 +1188,8 @@ bool BackupStoreFile::DecodedStream::IsSymLink() // Created: 9/12/03 // // -------------------------------------------------------------------------- -void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes) +void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes, + int Timeout) { THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream) } @@ -916,7 +1269,7 @@ void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength) sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); sAESDecrypt.Reset(); sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); - + // Set encryption to use this key, instead of the "default" blowfish key spEncrypt = &sAESEncrypt; sEncryptCipherType = HEADER_AES_ENCODING; @@ -961,9 +1314,9 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi { rOutput.Reallocate(256); } - + // Check alignment of the block - ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + ASSERT((((uint64_t)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); // Want to compress it? bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE); @@ -981,10 +1334,10 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi const void *iv = spEncrypt->SetRandomIV(ivLen); ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen); outOffset += ivLen; - + // Start encryption process spEncrypt->Begin(); - + #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \ { \ if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \ @@ -992,13 +1345,13 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \ } \ } - + // Encode the chunk if(compressChunk) { // buffer to compress into uint8_t buffer[2048]; - + // Set compressor with all the chunk as an input Compress<true> compress; compress.Input(Chunk, ChunkSize); @@ -1011,7 +1364,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi if(s > 0) { ENCODECHUNK_CHECK_SPACE(s) - outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); } else { @@ -1031,7 +1384,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi ENCODECHUNK_CHECK_SPACE(16) outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); } - + ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check return outOffset; @@ -1051,7 +1404,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize) { // Check alignment of the encoded block - ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + ASSERT((((uint64_t)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); // First check if(EncodedSize < 1) @@ -1060,7 +1413,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } const uint8_t *input = (uint8_t*)Encoded; - + // Get header, make checks, etc uint8_t header = input[0]; bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED; @@ -1069,7 +1422,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out { THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding) } - + #ifndef HAVE_OLD_SSL // Choose cipher CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt); @@ -1081,7 +1434,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } CipherContext &cipher(sBlowfishDecrypt); #endif - + // Check enough space for header, an IV and one byte of input int ivLen = cipher.GetIVLength(); if(EncodedSize < (1 + ivLen + 1)) @@ -1092,7 +1445,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out // Set IV in decrypt context, and start cipher.SetIV(input + 1); cipher.Begin(); - + // Setup vars for code int inOffset = 1 + ivLen; uint8_t *output = (uint8_t*)Output; @@ -1104,10 +1457,10 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out // Do things in chunks uint8_t buffer[2048]; int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer)); - + // Decompressor Compress<false> decompress; - + while(inOffset < EncodedSize) { // Decrypt a block @@ -1115,7 +1468,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl); inOffset += bl; - + // Decompress the decrypted data if(s > 0) { @@ -1126,7 +1479,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out os = decompress.Output(output + outOffset, OutputSize - outOffset); outOffset += os; } while(os > 0); - + // Check that there's space left in the output buffer -- there always should be if(outOffset >= OutputSize) { @@ -1134,7 +1487,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } } } - + // Get any compressed data remaining in the cipher context and compression int s = cipher.Final(buffer, sizeof(buffer)); decompress.Input(buffer, s); @@ -1157,7 +1510,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset); outOffset += cipher.Final(output + outOffset, OutputSize - outOffset); } - + return outOffset; } @@ -1209,10 +1562,10 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Get number of blocks int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - + // Calculate where the block index will be, check it's reasonable int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); int64_t blockIndexLoc = fileSize - blockIndexSize; @@ -1221,10 +1574,10 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr // Doesn't look good! THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Build a reordered stream std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership)); - + // Set it up... ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get())); int component = rreordered.AddComponent(pStream); @@ -1232,7 +1585,7 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc); // And then the rest of the file rreordered.AddBlock(component, blockIndexLoc, true, 0); - + return reordered; } @@ -1287,7 +1640,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, { in.reset(new FileStream(Filename)); } - + // Read header file_BlockIndexHeader hdr; if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) @@ -1313,17 +1666,17 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, // Get basic information int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase); - + //TODO: Verify that these sizes look reasonable - + // setup void *data = 0; int32_t dataSize = -1; bool matches = true; int64_t totalSizeInBlockIndex = 0; - + try - { + { for(int64_t b = 0; b < numBlocks; ++b) { // Read an entry from the stream @@ -1332,8 +1685,8 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, { // Couldn't read entry THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - + } + // Calculate IV for this entry uint64_t iv = entryIVBase; iv += b; @@ -1345,8 +1698,8 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, iv = box_swap64(iv); } #endif - sBlowfishDecryptBlockEntry.SetIV(&iv); - + sBlowfishDecryptBlockEntry.SetIV(&iv); + // Decrypt the encrypted section file_BlockIndexEntryEnc entryEnc; int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), @@ -1381,7 +1734,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } dataSize = blockClearSize + 128; } - + // Load in the block from the file, if it's not a symlink if(!sourceIsSymlink) { @@ -1403,7 +1756,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } } } - + // Keep on going regardless, to make sure the entire block index stream is read // -- must always be consistent about what happens with the stream. } @@ -1418,14 +1771,14 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } throw; } - + // free block if(data != 0) { ::free(data); data = 0; } - + // Check for data left over if it's not a symlink if(!sourceIsSymlink) { @@ -1436,13 +1789,13 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, matches = false; } } - + // Symlinks must have zero size on server if(sourceIsSymlink) { matches = (totalSizeInBlockIndex == 0); } - + return matches; } @@ -1522,10 +1875,10 @@ void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) } // Copy data ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize); - + // Free old BackupStoreFile::CodingChunkFree(mpBuffer); - + // Store new buffer mpBuffer = buffer; mBufferSize = NewSize; @@ -1554,5 +1907,44 @@ DiffTimer::DiffTimer() // // -------------------------------------------------------------------------- DiffTimer::~DiffTimer() -{ +{ +} + +// Shortcut interface +int64_t BackupStoreFile::QueryStoreFileDiff(BackupProtocolCallable& protocol, + const std::string& LocalFilename, int64_t DirectoryObjectID, + int64_t DiffFromFileID, int64_t AttributesHash, + const BackupStoreFilenameClear& StoreFilename, int Timeout, + DiffTimer *pDiffTimer, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) +{ + int64_t ModificationTime; + std::auto_ptr<BackupStoreFileEncodeStream> pStream; + + if(DiffFromFileID) + { + // Fetch the block index for this one + std::auto_ptr<BackupProtocolSuccess> getblockindex = + protocol.QueryGetBlockIndexByName(DirectoryObjectID, + StoreFilename); + ASSERT(getblockindex->GetObjectID() == DiffFromFileID); + std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream()); + + pStream = EncodeFileDiff(LocalFilename, + DirectoryObjectID, StoreFilename, DiffFromFileID, + *(blockIndexStream.get()), Timeout, pDiffTimer, + &ModificationTime, NULL // pIsCompletelyDifferent + ); + } + else + { + pStream = BackupStoreFile::EncodeFile(LocalFilename, + DirectoryObjectID, StoreFilename, &ModificationTime); + } + + std::auto_ptr<IOStream> upload(pStream.release()); + return protocol.QueryStoreFile(DirectoryObjectID, + ModificationTime, AttributesHash, DiffFromFileID, + StoreFilename, upload)->GetObjectID(); } + diff --git a/lib/backupstore/BackupStoreFile.h b/lib/backupstore/BackupStoreFile.h index 7c72e010..fe69caeb 100644 --- a/lib/backupstore/BackupStoreFile.h +++ b/lib/backupstore/BackupStoreFile.h @@ -14,8 +14,11 @@ #include <memory> #include <cstdlib> +#include "autogen_BackupProtocol.h" #include "BackupClientFileAttributes.h" +#include "BackupStoreFileWire.h" #include "BackupStoreFilename.h" +#include "CollectInBufferStream.h" #include "IOStream.h" #include "ReadLoggingStream.h" @@ -26,6 +29,7 @@ typedef struct int64_t mTotalFileStreamSize; } BackupStoreFileStats; +class BackgroundTask; class RunStatusProvider; // Uncomment to disable backwards compatibility @@ -40,6 +44,8 @@ class RunStatusProvider; // Have some memory allocation commands, note closing "Off" at end of file. #include "MemLeakFindOn.h" +class BackupStoreFileEncodeStream; + // -------------------------------------------------------------------------- // // Class @@ -60,8 +66,6 @@ public: virtual bool IsManaged() = 0; }; -class BackupStoreFileEncodeStream; - // -------------------------------------------------------------------------- // // Class @@ -83,9 +87,10 @@ public: public: ~DecodedStream(); - // Stream functions + // Stream functions virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); @@ -119,6 +124,64 @@ public: #endif }; + class VerifyStream : public IOStream + { + private: + enum + { + State_Header = 0, + State_FilenameHeader, + State_Filename, + State_AttributesSize, + State_Attributes, + State_Blocks, + }; + + int mState; + IOStream* mpCopyToStream; + CollectInBufferStream mCurrentUnitData; + size_t mCurrentUnitSize; + int64_t mNumBlocks; + int64_t mBlockIndexSize; + int64_t mCurrentPosition; + int64_t mBlockDataPosition; + bool mCurrentBufferIsAlternate; + CollectInBufferStream mAlternateData; + bool mBlockFromOtherFileReferenced; + int64_t mContainerID; + int64_t mDiffFromObjectID; + + public: + VerifyStream(IOStream* pCopyToStream = NULL) + : mState(State_Header), + mpCopyToStream(pCopyToStream), + mCurrentUnitSize(sizeof(file_StreamFormat)), + mNumBlocks(0), + mBlockIndexSize(0), + mCurrentPosition(0), + mBlockDataPosition(0), + mCurrentBufferIsAlternate(false), + mBlockFromOtherFileReferenced(false), + mContainerID(0), + mDiffFromObjectID(0) + { } + virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); + virtual void Close(bool CloseCopyStream = true); + virtual bool StreamDataLeft() + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual bool StreamClosed() + { + THROW_EXCEPTION(CommonException, NotSupported); + } + }; // Main interface static std::auto_ptr<BackupStoreFileEncodeStream> EncodeFile @@ -127,7 +190,8 @@ public: int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0, ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL + RunStatusProvider* pRunStatusProvider = NULL, + BackgroundTask* pBackgroundTask = NULL ); static std::auto_ptr<BackupStoreFileEncodeStream> EncodeFileDiff ( @@ -137,8 +201,19 @@ public: int Timeout, DiffTimer *pDiffTimer, int64_t *pModificationTime = 0, - bool *pIsCompletelyDifferent = 0 + bool *pIsCompletelyDifferent = 0, + BackgroundTask* pBackgroundTask = NULL ); + // Shortcut interface + static int64_t QueryStoreFileDiff(BackupProtocolCallable& protocol, + const std::string& LocalFilename, int64_t DirectoryObjectID, + int64_t DiffFromFileID, int64_t AttributesHash, + const BackupStoreFilenameClear& StoreFilename, + int Timeout = IOStream::TimeOutInfinite, + DiffTimer *pDiffTimer = NULL, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL); + static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); diff --git a/lib/backupstore/BackupStoreFileCmbIdx.cpp b/lib/backupstore/BackupStoreFileCmbIdx.cpp index c8bcc3b9..0eec3872 100644 --- a/lib/backupstore/BackupStoreFileCmbIdx.cpp +++ b/lib/backupstore/BackupStoreFileCmbIdx.cpp @@ -32,7 +32,8 @@ public: ~BSFCombinedIndexStream(); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); virtual void Initialise(IOStream &rFrom); @@ -289,7 +290,8 @@ int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout) // Created: 8/7/04 // // -------------------------------------------------------------------------- -void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes) +void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes, + int Timeout) { THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) } diff --git a/lib/backupstore/BackupStoreFileDiff.cpp b/lib/backupstore/BackupStoreFileDiff.cpp index fa8cb892..e6df11a6 100644 --- a/lib/backupstore/BackupStoreFileDiff.cpp +++ b/lib/backupstore/BackupStoreFileDiff.cpp @@ -16,7 +16,7 @@ #ifdef HAVE_TIME_H #include <time.h> -#elif HAVE_SYS_TIME_H +#elif defined HAVE_SYS_TIME_H #include <sys/time.h> #endif @@ -114,21 +114,21 @@ void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream) // Function // Name: BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *) // Purpose: Similar to EncodeFile, but takes the object ID of the file it's -// diffing from, and the index of the blocks in a stream. It'll then -// calculate which blocks can be reused from that old file. -// The timeout is the timeout value for reading the diff block index. -// If pIsCompletelyDifferent != 0, it will be set to true if the -// the two files are completely different (do not share any block), false otherwise. -// +// diffing from, and the index of the blocks in a stream. It'll then +// calculate which blocks can be reused from that old file. +// The timeout is the timeout value for reading the diff block index. +// If pIsCompletelyDifferent != 0, it will be set to true if the +// the two files are completely different (do not share any block), false otherwise. // Created: 12/1/04 // // -------------------------------------------------------------------------- std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff ( const std::string& Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, - IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, - int64_t *pModificationTime, bool *pIsCompletelyDifferent) + const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, + IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, + int64_t *pModificationTime, bool *pIsCompletelyDifferent, + BackgroundTask* pBackgroundTask) { // Is it a symlink? { @@ -144,7 +144,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff { *pIsCompletelyDifferent = true; } - return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + return EncodeFile(Filename, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); // BackgroundTask } } @@ -162,7 +166,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff { *pIsCompletelyDifferent = true; } - return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + return EncodeFile(Filename, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); // BackgroundTask } // Pointer to recipe we're going to create @@ -210,7 +218,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff new BackupStoreFileEncodeStream); // Do the initial setup - stream->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime); + stream->Setup(Filename, precipe, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); precipe = 0; // Stream has taken ownership of this // Tell user about completely different status? @@ -515,7 +527,7 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> // Search for each block size in turn // NOTE: Do the smallest size first, so that the scheme for adding - // entries in the found list works as expected and replaces smallers block + // entries in the found list works as expected and replaces smaller blocks // with larger blocks when it finds matches at the same offset in the file. for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s) { @@ -629,7 +641,7 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> { if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks)) { - BOX_TRACE("Found block match for " << hash << " of " << Sizes[s] << " bytes at offset " << fileOffset); + BOX_TRACE("Found block match of " << Sizes[s] << " bytes with hash " << hash << " at offset " << fileOffset); goodnessOfFit[fileOffset] = Sizes[s]; // Block matched, roll the checksum forward to the next block without doing @@ -657,7 +669,8 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> } else { - BOX_TRACE("False alarm match for " << hash << " of " << Sizes[s] << " bytes at offset " << fileOffset); + // Too many to log + // BOX_TRACE("False alarm match of " << Sizes[s] << " bytes with hash " << hash << " at offset " << fileOffset); } int64_t NumBlocksFound = static_cast<int64_t>( @@ -1029,9 +1042,11 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA { char b[64]; #ifdef WIN32 - sprintf(b, "%8I64d", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); + snprintf(b, sizeof(b), "%8I64d", (int64_t) + (rRecipe[e].mpStartBlock - pIndex)); #else - sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); + snprintf(b, sizeof(b), "%8lld", (int64_t) + (rRecipe[e].mpStartBlock - pIndex)); #endif BOX_TRACE(std::setw(8) << rRecipe[e].mSpaceBefore << diff --git a/lib/backupstore/BackupStoreFileEncodeStream.cpp b/lib/backupstore/BackupStoreFileEncodeStream.cpp index b53c4c26..83333a5b 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.cpp +++ b/lib/backupstore/BackupStoreFileEncodeStream.cpp @@ -11,6 +11,7 @@ #include <string.h> +#include "BackgroundTask.h" #include "BackupClientFileAttributes.h" #include "BackupStoreConstants.h" #include "BackupStoreException.h" @@ -40,25 +41,28 @@ using namespace BackupStoreFileCryptVar; // // -------------------------------------------------------------------------- BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() - : mpRecipe(0), - mpFile(0), - mpLogging(0), - mpRunStatusProvider(NULL), - mStatus(Status_Header), - mSendData(true), - mTotalBlocks(0), - mAbsoluteBlockNumber(-1), - mInstructionNumber(-1), - mNumBlocks(0), - mCurrentBlock(-1), - mCurrentBlockEncodedSize(0), - mPositionInCurrentBlock(0), - mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), - mLastBlockSize(0), - mTotalBytesSent(0), - mpRawBuffer(0), - mAllocatedBufferSize(0), - mEntryIVBase(0) +: mpRecipe(0), + mpFile(0), + mpLogging(0), + mpRunStatusProvider(NULL), + mpBackgroundTask(NULL), + mStatus(Status_Header), + mSendData(true), + mTotalBlocks(0), + mBytesToUpload(0), + mBytesUploaded(0), + mAbsoluteBlockNumber(-1), + mInstructionNumber(-1), + mNumBlocks(0), + mCurrentBlock(-1), + mCurrentBlockEncodedSize(0), + mPositionInCurrentBlock(0), + mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), + mLastBlockSize(0), + mTotalBytesSent(0), + mpRawBuffer(0), + mAllocatedBufferSize(0), + mEntryIVBase(0) { } @@ -78,21 +82,21 @@ BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() ::free(mpRawBuffer); mpRawBuffer = 0; } - + // Close the file, which we might have open if(mpFile) { delete mpFile; mpFile = 0; } - + // Clear up logging stream if(mpLogging) { delete mpLogging; mpLogging = 0; } - + // Free the recipe if(mpRecipe != 0) { @@ -115,7 +119,8 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, BackupStoreFileEncodeStream::Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) + RunStatusProvider* pRunStatusProvider, + BackgroundTask* pBackgroundTask) { // Pointer to a blank recipe which we might create BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; @@ -128,27 +133,27 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, BackupClientFileAttributes attr; attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime, 0 /* not interested in attr mod time */, &fileSize); - + // Might need to create a blank recipe... if(pRecipe == 0) { pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); - + BackupStoreFileEncodeStream::RecipeInstruction instruction; instruction.mSpaceBefore = fileSize; // whole file instruction.mBlocks = 0; // no blocks instruction.mpStartBlock = 0; // no block - pblankRecipe->push_back(instruction); + pblankRecipe->push_back(instruction); pRecipe = pblankRecipe; } - + // Tell caller? if(pModificationTime != 0) { *pModificationTime = modTime; } - + // Go through each instruction in the recipe and work out how many blocks // it will add, and the max clear size of these blocks int maxBlockClearSize = 0; @@ -162,14 +167,15 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize); // Add to accumlated total mTotalBlocks += numBlocks; + mBytesToUpload += (*pRecipe)[inst].mSpaceBefore; // Update maximum clear size if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize; if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize; } - + // Add number of blocks copied from the previous file mTotalBlocks += (*pRecipe)[inst].mBlocks; - + // Check for bad things if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0)) { @@ -182,7 +188,7 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize; } } - + // Send data? (symlinks don't have any data in them) mSendData = !attr.IsSymLink(); @@ -191,7 +197,7 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, { maxBlockClearSize = 0; } - + // Header file_StreamFormat hdr; hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1); @@ -201,16 +207,16 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128); hdr.mOptions = 0; // no options defined yet - + // Write header to stream mData.Write(&hdr, sizeof(hdr)); - + // Write filename to stream rStoreFilename.WriteToStream(mData); - + // Write attributes to stream attr.WriteToStream(mData); - + // Allocate some buffers for writing data if(mSendData) { @@ -229,10 +235,10 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, mpLogging = mpFile; mpFile = NULL; } - + // Work out the largest possible block required for the encoded data mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); - + // Then allocate two blocks of this size mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize); if(mpRawBuffer == 0) @@ -256,13 +262,13 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, blkhdr.mNumBlocks = box_hton64(0); mData.Write(&blkhdr, sizeof(blkhdr)); } - + // Ready for reading mData.SetForReading(); - + // Update stats BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize; - + // Finally, store the pointer to the recipe, when we know exceptions won't occur mpRecipe = pRecipe; } @@ -276,8 +282,9 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, } throw; } - + mpRunStatusProvider = pRunStatusProvider; + mpBackgroundTask = pBackgroundTask; } @@ -296,14 +303,14 @@ void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t do { rBlockSizeOut *= 2; - + rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut; - + } while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); - + // Last block size rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut); - + // Avoid small blocks? if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN && rNumBlocksOut > 1) @@ -312,7 +319,7 @@ void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t --rNumBlocksOut; rLastBlockSizeOut += rBlockSizeOut; } - + // checks! ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize); //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut); @@ -335,26 +342,39 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { return 0; } - + if(mpRunStatusProvider && mpRunStatusProvider->StopRun()) { THROW_EXCEPTION(BackupStoreException, SignalReceived); } + if(mpBackgroundTask) + { + BackgroundTask::State state = (mpRecipe->at(0).mBlocks == 0) + ? BackgroundTask::Uploading_Full + : BackgroundTask::Uploading_Patch; + if(!mpBackgroundTask->RunBackgroundTask(state, mBytesUploaded, + mBytesToUpload)) + { + THROW_EXCEPTION(BackupStoreException, + CancelledByBackgroundTask); + } + } + int bytesToRead = NBytes; uint8_t *buffer = (uint8_t*)pBuffer; - + while(bytesToRead > 0 && mStatus != Status_Finished) { if(mStatus == Status_Header || mStatus == Status_BlockListing) { // Header or block listing phase -- send from the buffered stream - + // Send bytes from the data buffer int b = mData.Read(buffer, bytesToRead, Timeout); bytesToRead -= b; buffer += b; - + // Check to see if all the data has been used from this stream if(!mData.StreamDataLeft()) { @@ -367,7 +387,7 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { // Reset the buffer so it can be used for the next phase mData.Reset(); - + // Get buffer ready for index? if(mStatus == Status_Header) { @@ -377,14 +397,14 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) ASSERT(mpRecipe != 0); blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID()); blkhdr.mNumBlocks = box_hton64(mTotalBlocks); - + // Generate the IV base Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase)); blkhdr.mEntryIVBase = box_hton64(mEntryIVBase); - + mData.Write(&blkhdr, sizeof(blkhdr)); } - + ++mStatus; } } @@ -392,7 +412,7 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) else if(mStatus == Status_Blocks) { // Block sending phase - + if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize) { // Next block! @@ -405,10 +425,10 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { SkipPreviousBlocksInInstruction(); } - + // Is there another instruction to go? ++mInstructionNumber; - + // Skip instructions which don't contain any data while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size()) && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0) @@ -416,12 +436,12 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) SkipPreviousBlocksInInstruction(); ++mInstructionNumber; } - + if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size())) { // End of blocks, go to next phase ++mStatus; - + // Set the data to reading so the index can be written mData.SetForReading(); } @@ -438,17 +458,17 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) EncodeCurrentBlock(); } } - + // Send data from the current block (if there's data to send) if(mPositionInCurrentBlock < mCurrentBlockEncodedSize) { // How much data to put in the buffer? int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock; if(s > bytesToRead) s = bytesToRead; - + // Copy it in ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s); - + // Update variables bytesToRead -= s; buffer += s; @@ -461,11 +481,11 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) ASSERT(false); } } - + // Add encoded size to stats BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead); mTotalBytesSent += (NBytes - bytesToRead); - + // Return size of data to caller return NBytes - bytesToRead; } @@ -490,27 +510,27 @@ void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction() // Index of the first block in old file (being diffed from) int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock); - + int64_t sizeToSkip = 0; for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b) { // Update stats BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; - + // Store the entry StoreBlockIndexEntry(0 - (firstIndex + b), (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize, (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum, - (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); // Increment the absolute block number -- kept encryption IV in sync ++mAbsoluteBlockNumber; - + // Add the size of this block to the size to skip sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; } - + // Move forward in the stream mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative); } @@ -528,7 +548,7 @@ void BackupStoreFileEncodeStream::SetForInstruction() { // Calculate block sizes CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize); - + // Set variables mCurrentBlock = 0; mCurrentBlockEncodedSize = 0; @@ -561,7 +581,7 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() // File should be open, but isn't. So logical error. THROW_EXCEPTION(BackupStoreException, Internal) } - + // Read the data in if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */)) @@ -571,13 +591,15 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer) } - + // Encode it mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer); - + + mBytesUploaded += blockRawSize; + //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); - + // Create block listing data -- generate checksums RollingChecksum weakChecksum(mpRawBuffer, blockRawSize); MD5Digest strongChecksum; @@ -587,7 +609,7 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() // Add entry to the index StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); - + // Set vars to reading this block mPositionInCurrentBlock = 0; } @@ -611,7 +633,7 @@ void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex // Then the clear section file_BlockIndexEntry entry; entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex)); - + // Then encrypt the encryted section // Generate the IV from the block number if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase)) @@ -645,7 +667,8 @@ void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex // Created: 8/12/03 // // -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes) +void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes, + int Timeout) { THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream) } @@ -686,11 +709,12 @@ bool BackupStoreFileEncodeStream::StreamClosed() // Created: 15/1/04 // // -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, - int64_t NumBlocksInIndex, int64_t OtherFileID) - : mpBlockIndex(pBlockIndex), - mNumBlocksInIndex(NumBlocksInIndex), - mOtherFileID(OtherFileID) +BackupStoreFileEncodeStream::Recipe::Recipe( + BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, + int64_t NumBlocksInIndex, int64_t OtherFileID) +: mpBlockIndex(pBlockIndex), + mNumBlocksInIndex(NumBlocksInIndex), + mOtherFileID(OtherFileID) { ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) } diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h index a169e036..5b9b4a61 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.h +++ b/lib/backupstore/BackupStoreFileEncodeStream.h @@ -79,14 +79,20 @@ public: const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL); + RunStatusProvider* pRunStatusProvider = NULL, + BackgroundTask* pBackgroundTask = NULL); virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); + int64_t GetBytesToUpload() { return mBytesToUpload; } int64_t GetTotalBytesSent() { return mTotalBytesSent; } + static void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, + int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); + private: enum { @@ -95,28 +101,29 @@ private: Status_BlockListing = 2, Status_Finished = 3 }; - -private: + void EncodeCurrentBlock(); - void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); void SkipPreviousBlocksInInstruction(); void SetForInstruction(); void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum); -private: Recipe *mpRecipe; IOStream *mpFile; // source file CollectInBufferStream mData; // buffer for header and index entries IOStream *mpLogging; RunStatusProvider* mpRunStatusProvider; + BackgroundTask* mpBackgroundTask; int mStatus; bool mSendData; // true if there's file data to send (ie not a symlink) int64_t mTotalBlocks; // Total number of blocks in the file + int64_t mBytesToUpload; // Total number of clear bytes to encode and upload + int64_t mBytesUploaded; // Total number of clear bytes already encoded + // excluding reused blocks already on the server. int64_t mAbsoluteBlockNumber; // The absolute block number currently being output // Instruction number int64_t mInstructionNumber; // All the below are within the current instruction - int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases + int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases int64_t mCurrentBlock; int32_t mCurrentBlockEncodedSize; int32_t mPositionInCurrentBlock; // for reading out diff --git a/lib/backupstore/BackupStoreFilenameClear.h b/lib/backupstore/BackupStoreFilenameClear.h index 595d1158..b7cf555f 100644 --- a/lib/backupstore/BackupStoreFilenameClear.h +++ b/lib/backupstore/BackupStoreFilenameClear.h @@ -42,7 +42,7 @@ public: #endif void SetClearFilename(const std::string &rToEncode); - // Setup for encryption of filenames + // Setup for encryption of filenames static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength); static void SetEncodingMethod(int Method); diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp index b6714709..efe3f7bb 100644 --- a/lib/backupstore/BackupStoreInfo.cpp +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -34,22 +34,24 @@ // // -------------------------------------------------------------------------- BackupStoreInfo::BackupStoreInfo() - : mAccountID(-1), - mDiscSet(-1), - mReadOnly(true), - mIsModified(false), - mClientStoreMarker(0), - mLastObjectIDUsed(-1), - mBlocksUsed(0), - mBlocksInCurrentFiles(0), - mBlocksInOldFiles(0), - mBlocksInDeletedFiles(0), - mBlocksInDirectories(0), - mNumFiles(0), - mNumOldFiles(0), - mNumDeletedFiles(0), - mNumDirectories(0), - mAccountEnabled(true) +: mAccountID(-1), + mDiscSet(-1), + mReadOnly(true), + mIsModified(false), + mClientStoreMarker(0), + mLastObjectIDUsed(-1), + mBlocksUsed(0), + mBlocksInCurrentFiles(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0), + mBlocksSoftLimit(0), + mBlocksHardLimit(0), + mNumCurrentFiles(0), + mNumOldFiles(0), + mNumDeletedFiles(0), + mNumDirectories(0), + mAccountEnabled(true) { } @@ -92,6 +94,31 @@ void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, info.Save(false); } +BackupStoreInfo::BackupStoreInfo(int32_t AccountID, const std::string &FileName, + int64_t BlockSoftLimit, int64_t BlockHardLimit) +: mAccountID(AccountID), + mDiscSet(-1), + mFilename(FileName), + mReadOnly(false), + mIsModified(false), + mClientStoreMarker(0), + mLastObjectIDUsed(0), + mBlocksUsed(0), + mBlocksInCurrentFiles(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0), + mBlocksSoftLimit(BlockSoftLimit), + mBlocksHardLimit(BlockHardLimit), + mNumCurrentFiles(0), + mNumOldFiles(0), + mNumDeletedFiles(0), + mNumDirectories(0), + mAccountEnabled(true) +{ + mExtraData.SetForReading(); // extra data is empty in this case +} + // -------------------------------------------------------------------------- // // Function @@ -108,16 +135,31 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { // Generate the filename std::string fn(rRootDir + INFO_FILENAME); - + // Open the file for reading (passing on optional request for revision ID) std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID)); + std::auto_ptr<BackupStoreInfo> info = Load(*rf, fn, ReadOnly); + + // Check it + if(info->GetAccountID() != AccountID) + { + THROW_FILE_ERROR("Found wrong account ID in store info", + fn, BackupStoreException, BadStoreInfoOnLoad); + } + + info->mDiscSet = DiscSet; + return info; +} +std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(IOStream& rStream, + const std::string FileName, bool ReadOnly) +{ // Read in format and version int32_t magic; - if(!rf->ReadFullBuffer(&magic, sizeof(magic), 0)) + if(!rStream.ReadFullBuffer(&magic, sizeof(magic), 0)) { THROW_FILE_ERROR("Failed to read store info file: " - "short read of magic number", fn, + "short read of magic number", FileName, BackupStoreException, CouldNotLoadStoreInfo); } @@ -135,16 +177,14 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { THROW_FILE_ERROR("Failed to read store info file: " "unknown magic " << BOX_FORMAT_HEX32(ntohl(magic)), - fn, BackupStoreException, BadStoreInfoOnLoad); + FileName, BackupStoreException, BadStoreInfoOnLoad); } // Make new object std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); - + // Put in basic location info - info->mAccountID = AccountID; - info->mDiscSet = DiscSet; - info->mFilename = fn; + info->mFilename = FileName; info->mReadOnly = ReadOnly; int64_t numDelObj = 0; @@ -152,23 +192,17 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { // Read in a header info_StreamFormat_1 hdr; - rf->Seek(0, IOStream::SeekType_Absolute); + rStream.Seek(0, IOStream::SeekType_Absolute); - if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), + if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) { THROW_FILE_ERROR("Failed to read store info header", - fn, BackupStoreException, CouldNotLoadStoreInfo); - } - - // Check it - if((int32_t)ntohl(hdr.mAccountID) != AccountID) - { - THROW_FILE_ERROR("Found wrong account ID in store info", - fn, BackupStoreException, BadStoreInfoOnLoad); + FileName, BackupStoreException, CouldNotLoadStoreInfo); } - + // Insert info from file + info->mAccountID = ntohl(hdr.mAccountID); info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker); info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed); info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); @@ -177,24 +211,16 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories); info->mBlocksSoftLimit = box_ntoh64(hdr.mBlocksSoftLimit); info->mBlocksHardLimit = box_ntoh64(hdr.mBlocksHardLimit); - + // Load up array of deleted objects numDelObj = box_ntoh64(hdr.mNumberDeletedDirectories); } else if(v2) { - Archive archive(*rf, IOStream::TimeOutInfinite); + Archive archive(rStream, IOStream::TimeOutInfinite); // Check it - int32_t FileAccountID; - archive.Read(FileAccountID); - if (FileAccountID != AccountID) - { - THROW_FILE_ERROR("Found wrong account ID in store " - "info: " << BOX_FORMAT_HEX32(FileAccountID), - fn, BackupStoreException, BadStoreInfoOnLoad); - } - + archive.Read(info->mAccountID); archive.Read(info->mAccountName); archive.Read(info->mClientStoreMarker); archive.Read(info->mLastObjectIDUsed); @@ -205,35 +231,35 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, archive.Read(info->mBlocksInDirectories); archive.Read(info->mBlocksSoftLimit); archive.Read(info->mBlocksHardLimit); - archive.Read(info->mNumFiles); - archive.Read(info->mNumOldFiles); - archive.Read(info->mNumDeletedFiles); - archive.Read(info->mNumDirectories); - archive.Read(numDelObj); + archive.Read(info->mNumCurrentFiles); + archive.Read(info->mNumOldFiles); + archive.Read(info->mNumDeletedFiles); + archive.Read(info->mNumDirectories); + archive.Read(numDelObj); } // Then load the list of deleted directories if(numDelObj > 0) { int64_t objs[NUM_DELETED_DIRS_BLOCK]; - + int64_t toload = numDelObj; while(toload > 0) { // How many in this one? int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload)); - - if(!rf->ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */)) + + if(!rStream.ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */)) { THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) } - + // Add them for(int t = 0; t < b; ++t) { info->mDeletedDirectories.push_back(box_ntoh64(objs[t])); } - + // Number loaded toload -= b; } @@ -247,24 +273,24 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, if(v2) { - Archive archive(*rf, IOStream::TimeOutInfinite); + Archive archive(rStream, IOStream::TimeOutInfinite); archive.ReadIfPresent(info->mAccountEnabled, true); } else { info->mAccountEnabled = true; } - + // If there's any data left in the info file, from future additions to // the file format, then we need to load it so that it won't be lost when // we resave the file. - IOStream::pos_type bytesLeft = rf->BytesLeftToRead(); + IOStream::pos_type bytesLeft = rStream.BytesLeftToRead(); if (bytesLeft > 0) { - rf->CopyStreamTo(info->mExtraData); + rStream.CopyStreamTo(info->mExtraData); } info->mExtraData.SetForReading(); - + // return it to caller return info; } @@ -289,10 +315,10 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( { // Generate the filename std::string fn(rRootDir + INFO_FILENAME); - + // Make new object std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); - + // Put in basic info info->mAccountID = AccountID; info->mAccountName = rAccountName; @@ -311,7 +337,7 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( info->mBlocksSoftLimit = BlockSoftLimit; info->mBlocksHardLimit = BlockHardLimit; info->mAccountEnabled = AccountEnabled; - + ExtraData.CopyStreamTo(info->mExtraData); info->mExtraData.SetForReading(); @@ -341,15 +367,22 @@ void BackupStoreInfo::Save(bool allowOverwrite) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + // Then... open a write file RaidFileWrite rf(mDiscSet, mFilename); rf.Open(allowOverwrite); - + Save(rf); + + // Commit it to disc, converting it to RAID now + rf.Commit(true); +} + +void BackupStoreInfo::Save(IOStream& rOutStream) +{ // Make header int32_t magic = htonl(INFO_MAGIC_VALUE_2); - rf.Write(&magic, sizeof(magic)); - Archive archive(rf, IOStream::TimeOutInfinite); + rOutStream.Write(&magic, sizeof(magic)); + Archive archive(rOutStream, IOStream::TimeOutInfinite); archive.Write(mAccountID); archive.Write(mAccountName); @@ -362,7 +395,7 @@ void BackupStoreInfo::Save(bool allowOverwrite) archive.Write(mBlocksInDirectories); archive.Write(mBlocksSoftLimit); archive.Write(mBlocksHardLimit); - archive.Write(mNumFiles); + archive.Write(mNumCurrentFiles); archive.Write(mNumOldFiles); archive.Write(mNumDeletedFiles); archive.Write(mNumDirectories); @@ -374,14 +407,14 @@ void BackupStoreInfo::Save(bool allowOverwrite) if(mDeletedDirectories.size() > 0) { int64_t objs[NUM_DELETED_DIRS_BLOCK]; - + int tosave = mDeletedDirectories.size(); std::vector<int64_t>::iterator i(mDeletedDirectories.begin()); while(tosave > 0) { // How many in this one? int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); - + // Add them for(int t = 0; t < b; ++t) { @@ -390,23 +423,20 @@ void BackupStoreInfo::Save(bool allowOverwrite) i++; } - // Write - rf.Write(objs, b * sizeof(int64_t)); - + // Write + rOutStream.Write(objs, b * sizeof(int64_t)); + // Number saved tosave -= b; } } - + archive.Write(mAccountEnabled); - + mExtraData.Seek(0, IOStream::SeekType_Absolute); - mExtraData.CopyStreamTo(rf); + mExtraData.CopyStreamTo(rOutStream); mExtraData.Seek(0, IOStream::SeekType_Absolute); - // Commit it to disc, converting it to RAID now - rf.Commit(true); - // Mark is as not modified mIsModified = false; } @@ -426,7 +456,6 @@ int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) COMPARE(AccountID); COMPARE(AccountName); - COMPARE(LastObjectIDUsed); COMPARE(BlocksUsed); COMPARE(BlocksInCurrentFiles); COMPARE(BlocksInOldFiles); @@ -434,32 +463,47 @@ int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) COMPARE(BlocksInDirectories); COMPARE(BlocksSoftLimit); COMPARE(BlocksHardLimit); - COMPARE(NumFiles); + COMPARE(NumCurrentFiles); COMPARE(NumOldFiles); COMPARE(NumDeletedFiles); COMPARE(NumDirectories); #undef COMPARE + if (rOldInfo.GetLastObjectIDUsed() != GetLastObjectIDUsed()) + { + BOX_NOTICE("LastObjectIDUsed changed from " << + rOldInfo.GetLastObjectIDUsed() << " to " << + GetLastObjectIDUsed()); + // Not important enough to be an error + // numChanges++; + } + return numChanges; } -#define APPLY_DELTA(field, delta) \ - if(mReadOnly) \ - { \ - THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) \ - } \ - \ - if((field + delta) < 0) \ - { \ - THROW_EXCEPTION_MESSAGE(BackupStoreException, \ - StoreInfoBlockDeltaMakesValueNegative, \ - "Failed to reduce " << #field << " from " << \ - field << " by " << delta); \ - } \ - \ - field += delta; \ +void BackupStoreInfo::ApplyDelta(int64_t& field, const std::string& field_name, + const int64_t delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly); + } + + if((field + delta) < 0) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + StoreInfoBlockDeltaMakesValueNegative, + "Failed to reduce " << field_name << " from " << + field << " by " << delta); + } + + field += delta; mIsModified = true; +} + +#define APPLY_DELTA(field, delta) \ + ApplyDelta(field, #field, delta) // -------------------------------------------------------------------------- // @@ -527,9 +571,9 @@ void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta) APPLY_DELTA(mBlocksInDirectories, Delta); } -void BackupStoreInfo::AdjustNumFiles(int64_t increase) +void BackupStoreInfo::AdjustNumCurrentFiles(int64_t increase) { - APPLY_DELTA(mNumFiles, increase); + APPLY_DELTA(mNumCurrentFiles, increase); } void BackupStoreInfo::AdjustNumOldFiles(int64_t increase) @@ -563,13 +607,13 @@ void BackupStoreInfo::CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + // Set the values mBlocksUsed = Used; mBlocksInOldFiles = InOldFiles; mBlocksInDeletedFiles = InDeletedFiles; mBlocksInDirectories = InDirectories; - + mIsModified = true; } @@ -588,9 +632,8 @@ void BackupStoreInfo::AddDeletedDirectory(int64_t DirID) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + mDeletedDirectories.push_back(DirID); - mIsModified = true; } @@ -608,14 +651,14 @@ void BackupStoreInfo::RemovedDeletedDirectory(int64_t DirID) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + std::vector<int64_t>::iterator i(std::find(mDeletedDirectories.begin(), mDeletedDirectories.end(), DirID)); if(i == mDeletedDirectories.end()) { THROW_EXCEPTION(BackupStoreException, StoreInfoDirNotInList) } + mDeletedDirectories.erase(i); - mIsModified = true; } @@ -636,7 +679,7 @@ void BackupStoreInfo::ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimi mBlocksSoftLimit = BlockSoftLimit; mBlocksHardLimit = BlockHardLimit; - + mIsModified = true; } @@ -655,15 +698,16 @@ int64_t BackupStoreInfo::AllocateObjectID() { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } + if(mLastObjectIDUsed < 0) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotInitialised) } - + + mIsModified = true; + // Return the next object ID return ++mLastObjectIDUsed; - - mIsModified = true; } @@ -682,9 +726,8 @@ void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + mClientStoreMarker = ClientStoreMarker; - mIsModified = true; } @@ -703,9 +746,8 @@ void BackupStoreInfo::SetAccountName(const std::string& rName) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + mAccountName = rName; - mIsModified = true; } diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h index 752cc44a..ec6cd2cb 100644 --- a/lib/backupstore/BackupStoreInfo.h +++ b/lib/backupstore/BackupStoreInfo.h @@ -77,20 +77,28 @@ private: BackupStoreInfo(); // No copying allowed BackupStoreInfo(const BackupStoreInfo &); - + public: // Create a New account, saving a blank info object to the disc - static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit); - + static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, + int64_t BlockSoftLimit, int64_t BlockHardLimit); + BackupStoreInfo(int32_t AccountID, const std::string &FileName, + int64_t BlockSoftLimit, int64_t BlockHardLimit); + // Load it from the store static std::auto_ptr<BackupStoreInfo> Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0); - + + // Load it from a stream (file or RaidFile) + static std::auto_ptr<BackupStoreInfo> Load(IOStream& rStream, + const std::string FileName, bool ReadOnly); + // Has info been modified? bool IsModified() const {return mIsModified;} - + // Save modified infomation back to store void Save(bool allowOverwrite = true); - + void Save(IOStream& rOutStream); + // Data access functions int32_t GetAccountID() const {return mAccountID;} int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;} @@ -102,7 +110,7 @@ public: const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;} int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;} int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;} - int64_t GetNumFiles() const {return mNumFiles;} + int64_t GetNumCurrentFiles() const {return mNumCurrentFiles;} int64_t GetNumOldFiles() const {return mNumOldFiles;} int64_t GetNumDeletedFiles() const {return mNumDeletedFiles;} int64_t GetNumDirectories() const {return mNumDirectories;} @@ -111,7 +119,7 @@ public: int GetDiscSetNumber() const {return mDiscSet;} int ReportChangesTo(BackupStoreInfo& rOldInfo); - + // Data modification functions void ChangeBlocksUsed(int64_t Delta); void ChangeBlocksInCurrentFiles(int64_t Delta); @@ -122,14 +130,14 @@ public: void AddDeletedDirectory(int64_t DirID); void RemovedDeletedDirectory(int64_t DirID); void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit); - void AdjustNumFiles(int64_t increase); + void AdjustNumCurrentFiles(int64_t increase); void AdjustNumOldFiles(int64_t increase); void AdjustNumDeletedFiles(int64_t increase); void AdjustNumDirectories(int64_t increase); - + // Object IDs int64_t AllocateObjectID(); - + // Client marker set and get int64_t GetClientStoreMarker() const {return mClientStoreMarker;} void SetClientStoreMarker(int64_t ClientStoreMarker); @@ -152,6 +160,20 @@ public: int64_t BlockSoftLimit, int64_t BlockHardLimit, bool AccountEnabled, IOStream& ExtraData); + typedef struct + { + int64_t mLastObjectIDUsed; + int64_t mBlocksUsed; + int64_t mBlocksInCurrentFiles; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; + int64_t mNumCurrentFiles; + int64_t mNumOldFiles; + int64_t mNumDeletedFiles; + int64_t mNumDirectories; + } Adjustment; + private: // Location information // Be VERY careful about changing types of these values, as @@ -162,10 +184,10 @@ private: std::string mFilename; bool mReadOnly; bool mIsModified; - + // Client infomation int64_t mClientStoreMarker; - + // Account information int64_t mLastObjectIDUsed; int64_t mBlocksUsed; @@ -175,13 +197,16 @@ private: int64_t mBlocksInDirectories; int64_t mBlocksSoftLimit; int64_t mBlocksHardLimit; - int64_t mNumFiles; + int64_t mNumCurrentFiles; int64_t mNumOldFiles; int64_t mNumDeletedFiles; int64_t mNumDirectories; std::vector<int64_t> mDeletedDirectories; bool mAccountEnabled; CollectInBufferStream mExtraData; + + void ApplyDelta(int64_t& field, const std::string& field_name, + const int64_t delta); }; #endif // BACKUPSTOREINFO__H diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp index 26f9acca..b2ea1abd 100644 --- a/lib/backupstore/BackupStoreRefCountDatabase.cpp +++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp @@ -9,6 +9,8 @@ #include "Box.h" +#include <stdio.h> + #include <algorithm> #include "BackupStoreRefCountDatabase.h" @@ -34,12 +36,85 @@ // // -------------------------------------------------------------------------- BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const - BackupStoreAccountDatabase::Entry& rAccount) + BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly, + bool Temporary, std::auto_ptr<FileStream> apDatabaseFile) : mAccount(rAccount), - mFilename(GetFilename(rAccount)), - mReadOnly(true), - mIsModified(false) + mFilename(GetFilename(rAccount, Temporary)), + mReadOnly(ReadOnly), + mIsModified(false), + mIsTemporaryFile(Temporary), + mapDatabaseFile(apDatabaseFile) +{ + ASSERT(!(ReadOnly && Temporary)); // being both doesn't make sense +} + +void BackupStoreRefCountDatabase::Commit() { + if (!mIsTemporaryFile) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Cannot commit a permanent reference count database"); + } + + if (!mapDatabaseFile.get()) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Reference count database is already closed"); + } + + mapDatabaseFile->Close(); + mapDatabaseFile.reset(); + + std::string Final_Filename = GetFilename(mAccount, false); + + #ifdef WIN32 + if(FileExists(Final_Filename) && unlink(Final_Filename.c_str()) != 0) + { + THROW_EMU_FILE_ERROR("Failed to delete old permanent refcount " + "database file", mFilename, CommonException, + OSFileError); + } + #endif + + if(rename(mFilename.c_str(), Final_Filename.c_str()) != 0) + { + THROW_EMU_ERROR("Failed to rename temporary refcount database " + "file from " << mFilename << " to " << + Final_Filename, CommonException, OSFileError); + } + + mFilename = Final_Filename; + mIsModified = false; + mIsTemporaryFile = false; +} + +void BackupStoreRefCountDatabase::Discard() +{ + if (!mIsTemporaryFile) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Cannot discard a permanent reference count database"); + } + + // Under normal conditions, we should know whether the file is still + // open or not, and not Discard it unless it's open. However if the + // final rename() fails during Commit(), the file will already be + // closed, and we don't want to blow up here in that case. + if (mapDatabaseFile.get()) + { + mapDatabaseFile->Close(); + mapDatabaseFile.reset(); + } + + if(unlink(mFilename.c_str()) != 0) + { + THROW_EMU_FILE_ERROR("Failed to delete temporary refcount " + "database file", mFilename, CommonException, + OSFileError); + } + + mIsModified = false; + mIsTemporaryFile = false; } // -------------------------------------------------------------------------- @@ -52,16 +127,35 @@ BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const // -------------------------------------------------------------------------- BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase() { + if (mIsTemporaryFile) + { + // Don't throw exceptions in a destructor. + BOX_ERROR("BackupStoreRefCountDatabase destroyed without " + "explicit commit or discard"); + try + { + Discard(); + } + catch(BoxException &e) + { + BOX_LOG_SYS_ERROR("Failed to discard BackupStoreRefCountDatabase " + "in destructor: " << e.what()); + } + } } std::string BackupStoreRefCountDatabase::GetFilename(const - BackupStoreAccountDatabase::Entry& rAccount) + BackupStoreAccountDatabase::Entry& rAccount, bool Temporary) { std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount); ASSERT(RootDir[RootDir.size() - 1] == '/' || RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); - std::string fn(RootDir + REFCOUNT_FILENAME ".db"); + std::string fn(RootDir + REFCOUNT_FILENAME ".rdb"); + if(Temporary) + { + fn += "X"; + } RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet())); return RaidFileUtil::MakeWriteFileName(rdiscSet, fn); @@ -72,40 +166,53 @@ std::string BackupStoreRefCountDatabase::GetFilename(const // Function // Name: BackupStoreRefCountDatabase::Create(int32_t, // const std::string &, int, bool) -// Purpose: Create a new database, overwriting an existing -// one only if AllowOverwrite is true. +// Purpose: Create a blank database, using a temporary file that +// you must Discard() or Commit() to make permanent. // Created: 2003/08/28 // // -------------------------------------------------------------------------- -void BackupStoreRefCountDatabase::Create(const - BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite) +std::auto_ptr<BackupStoreRefCountDatabase> + BackupStoreRefCountDatabase::Create + (const BackupStoreAccountDatabase::Entry& rAccount) { // Initial header refcount_StreamFormat hdr; hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE); hdr.mAccountID = htonl(rAccount.GetID()); - // Generate the filename - std::string Filename = GetFilename(rAccount); + std::string Filename = GetFilename(rAccount, true); // temporary // Open the file for writing - if (FileExists(Filename) && !AllowOverwrite) - { - THROW_FILE_ERROR("Failed to overwrite refcount database: " - "not allowed here", Filename, RaidFileException, - CannotOverwriteExistingFile); - } - - int flags = O_CREAT | O_BINARY | O_RDWR; - if (!AllowOverwrite) + if (FileExists(Filename)) { - flags |= O_EXCL; + BOX_WARNING(BOX_FILE_MESSAGE(Filename, "Overwriting existing " + "temporary reference count database")); + if (unlink(Filename.c_str()) != 0) + { + THROW_SYS_FILE_ERROR("Failed to delete old temporary " + "reference count database file", Filename, + CommonException, OSFileError); + } } + int flags = O_CREAT | O_BINARY | O_RDWR | O_EXCL; std::auto_ptr<FileStream> DatabaseFile(new FileStream(Filename, flags)); // Write header DatabaseFile->Write(&hdr, sizeof(hdr)); + + // Make new object + std::auto_ptr<BackupStoreRefCountDatabase> refcount( + new BackupStoreRefCountDatabase(rAccount, false, true, + DatabaseFile)); + + // The root directory must always have one reference for a database + // to be valid, so set that now on the new database. This will leave + // mIsModified set to true. + refcount->SetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); + + // return it to caller + return refcount; } // -------------------------------------------------------------------------- @@ -122,12 +229,13 @@ void BackupStoreRefCountDatabase::Create(const std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly) { - // Generate the filename - std::string filename = GetFilename(rAccount); + // Generate the filename. Cannot open a temporary database, so it must + // be a permanent one. + std::string Filename = GetFilename(rAccount, false); int flags = ReadOnly ? O_RDONLY : O_RDWR; // Open the file for read/write - std::auto_ptr<FileStream> dbfile(new FileStream(filename, + std::auto_ptr<FileStream> dbfile(new FileStream(Filename, flags | O_BINARY)); // Read in a header @@ -135,7 +243,7 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) { THROW_FILE_ERROR("Failed to read refcount database: " - "short read", filename, BackupStoreException, + "short read", Filename, BackupStoreException, CouldNotLoadStoreInfo); } @@ -144,16 +252,14 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID()) { THROW_FILE_ERROR("Failed to read refcount database: " - "bad magic number", filename, BackupStoreException, + "bad magic number", Filename, BackupStoreException, BadStoreInfoOnLoad); } // Make new object - std::auto_ptr<BackupStoreRefCountDatabase> refcount(new BackupStoreRefCountDatabase(rAccount)); - - // Put in basic location info - refcount->mReadOnly = ReadOnly; - refcount->mapDatabaseFile = dbfile; + std::auto_ptr<BackupStoreRefCountDatabase> refcount( + new BackupStoreRefCountDatabase(rAccount, ReadOnly, false, + dbfile)); // return it to caller return refcount; @@ -229,6 +335,7 @@ void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID, mapDatabaseFile->Seek(offset, SEEK_SET); refcount_t RefCountNetOrder = htonl(NewRefCount); mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder)); + mIsModified = true; } bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) @@ -240,3 +347,31 @@ bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) return (refcount > 0); } +int BackupStoreRefCountDatabase::ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs) +{ + int ErrorCount = 0; + int64_t MaxOldObjectId = rOldRefs.GetLastObjectIDUsed(); + int64_t MaxNewObjectId = GetLastObjectIDUsed(); + + for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID; + ObjectID < std::max(MaxOldObjectId, MaxNewObjectId); + ObjectID++) + { + typedef BackupStoreRefCountDatabase::refcount_t refcount_t; + refcount_t OldRefs = (ObjectID <= MaxOldObjectId) ? + rOldRefs.GetRefCount(ObjectID) : 0; + refcount_t NewRefs = (ObjectID <= MaxNewObjectId) ? + this->GetRefCount(ObjectID) : 0; + + if (OldRefs != NewRefs) + { + BOX_WARNING("Reference count of object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " changed from " << OldRefs << + " to " << NewRefs); + ErrorCount++; + } + } + + return ErrorCount; +} diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h index 93c79afb..915653a4 100644 --- a/lib/backupstore/BackupStoreRefCountDatabase.h +++ b/lib/backupstore/BackupStoreRefCountDatabase.h @@ -15,6 +15,7 @@ #include <vector> #include "BackupStoreAccountDatabase.h" +#include "BackupStoreConstants.h" #include "FileStream.h" class BackupStoreCheck; @@ -59,48 +60,39 @@ public: private: // Creation through static functions only BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry& - rAccount); + rAccount, bool ReadOnly, bool Temporary, + std::auto_ptr<FileStream> apDatabaseFile); // No copying allowed BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &); public: - // Create a new database for a new account. This method will refuse - // to overwrite any existing file. - static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount) - { - Create(rAccount, false); - } - + // Create a blank database, using a temporary file that you must + // Discard() or Commit() to make permanent. + static std::auto_ptr<BackupStoreRefCountDatabase> Create + (const BackupStoreAccountDatabase::Entry& rAccount); + void Commit(); + void Discard(); + // Load it from the store static std::auto_ptr<BackupStoreRefCountDatabase> Load(const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); - + typedef uint32_t refcount_t; // Data access functions refcount_t GetRefCount(int64_t ObjectID) const; int64_t GetLastObjectIDUsed() const; - + // Data modification functions void AddReference(int64_t ObjectID); // RemoveReference returns false if refcount drops to zero bool RemoveReference(int64_t ObjectID); + int ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs); private: - // Create a new database for an existing account. Used during - // account checking if opening the old database throws an exception. - // This method will overwrite any existing file. - static void CreateForRegeneration(const - BackupStoreAccountDatabase::Entry& rAccount) - { - Create(rAccount, true); - } - - static void Create(const BackupStoreAccountDatabase::Entry& rAccount, - bool AllowOverwrite); - static std::string GetFilename(const BackupStoreAccountDatabase::Entry& - rAccount); + rAccount, bool Temporary); + IOStream::pos_type GetSize() const { return mapDatabaseFile->GetPosition() + @@ -122,7 +114,13 @@ private: std::string mFilename; bool mReadOnly; bool mIsModified; + bool mIsTemporaryFile; std::auto_ptr<FileStream> mapDatabaseFile; + + bool NeedsCommitOrDiscard() + { + return mapDatabaseFile.get() && mIsModified && mIsTemporaryFile; + } }; #endif // BACKUPSTOREREFCOUNTDATABASE__H diff --git a/lib/backupstore/HousekeepStoreAccount.cpp b/lib/backupstore/HousekeepStoreAccount.cpp index 75feda7f..d5acf62c 100644 --- a/lib/backupstore/HousekeepStoreAccount.cpp +++ b/lib/backupstore/HousekeepStoreAccount.cpp @@ -2,7 +2,7 @@ // // File // Name: HousekeepStoreAccount.cpp -// Purpose: +// Purpose: // Created: 11/12/03 // // -------------------------------------------------------------------------- @@ -51,6 +51,7 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, mDeletionSizeTarget(0), mPotentialDeletionsTotalSize(0), mMaxSizeInPotentialDeletions(0), + mErrorCount(0), mBlocksUsed(0), mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), @@ -61,8 +62,6 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, mBlocksInDirectoriesDelta(0), mFilesDeleted(0), mEmptyDirectoriesDeleted(0), - mSuppressRefCountChangeWarnings(false), - mRefCountsAdjusted(0), mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY) { std::ostringstream tag; @@ -80,6 +79,20 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, // -------------------------------------------------------------------------- HousekeepStoreAccount::~HousekeepStoreAccount() { + if(mapNewRefs.get()) + { + // Discard() can throw exception, but destructors aren't supposed to do that, so + // just catch and log them. + try + { + mapNewRefs->Discard(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to destroy housekeeper: discarding the refcount " + "database threw an exception: " << e.what()); + } + } } // -------------------------------------------------------------------------- @@ -105,7 +118,7 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) { if(KeepTryingForever) { - BOX_WARNING("Failed to lock account for housekeeping, " + BOX_INFO("Failed to lock account for housekeeping, " "still trying..."); while(!writeLock.TryAndGetLock(writeLockFilename, 0600 /* restrictive file permissions */)) @@ -143,204 +156,108 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) mDeletionSizeTarget = 0; } - // initialise the refcount database - mNewRefCounts.clear(); - // try to pre-allocate as much memory as we need - mNewRefCounts.reserve(info->GetLastObjectIDUsed()); - // initialise the refcount of the root entry - mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0); - mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1; + BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); + mapNewRefs = BackupStoreRefCountDatabase::Create(account); // Scan the directory for potential things to delete // This will also remove eligible items marked with RemoveASAP - bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID); + bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, + *info); - // If scan directory stopped for some reason, probably parent - // instructed to terminate, stop now. if(!continueHousekeeping) { - // If any files were marked "delete now", then update - // the size of the store. - if(mBlocksUsedDelta != 0 || - mBlocksInOldFilesDelta != 0 || - mBlocksInDeletedFilesDelta != 0) - { - info->ChangeBlocksUsed(mBlocksUsedDelta); - info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); - info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); - - // Save the store info back - info->ReportChangesTo(*pOldInfo); - info->Save(); - } - - return false; + // The scan was incomplete, so the new block counts are + // incorrect, we can't rely on them. It's better to discard + // the new info and adjust the old one instead. + info = pOldInfo; + + // We're about to reset counters and exit, so report what + // happened now. + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " removed " << + (0 - mBlocksUsedDelta) << " blocks (" << + mFilesDeleted << " files, " << + mEmptyDirectoriesDeleted << " dirs) and the directory " + "scan was interrupted"); } - // Log any difference in opinion between the values recorded in - // the store info, and the values just calculated for space usage. - // BLOCK - { - int64_t used = info->GetBlocksUsed(); - int64_t usedOld = info->GetBlocksInOldFiles(); - int64_t usedDeleted = info->GetBlocksInDeletedFiles(); - int64_t usedDirectories = info->GetBlocksInDirectories(); - - // If the counts were wrong, taking into account RemoveASAP - // items deleted, log a message - if((used + mBlocksUsedDelta) != mBlocksUsed - || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles - || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles - || usedDirectories != mBlocksInDirectories) - { - // Log this - BOX_ERROR("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " found " - "and fixed wrong block counts: " - "used (" << - (used + mBlocksUsedDelta) << "," << - mBlocksUsed << "), old (" << - (usedOld + mBlocksInOldFilesDelta) << "," << - mBlocksInOldFiles << "), deleted (" << - (usedDeleted + mBlocksInDeletedFilesDelta) << - "," << mBlocksInDeletedFiles << "), dirs (" << - usedDirectories << "," << mBlocksInDirectories - << ")"); - } - - // If the current values don't match, store them - if(used != mBlocksUsed - || usedOld != mBlocksInOldFiles - || usedDeleted != mBlocksInDeletedFiles - || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta)) - { - // Set corrected values in store info - info->CorrectAllUsedValues(mBlocksUsed, - mBlocksInOldFiles, mBlocksInDeletedFiles, - mBlocksInDirectories + mBlocksInDirectoriesDelta); - - info->ReportChangesTo(*pOldInfo); - info->Save(); - } - } - - // Reset the delta counts for files, as they will include - // RemoveASAP flagged files deleted during the initial scan. + // If housekeeping made any changes, such as deleting RemoveASAP files, + // the differences in block counts will be recorded in the deltas. + info->ChangeBlocksUsed(mBlocksUsedDelta); + info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); + info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); - // keep for reporting + // Reset the delta counts for files, as they will include + // RemoveASAP flagged files deleted during the initial scan. + // keep removeASAPBlocksUsedDelta for reporting int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta; - mBlocksUsedDelta = 0; mBlocksInOldFilesDelta = 0; mBlocksInDeletedFilesDelta = 0; - - // Go and delete items from the accounts - bool deleteInterrupted = DeleteFiles(); - - // If that wasn't interrupted, remove any empty directories which - // are also marked as deleted in their containing directory - if(!deleteInterrupted) - { - deleteInterrupted = DeleteEmptyDirectories(); - } - - // Log deletion if anything was deleted - if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) - { - BOX_INFO("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " " - "removed " << - (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << - " blocks (" << mFilesDeleted << " files, " << - mEmptyDirectoriesDeleted << " dirs)" << - (deleteInterrupted?" and was interrupted":"")); - } + // If scan directory stopped for some reason, probably parent + // instructed to terminate, stop now. + // // We can only update the refcount database if we successfully // finished our scan of all directories, otherwise we don't actually // know which of the new counts are valid and which aren't - // (we might not have seen second references to some objects, etc.) + // (we might not have seen second references to some objects, etc.). - BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); - std::auto_ptr<BackupStoreRefCountDatabase> apReferences; + if(!continueHousekeeping) + { + mapNewRefs->Discard(); + info->Save(); + return false; + } + + // Report any UNexpected changes, and consider them to be errors. + // Do this before applying the expected changes below. + mErrorCount += info->ReportChangesTo(*pOldInfo); + info->Save(); + + // Try to load the old reference count database and check whether + // any counts have changed. We want to compare the mapNewRefs to + // apOldRefs before we delete any files, because that will also change + // the reference count in a way that's not an error. - // try to load the reference count database try { - apReferences = BackupStoreRefCountDatabase::Load(account, - false); + std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs = + BackupStoreRefCountDatabase::Load(account, false); + mErrorCount += mapNewRefs->ReportChangesTo(*apOldRefs); } catch(BoxException &e) { - BOX_WARNING("Reference count database is missing or corrupted " - "during housekeeping, creating a new one."); - mSuppressRefCountChangeWarnings = true; - BackupStoreRefCountDatabase::CreateForRegeneration(account); - apReferences = BackupStoreRefCountDatabase::Load(account, - false); + BOX_WARNING("Reference count database was missing or " + "corrupted during housekeeping, cannot check it for " + "errors."); + mErrorCount++; } - int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed(); + // Go and delete items from the accounts + bool deleteInterrupted = DeleteFiles(*info); - for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID; - ObjectID < mNewRefCounts.size(); ObjectID++) + // If that wasn't interrupted, remove any empty directories which + // are also marked as deleted in their containing directory + if(!deleteInterrupted) { - if (ObjectID > LastUsedObjectIdOnDisk) - { - if (!mSuppressRefCountChangeWarnings) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " not found in database, added" - " with " << mNewRefCounts[ObjectID] << - " references"); - } - apReferences->SetRefCount(ObjectID, - mNewRefCounts[ObjectID]); - mRefCountsAdjusted++; - LastUsedObjectIdOnDisk = ObjectID; - continue; - } - - BackupStoreRefCountDatabase::refcount_t OldRefCount = - apReferences->GetRefCount(ObjectID); - - if (OldRefCount != mNewRefCounts[ObjectID]) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " changed from " << OldRefCount << - " to " << mNewRefCounts[ObjectID]); - apReferences->SetRefCount(ObjectID, - mNewRefCounts[ObjectID]); - mRefCountsAdjusted++; - } + deleteInterrupted = DeleteEmptyDirectories(*info); } - // zero excess references in the database - for (int64_t ObjectID = mNewRefCounts.size(); - ObjectID <= LastUsedObjectIdOnDisk; ObjectID++) + // Log deletion if anything was deleted + if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) { - BackupStoreRefCountDatabase::refcount_t OldRefCount = - apReferences->GetRefCount(ObjectID); - BackupStoreRefCountDatabase::refcount_t NewRefCount = 0; - - if (OldRefCount != NewRefCount) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " changed from " << OldRefCount << - " to " << NewRefCount << " (not found)"); - apReferences->SetRefCount(ObjectID, NewRefCount); - mRefCountsAdjusted++; - } + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " " + "removed " << + (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << + " blocks (" << mFilesDeleted << " files, " << + mEmptyDirectoriesDeleted << " dirs)" << + (deleteInterrupted?" and was interrupted":"")); } - // force file to be saved and closed before releasing the lock below - apReferences.reset(); - - // Make sure the delta's won't cause problems if the counts are - // really wrong, and it wasn't fixed because the store was + // Make sure the delta's won't cause problems if the counts are + // really wrong, and it wasn't fixed because the store was // updated during the scan. if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) { @@ -348,28 +265,31 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) } if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles())) { - mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); + mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); } if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles())) { - mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); + mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); } if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories())) { mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories()); } - + // Update the usage counts in the store info->ChangeBlocksUsed(mBlocksUsedDelta); info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta); - + // Save the store info back - info->ReportChangesTo(*pOldInfo); info->Save(); - - // Explicity release the lock (would happen automatically on + + // force file to be saved and closed before releasing the lock below + mapNewRefs->Commit(); + mapNewRefs.reset(); + + // Explicity release the lock (would happen automatically on // going out of scope, included for code clarity) writeLock.ReleaseLock(); @@ -405,7 +325,8 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF // Created: 11/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) +bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID, + BackupStoreInfo& rBackupStoreInfo) { #ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) @@ -430,18 +351,19 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Open it. std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, objectFilename)); - + // Add the size of the directory on disc to the size being calculated int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); mBlocksInDirectories += originalDirSizeInBlocks; mBlocksUsed += originalDirSizeInBlocks; - + // Read the directory in BackupStoreDirectory dir; BufferedStream buf(*dirStream); dir.ReadFromStream(buf, IOStream::TimeOutInfinite); + dir.SetUserInfo1_SizeInBlocks(originalDirSizeInBlocks); dirStream->Close(); - + // Is it empty? if(dir.GetNumberOfEntries() == 0) { @@ -459,11 +381,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) while((en = i.Next()) != 0) { // This directory references this object - if (mNewRefCounts.size() <= en->GetObjectID()) - { - mNewRefCounts.resize(en->GetObjectID() + 1, 0); - } - mNewRefCounts[en->GetObjectID()]++; + mapNewRefs->AddReference(en->GetObjectID()); } } @@ -482,14 +400,15 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { int16_t enFlags = en->GetFlags(); if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0 - && (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) + && (en->IsDeleted() || en->IsOld())) { // Delete this immediately. - DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks); - + DeleteFile(ObjectID, en->GetObjectID(), dir, + objectFilename, rBackupStoreInfo); + // flag as having done something deletedSomething = true; - + // Must start the loop from the beginning again, as iterator is now // probably invalid. break; @@ -497,7 +416,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } } while(deletedSomething); } - + // BLOCK { // Add files to the list of potential deletions @@ -517,9 +436,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) int16_t enFlags = en->GetFlags(); int64_t enSizeInBlocks = en->GetSizeInBlocks(); mBlocksUsed += enSizeInBlocks; - if(enFlags & BackupStoreDirectory::Entry::Flags_OldVersion) mBlocksInOldFiles += enSizeInBlocks; - if(enFlags & BackupStoreDirectory::Entry::Flags_Deleted) mBlocksInDeletedFiles += enSizeInBlocks; - + if(en->IsOld()) mBlocksInOldFiles += enSizeInBlocks; + if(en->IsDeleted()) mBlocksInDeletedFiles += enSizeInBlocks; + // Work out ages of this version from the last mark int32_t enVersionAge = 0; std::map<version_t, int32_t>::iterator enVersionAgeI( @@ -536,9 +455,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) markVersionAges[version_t(en->GetName().GetEncodedFilename(), en->GetMarkNumber())] = enVersionAge; } // enVersionAge is now the age of this version. - + // Potentially add it to the list if it's deleted, if it's an old version or deleted - if((enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) + if(en->IsOld() || en->IsDeleted()) { // Is deleted / old version. DelEn d; @@ -547,17 +466,15 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) d.mSizeInBlocks = en->GetSizeInBlocks(); d.mMarkNumber = en->GetMarkNumber(); d.mVersionAgeWithinMark = enVersionAge; - d.mIsFlagDeleted = (enFlags & - BackupStoreDirectory::Entry::Flags_Deleted) - ? true : false; - + d.mIsFlagDeleted = en->IsDeleted(); + // Add it to the list mPotentialDeletions.insert(d); - + // Update various counts mPotentialDeletionsTotalSize += d.mSizeInBlocks; if(d.mSizeInBlocks > mMaxSizeInPotentialDeletions) mMaxSizeInPotentialDeletions = d.mSizeInBlocks; - + // Too much in the list of potential deletions? // (check against the deletion target + the max size in deletions, so that we never delete things // and take the total size below the deletion size target) @@ -565,7 +482,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { int64_t sizeToRemove = mPotentialDeletionsTotalSize - (mDeletionSizeTarget + mMaxSizeInPotentialDeletions); bool recalcMaxSize = false; - + while(sizeToRemove > 0) { // Make iterator for the last element, while checking that there's something there in the first place. @@ -577,7 +494,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } // Make this into an iterator pointing to the last element in the set --i; - + // Delete this one? if(sizeToRemove > i->mSizeInBlocks) { @@ -595,7 +512,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) break; } } - + if(recalcMaxSize) { // Because an object which was the maximum size recorded was deleted from the set @@ -615,23 +532,22 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } } + // Recurse into subdirectories { - // Recurse into subdirectories BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0) { - // Next level - ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir); - - if(!ScanDirectory(en->GetObjectID())) + ASSERT(en->IsDir()); + + if(!ScanDirectory(en->GetObjectID(), rBackupStoreInfo)) { // Halting operation return false; } } } - + return true; } @@ -648,11 +564,11 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &x, const HousekeepStoreAccount::DelEn &y) { // STL spec says this: - // A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. + // A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. // The sort order here is intended to preserve the entries of most value, that is, the newest objects // which are on a mark boundary. - + // Reverse order age, so oldest goes first if(x.mVersionAgeWithinMark > y.mVersionAgeWithinMark) { @@ -687,7 +603,7 @@ bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount // Created: 15/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::DeleteFiles() +bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo) { // Only delete files if the deletion target is greater than zero // (otherwise we delete one file each time round, which gradually deletes the old versions) @@ -718,21 +634,33 @@ bool HousekeepStoreAccount::DeleteFiles() // Get the filename std::string dirFilename; BackupStoreDirectory dir; - int64_t dirSizeInBlocksOrig = 0; { MakeObjectFilename(i->mInDirectory, dirFilename); std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename)); - dirSizeInBlocksOrig = dirStream->GetDiscUsageInBlocks(); dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + dir.SetUserInfo1_SizeInBlocks(dirStream->GetDiscUsageInBlocks()); } - + // Delete the file - DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig); - BOX_INFO("Housekeeping removed " << - (i->mIsFlagDeleted ? "deleted" : "old") << - " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << - " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); - + BackupStoreRefCountDatabase::refcount_t refs = + DeleteFile(i->mInDirectory, i->mObjectID, dir, + dirFilename, rBackupStoreInfo); + if(refs == 0) + { + BOX_INFO("Housekeeping removed " << + (i->mIsFlagDeleted ? "deleted" : "old") << + " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << + " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); + } + else + { + BOX_TRACE("Housekeeping preserved " << + (i->mIsFlagDeleted ? "deleted" : "old") << + " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << + " in dir " << BOX_FORMAT_OBJECTID(i->mInDirectory) << + " with " << refs << " references"); + } + // Stop if the deletion target has been matched or exceeded // (checking here rather than at the beginning will tend to reduce the // space to slightly less than the soft limit, which will allow the backup @@ -754,11 +682,17 @@ bool HousekeepStoreAccount::DeleteFiles() // BackupStoreDirectory &, const std::string &, int64_t) // Purpose: Delete a file. Takes the directory already loaded // in and the filename, for efficiency in both the -// usage scenarios. +// usage scenarios. Returns the number of references +// remaining. If it's zero, the file was removed from +// disk as unused. // Created: 15/7/04 // // -------------------------------------------------------------------------- -void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks) + +BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( + int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, + const std::string &rDirectoryFilename, + BackupStoreInfo& rBackupStoreInfo) { // Find the entry inside the directory bool wasDeleted = false; @@ -768,6 +702,9 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba std::auto_ptr<RaidFileWrite> padjustedEntry; // BLOCK { + BackupStoreRefCountDatabase::refcount_t refs = + mapNewRefs->GetRefCount(ObjectID); + BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID); if(pentry == 0) { @@ -775,26 +712,47 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba BOX_FORMAT_ACCOUNT(mAccountID) << " " "found error: object " << BOX_FORMAT_OBJECTID(ObjectID) << " " - "not found in dir " << + "not found in dir " << BOX_FORMAT_OBJECTID(InDirectory) << ", " "indicates logic error/corruption? Run " "bbstoreaccounts check <accid> fix"); - return; + mErrorCount++; + return refs; } - + // Record the flags it's got set - wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0); - wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0); + wasDeleted = pentry->IsDeleted(); + wasOldVersion = pentry->IsOld(); // Check this should be deleted if(!wasDeleted && !wasOldVersion) { - // Things changed size we were last around - return; + // Things changed since we were last around + return refs; } - + // Record size deletedFileSizeInBlocks = pentry->GetSizeInBlocks(); - + + if(refs > 1) + { + // Not safe to merge patches if someone else has a + // reference to this object, so just remove the + // directory entry and return. + rDirectory.DeleteEntry(ObjectID); + if(wasDeleted) + { + rBackupStoreInfo.AdjustNumDeletedFiles(-1); + } + + if(wasOldVersion) + { + rBackupStoreInfo.AdjustNumOldFiles(-1); + } + + mapNewRefs->RemoveReference(ObjectID); + return refs - 1; + } + // If the entry is involved in a chain of patches, it needs to be handled // a bit more carefully. if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0) @@ -826,7 +784,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba else { // This entry is in the middle of a chain, and two patches need combining. - + // First, adjust the directory entries BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer()); if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID @@ -838,7 +796,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba pnewer->SetDependsOlder(pentry->GetDependsOlder()); polder->SetDependsNewer(pentry->GetDependsNewer()); } - + // COMMON CODE to both cases // Generate the filename of the older version @@ -853,7 +811,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename)); // And open a write file to overwrite the other directory entry padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, - objFilenameOlder, mNewRefCounts[ObjectID])); + objFilenameOlder, mapNewRefs->GetRefCount(ObjectID))); padjustedEntry->Open(true /* allow overwriting */); if(pentry->GetDependsNewer() == 0) @@ -867,84 +825,86 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry); } // The file will be committed later when the directory is safely commited. - + // Work out the adjusted size int64_t newSize = padjustedEntry->GetDiscUsageInBlocks(); int64_t sizeDelta = newSize - polder->GetSizeInBlocks(); mBlocksUsedDelta += sizeDelta; - if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0) + if(polder->IsDeleted()) { mBlocksInDeletedFilesDelta += sizeDelta; } - if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0) + if(polder->IsOld()) { mBlocksInOldFilesDelta += sizeDelta; } polder->SetSizeInBlocks(newSize); } - + // pentry no longer valid } - + // Delete it from the directory rDirectory.DeleteEntry(ObjectID); - + // Save directory back to disc // BLOCK - int64_t dirRevisedSize = 0; { RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename, - mNewRefCounts[InDirectory]); + mapNewRefs->GetRefCount(InDirectory)); writeDir.Open(true /* allow overwriting */); rDirectory.WriteToStream(writeDir); - // get the disc usage (must do this before commiting it) - dirRevisedSize = writeDir.GetDiscUsageInBlocks(); + // Get the disc usage (must do this before commiting it) + int64_t new_size = writeDir.GetDiscUsageInBlocks(); // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - // adjust usage counts for this directory - if(dirRevisedSize > 0) - { - int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks; - mBlocksUsedDelta += adjust; - mBlocksInDirectoriesDelta += adjust; - } + // Adjust block counts if the directory itself changed in size + int64_t original_size = rDirectory.GetUserInfo1_SizeInBlocks(); + int64_t adjust = new_size - original_size; + mBlocksUsedDelta += adjust; + mBlocksInDirectoriesDelta += adjust; + + UpdateDirectorySize(rDirectory, new_size); } // Commit any new adjusted entry if(padjustedEntry.get() != 0) { padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - padjustedEntry.reset(); // delete it now + padjustedEntry.reset(); // delete it now } - // Drop reference count by one. If it reaches zero, delete the file. - if(--mNewRefCounts[ObjectID] == 0) + // Drop reference count by one. Must now be zero, to delete the file. + bool remaining_refs = mapNewRefs->RemoveReference(ObjectID); + ASSERT(!remaining_refs); + + // Delete from disc + BOX_TRACE("Removing unreferenced object " << + BOX_FORMAT_OBJECTID(ObjectID)); + std::string objFilename; + MakeObjectFilename(ObjectID, objFilename); + RaidFileWrite del(mStoreDiscSet, objFilename, mapNewRefs->GetRefCount(ObjectID)); + del.Delete(); + + // Adjust counts for the file + ++mFilesDeleted; + mBlocksUsedDelta -= deletedFileSizeInBlocks; + + if(wasDeleted) { - // Delete from disc - BOX_TRACE("Removing unreferenced object " << - BOX_FORMAT_OBJECTID(ObjectID)); - std::string objFilename; - MakeObjectFilename(ObjectID, objFilename); - RaidFileWrite del(mStoreDiscSet, objFilename, - mNewRefCounts[ObjectID]); - del.Delete(); + mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks; + rBackupStoreInfo.AdjustNumDeletedFiles(-1); } - else + + if(wasOldVersion) { - BOX_TRACE("Preserving object " << - BOX_FORMAT_OBJECTID(ObjectID) << " with " << - mNewRefCounts[ObjectID] << " references"); + mBlocksInOldFilesDelta -= deletedFileSizeInBlocks; + rBackupStoreInfo.AdjustNumOldFiles(-1); } - // Adjust counts for the file - ++mFilesDeleted; - mBlocksUsedDelta -= deletedFileSizeInBlocks; - if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks; - if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks; - // Delete the directory? // Do this if... dir has zero entries, and is marked as deleted in it's containing directory if(rDirectory.GetNumberOfEntries() == 0) @@ -952,8 +912,81 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba // Candidate for deletion mEmptyDirectories.push_back(InDirectory); } + + return 0; } +// -------------------------------------------------------------------------- +// +// Function +// Name: HousekeepStoreAccount::UpdateDirectorySize( +// BackupStoreDirectory& rDirectory, +// IOStream::pos_type new_size_in_blocks) +// Purpose: Update the directory size, modifying the parent +// directory's entry for this directory if necessary. +// Created: 05/03/14 +// +// -------------------------------------------------------------------------- + +void HousekeepStoreAccount::UpdateDirectorySize( + BackupStoreDirectory& rDirectory, + IOStream::pos_type new_size_in_blocks) +{ +#ifndef BOX_RELEASE_BUILD + { + std::string dirFilename; + MakeObjectFilename(rDirectory.GetObjectID(), dirFilename); + std::auto_ptr<RaidFileRead> dirStream( + RaidFileRead::Open(mStoreDiscSet, dirFilename)); + ASSERT(new_size_in_blocks == dirStream->GetDiscUsageInBlocks()); + } +#endif + + IOStream::pos_type old_size_in_blocks = + rDirectory.GetUserInfo1_SizeInBlocks(); + + if(new_size_in_blocks == old_size_in_blocks) + { + return; + } + + rDirectory.SetUserInfo1_SizeInBlocks(new_size_in_blocks); + + if (rDirectory.GetObjectID() == BACKUPSTORE_ROOT_DIRECTORY_ID) + { + return; + } + + std::string parentFilename; + MakeObjectFilename(rDirectory.GetContainerID(), parentFilename); + std::auto_ptr<RaidFileRead> parentStream( + RaidFileRead::Open(mStoreDiscSet, parentFilename)); + BackupStoreDirectory parent(*parentStream); + parentStream.reset(); + + BackupStoreDirectory::Entry* en = + parent.FindEntryByID(rDirectory.GetObjectID()); + ASSERT(en); + + if (en->GetSizeInBlocks() != old_size_in_blocks) + { + BOX_WARNING("Directory " << + BOX_FORMAT_OBJECTID(rDirectory.GetObjectID()) << + " entry in directory " << + BOX_FORMAT_OBJECTID(rDirectory.GetContainerID()) << + " had incorrect size " << en->GetSizeInBlocks() << + ", should have been " << old_size_in_blocks); + mErrorCount++; + } + + en->SetSizeInBlocks(new_size_in_blocks); + + RaidFileWrite writeDir(mStoreDiscSet, parentFilename, + mapNewRefs->GetRefCount(rDirectory.GetContainerID())); + writeDir.Open(true /* allow overwriting */); + parent.WriteToStream(writeDir); + writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); +} // -------------------------------------------------------------------------- // @@ -964,7 +997,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba // Created: 15/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::DeleteEmptyDirectories() +bool HousekeepStoreAccount::DeleteEmptyDirectories(BackupStoreInfo& rBackupStoreInfo) { while(mEmptyDirectories.size() > 0) { @@ -992,7 +1025,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() continue; } - DeleteEmptyDirectory(*i, toExamine); + DeleteEmptyDirectory(*i, toExamine, rBackupStoreInfo); } // Remove contents of empty directories @@ -1000,13 +1033,13 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() // Swap in new, so it's examined next time round mEmptyDirectories.swap(toExamine); } - + // Not interrupted return false; } void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, - std::vector<int64_t>& rToExamine) + std::vector<int64_t>& rToExamine, BackupStoreInfo& rBackupStoreInfo) { // Load up the directory to potentially delete std::string dirFilename; @@ -1046,16 +1079,18 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, std::auto_ptr<RaidFileRead> containingDirStream( RaidFileRead::Open(mStoreDiscSet, containingDirFilename)); - containingDirSizeInBlocksOrig = + containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks(); containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite); + containingDir.SetUserInfo1_SizeInBlocks(containingDirSizeInBlocksOrig); } // Find the entry - BackupStoreDirectory::Entry *pdirentry = + BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID()); - if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)) + // TODO FIXME invert test and reduce indentation + if((pdirentry != 0) && pdirentry->IsDeleted()) { // Should be deleted containingDir.DeleteEntry(dir.GetObjectID()); @@ -1068,7 +1103,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Write revised parent directory RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename, - mNewRefCounts[containingDir.GetObjectID()]); + mapNewRefs->GetRefCount(containingDir.GetObjectID())); writeDir.Open(true /* allow overwriting */); containingDir.WriteToStream(writeDir); @@ -1077,6 +1112,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + UpdateDirectorySize(containingDir, dirSize); // adjust usage counts for this directory if(dirSize > 0) @@ -1086,17 +1122,22 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, mBlocksInDirectoriesDelta += adjust; } - - if (--mNewRefCounts[dir.GetObjectID()] == 0) + if (mapNewRefs->RemoveReference(dir.GetObjectID())) { - // Delete the directory itself - RaidFileWrite del(mStoreDiscSet, dirFilename, - mNewRefCounts[dir.GetObjectID()]); - del.Delete(); + // Still referenced + BOX_TRACE("Housekeeping spared empty deleted dir " << + BOX_FORMAT_OBJECTID(dirId) << " due to " << + mapNewRefs->GetRefCount(dir.GetObjectID()) << + "remaining references"); + return; } - BOX_INFO("Housekeeping removed empty deleted dir " << + // Delete the directory itself + BOX_INFO("Housekeeping removing empty deleted dir " << BOX_FORMAT_OBJECTID(dirId)); + RaidFileWrite del(mStoreDiscSet, dirFilename, + mapNewRefs->GetRefCount(dir.GetObjectID())); + del.Delete(); // And adjust usage counts for the directory that's // just been deleted @@ -1105,6 +1146,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Update count ++mEmptyDirectoriesDeleted; + rBackupStoreInfo.AdjustNumDirectories(-1); } } diff --git a/lib/backupstore/HousekeepStoreAccount.h b/lib/backupstore/HousekeepStoreAccount.h index cfa05a2e..ff9e9ffe 100644 --- a/lib/backupstore/HousekeepStoreAccount.h +++ b/lib/backupstore/HousekeepStoreAccount.h @@ -14,6 +14,8 @@ #include <set> #include <vector> +#include "BackupStoreRefCountDatabase.h" + class BackupStoreDirectory; class HousekeepingCallback @@ -39,20 +41,25 @@ public: ~HousekeepStoreAccount(); bool DoHousekeeping(bool KeepTryingForever = false); - int GetRefCountsAdjusted() { return mRefCountsAdjusted; } + int GetErrorCount() { return mErrorCount; } private: // utility functions void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut); - bool ScanDirectory(int64_t ObjectID); - bool DeleteFiles(); - bool DeleteEmptyDirectories(); - void DeleteEmptyDirectory(int64_t dirId, - std::vector<int64_t>& rToExamine); - void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks); + bool ScanDirectory(int64_t ObjectID, BackupStoreInfo& rBackupStoreInfo); + bool DeleteFiles(BackupStoreInfo& rBackupStoreInfo); + bool DeleteEmptyDirectories(BackupStoreInfo& rBackupStoreInfo); + void DeleteEmptyDirectory(int64_t dirId, std::vector<int64_t>& rToExamine, + BackupStoreInfo& rBackupStoreInfo); + BackupStoreRefCountDatabase::refcount_t DeleteFile(int64_t InDirectory, + int64_t ObjectID, + BackupStoreDirectory &rDirectory, + const std::string &rDirectoryFilename, + BackupStoreInfo& rBackupStoreInfo); + void UpdateDirectorySize(BackupStoreDirectory &rDirectory, + IOStream::pos_type new_size_in_blocks); -private: typedef struct { int64_t mObjectID; @@ -81,6 +88,9 @@ private: // List of directories which are empty, and might be good for deleting std::vector<int64_t> mEmptyDirectories; + + // Count of errors found and fixed + int64_t mErrorCount; // The re-calculated blocks used stats int64_t mBlocksUsed; @@ -99,9 +109,7 @@ private: int64_t mEmptyDirectoriesDeleted; // New reference count list - std::vector<uint32_t> mNewRefCounts; - bool mSuppressRefCountChangeWarnings; - int mRefCountsAdjusted; + std::auto_ptr<BackupStoreRefCountDatabase> mapNewRefs; // Poll frequency int mCountUntilNextInterprocessMsgCheck; diff --git a/lib/backupstore/Makefile.extra b/lib/backupstore/Makefile.extra index c55fd549..6f181abd 100644 --- a/lib/backupstore/Makefile.extra +++ b/lib/backupstore/Makefile.extra @@ -1,9 +1,9 @@ MAKEPROTOCOL = ../../lib/server/makeprotocol.pl -GEN_CMD = $(MAKEPROTOCOL) backupprotocol.txt +GEN_CMD = $(MAKEPROTOCOL) BackupProtocol.txt # AUTOGEN SEEDING -autogen_BackupProtocol.cpp autogen_BackupProtocol.h: $(MAKEPROTOCOL) backupprotocol.txt +autogen_BackupProtocol.cpp autogen_BackupProtocol.h: $(MAKEPROTOCOL) BackupProtocol.txt $(_PERL) $(GEN_CMD) diff --git a/lib/backupstore/StoreTestUtils.cpp b/lib/backupstore/StoreTestUtils.cpp new file mode 100644 index 00000000..2b773cb1 --- /dev/null +++ b/lib/backupstore/StoreTestUtils.cpp @@ -0,0 +1,300 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreTestUtils.cpp +// Purpose: Utilities for housekeeping and checking a test store +// Created: 18/02/14 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <cstdio> +#include <vector> + +#include "autogen_BackupProtocol.h" +#include "BoxPortsAndFiles.h" +#include "BackupStoreAccounts.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreConfigVerify.h" +#include "BackupStoreConstants.h" +#include "BackupStoreInfo.h" +#include "HousekeepStoreAccount.h" +#include "Logging.h" +#include "ServerControl.h" +#include "SocketStreamTLS.h" +#include "StoreTestUtils.h" +#include "TLSContext.h" +#include "Test.h" + +bool create_account(int soft, int hard) +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + + Logger::LevelGuard guard(Logging::GetConsole(), Log::WARNING); + int result = control.CreateAccount(0x01234567, 0, soft, hard); + TEST_EQUAL(0, result); + return (result == 0); +} + +bool delete_account() +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + Logger::LevelGuard guard(Logging::GetConsole(), Log::WARNING); + TEST_THAT_THROWONFAIL(control.DeleteAccount(0x01234567, false) == 0); + return true; +} + +std::vector<uint32_t> ExpectedRefCounts; +int bbstored_pid = 0, bbackupd_pid = 0; + +void set_refcount(int64_t ObjectID, uint32_t RefCount) +{ + if ((int64_t)ExpectedRefCounts.size() <= ObjectID) + { + ExpectedRefCounts.resize(ObjectID + 1, 0); + } + ExpectedRefCounts[ObjectID] = RefCount; + for (size_t i = ExpectedRefCounts.size() - 1; i >= 1; i--) + { + if (ExpectedRefCounts[i] == 0) + { + // BackupStoreCheck and housekeeping will both + // regenerate the refcount DB without any missing + // items at the end, so we need to prune ourselves + // of all items with no references to match. + ExpectedRefCounts.resize(i); + } + } +} + +void init_context(TLSContext& rContext) +{ + rContext.Initialise(false /* client */, + "testfiles/clientCerts.pem", + "testfiles/clientPrivKey.pem", + "testfiles/clientTrustedCAs.pem"); +} + +std::auto_ptr<SocketStream> open_conn(const char *hostname, + TLSContext& rContext) +{ + init_context(rContext); + std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS); + conn->Open(rContext, Socket::TypeINET, hostname, + BOX_PORT_BBSTORED_TEST); + return static_cast<std::auto_ptr<SocketStream> >(conn); +} + +std::auto_ptr<BackupProtocolCallable> connect_to_bbstored(TLSContext& rContext) +{ + // Make a protocol + std::auto_ptr<BackupProtocolCallable> protocol(new + BackupProtocolClient(open_conn("localhost", rContext))); + + // Check the version + std::auto_ptr<BackupProtocolVersion> serverVersion( + protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION)); + TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + + return protocol; +} + +std::auto_ptr<BackupProtocolCallable> connect_and_login(TLSContext& rContext, + int flags) +{ + // Make a protocol + std::auto_ptr<BackupProtocolCallable> protocol = + connect_to_bbstored(rContext); + + // Login + protocol->QueryLogin(0x01234567, flags); + + return protocol; +} + +bool check_num_files(int files, int old, int deleted, int dirs) +{ + std::auto_ptr<BackupStoreInfo> apInfo = + BackupStoreInfo::Load(0x1234567, + "backup/01234567/", 0, true); + TEST_EQUAL_LINE(files, apInfo->GetNumCurrentFiles(), + "current files"); + TEST_EQUAL_LINE(old, apInfo->GetNumOldFiles(), + "old files"); + TEST_EQUAL_LINE(deleted, apInfo->GetNumDeletedFiles(), + "deleted files"); + TEST_EQUAL_LINE(dirs, apInfo->GetNumDirectories(), + "directories"); + + return (files == apInfo->GetNumCurrentFiles() && + old == apInfo->GetNumOldFiles() && + deleted == apInfo->GetNumDeletedFiles() && + dirs == apInfo->GetNumDirectories()); +} + +bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, + int Deleted, int Dirs, int Total) +{ + std::auto_ptr<BackupProtocolAccountUsage2> usage = + Client.QueryGetAccountUsage2(); + TEST_EQUAL_LINE(Total, usage->GetBlocksUsed(), "wrong BlocksUsed"); + TEST_EQUAL_LINE(Current, usage->GetBlocksInCurrentFiles(), + "wrong BlocksInCurrentFiles"); + TEST_EQUAL_LINE(Old, usage->GetBlocksInOldFiles(), + "wrong BlocksInOldFiles"); + TEST_EQUAL_LINE(Deleted, usage->GetBlocksInDeletedFiles(), + "wrong BlocksInDeletedFiles"); + TEST_EQUAL_LINE(Dirs, usage->GetBlocksInDirectories(), + "wrong BlocksInDirectories"); + return (Total == usage->GetBlocksUsed() && + Current == usage->GetBlocksInCurrentFiles() && + Old == usage->GetBlocksInOldFiles() && + Deleted == usage->GetBlocksInDeletedFiles() && + Dirs == usage->GetBlocksInDirectories()); +} + +bool change_account_limits(const char* soft, const char* hard) +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + int result = control.SetLimit(0x01234567, soft, hard); + TEST_EQUAL(0, result); + return (result == 0); +} + +int check_account_for_errors(Log::Level log_level) +{ + Logger::LevelGuard guard(Logging::GetConsole(), log_level); + Logging::Tagger tag("check fix", true); + Logging::ShowTagOnConsole show; + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify("testfiles/bbstored.conf", + &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + int errors_fixed = control.CheckAccount(0x01234567, + true, // FixErrors + false, // Quiet + true); // ReturnNumErrorsFound + return errors_fixed; +} + +bool check_account(Log::Level log_level) +{ + int errors_fixed = check_account_for_errors(log_level); + TEST_EQUAL(0, errors_fixed); + return (errors_fixed == 0); +} + +int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount) +{ + std::string rootDir = BackupStoreAccounts::GetAccountRoot(rAccount); + int discSet = rAccount.GetDiscSet(); + + // Do housekeeping on this account + HousekeepStoreAccount housekeeping(rAccount.GetID(), rootDir, + discSet, NULL); + TEST_THAT(housekeeping.DoHousekeeping(true /* keep trying forever */)); + return housekeeping.GetErrorCount(); +} + +// Run housekeeping (for which we need to disconnect ourselves) and check +// that it doesn't change the numbers of files. +// +// Also check that bbstoreaccounts doesn't change anything + +bool run_housekeeping_and_check_account() +{ + int error_count; + + { + Logging::Tagger tag("", true); + Logging::ShowTagOnConsole show; + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + BackupStoreAccountDatabase::Entry account = + apAccounts->GetEntry(0x1234567); + error_count = run_housekeeping(account); + } + + TEST_EQUAL_LINE(0, error_count, "housekeeping errors"); + + bool check_account_is_ok = check_account(); + TEST_THAT(check_account_is_ok); + + return error_count == 0 && check_account_is_ok; +} + +bool check_reference_counts() +{ + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + BackupStoreAccountDatabase::Entry account = + apAccounts->GetEntry(0x1234567); + + std::auto_ptr<BackupStoreRefCountDatabase> apReferences( + BackupStoreRefCountDatabase::Load(account, true)); + TEST_EQUAL(ExpectedRefCounts.size(), + apReferences->GetLastObjectIDUsed() + 1); + + bool counts_ok = true; + + for (unsigned int i = BackupProtocolListDirectory::RootDirectory; + i < ExpectedRefCounts.size(); i++) + { + TEST_EQUAL_LINE(ExpectedRefCounts[i], + apReferences->GetRefCount(i), + "object " << BOX_FORMAT_OBJECTID(i)); + if (ExpectedRefCounts[i] != apReferences->GetRefCount(i)) + { + counts_ok = false; + } + } + + return counts_ok; +} + +bool StartServer() +{ + bbstored_pid = StartDaemon(bbstored_pid, + BBSTORED " " + bbstored_args + " testfiles/bbstored.conf", + "testfiles/bbstored.pid"); + return bbstored_pid != 0; +} + +bool StopServer(bool wait_for_process) +{ + bool result = StopDaemon(bbstored_pid, "testfiles/bbstored.pid", + "bbstored.memleaks", wait_for_process); + bbstored_pid = 0; + return result; +} + +bool StartClient(const std::string& bbackupd_conf_file) +{ + bbackupd_pid = StartDaemon(bbackupd_pid, + BBACKUPD " " + bbackupd_args + " " + bbackupd_conf_file, + "testfiles/bbackupd.pid"); + return bbackupd_pid != 0; +} + +bool StopClient(bool wait_for_process) +{ + bool result = StopDaemon(bbackupd_pid, "testfiles/bbackupd.pid", + "bbackupd.memleaks", wait_for_process); + bbackupd_pid = 0; + return result; +} + diff --git a/lib/backupstore/StoreTestUtils.h b/lib/backupstore/StoreTestUtils.h new file mode 100644 index 00000000..b3faebb5 --- /dev/null +++ b/lib/backupstore/StoreTestUtils.h @@ -0,0 +1,118 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreTestUtils.h +// Purpose: Utilities for housekeeping and checking a test store +// Created: 18/02/14 +// +// -------------------------------------------------------------------------- + +#ifndef STORETESTUTILS__H +#define STORETESTUTILS__H + +#include "Test.h" + +class BackupProtocolCallable; +class BackupProtocolClient; +class SocketStreamTLS; +class TLSContext; + +//! Holds the expected reference counts of each object. +extern std::vector<uint32_t> ExpectedRefCounts; + +//! Holds the PID of the currently running bbstored test server. +extern int bbstored_pid, bbackupd_pid; + +//! Sets the expected refcount of an object, resizing vector if necessary. +void set_refcount(int64_t ObjectID, uint32_t RefCount = 1); + +//! Initialises a TLSContext object using the standard certficate filenames. +void init_context(TLSContext& rContext); + +//! Opens a connection to the server (bbstored). +std::auto_ptr<SocketStream> open_conn(const char *hostname, + TLSContext& rContext); + +//! Opens a connection to the server (bbstored) without logging in. +std::auto_ptr<BackupProtocolCallable> connect_to_bbstored(TLSContext& rContext); + +//! Opens a connection to the server (bbstored) and logs in. +std::auto_ptr<BackupProtocolCallable> connect_and_login(TLSContext& rContext, + int flags = 0); + +//! Checks the number of files of each type in the store against expectations. +bool check_num_files(int files, int old, int deleted, int dirs); + +//! Checks the number of blocks in files of each type against expectations. +bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, + int Deleted, int Dirs, int Total); + +//! Change the soft and hard limits on the test account. +bool change_account_limits(const char* soft, const char* hard); + +//! Checks an account for errors, returning the number of errors found and fixed. +int check_account_for_errors(Log::Level log_level = Log::WARNING); + +//! Checks an account for errors, returning true if it's OK, for use in assertions. +bool check_account(Log::Level log_level = Log::WARNING); + +//! Runs housekeeping on an account, to remove old and deleted files if necessary. +int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount); + +//! Runs housekeeping and checks the account, returning true if it's OK. +bool run_housekeeping_and_check_account(); + +//! Tests that all object reference counts have the expected values. +bool check_reference_counts(); + +//! Starts the bbstored test server running, which must not already be running. +bool StartServer(); + +//! Stops the currently running bbstored test server. +bool StopServer(bool wait_for_process = false); + +//! Starts the bbackupd client running, which must not already be running. +bool StartClient(const std::string& bbackupd_conf_file = "testfiles/bbackupd.conf"); + +//! Stops the currently running bbackupd client. +bool StopClient(bool wait_for_process = false); + +//! Creates the standard test account, for example after delete_account(). +bool create_account(int soft, int hard); + +//! Deletes the standard test account, for testing behaviour with no account. +bool delete_account(); + +#define TEST_PROTOCOL_ERROR_OR(protocol, error, or_statements) \ + { \ + int type, subtype; \ + (protocol).GetLastError(type, subtype); \ + if (type == BackupProtocolError::ErrorType) \ + { \ + TEST_EQUAL_LINE(BackupProtocolError::error, subtype, \ + "command returned error: " << \ + BackupProtocolError::GetMessage(subtype)); \ + if (subtype != BackupProtocolError::error) \ + { \ + or_statements; \ + } \ + } \ + else \ + { \ + TEST_FAIL_WITH_MESSAGE("command did not return an error, but a " \ + "response of type " << type << ", subtype " << subtype << \ + " instead"); \ + or_statements; \ + } \ + } + +#define TEST_COMMAND_RETURNS_ERROR_OR(protocol, command, error, or_statements) \ + TEST_CHECK_THROWS_AND_OR((protocol) . command, ConnectionException, \ + Protocol_UnexpectedReply, /* and_command */, or_statements); \ + TEST_PROTOCOL_ERROR_OR(protocol, error, or_statements) + +#define TEST_COMMAND_RETURNS_ERROR(protocol, command, error) \ + TEST_COMMAND_RETURNS_ERROR_OR(protocol, command, error,) + +#endif // STORETESTUTILS__H + diff --git a/lib/bbackupd/BackupClientContext.cpp b/lib/bbackupd/BackupClientContext.cpp new file mode 100644 index 00000000..4c0b01ce --- /dev/null +++ b/lib/bbackupd/BackupClientContext.cpp @@ -0,0 +1,578 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientContext.cpp +// Purpose: Keep track of context +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_SIGNAL_H + #include <signal.h> +#endif + +#ifdef HAVE_SYS_TIME_H + #include <sys/time.h> +#endif + +#include "BoxPortsAndFiles.h" +#include "BoxTime.h" +#include "BackupClientContext.h" +#include "SocketStreamTLS.h" +#include "Socket.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" +#include "BackupDaemon.h" +#include "autogen_BackupProtocol.h" +#include "BackupStoreFile.h" +#include "Logging.h" +#include "TcpNice.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::BackupClientContext(BackupDaemon &, TLSContext &, const std::string &, int32_t, bool, bool, std::string) +// Purpose: Constructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientContext::BackupClientContext +( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier& rProgressNotifier, + bool TcpNiceMode +) +: mExperimentalSnapshotMode(false), + mrResolver(rResolver), + mrTLSContext(rTLSContext), + mHostname(rHostname), + mPort(Port), + mAccountNumber(AccountNumber), + mExtendedLogging(ExtendedLogging), + mExtendedLogToFile(ExtendedLogToFile), + mExtendedLogFile(ExtendedLogFile), + mpExtendedLogFileHandle(NULL), + mClientStoreMarker(ClientStoreMarker_NotKnown), + mpDeleteList(0), + mpCurrentIDMap(0), + mpNewIDMap(0), + mStorageLimitExceeded(false), + mpExcludeFiles(0), + mpExcludeDirs(0), + mKeepAliveTimer(0, "KeepAliveTime"), + mbIsManaged(false), + mrProgressNotifier(rProgressNotifier), + mTcpNiceMode(TcpNiceMode), + mpNice(NULL) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::~BackupClientContext() +// Purpose: Destructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientContext::~BackupClientContext() +{ + CloseAnyOpenConnection(); + + // Delete delete list + if(mpDeleteList != 0) + { + delete mpDeleteList; + mpDeleteList = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetConnection() +// Purpose: Returns the connection, making the connection and logging into +// the backup store if necessary. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupProtocolCallable &BackupClientContext::GetConnection() +{ + // Already got it? Just return it. + if(mapConnection.get()) + { + return *mapConnection; + } + + // Defensive. Must close connection before releasing any old socket. + mapConnection.reset(); + + std::auto_ptr<SocketStream> apSocket(new SocketStreamTLS); + + try + { + // Defensive. + mapConnection.reset(); + + // Log intention + BOX_INFO("Opening connection to server '" << mHostname << + "'..."); + + // Connect! + ((SocketStreamTLS *)(apSocket.get()))->Open(mrTLSContext, + Socket::TypeINET, mHostname, mPort); + + if(mTcpNiceMode) + { + // Pass control of apSocket to NiceSocketStream, + // which will take care of destroying it for us. + // But we need to hang onto a pointer to the nice + // socket, so we can enable and disable nice mode. + // This is scary, it could be deallocated under us. + mpNice = new NiceSocketStream(apSocket); + apSocket.reset(mpNice); + } + + // We need to call some methods that aren't defined in + // BackupProtocolCallable, so we need to hang onto a more + // strongly typed pointer (to avoid far too many casts). + BackupProtocolClient *pClient = new BackupProtocolClient(apSocket); + mapConnection.reset(pClient); + + // Set logging option + pClient->SetLogToSysLog(mExtendedLogging); + + if (mExtendedLogToFile) + { + ASSERT(mpExtendedLogFileHandle == NULL); + + mpExtendedLogFileHandle = fopen( + mExtendedLogFile.c_str(), "a+"); + + if (!mpExtendedLogFileHandle) + { + BOX_LOG_SYS_ERROR("Failed to open extended " + "log file: " << mExtendedLogFile); + } + else + { + pClient->SetLogToFile(mpExtendedLogFileHandle); + } + } + + // Handshake + pClient->Handshake(); + + // Check the version of the server + { + std::auto_ptr<BackupProtocolVersion> serverVersion( + mapConnection->QueryVersion(BACKUP_STORE_SERVER_VERSION)); + if(serverVersion->GetVersion() != BACKUP_STORE_SERVER_VERSION) + { + THROW_EXCEPTION(BackupStoreException, WrongServerVersion) + } + } + + // Login -- if this fails, the Protocol will exception + std::auto_ptr<BackupProtocolLoginConfirmed> loginConf( + mapConnection->QueryLogin(mAccountNumber, 0 /* read/write */)); + + // Check that the client store marker is the one we expect + if(mClientStoreMarker != ClientStoreMarker_NotKnown) + { + if(loginConf->GetClientStoreMarker() != mClientStoreMarker) + { + // Not good... finish the connection, abort, etc, ignoring errors + try + { + mapConnection->QueryFinished(); + } + catch(...) + { + // IGNORE + } + + // Then throw an exception about this + THROW_EXCEPTION_MESSAGE(BackupStoreException, + ClientMarkerNotAsExpected, + "Expected " << mClientStoreMarker << + " but found " << loginConf->GetClientStoreMarker() << + ": is someone else writing to the " + "same account?"); + } + } + else // mClientStoreMarker == ClientStoreMarker_NotKnown + { + // Yes, choose one, the current time will do + box_time_t marker = GetCurrentBoxTime(); + + // Set it on the store + mapConnection->QuerySetClientStoreMarker(marker); + + // Record it so that it can be picked up later. + mClientStoreMarker = marker; + } + + // Log success + BOX_INFO("Connection made, login successful"); + + // Check to see if there is any space available on the server + if(loginConf->GetBlocksUsed() >= loginConf->GetBlocksHardLimit()) + { + // no -- flag so only things like deletions happen + mStorageLimitExceeded = true; + // Log + BOX_WARNING("Exceeded storage hard-limit on server, " + "not uploading changes to files"); + } + } + catch(...) + { + // Clean up. + mapConnection.reset(); + throw; + } + + return *mapConnection; +} + +BackupProtocolCallable* BackupClientContext::GetOpenConnection() const +{ + return mapConnection.get(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::CloseAnyOpenConnection() +// Purpose: Closes a connection, if it's open +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupClientContext::CloseAnyOpenConnection() +{ + BackupProtocolCallable* pConnection(GetOpenConnection()); + if(pConnection) + { + try + { + // Quit nicely + pConnection->QueryFinished(); + } + catch(...) + { + // Ignore errors here + } + + // Delete it anyway. + mapConnection.reset(); + } + + // Delete any pending list + if(mpDeleteList != 0) + { + delete mpDeleteList; + mpDeleteList = 0; + } + + if (mpExtendedLogFileHandle != NULL) + { + fclose(mpExtendedLogFileHandle); + mpExtendedLogFileHandle = NULL; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetTimeout() +// Purpose: Gets the current timeout time. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +int BackupClientContext::GetTimeout() const +{ + BackupProtocolCallable* pConnection(GetOpenConnection()); + if(pConnection) + { + return pConnection->GetTimeout(); + } + + return (15*60*1000); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetDeleteList() +// Purpose: Returns the delete list, creating one if necessary +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +BackupClientDeleteList &BackupClientContext::GetDeleteList() +{ + // Already created? + if(mpDeleteList == 0) + { + mpDeleteList = new BackupClientDeleteList; + } + + // Return reference to object + return *mpDeleteList; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::PerformDeletions() +// Purpose: Perform any pending file deletions. +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientContext::PerformDeletions() +{ + // Got a list? + if(mpDeleteList == 0) + { + // Nothing to do + return; + } + + // Delegate to the delete list object + mpDeleteList->PerformDeletions(*this); + + // Delete the object + delete mpDeleteList; + mpDeleteList = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetCurrentIDMap() const +// Purpose: Return a (const) reference to the current ID map +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +const BackupClientInodeToIDMap &BackupClientContext::GetCurrentIDMap() const +{ + ASSERT(mpCurrentIDMap != 0); + if(mpCurrentIDMap == 0) + { + THROW_EXCEPTION(CommonException, Internal) + } + return *mpCurrentIDMap; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetNewIDMap() const +// Purpose: Return a reference to the new ID map +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +BackupClientInodeToIDMap &BackupClientContext::GetNewIDMap() const +{ + ASSERT(mpNewIDMap != 0); + if(mpNewIDMap == 0) + { + THROW_EXCEPTION(CommonException, Internal) + } + return *mpNewIDMap; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::FindFilename(int64_t, int64_t, std::string &, bool &) const +// Purpose: Attempts to find the pathname of an object with a given ID on the server. +// Returns true if it can be found, in which case rPathOut is the local filename, +// and rIsDirectoryOut == true if the local object is a directory. +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut, + bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer, box_time_t *pAttributesHashOnServer, BackupStoreFilenameClear *pLeafname) +{ + // Make a connection to the server + BackupProtocolCallable &connection(GetConnection()); + + // Request filenames from the server, in a "safe" manner to ignore errors properly + { + BackupProtocolGetObjectName send(ObjectID, ContainingDirectory); + connection.Send(send); + } + std::auto_ptr<BackupProtocolMessage> preply(connection.Receive()); + + // Is it of the right type? + if(preply->GetType() != BackupProtocolObjectName::TypeID) + { + // Was an error or something + return false; + } + + // Cast to expected type. + BackupProtocolObjectName *names = (BackupProtocolObjectName *)(preply.get()); + + // Anything found? + int32_t numElements = names->GetNumNameElements(); + if(numElements <= 0) + { + // No. + return false; + } + + // Get the stream containing all the names + std::auto_ptr<IOStream> nameStream(connection.ReceiveStream()); + + // Path + std::string path; + + // Remember this is in reverse order! + for(int l = 0; l < numElements; ++l) + { + BackupStoreFilenameClear elementName; + elementName.ReadFromStream(*nameStream, GetTimeout()); + + // Store leafname for caller? + if(l == 0 && pLeafname) + { + *pLeafname = elementName; + } + + // Is it part of the filename in the location? + if(l < (numElements - 1)) + { + // Part of filename within + path = (path.empty())?(elementName.GetClearFilename()):(elementName.GetClearFilename() + DIRECTORY_SEPARATOR_ASCHAR + path); + } + else + { + // Location name -- look up in daemon's records + std::string locPath; + if(!mrResolver.FindLocationPathName(elementName.GetClearFilename(), locPath)) + { + // Didn't find the location... so can't give the local filename + return false; + } + + // Add in location path + path = (path.empty())?(locPath):(locPath + DIRECTORY_SEPARATOR_ASCHAR + path); + } + } + + // Is it a directory? + rIsDirectoryOut = ((names->GetFlags() & BackupProtocolListDirectory::Flags_Dir) == BackupProtocolListDirectory::Flags_Dir); + + // Is it the current version? + rIsCurrentVersionOut = ((names->GetFlags() & (BackupProtocolListDirectory::Flags_OldVersion | BackupProtocolListDirectory::Flags_Deleted)) == 0); + + // And other information which may be required + if(pModTimeOnServer) *pModTimeOnServer = names->GetModificationTime(); + if(pAttributesHashOnServer) *pAttributesHashOnServer = names->GetAttributesHash(); + + // Tell caller about the pathname + rPathOut = path; + + // Found + return true; +} + +void BackupClientContext::SetMaximumDiffingTime(int iSeconds) +{ + mMaximumDiffingTime = iSeconds < 0 ? 0 : iSeconds; + BOX_TRACE("Set maximum diffing time to " << mMaximumDiffingTime << + " seconds"); +} + +void BackupClientContext::SetKeepAliveTime(int iSeconds) +{ + mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; + BOX_TRACE("Set keep-alive time to " << mKeepAliveTime << " seconds"); + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::ManageDiffProcess() +// Purpose: Initiates a file diff control timer +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::ManageDiffProcess() +{ + ASSERT(!mbIsManaged); + mbIsManaged = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::UnManageDiffProcess() +// Purpose: suspends file diff control timer +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::UnManageDiffProcess() +{ + // ASSERT(mbIsManaged); + mbIsManaged = false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::DoKeepAlive() +// Purpose: Check whether it's time to send a KeepAlive +// message over the SSL link, and if so, send it. +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::DoKeepAlive() +{ + BackupProtocolCallable* pConnection(GetOpenConnection()); + if (!pConnection) + { + return; + } + + if (mKeepAliveTime == 0) + { + return; + } + + if (!mKeepAliveTimer.HasExpired()) + { + return; + } + + BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); + pConnection->QueryGetIsAlive(); + + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); +} + +int BackupClientContext::GetMaximumDiffingTime() +{ + return mMaximumDiffingTime; +} diff --git a/lib/bbackupd/BackupClientContext.h b/lib/bbackupd/BackupClientContext.h new file mode 100644 index 00000000..df43a232 --- /dev/null +++ b/lib/bbackupd/BackupClientContext.h @@ -0,0 +1,252 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientContext.h +// Purpose: Keep track of context +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTCONTEXT__H +#define BACKUPCLIENTCONTEXT__H + +#include "BoxTime.h" +#include "BackupClientDeleteList.h" +#include "BackupClientDirectoryRecord.h" +#include "BackupDaemonInterface.h" +#include "BackupStoreFile.h" +#include "ExcludeList.h" +#include "TcpNice.h" +#include "Timer.h" + +class TLSContext; +class BackupProtocolClient; +class SocketStreamTLS; +class BackupClientInodeToIDMap; +class BackupDaemon; +class BackupStoreFilenameClear; + +#include <string> + + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientContext +// Purpose: +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class BackupClientContext : public DiffTimer +{ +public: + BackupClientContext + ( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode + ); + virtual ~BackupClientContext(); + +private: + BackupClientContext(const BackupClientContext &); + +public: + // GetConnection() will open a connection if none is currently open. + virtual BackupProtocolCallable& GetConnection(); + // GetOpenConnection() will not open a connection, just return NULL if there is + // no connection already open. + virtual BackupProtocolCallable* GetOpenConnection() const; + void CloseAnyOpenConnection(); + int GetTimeout() const; + BackupClientDeleteList &GetDeleteList(); + void PerformDeletions(); + + enum + { + ClientStoreMarker_NotKnown = 0 + }; + + void SetClientStoreMarker(int64_t ClientStoreMarker) {mClientStoreMarker = ClientStoreMarker;} + int64_t GetClientStoreMarker() const {return mClientStoreMarker;} + + bool StorageLimitExceeded() {return mStorageLimitExceeded;} + void SetStorageLimitExceeded() {mStorageLimitExceeded = true;} + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetIDMaps(const BackupClientInodeToIDMap *, BackupClientInodeToIDMap *) + // Purpose: Store pointers to the Current and New ID maps + // Created: 11/11/03 + // + // -------------------------------------------------------------------------- + void SetIDMaps(const BackupClientInodeToIDMap *pCurrent, BackupClientInodeToIDMap *pNew) + { + ASSERT(pCurrent != 0); + ASSERT(pNew != 0); + mpCurrentIDMap = pCurrent; + mpNewIDMap = pNew; + } + const BackupClientInodeToIDMap &GetCurrentIDMap() const; + BackupClientInodeToIDMap &GetNewIDMap() const; + + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetExcludeLists(ExcludeList *, ExcludeList *) + // Purpose: Sets the exclude lists for the operation. Can be 0. + // Created: 28/1/04 + // + // -------------------------------------------------------------------------- + void SetExcludeLists(ExcludeList *pExcludeFiles, ExcludeList *pExcludeDirs) + { + mpExcludeFiles = pExcludeFiles; + mpExcludeDirs = pExcludeDirs; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::ExcludeFile(const std::string &) + // Purpose: Returns true is this file should be excluded from the backup + // Created: 28/1/04 + // + // -------------------------------------------------------------------------- + inline bool ExcludeFile(const std::string &rFullFilename) + { + if(mpExcludeFiles != 0) + { + return mpExcludeFiles->IsExcluded(rFullFilename); + } + // If no list, don't exclude anything + return false; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::ExcludeDir(const std::string &) + // Purpose: Returns true is this directory should be excluded from the backup + // Created: 28/1/04 + // + // -------------------------------------------------------------------------- + inline bool ExcludeDir(const std::string &rFullDirName) + { + if(mpExcludeDirs != 0) + { + return mpExcludeDirs->IsExcluded(rFullDirName); + } + // If no list, don't exclude anything + return false; + } + + // Utility functions -- may do a lot of work + bool FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut, + bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer = 0, box_time_t *pAttributesHashOnServer = 0, + BackupStoreFilenameClear *pLeafname = 0); // not const as may connect to server + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetMaximumDiffingTime() + // Purpose: Sets the maximum time that will be spent diffing a file + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void SetMaximumDiffingTime(int iSeconds); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetKeepAliveTime() + // Purpose: Sets the time interval for repetitive keep-alive operation + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + virtual void SetKeepAliveTime(int iSeconds); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::ManageDiffProcess() + // Purpose: Initiates an SSL connection/session keep-alive process + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void ManageDiffProcess(); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::UnManageDiffProcess() + // Purpose: Suspends an SSL connection/session keep-alive process + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void UnManageDiffProcess(); + + // ------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::DoKeepAlive() + // Purpose: Check whether it's time to send a KeepAlive + // message over the SSL link, and if so, send it. + // Created: 04/19/2005 + // + // ------------------------------------------------------------------- + virtual void DoKeepAlive(); + virtual int GetMaximumDiffingTime(); + virtual bool IsManaged() { return mbIsManaged; } + + ProgressNotifier& GetProgressNotifier() const + { + return mrProgressNotifier; + } + + void SetNiceMode(bool enabled) + { + if(mTcpNiceMode) + { + mpNice->SetEnabled(enabled); + } + } + + bool mExperimentalSnapshotMode; + +private: + LocationResolver &mrResolver; + TLSContext &mrTLSContext; + std::string mHostname; + int mPort; + uint32_t mAccountNumber; + std::auto_ptr<BackupProtocolCallable> mapConnection; + bool mExtendedLogging; + bool mExtendedLogToFile; + std::string mExtendedLogFile; + FILE* mpExtendedLogFileHandle; + int64_t mClientStoreMarker; + BackupClientDeleteList *mpDeleteList; + const BackupClientInodeToIDMap *mpCurrentIDMap; + BackupClientInodeToIDMap *mpNewIDMap; + bool mStorageLimitExceeded; + ExcludeList *mpExcludeFiles; + ExcludeList *mpExcludeDirs; + Timer mKeepAliveTimer; + bool mbIsManaged; + int mKeepAliveTime; + int mMaximumDiffingTime; + ProgressNotifier &mrProgressNotifier; + bool mTcpNiceMode; + NiceSocketStream *mpNice; +}; + +#endif // BACKUPCLIENTCONTEXT__H diff --git a/lib/bbackupd/BackupClientDeleteList.cpp b/lib/bbackupd/BackupClientDeleteList.cpp new file mode 100644 index 00000000..ce5e6264 --- /dev/null +++ b/lib/bbackupd/BackupClientDeleteList.cpp @@ -0,0 +1,229 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDeleteList.cpp +// Purpose: List of pending deletes for backup +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <algorithm> + +#include "BackupClientDeleteList.h" +#include "BackupClientContext.h" +#include "autogen_BackupProtocol.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::BackupClientDeleteList() +// Purpose: Constructor +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +BackupClientDeleteList::BackupClientDeleteList() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::~BackupClientDeleteList() +// Purpose: Destructor +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +BackupClientDeleteList::~BackupClientDeleteList() +{ +} + +BackupClientDeleteList::FileToDelete::FileToDelete(int64_t DirectoryID, + const BackupStoreFilename& rFilename, + const std::string& rLocalPath) +: mDirectoryID(DirectoryID), + mFilename(rFilename), + mLocalPath(rLocalPath) +{ } + +BackupClientDeleteList::DirToDelete::DirToDelete(int64_t ObjectID, + const std::string& rLocalPath) +: mObjectID(ObjectID), + mLocalPath(rLocalPath) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::AddDirectoryDelete(int64_t, +// const BackupStoreFilename&) +// Purpose: Add a directory to the list of directories to be deleted. +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID, + const std::string& rLocalPath) +{ + // Only add the delete to the list if it's not in the "no delete" set + if(mDirectoryNoDeleteList.find(ObjectID) == + mDirectoryNoDeleteList.end()) + { + // Not in the list, so should delete it + mDirectoryList.push_back(DirToDelete(ObjectID, rLocalPath)); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::AddFileDelete(int64_t, +// const BackupStoreFilename &) +// Purpose: +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, + const BackupStoreFilename &rFilename, const std::string& rLocalPath) +{ + // Try to find it in the no delete list + std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator + delEntry(mFileNoDeleteList.begin()); + while(delEntry != mFileNoDeleteList.end()) + { + if((delEntry)->first == DirectoryID + && (delEntry)->second == rFilename) + { + // Found! + break; + } + ++delEntry; + } + + // Only add it to the delete list if it wasn't in the no delete list + if(delEntry == mFileNoDeleteList.end()) + { + mFileList.push_back(FileToDelete(DirectoryID, rFilename, + rLocalPath)); + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) +// Purpose: Perform all the pending deletes +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) +{ + // Anything to do? + if(mDirectoryList.empty() && mFileList.empty()) + { + // Nothing! + return; + } + + // Get a connection + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Do the deletes + for(std::vector<DirToDelete>::iterator i(mDirectoryList.begin()); + i != mDirectoryList.end(); ++i) + { + connection.QueryDeleteDirectory(i->mObjectID); + rContext.GetProgressNotifier().NotifyDirectoryDeleted( + i->mObjectID, i->mLocalPath); + } + + // Clear the directory list + mDirectoryList.clear(); + + // Delete the files + for(std::vector<FileToDelete>::iterator i(mFileList.begin()); + i != mFileList.end(); ++i) + { + connection.QueryDeleteFile(i->mDirectoryID, i->mFilename); + rContext.GetProgressNotifier().NotifyFileDeleted( + i->mDirectoryID, i->mLocalPath); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::StopDirectoryDeletion(int64_t) +// Purpose: Stop a directory being deleted +// Created: 19/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::StopDirectoryDeletion(int64_t ObjectID) +{ + // First of all, is it in the delete vector? + std::vector<DirToDelete>::iterator delEntry(mDirectoryList.begin()); + for(; delEntry != mDirectoryList.end(); delEntry++) + { + if(delEntry->mObjectID == ObjectID) + { + // Found! + break; + } + } + if(delEntry != mDirectoryList.end()) + { + // erase this entry + mDirectoryList.erase(delEntry); + } + else + { + // Haven't been asked to delete it yet, put it in the + // no delete list + mDirectoryNoDeleteList.insert(ObjectID); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::StopFileDeletion(int64_t, const BackupStoreFilename &) +// Purpose: Stop a file from being deleted +// Created: 19/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, + const BackupStoreFilename &rFilename) +{ + // Find this in the delete list + std::vector<FileToDelete>::iterator delEntry(mFileList.begin()); + while(delEntry != mFileList.end()) + { + if(delEntry->mDirectoryID == DirectoryID + && delEntry->mFilename == rFilename) + { + // Found! + break; + } + ++delEntry; + } + + if(delEntry != mFileList.end()) + { + // erase this entry + mFileList.erase(delEntry); + } + else + { + // Haven't been asked to delete it yet, put it in the no delete list + mFileNoDeleteList.push_back(std::pair<int64_t, BackupStoreFilename>(DirectoryID, rFilename)); + } +} + diff --git a/lib/bbackupd/BackupClientDeleteList.h b/lib/bbackupd/BackupClientDeleteList.h new file mode 100644 index 00000000..b0fbf51a --- /dev/null +++ b/lib/bbackupd/BackupClientDeleteList.h @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDeleteList.h +// Purpose: List of pending deletes for backup +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTDELETELIST__H +#define BACKUPCLIENTDELETELIST__H + +#include "BackupStoreFilename.h" + +class BackupClientContext; + +#include <vector> +#include <utility> +#include <set> + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientDeleteList +// Purpose: List of pending deletes for backup +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +class BackupClientDeleteList +{ +private: + class FileToDelete + { + public: + int64_t mDirectoryID; + BackupStoreFilename mFilename; + std::string mLocalPath; + FileToDelete(int64_t DirectoryID, + const BackupStoreFilename& rFilename, + const std::string& rLocalPath); + }; + + class DirToDelete + { + public: + int64_t mObjectID; + std::string mLocalPath; + DirToDelete(int64_t ObjectID, const std::string& rLocalPath); + }; + +public: + BackupClientDeleteList(); + ~BackupClientDeleteList(); + + void AddDirectoryDelete(int64_t ObjectID, + const std::string& rLocalPath); + void AddFileDelete(int64_t DirectoryID, + const BackupStoreFilename &rFilename, + const std::string& rLocalPath); + + void StopDirectoryDeletion(int64_t ObjectID); + void StopFileDeletion(int64_t DirectoryID, + const BackupStoreFilename &rFilename); + + void PerformDeletions(BackupClientContext &rContext); + +private: + std::vector<DirToDelete> mDirectoryList; + std::set<int64_t> mDirectoryNoDeleteList; // note: things only get in this list if they're not present in mDirectoryList when they are 'added' + std::vector<FileToDelete> mFileList; + std::vector<std::pair<int64_t, BackupStoreFilename> > mFileNoDeleteList; +}; + +#endif // BACKUPCLIENTDELETELIST__H + diff --git a/lib/bbackupd/BackupClientDirectoryRecord.cpp b/lib/bbackupd/BackupClientDirectoryRecord.cpp new file mode 100644 index 00000000..94cb7965 --- /dev/null +++ b/lib/bbackupd/BackupClientDirectoryRecord.cpp @@ -0,0 +1,2302 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDirectoryRecord.cpp +// Purpose: Implementation of record about directory for +// backup client +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_DIRENT_H + #include <dirent.h> +#endif + +#include <errno.h> +#include <string.h> + +#include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" +#include "autogen_ClientException.h" +#include "Archive.h" +#include "BackupClientContext.h" +#include "BackupClientDirectoryRecord.h" +#include "BackupClientInodeToIDMap.h" +#include "BackupDaemon.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileEncodeStream.h" +#include "BufferedStream.h" +#include "CommonException.h" +#include "CollectInBufferStream.h" +#include "FileModificationTime.h" +#include "IOStream.h" +#include "Logging.h" +#include "MemBlockStream.h" +#include "PathUtils.h" +#include "RateLimitingStream.h" +#include "ReadLoggingStream.h" + +#include "MemLeakFindOn.h" + +typedef std::map<std::string, BackupStoreDirectory::Entry *> DecryptedEntriesMap_t; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::BackupClientDirectoryRecord() +// Purpose: Constructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName) + : mObjectID(ObjectID), + mSubDirName(rSubDirName), + mInitialSyncDone(false), + mSyncDone(false), + mpPendingEntries(0) +{ + ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::~BackupClientDirectoryRecord() +// Purpose: Destructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::~BackupClientDirectoryRecord() +{ + // Make deletion recursive + DeleteSubDirectories(); + + // Delete maps + if(mpPendingEntries != 0) + { + delete mpPendingEntries; + mpPendingEntries = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::DeleteSubDirectories(); +// Purpose: Delete all sub directory entries +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::DeleteSubDirectories() +{ + // Delete all pointers + for(std::map<std::string, BackupClientDirectoryRecord *>::iterator i = mSubDirectories.begin(); + i != mSubDirectories.end(); ++i) + { + delete i->second; + } + + // Empty list + mSubDirectories.clear(); +} + +std::string BackupClientDirectoryRecord::ConvertVssPathToRealPath( + const std::string &rVssPath, + const Location& rBackupLocation) +{ +#ifdef ENABLE_VSS + BOX_TRACE("VSS: ConvertVssPathToRealPath: mIsSnapshotCreated = " << + rBackupLocation.mIsSnapshotCreated); + BOX_TRACE("VSS: ConvertVssPathToRealPath: File/Directory Path = " << + rVssPath.substr(0, rBackupLocation.mSnapshotPath.length())); + BOX_TRACE("VSS: ConvertVssPathToRealPath: Snapshot Path = " << + rBackupLocation.mSnapshotPath); + if (rBackupLocation.mIsSnapshotCreated && + rVssPath.substr(0, rBackupLocation.mSnapshotPath.length()) == + rBackupLocation.mSnapshotPath) + { + std::string convertedPath = rBackupLocation.mPath + + rVssPath.substr(rBackupLocation.mSnapshotPath.length()); + BOX_TRACE("VSS: ConvertVssPathToRealPath: Converted Path = " << + convertedPath); + return convertedPath; + } +#endif + + return rVssPath; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncDirectory(i +// BackupClientDirectoryRecord::SyncParams &, +// int64_t, const std::string &, +// const std::string &, bool) +// Purpose: Recursively synchronise a local directory +// with the server. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::SyncDirectory( + BackupClientDirectoryRecord::SyncParams &rParams, + int64_t ContainingDirectoryID, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + bool ThisDirHasJustBeenCreated) +{ + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + + // Signal received by daemon? + if(rParams.mrRunStatusProvider.StopRun()) + { + // Yes. Stop now. + THROW_EXCEPTION(BackupStoreException, SignalReceived) + } + + // Start by making some flag changes, marking this sync as not done, + // and on the immediate sub directories. + mSyncDone = false; + for(std::map<std::string, BackupClientDirectoryRecord *>::iterator + i = mSubDirectories.begin(); + i != mSubDirectories.end(); ++i) + { + i->second->mSyncDone = false; + } + + // Work out the time in the future after which the file should + // be uploaded regardless. This is a simple way to avoid having + // too many problems with file servers when they have clients + // with badly out of sync clocks. + rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + + rParams.mMaxFileTimeInFuture; + + // Build the current state checksum to compare against while + // getting info from dirs. Note checksum is used locally only, + // so byte order isn't considered. + MD5Digest currentStateChecksum; + + EMU_STRUCT_STAT dest_st; + // Stat the directory, to get attribute info + // If it's a symbolic link, we want the link target here + // (as we're about to back up the contents of the directory) + { + if(EMU_STAT(rLocalPath.c_str(), &dest_st) != 0) + { + // The directory has probably been deleted, so + // just ignore this error. In a future scan, this + // deletion will be noticed, deleted from server, + // and this object deleted. + rNotifier.NotifyDirStatFailed(this, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation), + strerror(errno)); + return; + } + + BOX_TRACE("Stat dir '" << rLocalPath << "' " + "found device/inode " << + dest_st.st_dev << "/" << dest_st.st_ino); + + // Store inode number in map so directories are tracked + // in case they're renamed + { + BackupClientInodeToIDMap &idMap( + rParams.mrContext.GetNewIDMap()); + idMap.AddToMap(dest_st.st_ino, mObjectID, ContainingDirectoryID, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation)); + } + // Add attributes to checksum + currentStateChecksum.Add(&dest_st.st_mode, + sizeof(dest_st.st_mode)); + currentStateChecksum.Add(&dest_st.st_uid, + sizeof(dest_st.st_uid)); + currentStateChecksum.Add(&dest_st.st_gid, + sizeof(dest_st.st_gid)); + // Inode to be paranoid about things moving around + currentStateChecksum.Add(&dest_st.st_ino, + sizeof(dest_st.st_ino)); +#ifdef HAVE_STRUCT_STAT_ST_FLAGS + currentStateChecksum.Add(&dest_st.st_flags, + sizeof(dest_st.st_flags)); +#endif + + StreamableMemBlock xattr; + BackupClientFileAttributes::FillExtendedAttr(xattr, + rLocalPath.c_str()); + currentStateChecksum.Add(xattr.GetBuffer(), xattr.GetSize()); + } + + // Read directory entries, building arrays of names + // First, need to read the contents of the directory. + std::vector<std::string> dirs; + std::vector<std::string> files; + bool downloadDirectoryRecordBecauseOfFutureFiles = false; + + // BLOCK + { + // read the contents... + DIR *dirHandle = 0; + try + { + std::string nonVssDirPath = ConvertVssPathToRealPath(rLocalPath, + rBackupLocation); + rNotifier.NotifyScanDirectory(this, nonVssDirPath); + + dirHandle = ::opendir(rLocalPath.c_str()); + if(dirHandle == 0) + { + // Report the error (logs and eventual email to administrator) + if (errno == EACCES) + { + rNotifier.NotifyDirListFailed(this, + nonVssDirPath, + "Access denied"); + } + else + { + rNotifier.NotifyDirListFailed(this, + nonVssDirPath, + strerror(errno)); + } + + // Report the error (logs and eventual email + // to administrator) + SetErrorWhenReadingFilesystemObject(rParams, + nonVssDirPath); + // Ignore this directory for now. + return; + } + + struct dirent *en = 0; + int num_entries_found = 0; + + while((en = ::readdir(dirHandle)) != 0) + { + num_entries_found++; + rParams.mrContext.DoKeepAlive(); + if(rParams.mpBackgroundTask) + { + rParams.mpBackgroundTask->RunBackgroundTask( + BackgroundTask::Scanning_Dirs, + num_entries_found, 0); + } + + if (!SyncDirectoryEntry(rParams, rNotifier, + rBackupLocation, rLocalPath, + currentStateChecksum, en, dest_st, dirs, + files, downloadDirectoryRecordBecauseOfFutureFiles)) + { + // This entry is not to be backed up. + continue; + } + } + + if(::closedir(dirHandle) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + dirHandle = 0; + } + catch(...) + { + if(dirHandle != 0) + { + ::closedir(dirHandle); + } + throw; + } + } + + // Finish off the checksum, and compare with the one currently stored + bool checksumDifferent = true; + currentStateChecksum.Finish(); + if(mInitialSyncDone && currentStateChecksum.DigestMatches(mStateChecksum)) + { + // The checksum is the same, and there was one to compare with + checksumDifferent = false; + } + + // Pointer to potentially downloaded store directory info + std::auto_ptr<BackupStoreDirectory> apDirOnStore; + + try + { + // Want to get the directory listing? + if(ThisDirHasJustBeenCreated) + { + // Avoid sending another command to the server when we know it's empty + apDirOnStore.reset(new BackupStoreDirectory(mObjectID, + ContainingDirectoryID)); + } + // Consider asking the store for it + else if(!mInitialSyncDone || checksumDifferent || + downloadDirectoryRecordBecauseOfFutureFiles) + { + apDirOnStore = FetchDirectoryListing(rParams); + } + + // Make sure the attributes are up to date -- if there's space + // on the server and this directory has not just been created + // (because it's attributes will be correct in this case) and + // the checksum is different, implying they *MIGHT* be + // different. + if((!ThisDirHasJustBeenCreated) && checksumDifferent && + !rParams.mrContext.StorageLimitExceeded()) + { + UpdateAttributes(rParams, apDirOnStore.get(), rLocalPath); + } + + // Create the list of pointers to directory entries + std::vector<BackupStoreDirectory::Entry *> entriesLeftOver; + if(apDirOnStore.get()) + { + entriesLeftOver.resize(apDirOnStore->GetNumberOfEntries(), 0); + BackupStoreDirectory::Iterator i(*apDirOnStore); + // Copy in pointers to all the entries + for(unsigned int l = 0; l < apDirOnStore->GetNumberOfEntries(); ++l) + { + entriesLeftOver[l] = i.Next(); + } + } + + // Do the directory reading + bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, + rRemotePath, rBackupLocation, apDirOnStore.get(), + entriesLeftOver, files, dirs); + + // LAST THING! (think exception safety) + // Store the new checksum -- don't fetch things unnecessarily + // in the future But... only if 1) the storage limit isn't + // exceeded -- make sure things are done again if the directory + // is modified later and 2) All the objects within the + // directory were stored successfully. + if(!rParams.mrContext.StorageLimitExceeded() && + updateCompleteSuccess) + { + currentStateChecksum.CopyDigestTo(mStateChecksum); + } + } + catch(...) + { + // Bad things have happened -- clean up + // Set things so that we get a full go at stuff later + ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); + + throw; + } + + // Flag things as having happened. + mInitialSyncDone = true; + mSyncDone = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncDirectorEntry( +// BackupClientDirectoryRecord::SyncParams &, +// int64_t, const std::string &, +// const std::string &, bool) +// Purpose: Recursively synchronise a local directory +// with the server. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +bool BackupClientDirectoryRecord::SyncDirectoryEntry( + BackupClientDirectoryRecord::SyncParams &rParams, + ProgressNotifier& rNotifier, + const Location& rBackupLocation, + const std::string &rDirLocalPath, + MD5Digest& currentStateChecksum, + struct dirent *en, + EMU_STRUCT_STAT dir_st, + std::vector<std::string>& rDirs, + std::vector<std::string>& rFiles, + bool& rDownloadDirectoryRecordBecauseOfFutureFiles) +{ + std::string entry_name = en->d_name; + if(entry_name == "." || entry_name == "..") + { + // ignore parent directory entries + return false; + } + + // Stat file to get info + std::string filename = MakeFullPath(rDirLocalPath, entry_name); + std::string realFileName = ConvertVssPathToRealPath(filename, + rBackupLocation); + EMU_STRUCT_STAT file_st; + +#ifdef WIN32 + // Don't stat the file just yet, to ensure that users can exclude + // unreadable files to suppress warnings that they are not accessible. + // + // Our emulated readdir() abuses en->d_type, which would normally + // contain DT_REG, DT_DIR, etc, but we only use it here and prefer to + // have the full file attributes. + + int type; + if (en->d_type & FILE_ATTRIBUTE_DIRECTORY) + { + type = S_IFDIR; + } + else + { + type = S_IFREG; + } +#else // !WIN32 + if(EMU_LSTAT(filename.c_str(), &file_st) != 0) + { + // We don't know whether it's a file or a directory, so check + // both. This only affects whether a warning message is + // displayed; the file is not backed up in either case. + if(!(rParams.mrContext.ExcludeFile(filename)) && + !(rParams.mrContext.ExcludeDir(filename))) + { + // Report the error (logs and eventual email to + // administrator) + rNotifier.NotifyFileStatFailed(this, filename, + strerror(errno)); + + // FIXME move to NotifyFileStatFailed() + SetErrorWhenReadingFilesystemObject(rParams, filename); + } + + // Ignore this entry for now. + return false; + } + + BOX_TRACE("Stat entry '" << filename << "' found device/inode " << + file_st.st_dev << "/" << file_st.st_ino); + + // Workaround for apparent btrfs bug, where symlinks appear to be on + // a different filesystem than their containing directory, thanks to + // Toke Hoiland-Jorgensen. + + int type = file_st.st_mode & S_IFMT; + if(type == S_IFDIR && file_st.st_dev != dir_st.st_dev) + { + if(!(rParams.mrContext.ExcludeDir(filename))) + { + rNotifier.NotifyMountPointSkipped(this, filename); + } + return false; + } +#endif + + if(type == S_IFREG || type == S_IFLNK) + { + // File or symbolic link + + // Exclude it? + if(rParams.mrContext.ExcludeFile(realFileName)) + { + rNotifier.NotifyFileExcluded(this, realFileName); + // Next item! + return false; + } + } + else if(type == S_IFDIR) + { + // Directory + + // Exclude it? + if(rParams.mrContext.ExcludeDir(realFileName)) + { + rNotifier.NotifyDirExcluded(this, realFileName); + + // Next item! + return false; + } + + #ifdef WIN32 + // exclude reparse points, as Application Data points to the + // parent directory under Vista and later, and causes an + // infinite loop: + // http://social.msdn.microsoft.com/forums/en-US/windowscompatibility/thread/05d14368-25dd-41c8-bdba-5590bf762a68/ + if (en->d_type & FILE_ATTRIBUTE_REPARSE_POINT) + { + rNotifier.NotifyMountPointSkipped(this, realFileName); + return false; + } + #endif + } + else // not a file or directory, what is it? + { + if (type == S_IFSOCK +#ifndef WIN32 + || type == S_IFIFO +#endif + ) + { + // removed notification for these types + // see Debian bug 479145, no objections + } + else if(rParams.mrContext.ExcludeFile(realFileName)) + { + rNotifier.NotifyFileExcluded(this, realFileName); + } + else + { + rNotifier.NotifyUnsupportedFileType(this, realFileName); + SetErrorWhenReadingFilesystemObject(rParams, + realFileName); + } + + return false; + } + + // The object should be backed up (file, symlink or dir, not excluded). + // So make the information for adding to the checksum. + + #ifdef WIN32 + // We didn't stat the file before, but now we need the information. + if(emu_stat(filename.c_str(), &file_st) != 0) + { + rNotifier.NotifyFileStatFailed(this, + ConvertVssPathToRealPath(filename, rBackupLocation), + strerror(errno)); + + // Report the error (logs and eventual email to administrator) + SetErrorWhenReadingFilesystemObject(rParams, filename); + + // Ignore this entry for now. + return false; + } + + if(file_st.st_dev != dir_st.st_dev) + { + rNotifier.NotifyMountPointSkipped(this, + ConvertVssPathToRealPath(filename, rBackupLocation)); + return false; + } + #endif + + // Basic structure for checksum info + struct { + box_time_t mModificationTime; + box_time_t mAttributeModificationTime; + int64_t mSize; + // And then the name follows + } checksum_info; + + // Be paranoid about structure packing + ::memset(&checksum_info, 0, sizeof(checksum_info)); + + checksum_info.mModificationTime = FileModificationTime(file_st); + checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st); + checksum_info.mSize = file_st.st_size; + currentStateChecksum.Add(&checksum_info, sizeof(checksum_info)); + currentStateChecksum.Add(en->d_name, strlen(en->d_name)); + + // If the file has been modified madly into the future, download the + // directory record anyway to ensure that it doesn't get uploaded + // every single time the disc is scanned. + if(checksum_info.mModificationTime > rParams.mUploadAfterThisTimeInTheFuture) + { + rDownloadDirectoryRecordBecauseOfFutureFiles = true; + // Log that this has happened + if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) + { + rNotifier.NotifyFileModifiedInFuture(this, + ConvertVssPathToRealPath(filename, rBackupLocation)); + rParams.mHaveLoggedWarningAboutFutureFileTimes = true; + } + } + + // We've decided to back it up, so add to file or directory list. + if(type == S_IFREG || type == S_IFLNK) + { + rFiles.push_back(entry_name); + } + else if(type == S_IFDIR) + { + rDirs.push_back(entry_name); + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::FetchDirectoryListing( +// BackupClientDirectoryRecord::SyncParams &) +// Purpose: Fetch the directory listing of this directory from +// the store. +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreDirectory> +BackupClientDirectoryRecord::FetchDirectoryListing( + BackupClientDirectoryRecord::SyncParams &rParams) +{ + std::auto_ptr<BackupStoreDirectory> apDir; + + // Get connection to store + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Query the directory + std::auto_ptr<BackupProtocolSuccess> dirreply(connection.QueryListDirectory( + mObjectID, + // both files and directories + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // exclude old/deleted stuff + BackupProtocolListDirectory::Flags_Deleted | + BackupProtocolListDirectory::Flags_OldVersion, + true /* want attributes */)); + + // Retrieve the directory from the stream following + apDir.reset(new BackupStoreDirectory(connection.ReceiveStream(), + connection.GetTimeout())); + return apDir; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::UpdateAttributes( +// BackupClientDirectoryRecord::SyncParams &, +// const std::string &) +// Purpose: Sets the attributes of the directory on the store, +// if necessary. +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::UpdateAttributes( + BackupClientDirectoryRecord::SyncParams &rParams, + BackupStoreDirectory *pDirOnStore, + const std::string &rLocalPath) +{ + // Get attributes for the directory + BackupClientFileAttributes attr; + box_time_t attrModTime = 0; + attr.ReadAttributes(rLocalPath.c_str(), true /* directories have zero mod times */, + 0 /* no modification time */, &attrModTime); + + // Assume attributes need updating, unless proved otherwise + bool updateAttr = true; + + // Got a listing to compare with? + ASSERT(pDirOnStore == 0 || (pDirOnStore != 0 && pDirOnStore->HasAttributes())); + if(pDirOnStore != 0 && pDirOnStore->HasAttributes()) + { + const StreamableMemBlock &storeAttrEnc(pDirOnStore->GetAttributes()); + // Explict decryption + BackupClientFileAttributes storeAttr(storeAttrEnc); + + // Compare the attributes + if(attr.Compare(storeAttr, true, + true /* ignore both modification times */)) + { + // No update necessary + updateAttr = false; + } + } + + // Update them? + if(updateAttr) + { + // Get connection to store + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Exception thrown if this doesn't work + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); + connection.QueryChangeDirAttributes(mObjectID, attrModTime, attrStream); + } +} + +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath) +{ + BackupStoreFilenameClear fn(en->GetName()); + return DecryptFilename(fn, en->GetObjectID(), rRemoteDirectoryPath); +} + +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreFilenameClear fn, int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath) +{ + std::string filenameClear; + try + { + filenameClear = fn.GetClearFilename(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to decrypt filename for object " << + BOX_FORMAT_OBJECTID(filenameObjectID) << " in " + "directory " << BOX_FORMAT_OBJECTID(mObjectID) << + " (" << rRemoteDirectoryPath << ")"); + throw; + } + return filenameClear; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &, const std::string &, BackupStoreDirectory *, std::vector<BackupStoreDirectory::Entry *> &) +// Purpose: Update the items stored on the server. The rFiles vector will be erased after it's used to save space. +// Returns true if all items were updated successfully. (If not, the failures will have been logged). +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +bool BackupClientDirectoryRecord::UpdateItems( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + BackupStoreDirectory *pDirOnStore, + std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, + std::vector<std::string> &rFiles, + const std::vector<std::string> &rDirs) +{ + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + + bool allUpdatedSuccessfully = true; + + // Decrypt all the directory entries. + // It would be nice to be able to just compare the encrypted versions, however this doesn't work + // in practise because there can be multiple encodings of the same filename using different + // methods (although each method will result in the same string for the same filename.) This + // happens when the server fixes a broken store, and gives plain text generated filenames. + // So if we didn't do things like this, then you wouldn't be able to recover from bad things + // happening with the server. + DecryptedEntriesMap_t decryptedEntries; + if(pDirOnStore != 0) + { + BackupStoreDirectory::Iterator i(*pDirOnStore); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + std::string filenameClear; + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + decryptedEntries[filenameClear] = en; + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "pretending that the file doesn't " + "exist"); + } + } + } + + // Do files + for(std::vector<std::string>::const_iterator f = rFiles.begin(); + f != rFiles.end(); ++f) + { + // Send keep-alive message if needed + rContext.DoKeepAlive(); + + // Filename of this file + std::string filename(MakeFullPath(rLocalPath, *f)); + std::string nonVssFilePath = ConvertVssPathToRealPath(filename, + rBackupLocation); + + // Get relevant info about file + box_time_t modTime = 0; + uint64_t attributesHash = 0; + int64_t fileSize = 0; + InodeRefType inodeNum = 0; + // BLOCK + { + // Stat the file + EMU_STRUCT_STAT st; + if(EMU_LSTAT(filename.c_str(), &st) != 0) + { + rNotifier.NotifyFileStatFailed(this, nonVssFilePath, + strerror(errno)); + + // Report the error (logs and + // eventual email to administrator) + SetErrorWhenReadingFilesystemObject(rParams, nonVssFilePath); + + // Ignore this entry for now. + continue; + } + + // Extract required data + modTime = FileModificationTime(st); + fileSize = st.st_size; + inodeNum = st.st_ino; + attributesHash = BackupClientFileAttributes::GenerateAttributeHash(st, filename, *f); + } + + // See if it's in the listing (if we have one) + BackupStoreFilenameClear storeFilename(*f); + BackupStoreDirectory::Entry *en = 0; + int64_t latestObjectID = 0; + if(pDirOnStore != 0) + { + DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*f)); + if(i != decryptedEntries.end()) + { + en = i->second; + latestObjectID = en->GetObjectID(); + } + } + + // Check that the entry which might have been found is in fact a file + if((en != 0) && !(en->IsFile())) + { + // Directory exists in the place of this file -- sort it out + RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, + en, *f); + en = 0; + } + + // Check for renaming? + if(pDirOnStore != 0 && en == 0) + { + // We now know... + // 1) File has just been added + // 2) It's not in the store + + // Do we know about the inode number? + const BackupClientInodeToIDMap &idMap(rContext.GetCurrentIDMap()); + int64_t renameObjectID = 0, renameInDirectory = 0; + if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) + { + // Look up on the server to get the name, to build the local filename + std::string localPotentialOldName; + bool isDir = false; + bool isCurrentVersion = false; + box_time_t srvModTime = 0, srvAttributesHash = 0; + BackupStoreFilenameClear oldLeafname; + if(rContext.FindFilename(renameObjectID, renameInDirectory, + localPotentialOldName, isDir, isCurrentVersion, + &srvModTime, &srvAttributesHash, &oldLeafname)) + { + // Only interested if it's a file and the latest version + if(!isDir && isCurrentVersion) + { + // Check that the object we found in the ID map doesn't exist on disc + EMU_STRUCT_STAT st; + if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) + { + // Doesn't exist locally, but does exist on the server. + // Therefore we can safely rename it to this new file. + + // Get the connection to the server + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Only do this step if there is room on the server. + // This step will be repeated later when there is space available + if(!rContext.StorageLimitExceeded()) + { + // Rename the existing files (ie include old versions) on the server + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, + storeFilename); + + // Stop the attempt to delete the file in the original location + BackupClientDeleteList &rdelList(rContext.GetDeleteList()); + rdelList.StopFileDeletion(renameInDirectory, oldLeafname); + + // Create new entry in the directory for it + // -- will be near enough what's actually on the server for the rest to work. + en = pDirOnStore->AddEntry(storeFilename, + srvModTime, renameObjectID, + 0 /* size in blocks unknown, but not needed */, + BackupStoreDirectory::Entry::Flags_File, + srvAttributesHash); + + // Store the object ID for the inode lookup map later + latestObjectID = renameObjectID; + } + } + } + } + } + } + + // Is it in the mPendingEntries list? + box_time_t pendingFirstSeenTime = 0; // ie not seen + if(mpPendingEntries != 0) + { + std::map<std::string, box_time_t>::const_iterator i(mpPendingEntries->find(*f)); + if(i != mpPendingEntries->end()) + { + // found it -- set flag + pendingFirstSeenTime = i->second; + } + } + + // If pDirOnStore == 0, then this must have been after an initial sync: + ASSERT(pDirOnStore != 0 || mInitialSyncDone); + // So, if pDirOnStore == 0, then we know that everything before syncPeriodStart + // is either on the server, or in the toupload list. If the directory had changed, + // we'd have got a directory listing. + // + // At this point, if (pDirOnStore == 0 && en == 0), we can assume it's on the server with a + // mod time < syncPeriodStart, or didn't exist before that time. + // + // But if en != 0, then we need to compare modification times to avoid uploading it again. + + // Need to update? + // + // Condition for upload: + // modification time within sync period + // if it's been seen before but not uploaded, is the time from this first sight longer than the MaxUploadWait + // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store + + bool doUpload = false; + std::string decisionReason = "unknown"; + + // Only upload a file if the mod time locally is + // different to that on the server. + + if (en == 0 || en->GetModificationTime() != modTime) + { + // Check the file modified within the acceptable time period we're checking + // If the file isn't on the server, the acceptable time starts at zero. + // Check pDirOnStore and en, because if we didn't download a directory listing, + // pDirOnStore will be zero, but we know it's on the server. + if (modTime < rParams.mSyncPeriodEnd) + { + if (pDirOnStore != 0 && en == 0) + { + doUpload = true; + decisionReason = "not on server"; + } + else if (modTime >= rParams.mSyncPeriodStart) + { + doUpload = true; + decisionReason = "modified since last sync"; + } + } + + // However, just in case things are continually + // modified, we check the first seen time. + // The two compares of syncPeriodEnd and + // pendingFirstSeenTime are because the values + // are unsigned. + + if (!doUpload && + pendingFirstSeenTime != 0 && + rParams.mSyncPeriodEnd > pendingFirstSeenTime && + (rParams.mSyncPeriodEnd - pendingFirstSeenTime) + > rParams.mMaxUploadWait) + { + doUpload = true; + decisionReason = "continually modified"; + } + + // Then make sure that if files are added with a + // time less than the sync period start + // (which can easily happen on file server), it + // gets uploaded. The directory contents checksum + // will pick up the fact it has been added, so the + // store listing will be available when this happens. + + if (!doUpload && + modTime <= rParams.mSyncPeriodStart && + en != 0 && + en->GetModificationTime() != modTime) + { + doUpload = true; + decisionReason = "mod time changed"; + } + + // And just to catch really badly off clocks in + // the future for file server clients, + // just upload the file if it's madly in the future. + + if (!doUpload && modTime > + rParams.mUploadAfterThisTimeInTheFuture) + { + doUpload = true; + decisionReason = "mod time in the future"; + } + } + + if (en != 0 && en->GetModificationTime() == modTime) + { + doUpload = false; + decisionReason = "not modified since last upload"; + } + else if (!doUpload) + { + if (modTime > rParams.mSyncPeriodEnd) + { + box_time_t now = GetCurrentBoxTime(); + int age = BoxTimeToSeconds(now - + modTime); + std::ostringstream s; + s << "modified too recently: only " << + age << " seconds ago"; + decisionReason = s.str(); + } + else + { + std::ostringstream s; + s << "mod time is " << modTime << + " which is outside sync window, " + << rParams.mSyncPeriodStart << " to " + << rParams.mSyncPeriodEnd; + decisionReason = s.str(); + } + } + + BOX_TRACE("Upload decision: " << nonVssFilePath << ": " << + (doUpload ? "will upload" : "will not upload") << + " (" << decisionReason << ")"); + + bool fileSynced = true; + + if (doUpload) + { + // Upload needed, don't mark sync success until + // we've actually done it + fileSynced = false; + + // Make sure we're connected -- must connect here so we know whether + // the storage limit has been exceeded, and hence whether or not + // to actually upload the file. + rContext.GetConnection(); + + // Only do this step if there is room on the server. + // This step will be repeated later when there is space available + if(!rContext.StorageLimitExceeded()) + { + // Upload the file to the server, recording the + // object ID it returns + bool noPreviousVersionOnServer = + ((pDirOnStore != 0) && (en == 0)); + + // Surround this in a try/catch block, to + // catch errors, but still continue + bool uploadSuccess = false; + try + { + latestObjectID = UploadFile(rParams, + filename, + nonVssFilePath, + rRemotePath + "/" + *f, + storeFilename, + fileSize, modTime, + attributesHash, + noPreviousVersionOnServer); + + if (latestObjectID == 0) + { + // storage limit exceeded + rParams.mrContext.SetStorageLimitExceeded(); + uploadSuccess = false; + allUpdatedSuccessfully = false; + } + else + { + uploadSuccess = true; + } + } + catch(ConnectionException &e) + { + // Connection errors should just be + // passed on to the main handler, + // retries would probably just cause + // more problems. + rNotifier.NotifyFileUploadException( + this, nonVssFilePath, e); + throw; + } + catch(BoxException &e) + { + if (e.GetType() == BackupStoreException::ExceptionType && + e.GetSubType() == BackupStoreException::SignalReceived) + { + // abort requested, pass the + // exception on up. + throw; + } + + // an error occured -- make return + // code false, to show error in directory + allUpdatedSuccessfully = false; + // Log it. + SetErrorWhenReadingFilesystemObject(rParams, + nonVssFilePath); + rNotifier.NotifyFileUploadException(this, + nonVssFilePath, e); + } + + // Update structures if the file was uploaded + // successfully. + if(uploadSuccess) + { + fileSynced = true; + + // delete from pending entries + if(pendingFirstSeenTime != 0 && mpPendingEntries != 0) + { + mpPendingEntries->erase(*f); + } + } + } + else + { + rNotifier.NotifyFileSkippedServerFull(this, nonVssFilePath); + } + } + else if(en != 0 && en->GetAttributesHash() != attributesHash) + { + // Attributes have probably changed, upload them again. + // If the attributes have changed enough, the directory + // hash will have changed too, and so the dir will have + // been downloaded, and the entry will be available. + + // Get connection + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Only do this step if there is room on the server. + // This step will be repeated later when there is + // space available + if(!rContext.StorageLimitExceeded()) + { + try + { + rNotifier.NotifyFileUploadingAttributes(this, + nonVssFilePath); + + // Update store + BackupClientFileAttributes attr; + attr.ReadAttributes(filename, + false /* put mod times in the attributes, please */); + std::auto_ptr<IOStream> attrStream( + new MemBlockStream(attr)); + connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream); + fileSynced = true; + } + catch (BoxException &e) + { + BOX_ERROR("Failed to read or store file attributes " + "for '" << nonVssFilePath << "', will try again " + "later"); + } + } + } + + if(modTime >= rParams.mSyncPeriodEnd) + { + // Allocate? + if(mpPendingEntries == 0) + { + mpPendingEntries = new std::map<std::string, box_time_t>; + } + // Adding to mPendingEntries list + if(pendingFirstSeenTime == 0) + { + // Haven't seen this before -- add to list! + (*mpPendingEntries)[*f] = modTime; + } + } + + // Zero pointer in rEntriesLeftOver, if we have a pointer to zero + if(en != 0) + { + for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) + { + if(rEntriesLeftOver[l] == en) + { + rEntriesLeftOver[l] = 0; + break; + } + } + } + + // Does this file need an entry in the ID map? + if(fileSize >= rParams.mFileTrackingSizeThreshold) + { + // Get the map + BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap()); + + // Need to get an ID from somewhere... + if(latestObjectID == 0) + { + // Don't know it -- haven't sent anything to the store, and didn't get a listing. + // Look it up in the current map, and if it's there, use that. + const BackupClientInodeToIDMap ¤tIDMap(rContext.GetCurrentIDMap()); + int64_t objid = 0, dirid = 0; + if(currentIDMap.Lookup(inodeNum, objid, dirid)) + { + // Found + if (dirid != mObjectID) + { + BOX_WARNING("Found conflicting parent ID for " + "file ID " << inodeNum << " (" << + nonVssFilePath << "): expected " << + mObjectID << " but found " << dirid << + " (same directory used in two different " + "locations?)"); + } + + ASSERT(dirid == mObjectID); + + // NOTE: If the above assert fails, an inode number has been reused by the OS, + // or there is a problem somewhere. If this happened on a short test run, look + // into it. However, in a long running process this may happen occasionally and + // not indicate anything wrong. + // Run the release version for real life use, where this check is not made. + + latestObjectID = objid; + } + } + + if(latestObjectID != 0) + { + BOX_TRACE("Storing uploaded file ID " << + inodeNum << " (" << nonVssFilePath << ") " + "in ID map as object " << + latestObjectID << " with parent " << + mObjectID); + idMap.AddToMap(inodeNum, latestObjectID, + mObjectID /* containing directory */, + nonVssFilePath); + } + + } + + if (fileSynced) + { + rNotifier.NotifyFileSynchronised(this, nonVssFilePath, + fileSize); + } + } + + // Erase contents of files to save space when recursing + rFiles.clear(); + + // Delete the pending entries, if the map is empty + if(mpPendingEntries != 0 && mpPendingEntries->size() == 0) + { + BOX_TRACE("Deleting mpPendingEntries from dir ID " << + BOX_FORMAT_OBJECTID(mObjectID)); + delete mpPendingEntries; + mpPendingEntries = 0; + } + + // Do directories + for(std::vector<std::string>::const_iterator d = rDirs.begin(); + d != rDirs.end(); ++d) + { + // Send keep-alive message if needed + rContext.DoKeepAlive(); + + // Get the local filename + std::string dirname(MakeFullPath(rLocalPath, *d)); + std::string nonVssDirPath = ConvertVssPathToRealPath(dirname, + rBackupLocation); + + // See if it's in the listing (if we have one) + BackupStoreFilenameClear storeFilename(*d); + BackupStoreDirectory::Entry *en = 0; + if(pDirOnStore != 0) + { + DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*d)); + if(i != decryptedEntries.end()) + { + en = i->second; + } + } + + // Check that the entry which might have been found is in fact a directory + if((en != 0) && !(en->IsDir())) + { + // Entry exists, but is not a directory. Bad. + // Get rid of it. + BackupProtocolCallable &connection(rContext.GetConnection()); + connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); + + std::string filenameClear = DecryptFilename(en, + rRemotePath); + rNotifier.NotifyFileDeleted(en->GetObjectID(), + filenameClear); + + // Nothing found + en = 0; + } + + // Zero pointer in rEntriesLeftOver, if we have a pointer to zero + if(en != 0) + { + for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) + { + if(rEntriesLeftOver[l] == en) + { + rEntriesLeftOver[l] = 0; + break; + } + } + } + + // Flag for having created directory, so can optimise the + // recursive call not to read it again, because we know + // it's empty. + bool haveJustCreatedDirOnServer = false; + + // Next, see if it's in the list of sub directories + BackupClientDirectoryRecord *psubDirRecord = 0; + std::map<std::string, BackupClientDirectoryRecord *>::iterator + e(mSubDirectories.find(*d)); + + if(e != mSubDirectories.end()) + { + // In the list, just use this pointer + psubDirRecord = e->second; + } + else + { + // Note: if we have exceeded our storage limit, then + // we should not upload any more data, nor create any + // DirectoryRecord representing data that would have + // been uploaded. This step will be repeated when + // there is some space available. + bool doCreateDirectoryRecord = true; + + // Need to create the record. But do we need to create the directory on the server? + int64_t subDirObjectID = 0; + if(en != 0) + { + // No. Exists on the server, and we know about it from the listing. + subDirObjectID = en->GetObjectID(); + } + else if(rContext.StorageLimitExceeded()) + // know we've got a connection if we get this far, + // as dir will have been modified. + { + doCreateDirectoryRecord = false; + } + else + { + // Yes, creation required! + // It is known that it doesn't exist: + // + // if en == 0 and pDirOnStore == 0, then the + // directory has had an initial sync, and + // hasn't been modified (Really? then why + // are we here? TODO FIXME) + // so it has definitely been created already + // (so why create it again?) + // + // if en == 0 but pDirOnStore != 0, well... obviously it doesn't exist. + // + subDirObjectID = CreateRemoteDir(dirname, + nonVssDirPath, rRemotePath + "/" + *d, + storeFilename, &haveJustCreatedDirOnServer, + rParams); + doCreateDirectoryRecord = (subDirObjectID != 0); + } + + if (doCreateDirectoryRecord) + { + // New an object for this + psubDirRecord = new BackupClientDirectoryRecord(subDirObjectID, *d); + + // Store in list + try + { + mSubDirectories[*d] = psubDirRecord; + } + catch(...) + { + delete psubDirRecord; + psubDirRecord = 0; + throw; + } + } + } + + // ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded()); + // There's another possible reason now: the directory no longer + // existed when we finally got around to checking its + // attributes. See for example Brendon Baumgartner's reported + // error with Wordpress cache directories. + + if(psubDirRecord) + { + // Sync this sub directory too + psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, + rRemotePath + "/" + *d, rBackupLocation, + haveJustCreatedDirOnServer); + } + } + + // Delete everything which is on the store, but not on disc + for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) + { + if(rEntriesLeftOver[l] != 0) + { + BackupStoreDirectory::Entry *en = rEntriesLeftOver[l]; + + // These entries can't be deleted immediately, as it would prevent + // renaming and moving of objects working properly. So we add them + // to a list, which is actually deleted at the very end of the session. + // If there's an error during the process, it doesn't matter if things + // aren't actually deleted, as the whole state will be reset anyway. + BackupClientDeleteList &rdel(rContext.GetDeleteList()); + std::string filenameClear; + bool isCorruptFilename = false; + + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "scheduling that file for deletion"); + filenameClear = "<corrupt filename>"; + isCorruptFilename = true; + } + + std::string localName = MakeFullPath(rLocalPath, + filenameClear); + std::string nonVssLocalName = ConvertVssPathToRealPath(localName, + rBackupLocation); + + // Delete this entry -- file or directory? + if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) + { + // Set a pending deletion for the file + rdel.AddFileDelete(mObjectID, en->GetName(), + localName); + } + else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) + { + // Set as a pending deletion for the directory + rdel.AddDirectoryDelete(en->GetObjectID(), + localName); + + // If there's a directory record for it in + // the sub directory map, delete it now + BackupStoreFilenameClear dirname(en->GetName()); + std::map<std::string, BackupClientDirectoryRecord *>::iterator + e(mSubDirectories.find(filenameClear)); + if(e != mSubDirectories.end() && !isCorruptFilename) + { + // Carefully delete the entry from the map + BackupClientDirectoryRecord *rec = e->second; + mSubDirectories.erase(e); + delete rec; + + BOX_TRACE("Deleted directory record for " << + nonVssLocalName); + } + } + } + } + + // Return success flag (will be false if some files failed) + return allUpdatedSuccessfully; +} + +int64_t BackupClientDirectoryRecord::CreateRemoteDir(const std::string& localDirPath, + const std::string& nonVssDirPath, const std::string& remoteDirPath, + BackupStoreFilenameClear& storeFilename, bool* pHaveJustCreatedDirOnServer, + BackupClientDirectoryRecord::SyncParams &rParams) +{ + // Get attributes + box_time_t attrModTime = 0; + InodeRefType inodeNum = 0; + BackupClientFileAttributes attr; + *pHaveJustCreatedDirOnServer = false; + ProgressNotifier& rNotifier(rParams.mrContext.GetProgressNotifier()); + + try + { + attr.ReadAttributes(localDirPath, + true /* directories have zero mod times */, + 0 /* not interested in mod time */, + &attrModTime, 0 /* not file size */, + &inodeNum); + } + catch (BoxException &e) + { + // We used to try to recover from this, but we'd need an + // attributes block to upload to the server, so we have to + // skip creating the directory instead. + BOX_WARNING("Failed to read attributes of directory, " + "ignoring it for now: " << nonVssDirPath); + return 0; // no object ID + } + + // Check to see if the directory been renamed + // First, do we have a record in the ID map? + int64_t renameObjectID = 0, renameInDirectory = 0; + bool renameDir = false; + const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap()); + + if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) + { + // Look up on the server to get the name, to build the local filename + std::string localPotentialOldName; + bool isDir = false; + bool isCurrentVersion = false; + if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, + localPotentialOldName, isDir, isCurrentVersion)) + { + // Only interested if it's a directory + if(isDir && isCurrentVersion) + { + // Check that the object doesn't exist already + EMU_STRUCT_STAT st; + if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && + errno == ENOENT) + { + // Doesn't exist locally, but does exist + // on the server. Therefore we can + // safely rename it. + renameDir = true; + } + } + } + } + + // Get connection + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Don't do a check for storage limit exceeded here, because if we get to this + // stage, a connection will have been opened, and the status known, so the check + // in the else if(...) above will be correct. + + // Build attribute stream for sending + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); + + if(renameDir) + { + // Rename the existing directory on the server + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, + storeFilename); + + // Put the latest attributes on it + connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream); + + // Stop it being deleted later + BackupClientDeleteList &rdelList( + rParams.mrContext.GetDeleteList()); + rdelList.StopDirectoryDeletion(renameObjectID); + + // This is the ID for the renamed directory + return renameObjectID; + } + else + { + int64_t subDirObjectID = 0; // no object ID + + // Create a new directory + try + { + subDirObjectID = connection.QueryCreateDirectory( + mObjectID, attrModTime, storeFilename, + attrStream)->GetObjectID(); + // Flag as having done this for optimisation later + *pHaveJustCreatedDirOnServer = true; + } + catch(BoxException &e) + { + int type, subtype; + connection.GetLastError(type, subtype); + rNotifier.NotifyFileUploadServerError(this, nonVssDirPath, + type, subtype); + if(e.GetType() == ConnectionException::ExceptionType && + e.GetSubType() == ConnectionException::Protocol_UnexpectedReply && + type == BackupProtocolError::ErrorType && + subtype == BackupProtocolError::Err_StorageLimitExceeded) + { + // The hard limit was exceeded on the server, notify! + rParams.mrContext.SetStorageLimitExceeded(); + rParams.mrSysadminNotifier.NotifySysadmin( + SysadminNotifier::StoreFull); + } + else + { + throw; + } + } + + if(*pHaveJustCreatedDirOnServer) + { + rNotifier.NotifyDirectoryCreated(subDirObjectID, + nonVssDirPath, remoteDirPath); + } + + return subDirObjectID; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &, BackupStoreDirectory *, int64_t, const std::string &) +// Purpose: Called to resolve difficulties when a directory is found on the +// store where a file is to be uploaded. +// Created: 9/7/04 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile( + SyncParams &rParams, + BackupStoreDirectory* pDirOnStore, + BackupStoreDirectory::Entry* pEntry, + const std::string &rFilename) +{ + // First, delete the directory + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + connection.QueryDeleteDirectory(pEntry->GetObjectID()); + + BackupStoreFilenameClear clear(pEntry->GetName()); + rParams.mrContext.GetProgressNotifier().NotifyDirectoryDeleted( + pEntry->GetObjectID(), clear.GetClearFilename()); + + // Then, delete any directory record + std::map<std::string, BackupClientDirectoryRecord *>::iterator + e(mSubDirectories.find(rFilename)); + + if(e != mSubDirectories.end()) + { + // A record exists for this, remove it + BackupClientDirectoryRecord *psubDirRecord = e->second; + mSubDirectories.erase(e); + + // And delete the object + delete psubDirRecord; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::UploadFile( +// BackupClientDirectoryRecord::SyncParams &, +// const std::string &, +// const BackupStoreFilename &, +// int64_t, box_time_t, box_time_t, bool) +// Purpose: Private. Upload a file to the server. May send +// a patch instead of the whole thing +// Created: 20/1/04 +// +// -------------------------------------------------------------------------- +int64_t BackupClientDirectoryRecord::UploadFile( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string &rLocalPath, + const std::string &rNonVssFilePath, + const std::string &rRemotePath, + const BackupStoreFilenameClear &rStoreFilename, + int64_t FileSize, + box_time_t ModificationTime, + box_time_t AttributesHash, + bool NoPreviousVersionOnServer) +{ + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + + // Get the connection + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Info + int64_t objID = 0; + int64_t uploadedSize = -1; + + // Use a try block to catch store full errors + try + { + std::auto_ptr<BackupStoreFileEncodeStream> apStreamToUpload; + int64_t diffFromID = 0; + + // Might an old version be on the server, and is the file + // size over the diffing threshold? + if(!NoPreviousVersionOnServer && + FileSize >= rParams.mDiffingUploadSizeThreshold) + { + // YES -- try to do diff, if possible + // First, query the server to see if there's an old version available + std::auto_ptr<BackupProtocolSuccess> getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename)); + diffFromID = getBlockIndex->GetObjectID(); + + if(diffFromID != 0) + { + // Found an old version + + // Get the index + std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream()); + + // + // Diff the file + // + + rContext.ManageDiffProcess(); + + bool isCompletelyDifferent = false; + + apStreamToUpload = BackupStoreFile::EncodeFileDiff( + rLocalPath, + mObjectID, /* containing directory */ + rStoreFilename, diffFromID, *blockIndexStream, + connection.GetTimeout(), + &rContext, // DiffTimer implementation + 0 /* not interested in the modification time */, + &isCompletelyDifferent, + rParams.mpBackgroundTask); + + if(isCompletelyDifferent) + { + diffFromID = 0; + } + + rContext.UnManageDiffProcess(); + } + } + + if(apStreamToUpload.get()) + { + rNotifier.NotifyFileUploadingPatch(this, rNonVssFilePath, + apStreamToUpload->GetBytesToUpload()); + } + else // No patch upload, so do a normal upload + { + // below threshold or nothing to diff from, so upload whole + rNotifier.NotifyFileUploading(this, rNonVssFilePath); + + // Prepare to upload, getting a stream which will encode the file as we go along + apStreamToUpload = BackupStoreFile::EncodeFile( + rLocalPath, mObjectID, /* containing directory */ + rStoreFilename, NULL, &rParams, + &(rParams.mrRunStatusProvider), + rParams.mpBackgroundTask); + } + + rContext.SetNiceMode(true); + std::auto_ptr<IOStream> apWrappedStream; + + if(rParams.mMaxUploadRate > 0) + { + apWrappedStream.reset(new RateLimitingStream( + *apStreamToUpload, rParams.mMaxUploadRate)); + } + else + { + // Wrap the stream in *something*, so that + // QueryStoreFile() doesn't delete the original + // stream (upload object) and we can retrieve + // the byte counter. + apWrappedStream.reset(new BufferedStream( + *apStreamToUpload)); + } + + // Send to store + std::auto_ptr<BackupProtocolSuccess> stored( + connection.QueryStoreFile(mObjectID, ModificationTime, + AttributesHash, diffFromID, rStoreFilename, + apWrappedStream)); + + rContext.SetNiceMode(false); + + // Get object ID from the result + objID = stored->GetObjectID(); + uploadedSize = apStreamToUpload->GetTotalBytesSent(); + } + catch(BoxException &e) + { + rContext.UnManageDiffProcess(); + + if(e.GetType() == ConnectionException::ExceptionType && + e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) + { + // Check and see what error the protocol has, + // this is more useful to users than the exception. + int type, subtype; + if(connection.GetLastError(type, subtype)) + { + if(type == BackupProtocolError::ErrorType + && subtype == BackupProtocolError::Err_StorageLimitExceeded) + { + // The hard limit was exceeded on the server, notify! + rParams.mrSysadminNotifier.NotifySysadmin( + SysadminNotifier::StoreFull); + // return an error code instead of + // throwing an exception that we + // can't debug. + return 0; + } + rNotifier.NotifyFileUploadServerError(this, + rNonVssFilePath, type, subtype); + } + } + + // Send the error on it's way + throw; + } + + rNotifier.NotifyFileUploaded(this, rNonVssFilePath, FileSize, + uploadedSize, objID); + + // Return the new object ID of this file + return objID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( +// SyncParams &, const char *) +// Purpose: Sets the error state when there were problems +// reading an object from the filesystem. +// Created: 29/3/04 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string& rFilename) +{ + // Zero hash, so it gets synced properly next time round. + ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); + + // More detailed logging was already done by the caller, but if we + // have a read error reported, we need to be able to search the logs + // to find out which file it was, so we need to log a consistent and + // clear error message. + BOX_WARNING("Failed to backup file, see above for details: " << + rFilename); + + // Mark that an error occured in the parameters object + rParams.mReadErrorsOnFilesystemObjects = true; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncParams::SyncParams(BackupClientContext &) +// Purpose: Constructor +// Created: 8/3/04 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::SyncParams::SyncParams( + RunStatusProvider &rRunStatusProvider, + SysadminNotifier &rSysadminNotifier, + ProgressNotifier &rProgressNotifier, + BackupClientContext &rContext, + BackgroundTask *pBackgroundTask) +: mSyncPeriodStart(0), + mSyncPeriodEnd(0), + mMaxUploadWait(0), + mMaxFileTimeInFuture(99999999999999999LL), + mFileTrackingSizeThreshold(16*1024), + mDiffingUploadSizeThreshold(16*1024), + mpBackgroundTask(pBackgroundTask), + mrRunStatusProvider(rRunStatusProvider), + mrSysadminNotifier(rSysadminNotifier), + mrProgressNotifier(rProgressNotifier), + mrContext(rContext), + mReadErrorsOnFilesystemObjects(false), + mMaxUploadRate(0), + mUploadAfterThisTimeInTheFuture(99999999999999999LL), + mHaveLoggedWarningAboutFutureFileTimes(false) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncParams::~SyncParams() +// Purpose: Destructor +// Created: 8/3/04 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::SyncParams::~SyncParams() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::Deserialize(Archive & rArchive) +{ + // Make deletion recursive + DeleteSubDirectories(); + + // Delete maps + if(mpPendingEntries != 0) + { + delete mpPendingEntries; + mpPendingEntries = 0; + } + + // + // + // + rArchive.Read(mObjectID); + rArchive.Read(mSubDirName); + rArchive.Read(mInitialSyncDone); + rArchive.Read(mSyncDone); + + // + // + // + int64_t iCount = 0; + rArchive.Read(iCount); + + if (iCount != sizeof(mStateChecksum)/sizeof(mStateChecksum[0])) + { + // we have some kind of internal system representation change: throw for now + THROW_EXCEPTION(CommonException, Internal) + } + + for (int v = 0; v < iCount; v++) + { + // Load each checksum entry + rArchive.Read(mStateChecksum[v]); + } + + // + // + // + iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + // load each pending entry + mpPendingEntries = new std::map<std::string, box_time_t>; + if (!mpPendingEntries) + { + throw std::bad_alloc(); + } + + for (int v = 0; v < iCount; v++) + { + std::string strItem; + box_time_t btItem; + + rArchive.Read(strItem); + rArchive.Read(btItem); + (*mpPendingEntries)[strItem] = btItem; + } + } + + // + // + // + iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + for (int v = 0; v < iCount; v++) + { + std::string strItem; + rArchive.Read(strItem); + + BackupClientDirectoryRecord* pSubDirRecord = + new BackupClientDirectoryRecord(0, ""); + // will be deserialized anyway, give it id 0 for now + + if (!pSubDirRecord) + { + throw std::bad_alloc(); + } + + /***** RECURSE *****/ + pSubDirRecord->Deserialize(rArchive); + mSubDirectories[strItem] = pSubDirRecord; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mObjectID); + rArchive.Write(mSubDirName); + rArchive.Write(mInitialSyncDone); + rArchive.Write(mSyncDone); + + // + // + // + int64_t iCount = 0; + + // when reading back the archive, we will + // need to know how many items there are. + iCount = sizeof(mStateChecksum) / sizeof(mStateChecksum[0]); + rArchive.Write(iCount); + + for (int v = 0; v < iCount; v++) + { + rArchive.Write(mStateChecksum[v]); + } + + // + // + // + if (!mpPendingEntries) + { + iCount = 0; + rArchive.Write(iCount); + } + else + { + iCount = mpPendingEntries->size(); + rArchive.Write(iCount); + + for (std::map<std::string, box_time_t>::const_iterator + i = mpPendingEntries->begin(); + i != mpPendingEntries->end(); i++) + { + rArchive.Write(i->first); + rArchive.Write(i->second); + } + } + // + // + // + iCount = mSubDirectories.size(); + rArchive.Write(iCount); + + for (std::map<std::string, BackupClientDirectoryRecord*>::const_iterator + i = mSubDirectories.begin(); + i != mSubDirectories.end(); i++) + { + const BackupClientDirectoryRecord* pSubItem = i->second; + ASSERT(pSubItem); + + rArchive.Write(i->first); + pSubItem->Serialize(rArchive); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Location() +// Purpose: Constructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +Location::Location() +: mIDMapIndex(0) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::~Location() +// Purpose: Destructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +Location::~Location() +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, +// using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void Location::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mName); + rArchive.Write(mPath); + rArchive.Write(mIDMapIndex); + + // + // + // + if(!mapDirectoryRecord.get()) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mapDirectoryRecord->Serialize(rArchive); + } + + // + // + // + if(!mapExcludeFiles.get()) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mapExcludeFiles->Serialize(rArchive); + } + + // + // + // + if(!mapExcludeDirs.get()) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mapExcludeDirs->Serialize(rArchive); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void Location::Deserialize(Archive &rArchive) +{ + // + // + // + mapDirectoryRecord.reset(); + mapExcludeFiles.reset(); + mapExcludeDirs.reset(); + + // + // + // + rArchive.Read(mName); + rArchive.Read(mPath); + rArchive.Read(mIDMapIndex); + + // + // + // + int64_t aMagicMarker = 0; + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, ""); + if(!pSubRecord) + { + throw std::bad_alloc(); + } + + mapDirectoryRecord.reset(pSubRecord); + mapDirectoryRecord->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } + + // + // + // + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mapExcludeFiles.reset(new ExcludeList); + if(!mapExcludeFiles.get()) + { + throw std::bad_alloc(); + } + + mapExcludeFiles->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } + + // + // + // + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mapExcludeDirs.reset(new ExcludeList); + if(!mapExcludeDirs.get()) + { + throw std::bad_alloc(); + } + + mapExcludeDirs->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } +} diff --git a/lib/bbackupd/BackupClientDirectoryRecord.h b/lib/bbackupd/BackupClientDirectoryRecord.h new file mode 100644 index 00000000..865fc747 --- /dev/null +++ b/lib/bbackupd/BackupClientDirectoryRecord.h @@ -0,0 +1,244 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDirectoryRecord.h +// Purpose: Implementation of record about directory for backup client +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTDIRECTORYRECORD__H +#define BACKUPCLIENTDIRECTORYRECORD__H + +#include <string> +#include <map> +#include <memory> + +#include "BackgroundTask.h" +#include "BackupClientFileAttributes.h" +#include "BackupDaemonInterface.h" +#include "BackupStoreDirectory.h" +#include "BoxTime.h" +#include "MD5Digest.h" +#include "ReadLoggingStream.h" +#include "RunStatusProvider.h" + +#ifdef ENABLE_VSS +# include <comdef.h> +# include <Vss.h> +# include <VsWriter.h> +# include <VsBackup.h> +#endif + +class Archive; +class BackupClientContext; +class BackupDaemon; +class ExcludeList; +class Location; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientDirectoryRecord +// Purpose: Implementation of record about directory for backup client +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class BackupClientDirectoryRecord +{ +public: + BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName); + virtual ~BackupClientDirectoryRecord(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; +private: + BackupClientDirectoryRecord(const BackupClientDirectoryRecord &); +public: + + enum + { + UnknownDirectoryID = 0 + }; + + // -------------------------------------------------------------------------- + // + // Class + // Name: BackupClientDirectoryRecord::SyncParams + // Purpose: Holds parameters etc for directory syncing. Not passed as + // const, some parameters may be modified during sync. + // Created: 8/3/04 + // + // -------------------------------------------------------------------------- + class SyncParams : public ReadLoggingStream::Logger + { + public: + SyncParams( + RunStatusProvider &rRunStatusProvider, + SysadminNotifier &rSysadminNotifier, + ProgressNotifier &rProgressNotifier, + BackupClientContext &rContext, + BackgroundTask *pBackgroundTask); + ~SyncParams(); + private: + // No copying + SyncParams(const SyncParams&); + SyncParams &operator=(const SyncParams&); + + public: + // Data members are public, as accessors are not justified here + box_time_t mSyncPeriodStart; + box_time_t mSyncPeriodEnd; + box_time_t mMaxUploadWait; + box_time_t mMaxFileTimeInFuture; + int32_t mFileTrackingSizeThreshold; + int32_t mDiffingUploadSizeThreshold; + BackgroundTask *mpBackgroundTask; + RunStatusProvider &mrRunStatusProvider; + SysadminNotifier &mrSysadminNotifier; + ProgressNotifier &mrProgressNotifier; + BackupClientContext &mrContext; + bool mReadErrorsOnFilesystemObjects; + int64_t mMaxUploadRate; + + // Member variables modified by syncing process + box_time_t mUploadAfterThisTimeInTheFuture; + bool mHaveLoggedWarningAboutFutureFileTimes; + + bool StopRun() { return mrRunStatusProvider.StopRun(); } + void NotifySysadmin(SysadminNotifier::EventCode Event) + { + mrSysadminNotifier.NotifySysadmin(Event); + } + ProgressNotifier& GetProgressNotifier() const + { + return mrProgressNotifier; + } + + /* ReadLoggingStream::Logger implementation */ + virtual void Log(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset, + length, elapsed, finish); + } + virtual void Log(int64_t readSize, int64_t offset, + int64_t length) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset, + length); + } + virtual void Log(int64_t readSize, int64_t offset) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset); + } + }; + + void SyncDirectory(SyncParams &rParams, + int64_t ContainingDirectoryID, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + bool ThisDirHasJustBeenCreated = false); + + bool SyncDirectoryEntry(SyncParams &rParams, + ProgressNotifier& rNotifier, + const Location& rBackupLocation, + const std::string &rDirLocalPath, + MD5Digest& currentStateChecksum, + struct dirent *en, + EMU_STRUCT_STAT dir_st, + std::vector<std::string>& rDirs, + std::vector<std::string>& rFiles, + bool& rDownloadDirectoryRecordBecauseOfFutureFiles); + + std::string ConvertVssPathToRealPath(const std::string &rVssPath, + const Location& rBackupLocation); + + int64_t GetObjectID() const { return mObjectID; } + +private: + void DeleteSubDirectories(); + std::auto_ptr<BackupStoreDirectory> FetchDirectoryListing(SyncParams &rParams); + void UpdateAttributes(SyncParams &rParams, + BackupStoreDirectory *pDirOnStore, + const std::string &rLocalPath); +protected: // to allow tests to hook in before UpdateItems() runs + virtual bool UpdateItems(SyncParams &rParams, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + BackupStoreDirectory *pDirOnStore, + std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, + std::vector<std::string> &rFiles, + const std::vector<std::string> &rDirs); +private: + int64_t CreateRemoteDir(const std::string& localDirPath, + const std::string& nonVssDirPath, + const std::string& remoteDirPath, + BackupStoreFilenameClear& storeFilename, + bool* pHaveJustCreatedDirOnServer, + BackupClientDirectoryRecord::SyncParams &rParams); + int64_t UploadFile(SyncParams &rParams, + const std::string &rFilename, + const std::string &rNonVssFilePath, + const std::string &rRemotePath, + const BackupStoreFilenameClear &rStoreFilename, + int64_t FileSize, box_time_t ModificationTime, + box_time_t AttributesHash, bool NoPreviousVersionOnServer); + void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, + const std::string& rFilename); + void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, + BackupStoreDirectory* pDirOnStore, + BackupStoreDirectory::Entry* pEntry, + const std::string &rFilename); + std::string DecryptFilename(BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath); + std::string DecryptFilename(BackupStoreFilenameClear fn, + int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath); + + int64_t mObjectID; + std::string mSubDirName; + bool mInitialSyncDone; + bool mSyncDone; + + // Checksum of directory contents and attributes, used to detect changes + uint8_t mStateChecksum[MD5Digest::DigestLength]; + + std::map<std::string, box_time_t> *mpPendingEntries; + std::map<std::string, BackupClientDirectoryRecord *> mSubDirectories; + // mpPendingEntries is a pointer rather than simple a member + // variable, because most of the time it'll be empty. This would + // waste a lot of memory because of STL allocation policies. +}; + +class Location +{ +public: + Location(); + ~Location(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; +private: + Location(const Location &); // copy not allowed + Location &operator=(const Location &); +public: + std::string mName; + std::string mPath; + std::auto_ptr<BackupClientDirectoryRecord> mapDirectoryRecord; + std::auto_ptr<ExcludeList> mapExcludeFiles; + std::auto_ptr<ExcludeList> mapExcludeDirs; + int mIDMapIndex; + +#ifdef ENABLE_VSS + bool mIsSnapshotCreated; + VSS_ID mSnapshotVolumeId; + std::string mSnapshotPath; +#endif +}; + +#endif // BACKUPCLIENTDIRECTORYRECORD__H + + diff --git a/lib/bbackupd/BackupClientInodeToIDMap.cpp b/lib/bbackupd/BackupClientInodeToIDMap.cpp new file mode 100644 index 00000000..6eaf7394 --- /dev/null +++ b/lib/bbackupd/BackupClientInodeToIDMap.cpp @@ -0,0 +1,320 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientInodeToIDMap.cpp +// Purpose: Map of inode numbers to file IDs on the store +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> +#include <depot.h> + +#define BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION +#include "BackupClientInodeToIDMap.h" +#undef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION + +#include "Archive.h" +#include "BackupStoreException.h" +#include "CollectInBufferStream.h" +#include "MemBlockStream.h" +#include "autogen_CommonException.h" + +#include "MemLeakFindOn.h" + +#define BOX_DBM_INODE_DB_VERSION_KEY "BackupClientInodeToIDMap.Version" +#define BOX_DBM_INODE_DB_VERSION_CURRENT 2 + +#define BOX_DBM_MESSAGE(stuff) stuff << " (qdbm): " << dperrmsg(dpecode) + +#define BOX_LOG_DBM_ERROR(stuff) \ + BOX_ERROR(BOX_DBM_MESSAGE(stuff)) + +#define THROW_DBM_ERROR(message, filename, exception, subtype) \ + BOX_LOG_DBM_ERROR(message << ": " << filename); \ + THROW_EXCEPTION_MESSAGE(exception, subtype, \ + BOX_DBM_MESSAGE(message << ": " << filename)); + +#define ASSERT_DBM_OK(operation, message, filename, exception, subtype) \ + if(!(operation)) \ + { \ + THROW_DBM_ERROR(message, filename, exception, subtype); \ + } + +#define ASSERT_DBM_OPEN() \ + if(mpDepot == 0) \ + { \ + THROW_EXCEPTION_MESSAGE(BackupStoreException, InodeMapNotOpen, \ + "Inode database not open"); \ + } + +#define ASSERT_DBM_CLOSED() \ + if(mpDepot != 0) \ + { \ + THROW_EXCEPTION_MESSAGE(CommonException, Internal, \ + "Inode database already open: " << mFilename); \ + } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::BackupClientInodeToIDMap() +// Purpose: Constructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +BackupClientInodeToIDMap::BackupClientInodeToIDMap() + : mReadOnly(true), + mEmpty(false), + mpDepot(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::~BackupClientInodeToIDMap() +// Purpose: Destructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +BackupClientInodeToIDMap::~BackupClientInodeToIDMap() +{ + if(mpDepot != 0) + { + Close(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::Open(const char *, bool, bool) +// Purpose: Open the database map, creating a file on disc to store everything +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::Open(const char *Filename, bool ReadOnly, + bool CreateNew) +{ + mFilename = Filename; + + // Correct arguments? + ASSERT(!(CreateNew && ReadOnly)); + + // Correct usage? + ASSERT_DBM_CLOSED(); + ASSERT(!mEmpty); + + // Open the database file + int mode = ReadOnly ? DP_OREADER : DP_OWRITER; + if(CreateNew) + { + mode |= DP_OCREAT; + } + + mpDepot = dpopen(Filename, mode, 0); + + if(!mpDepot) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + BOX_DBM_MESSAGE("Failed to open inode database: " << + mFilename)); + } + + const char* version_key = BOX_DBM_INODE_DB_VERSION_KEY; + int32_t version = 0; + + if(CreateNew) + { + version = BOX_DBM_INODE_DB_VERSION_CURRENT; + + int ret = dpput(mpDepot, version_key, strlen(version_key), + (char *)(&version), sizeof(version), DP_DKEEP); + + if(!ret) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + BOX_DBM_MESSAGE("Failed to write version number to inode " + "database: " << mFilename)); + } + } + else + { + int ret = dpgetwb(mpDepot, version_key, strlen(version_key), 0, + sizeof(version), (char *)(&version)); + + if(ret == -1) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Missing version number in inode database. Perhaps it " + "needs to be recreated: " << mFilename); + } + + if(ret != sizeof(version)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Wrong size version number in inode database: expected " + << sizeof(version) << " bytes but found " << ret); + } + + if(version != BOX_DBM_INODE_DB_VERSION_CURRENT) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Wrong version number in inode database: expected " << + BOX_DBM_INODE_DB_VERSION_CURRENT << " but found " << + version << ". Perhaps it needs to be recreated: " << + mFilename); + } + + // By this point the version number has been checked and is OK. + } + + // Read only flag + mReadOnly = ReadOnly; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::OpenEmpty() +// Purpose: 'Open' this map. Not associated with a disc file. +// Useful for when a map is required, but is against +// an empty file on disc which shouldn't be created. +// Implies read only. +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::OpenEmpty() +{ + ASSERT_DBM_CLOSED(); + ASSERT(mpDepot == 0); + mEmpty = true; + mReadOnly = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::Close() +// Purpose: Close the database file +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::Close() +{ + ASSERT_DBM_OPEN(); + ASSERT_DBM_OK(dpclose(mpDepot), "Failed to close inode database", + mFilename, BackupStoreException, BerkelyDBFailure); + mpDepot = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::AddToMap(InodeRefType, +// int64_t, int64_t) +// Purpose: Adds an entry to the map. Overwrites any existing +// entry. +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, + int64_t InDirectory, const std::string& LocalPath) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, InodeMapIsReadOnly); + } + + if(mpDepot == 0) + { + THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); + } + + ASSERT_DBM_OPEN(); + + // Setup structures + CollectInBufferStream buf; + Archive arc(buf, IOStream::TimeOutInfinite); + arc.WriteExact((uint64_t)ObjectID); + arc.WriteExact((uint64_t)InDirectory); + arc.Write(LocalPath); + buf.SetForReading(); + + ASSERT_DBM_OK(dpput(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), + (const char *)buf.GetBuffer(), buf.GetSize(), DP_DOVER), + "Failed to add record to inode database", mFilename, + BackupStoreException, BerkelyDBFailure); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::Lookup(InodeRefType, +// int64_t &, int64_t &) const +// Purpose: Looks up an inode in the map, returning true if it +// exists, and the object ids of it and the directory +// it's in the reference arguments. +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, + int64_t &rInDirectoryOut, std::string* pLocalPathOut) const +{ + if(mEmpty) + { + // Map is empty + return false; + } + + if(mpDepot == 0) + { + THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); + } + + ASSERT_DBM_OPEN(); + int size; + char* data = dpget(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), + 0, -1, &size); + if(data == NULL) + { + // key not in file + return false; + } + + // Free data automatically when the guard goes out of scope. + MemoryBlockGuard<char *> guard(data); + MemBlockStream stream(data, size); + Archive arc(stream, IOStream::TimeOutInfinite); + + // Return data + try + { + arc.Read(rObjectIDOut); + arc.Read(rInDirectoryOut); + if(pLocalPathOut) + { + arc.Read(*pLocalPathOut); + } + } + catch(CommonException &e) + { + if(e.GetSubType() == CommonException::ArchiveBlockIncompleteRead) + { + THROW_FILE_ERROR("Failed to lookup record in inode database: " + << InodeRef << ": not enough data in record", mFilename, + BackupStoreException, BerkelyDBFailure); + // Need to throw precisely that exception to ensure that the + // invalid database is deleted, so that we don't hit the same + // error next time. + } + + throw; + } + + // Found + return true; +} diff --git a/lib/bbackupd/BackupClientInodeToIDMap.h b/lib/bbackupd/BackupClientInodeToIDMap.h new file mode 100644 index 00000000..4bb1e085 --- /dev/null +++ b/lib/bbackupd/BackupClientInodeToIDMap.h @@ -0,0 +1,59 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientInodeToIDMap.h +// Purpose: Map of inode numbers to file IDs on the store +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTINODETOIDMAP_H +#define BACKUPCLIENTINODETOIDMAP_H + +#include <sys/types.h> + +#include <map> +#include <utility> + +// avoid having to include the DB files when not necessary +#ifndef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION + class DEPOT; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientInodeToIDMap +// Purpose: Map of inode numbers to file IDs on the store +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +class BackupClientInodeToIDMap +{ +public: + BackupClientInodeToIDMap(); + ~BackupClientInodeToIDMap(); +private: + BackupClientInodeToIDMap(const BackupClientInodeToIDMap &rToCopy); // not allowed +public: + + void Open(const char *Filename, bool ReadOnly, bool CreateNew); + void OpenEmpty(); + + void AddToMap(InodeRefType InodeRef, int64_t ObjectID, + int64_t InDirectory, const std::string& LocalPath); + bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, + int64_t &rInDirectoryOut, std::string* pLocalPathOut = NULL) const; + + void Close(); + +private: + bool mReadOnly; + bool mEmpty; + std::string mFilename; + DEPOT *mpDepot; +}; + +#endif // BACKUPCLIENTINODETOIDMAP_H + + diff --git a/lib/bbackupd/BackupDaemon.cpp b/lib/bbackupd/BackupDaemon.cpp new file mode 100644 index 00000000..996c1919 --- /dev/null +++ b/lib/bbackupd/BackupDaemon.cpp @@ -0,0 +1,3646 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemon.cpp +// Purpose: Backup daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif +#ifdef HAVE_SIGNAL_H + #include <signal.h> +#endif +#ifdef HAVE_SYS_PARAM_H + #include <sys/param.h> +#endif +#ifdef HAVE_SYS_WAIT_H + #include <sys/wait.h> +#endif +#ifdef HAVE_SYS_MOUNT_H + #include <sys/mount.h> +#endif +#ifdef HAVE_MNTENT_H + #include <mntent.h> +#endif +#ifdef HAVE_SYS_MNTTAB_H + #include <cstdio> + #include <sys/mnttab.h> +#endif +#ifdef HAVE_PROCESS_H + #include <process.h> +#endif + +#include <iostream> +#include <set> +#include <sstream> + +#include "Configuration.h" +#include "IOStream.h" +#include "MemBlockStream.h" +#include "CommonException.h" +#include "BoxPortsAndFiles.h" + +#include "SSLLib.h" + +#include "autogen_BackupProtocol.h" +#include "autogen_ClientException.h" +#include "autogen_CommonException.h" +#include "autogen_ConversionException.h" +#include "Archive.h" +#include "BackupClientContext.h" +#include "BackupClientCryptoKeys.h" +#include "BackupClientDirectoryRecord.h" +#include "BackupClientFileAttributes.h" +#include "BackupClientInodeToIDMap.h" +#include "BackupClientMakeExcludeList.h" +#include "BackupConstants.h" +#include "BackupDaemon.h" +#include "BackupDaemonConfigVerify.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFilenameClear.h" +#include "BannerText.h" +#include "Conversion.h" +#include "ExcludeList.h" +#include "FileStream.h" +#include "IOStreamGetLine.h" +#include "LocalProcessStream.h" +#include "Logging.h" +#include "Random.h" +#include "Timer.h" +#include "Utils.h" + +#ifdef WIN32 + #include "Win32ServiceFunctions.h" + #include "Win32BackupService.h" + + extern Win32BackupService* gpDaemonService; + +# ifdef ENABLE_VSS +# include <comdef.h> +# include <Vss.h> +# include <VsWriter.h> +# include <VsBackup.h> + + // http://www.flounder.com/cstring.htm + std::string GetMsgForHresult(HRESULT hr) + { + std::ostringstream buf; + + if(hr == VSS_S_ASYNC_CANCELLED) + { + buf << "VSS async operation cancelled"; + } + else if(hr == VSS_S_ASYNC_FINISHED) + { + buf << "VSS async operation finished"; + } + else if(hr == VSS_S_ASYNC_PENDING) + { + buf << "VSS async operation pending"; + } + else + { + buf << _com_error(hr).ErrorMessage(); + } + + buf << " (" << BOX_FORMAT_HEX32(hr) << ")"; + return buf.str(); + } + + std::string WideStringToString(WCHAR *buf) + { + if (buf == NULL) + { + return "(null)"; + } + + char* pStr = ConvertFromWideString(buf, CP_UTF8); + + if(pStr == NULL) + { + return "(conversion failed)"; + } + + std::string result(pStr); + free(pStr); + return result; + } + + std::string GuidToString(GUID guid) + { + wchar_t buf[64]; + StringFromGUID2(guid, buf, sizeof(buf)); + return WideStringToString(buf); + } + + std::string BstrToString(const BSTR arg) + { + if(arg == NULL) + { + return std::string("(null)"); + } + else + { + // Extract the *long* before where the arg points to + long len = ((long *)arg)[-1] / 2; + std::wstring wstr((WCHAR *)arg, len); + std::string str; + if(!ConvertFromWideString(wstr, &str, CP_UTF8)) + { + throw std::exception("string conversion failed"); + } + return str; + } + } +# endif + + // Mutex support by Achim: see https://www.boxbackup.org/ticket/67 + + // Creates the two mutexes checked for by the installer/uninstaller to + // see if the program is still running. One of the mutexes is created + // in the global name space (which makes it possible to access the + // mutex across user sessions in Windows XP); the other is created in + // the session name space (because versions of Windows NT prior to + // 4.0 TSE don't have a global name space and don't support the + // 'Global\' prefix). + + void CreateMutexes(const std::string& rName) + { + SECURITY_DESCRIPTOR SecurityDesc; + SECURITY_ATTRIBUTES SecurityAttr; + + /* By default on Windows NT, created mutexes are accessible only by the user + running the process. We need our mutexes to be accessible to all users, so + that the mutex detection can work across user sessions in Windows XP. To + do this we use a security descriptor with a null DACL. + */ + + InitializeSecurityDescriptor(&SecurityDesc, SECURITY_DESCRIPTOR_REVISION); + SetSecurityDescriptorDacl(&SecurityDesc, TRUE, NULL, FALSE); + SecurityAttr.nLength = sizeof(SecurityAttr); + SecurityAttr.lpSecurityDescriptor = &SecurityDesc; + SecurityAttr.bInheritHandle = FALSE; + // We don't care if this succeeds or fails. It's only used to + // ensure that an installer can detect if Box Backup is running. + CreateMutexA(&SecurityAttr, FALSE, rName.c_str()); + std::string global_name = "Global\\" + rName; + CreateMutexA(&SecurityAttr, FALSE, global_name.c_str()); + } +#endif + +#include "MemLeakFindOn.h" + +static const time_t MAX_SLEEP_TIME = 1024; + +// Make the actual sync period have a little bit of extra time, up to a 64th of the main sync period. +// This prevents repetative cycles of load on the server +#define SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY 6 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::BackupDaemon() +// Purpose: constructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupDaemon::BackupDaemon() + : mState(BackupDaemon::State_Initialising), + mDeleteRedundantLocationsAfter(0), + mLastNotifiedEvent(SysadminNotifier::MAX), + mDeleteUnusedRootDirEntriesAfter(0), + mClientStoreMarker(BackupClientContext::ClientStoreMarker_NotKnown), + mStorageLimitExceeded(false), + mReadErrorsOnFilesystemObjects(false), + mLastSyncTime(0), + mNextSyncTime(0), + mCurrentSyncStartTime(0), + mUpdateStoreInterval(0), + mDeleteStoreObjectInfoFile(false), + mDoSyncForcedByPreviousSyncError(false), + mNumFilesUploaded(-1), + mNumDirsCreated(-1), + mMaxBandwidthFromSyncAllowScript(0), + mLogAllFileAccess(false), + mpProgressNotifier(this), + mpLocationResolver(this), + mpRunStatusProvider(this), + mpSysadminNotifier(this), + mapCommandSocketPollTimer(NULL) + #ifdef WIN32 + , mInstallService(false), + mRemoveService(false), + mRunAsService(false), + mServiceName("bbackupd") + #endif +#ifdef ENABLE_VSS + , mpVssBackupComponents(NULL) +#endif +{ + // Only ever one instance of a daemon + SSLLib::Initialise(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::~BackupDaemon() +// Purpose: Destructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupDaemon::~BackupDaemon() +{ + DeleteAllLocations(); + DeleteAllIDMaps(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DaemonName() +// Purpose: Get name of daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +const char *BackupDaemon::DaemonName() const +{ + return "bbackupd"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DaemonBanner() +// Purpose: Daemon banner +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +std::string BackupDaemon::DaemonBanner() const +{ + return BANNER_TEXT("Backup Client"); +} + +void BackupDaemon::Usage() +{ + this->Daemon::Usage(); + +#ifdef WIN32 + std::cout << + " -s Run as a Windows Service, for internal use only\n" + " -i Install Windows Service (you may want to specify a config file)\n" + " -r Remove Windows Service\n" + " -S <name> Service name for -i and -r options\n"; +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::GetConfigVerify() +// Purpose: Get configuration specification +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify *BackupDaemon::GetConfigVerify() const +{ + // Defined elsewhere + return &BackupDaemonConfigVerify; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetupInInitialProcess() +// Purpose: Platforms with non-checkable credentials on +// local sockets only. +// Prints a warning if the command socket is used. +// Created: 25/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetupInInitialProcess() +{ + const Configuration& config(GetConfiguration()); + + // These keys may or may not be required, depending on the configured + // store type (e.g. not when using Amazon S3 stores), so they can't be + // verified by BackupDaemonConfigVerify. + std::vector<std::string> requiredKeys; + requiredKeys.push_back("StoreHostname"); + requiredKeys.push_back("AccountNumber"); + requiredKeys.push_back("CertificateFile"); + requiredKeys.push_back("PrivateKeyFile"); + requiredKeys.push_back("TrustedCAsFile"); + bool missingRequiredKeys = false; + + for(std::vector<std::string>::const_iterator i = requiredKeys.begin(); + i != requiredKeys.end(); i++) + { + if(!config.KeyExists(*i)) + { + BOX_ERROR("Missing required configuration key: " << *i); + missingRequiredKeys = true; + } + } + + if(missingRequiredKeys) + { + THROW_EXCEPTION_MESSAGE(CommonException, InvalidConfiguration, + "Some required configuration keys are missing in " << + GetConfigFileName()); + } + +#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET + // Print a warning on this platform if the CommandSocket is used. + if(GetConfiguration().KeyExists("CommandSocket")) + { + BOX_WARNING( + "==============================================================================\n" + "SECURITY WARNING: This platform cannot check the credentials of connections to\n" + "the command socket. This is a potential DoS security problem.\n" + "Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl\n" + "is not used.\n" + "==============================================================================\n" + ); + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteAllLocations() +// Purpose: Deletes all records stored +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteAllLocations() +{ + // Run through, and delete everything + for(Locations::iterator i = mLocations.begin(); + i != mLocations.end(); ++i) + { + delete *i; + } + + // Clear the contents of the map, so it is empty + mLocations.clear(); + + // And delete everything from the associated mount vector + mIDMapMounts.clear(); +} + +#ifdef WIN32 +std::string BackupDaemon::GetOptionString() +{ + std::string oldOpts = this->Daemon::GetOptionString(); + ASSERT(oldOpts.find("s") == std::string::npos); + ASSERT(oldOpts.find("S") == std::string::npos); + ASSERT(oldOpts.find("i") == std::string::npos); + ASSERT(oldOpts.find("r") == std::string::npos); + return oldOpts + "sS:ir"; +} + +int BackupDaemon::ProcessOption(signed int option) +{ + switch(option) + { + case 's': + { + mRunAsService = true; + return 0; + } + + case 'S': + { + mServiceName = optarg; + Logging::SetProgramName(mServiceName); + return 0; + } + + case 'i': + { + mInstallService = true; + return 0; + } + + case 'r': + { + mRemoveService = true; + return 0; + } + + default: + { + return this->Daemon::ProcessOption(option); + } + } +} + +int BackupDaemon::Main(const std::string &rConfigFileName) +{ + if (mInstallService) + { + return InstallService(rConfigFileName.c_str(), mServiceName); + } + + if (mRemoveService) + { + return RemoveService(mServiceName); + } + +#ifdef ENABLE_VSS + HRESULT result = CoInitialize(NULL); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to initialize COM: " << + GetMsgForHresult(result)); + return 1; + } +#endif + + CreateMutexes("__boxbackup_mutex__"); + + int returnCode; + + if (mRunAsService) + { + // We will be called reentrantly by the Service Control + // Manager, and we had better not call OurService again! + mRunAsService = false; + + BOX_INFO("Box Backup service starting"); + returnCode = OurService(rConfigFileName.c_str()); + BOX_INFO("Box Backup service shut down"); + } + else + { + returnCode = this->Daemon::Main(rConfigFileName); + } + + return returnCode; +} +#endif // WIN32 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Run() +// Purpose: Run function for daemon +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Run() +{ + // initialise global timer mechanism + Timers::Init(); + + mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, + "CommandSocketPollTimer")); + + #ifndef WIN32 + // Ignore SIGPIPE so that if a command connection is broken, + // the daemon doesn't terminate. + ::signal(SIGPIPE, SIG_IGN); + #endif + + // Create a command socket? + const Configuration &conf(GetConfiguration()); + if(conf.KeyExists("CommandSocket")) + { + // Yes, create a local UNIX socket + mapCommandSocketInfo.reset(new CommandSocketInfo); + const char *socketName = + conf.GetKeyValue("CommandSocket").c_str(); + #ifdef WIN32 + mapCommandSocketInfo->mListeningSocket.Listen( + socketName); + #else + ::unlink(socketName); + mapCommandSocketInfo->mListeningSocket.Listen( + Socket::TypeUNIX, socketName); + #endif + } + + // Handle things nicely on exceptions + try + { + Run2(); + } + catch(...) + { + try + { + mapCommandSocketInfo.reset(); + } + catch(std::exception &e) + { + BOX_WARNING("Internal error while closing command " + "socket after another exception, ignored: " << + e.what()); + } + catch(...) + { + BOX_WARNING("Error closing command socket after " + "exception, ignored."); + } + + mapCommandSocketPollTimer.reset(); + Timers::Cleanup(); + + throw; + } + + // Clean up + mapCommandSocketInfo.reset(); + mapCommandSocketPollTimer.reset(); + Timers::Cleanup(); +} + +void BackupDaemon::InitCrypto() +{ + // Read in the certificates creating a TLS context + const Configuration &conf(GetConfiguration()); + std::string certFile(conf.GetKeyValue("CertificateFile")); + std::string keyFile(conf.GetKeyValue("PrivateKeyFile")); + std::string caFile(conf.GetKeyValue("TrustedCAsFile")); + mTlsContext.Initialise(false /* as client */, certFile.c_str(), + keyFile.c_str(), caFile.c_str()); + + // Set up the keys for various things + BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile")); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Run2() +// Purpose: Run function for daemon (second stage) +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Run2() +{ + InitCrypto(); + + const Configuration &conf(GetConfiguration()); + + // How often to connect to the store (approximate) + mUpdateStoreInterval = SecondsToBoxTime( + conf.GetKeyValueInt("UpdateStoreInterval")); + mBackupErrorDelay = conf.GetKeyValueInt("BackupErrorDelay"); + + // But are we connecting automatically? + bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup"); + + // When the next sync should take place -- which is ASAP + mNextSyncTime = 0; + + // When the last sync started (only updated if the store was not full when the sync ended) + mLastSyncTime = 0; + + // -------------------------------------------------------------------------------------------- + + mDeleteStoreObjectInfoFile = DeserializeStoreObjectInfo(mLastSyncTime, + mNextSyncTime); + + // -------------------------------------------------------------------------------------------- + + + // Set state + SetState(State_Idle); + + mDoSyncForcedByPreviousSyncError = false; + + // Loop around doing backups + do + { + // Flags used below + bool storageLimitExceeded = false; + bool doSync = false; + bool mDoSyncForcedByCommand = false; + + // Check whether we should be stopping, and if so, + // don't hang around waiting on the command socket. + if(StopRun()) + { + BOX_INFO("Skipping command socket polling " + "due to shutdown request"); + break; + } + + // Is a delay necessary? + box_time_t currentTime = GetCurrentBoxTime(); + box_time_t requiredDelay = (mNextSyncTime < currentTime) + ? (0) : (mNextSyncTime - currentTime); + mNextSyncTime = currentTime + requiredDelay; + + if (mDoSyncForcedByPreviousSyncError) + { + BOX_INFO("Last backup was not successful, " + "next one starting at " << + FormatTime(mNextSyncTime, false, true)); + } + else if (automaticBackup) + { + BOX_INFO("Automatic backups are enabled, " + "next one starting at " << + FormatTime(mNextSyncTime, false, true)); + } + else + { + BOX_INFO("No automatic backups, waiting for " + "bbackupctl snapshot command"); + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } + + if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME)) + { + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } + + // Only delay if necessary + if(requiredDelay == 0) + { + // No sleep necessary, so don't listen on the command + // socket at all right now. + } + else if(mapCommandSocketInfo.get() != 0) + { + // A command socket exists, so sleep by waiting for a + // connection or command on it. + WaitOnCommandSocket(requiredDelay, doSync, + mDoSyncForcedByCommand); + } + else + { + // No command socket or connection, just do a normal + // sleep. + time_t sleepSeconds = + BoxTimeToSeconds(requiredDelay); + ::sleep((sleepSeconds <= 0) + ? 1 : sleepSeconds); + } + + // We have now slept, so if automaticBackup is enabled then + // it's time for a backup now. + + if(StopRun()) + { + BOX_INFO("Stopping idle loop due to shutdown request"); + break; + } + else if(doSync) + { + BOX_INFO("Starting a backup immediately due to " + "bbackupctl sync command"); + } + else if(GetCurrentBoxTime() < mNextSyncTime) + { + BOX_TRACE("Deadline not reached, sleeping again"); + continue; + } + else if(mDoSyncForcedByPreviousSyncError) + { + BOX_INFO("Last backup was not successful, next one " + "starting now"); + } + else if(!automaticBackup) + { + BOX_TRACE("Sleeping again because automatic backups " + "are not enabled"); + continue; + } + else + { + BOX_INFO("Automatic backups are enabled, next one " + "starting now"); + } + + // If we pass this point, or exit the loop, we should have + // logged something at INFO level or higher to explain why. + + // Use a script to see if sync is allowed now? + if(mDoSyncForcedByCommand) + { + BOX_INFO("Skipping SyncAllowScript due to bbackupctl " + "force-sync command"); + } + else + { + int d = UseScriptToSeeIfSyncAllowed(); + if(d > 0) + { + // Script has asked for a delay + mNextSyncTime = GetCurrentBoxTime() + + SecondsToBoxTime(d); + BOX_INFO("Impending backup stopped by " + "SyncAllowScript, next attempt " + "scheduled for " << + FormatTime(mNextSyncTime, false)); + continue; + } + } + + mCurrentSyncStartTime = GetCurrentBoxTime(); + RunSyncNowWithExceptionHandling(); + + // Set state + SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle); + } + while(!StopRun()); + + // Make sure we have a clean start next time round (if restart) + DeleteAllLocations(); + DeleteAllIDMaps(); +} + +std::auto_ptr<BackupClientContext> BackupDaemon::RunSyncNowWithExceptionHandling() +{ + bool errorOccurred = false; + int errorCode = 0, errorSubCode = 0; + std::string errorString = "unknown"; + + try + { + OnBackupStart(); + // Do sync + RunSyncNow(); + } + catch(BoxException &e) + { + errorOccurred = true; + errorString = e.what(); + errorCode = e.GetType(); + errorSubCode = e.GetSubType(); + } + catch(std::exception &e) + { + BOX_ERROR("Internal error during backup run: " << e.what()); + errorOccurred = true; + errorString = e.what(); + } + catch(...) + { + // TODO: better handling of exceptions here... + // need to be very careful + errorOccurred = true; + } + + // do not retry immediately without a good reason + mDoSyncForcedByPreviousSyncError = false; + + // Is it a berkely db failure? + bool isBerkelyDbFailure = false; + + // Notify system administrator about the final state of the backup + if(errorOccurred) + { + if (errorCode == BackupStoreException::ExceptionType + && errorSubCode == BackupStoreException::BerkelyDBFailure) + { + isBerkelyDbFailure = true; + } + + if(isBerkelyDbFailure) + { + // Delete corrupt files + DeleteCorruptBerkelyDbFiles(); + } + + ResetCachedState(); + + // Handle restart? + if(StopRun()) + { + BOX_NOTICE("Exception (" << errorCode << "/" << + errorSubCode << ") due to signal"); + OnBackupFinish(); + return mapClientContext; // releases mapClientContext + } + + NotifySysadmin(SysadminNotifier::BackupError); + + // If the Berkely db files get corrupted, + // delete them and try again immediately. + if(isBerkelyDbFailure) + { + BOX_ERROR("Berkely db inode map files corrupted, " + "deleting and restarting scan. Renamed files " + "and directories will not be tracked until " + "after this scan."); + ::sleep(1); + } + else + { + // Not restart/terminate, pause and retry + // Notify administrator + SetState(State_Error); + BOX_ERROR("Exception caught (" << errorString << + " " << errorCode << "/" << errorSubCode << + "), reset state and waiting to retry..."); + ::sleep(10); + mNextSyncTime = GetCurrentBoxTime() + + SecondsToBoxTime(mBackupErrorDelay) + + Random::RandomInt(mUpdateStoreInterval >> + SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); + } + } + + if(mReadErrorsOnFilesystemObjects) + { + NotifySysadmin(SysadminNotifier::ReadError); + } + + if(mStorageLimitExceeded) + { + NotifySysadmin(SysadminNotifier::StoreFull); + } + + if (!errorOccurred && !mReadErrorsOnFilesystemObjects && + !mStorageLimitExceeded) + { + NotifySysadmin(SysadminNotifier::BackupOK); + } + + // If we were retrying after an error, and this backup succeeded, + // then now would be a good time to stop :-) + mDoSyncForcedByPreviousSyncError = errorOccurred && !isBerkelyDbFailure; + + OnBackupFinish(); + return mapClientContext; // releases mapClientContext +} + +void BackupDaemon::ResetCachedState() +{ + // Clear state data + // Go back to beginning of time + mLastSyncTime = 0; + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything + DeleteAllLocations(); + DeleteAllIDMaps(); +} + +std::auto_ptr<BackupClientContext> BackupDaemon::GetNewContext +( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode +) +{ + std::auto_ptr<BackupClientContext> context(new BackupClientContext( + rResolver, rTLSContext, rHostname, Port, AccountNumber, + ExtendedLogging, ExtendedLogToFile, ExtendedLogFile, + rProgressNotifier, TcpNiceMode)); + return context; +} + +// Returns the BackupClientContext so that tests can use it to hold the +// connection open and prevent housekeeping from running. Otherwise don't use +// it, let it be destroyed and close the connection. +std::auto_ptr<BackupClientContext> BackupDaemon::RunSyncNow() +{ + // Delete the serialised store object file, + // so that we don't try to reload it after a + // partially completed backup + if(mDeleteStoreObjectInfoFile && !DeleteStoreObjectInfo()) + { + BOX_ERROR("Failed to delete the StoreObjectInfoFile, " + "backup cannot continue safely."); + THROW_EXCEPTION(ClientException, + FailedToDeleteStoreObjectInfoFile); + } + + // In case the backup throws an exception, + // we should not try to delete the store info + // object file again. + mDeleteStoreObjectInfoFile = false; + + const Configuration &conf(GetConfiguration()); + + std::auto_ptr<FileLogger> fileLogger; + + if (conf.KeyExists("LogFile")) + { + bool overwrite = false; + if (conf.KeyExists("LogFileOverwrite")) + { + overwrite = conf.GetKeyValueBool("LogFileOverwrite"); + } + + Log::Level level = Log::INFO; + if (conf.KeyExists("LogFileLevel")) + { + level = Logging::GetNamedLevel( + conf.GetKeyValue("LogFileLevel")); + } + + fileLogger.reset(new FileLogger(conf.GetKeyValue("LogFile"), + level, !overwrite)); + } + + std::string extendedLogFile; + if (conf.KeyExists("ExtendedLogFile")) + { + extendedLogFile = conf.GetKeyValue("ExtendedLogFile"); + } + + if (conf.KeyExists("LogAllFileAccess")) + { + mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess"); + } + + // Then create a client context object (don't + // just connect, as this may be unnecessary) + mapClientContext = GetNewContext( + *mpLocationResolver, + mTlsContext, + conf.GetKeyValue("StoreHostname"), + conf.GetKeyValueInt("StorePort"), + conf.GetKeyValueUint32("AccountNumber"), + conf.GetKeyValueBool("ExtendedLogging"), + conf.KeyExists("ExtendedLogFile"), + extendedLogFile, + *mpProgressNotifier, + conf.GetKeyValueBool("TcpNice") + ); + + // The minimum age a file needs to be before it will be + // considered for uploading + box_time_t minimumFileAge = SecondsToBoxTime( + conf.GetKeyValueInt("MinimumFileAge")); + + // The maximum time we'll wait to upload a file, regardless + // of how often it's modified + box_time_t maxUploadWait = SecondsToBoxTime( + conf.GetKeyValueInt("MaxUploadWait")); + // Adjust by subtracting the minimum file age, so is relative + // to sync period end in comparisons + if (maxUploadWait > minimumFileAge) + { + maxUploadWait -= minimumFileAge; + } + else + { + maxUploadWait = 0; + } + + // Calculate the sync period of files to examine + box_time_t syncPeriodStart = mLastSyncTime; + box_time_t syncPeriodEnd = GetCurrentBoxTime() - minimumFileAge; + + if(syncPeriodStart >= syncPeriodEnd && + syncPeriodStart - syncPeriodEnd < minimumFileAge) + { + // This can happen if we receive a force-sync command less + // than minimumFileAge after the last sync. Deal with it by + // moving back syncPeriodStart, which should not do any + // damage. + syncPeriodStart = syncPeriodEnd - SecondsToBoxTime(1); + } + + if(syncPeriodStart >= syncPeriodEnd) + { + BOX_ERROR("Invalid (negative) sync period: perhaps your clock " + "is going backwards? (" << syncPeriodStart << " to " << + syncPeriodEnd << ")"); + THROW_EXCEPTION(ClientException, ClockWentBackwards); + } + + // Check logic + ASSERT(syncPeriodEnd > syncPeriodStart); + // Paranoid check on sync times + if(syncPeriodStart >= syncPeriodEnd) + { + return mapClientContext; // releases mapClientContext + } + + // Adjust syncPeriodEnd to emulate snapshot behaviour properly + box_time_t syncPeriodEndExtended = syncPeriodEnd; + + // Using zero min file age? + if(minimumFileAge == 0) + { + // Add a year on to the end of the end time, + // to make sure we sync files which are + // modified after the scan run started. + // Of course, they may be eligible to be + // synced again the next time round, + // but this should be OK, because the changes + // only upload should upload no data. + syncPeriodEndExtended += SecondsToBoxTime( + (time_t)(356*24*3600)); + } + + // Set up the sync parameters + BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider, + *mpSysadminNotifier, *mpProgressNotifier, *mapClientContext, this); + params.mSyncPeriodStart = syncPeriodStart; + params.mSyncPeriodEnd = syncPeriodEndExtended; + // use potentially extended end time + params.mMaxUploadWait = maxUploadWait; + params.mFileTrackingSizeThreshold = + conf.GetKeyValueInt("FileTrackingSizeThreshold"); + params.mDiffingUploadSizeThreshold = + conf.GetKeyValueInt("DiffingUploadSizeThreshold"); + params.mMaxFileTimeInFuture = + SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture")); + mNumFilesUploaded = 0; + mNumDirsCreated = 0; + + if(conf.KeyExists("MaxUploadRate")) + { + params.mMaxUploadRate = conf.GetKeyValueInt("MaxUploadRate"); + } + + if(mMaxBandwidthFromSyncAllowScript != 0) + { + params.mMaxUploadRate = mMaxBandwidthFromSyncAllowScript; + } + + mDeleteRedundantLocationsAfter = + conf.GetKeyValueInt("DeleteRedundantLocationsAfter"); + mStorageLimitExceeded = false; + mReadErrorsOnFilesystemObjects = false; + + // Setup various timings + int maximumDiffingTime = 600; + int keepAliveTime = 60; + + // max diffing time, keep-alive time + if(conf.KeyExists("MaximumDiffingTime")) + { + maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime"); + } + if(conf.KeyExists("KeepAliveTime")) + { + keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); + } + + mapClientContext->SetMaximumDiffingTime(maximumDiffingTime); + mapClientContext->SetKeepAliveTime(keepAliveTime); + + // Set store marker + mapClientContext->SetClientStoreMarker(mClientStoreMarker); + + // Set up the locations, if necessary -- need to do it here so we have + // a (potential) connection to use. + { + const Configuration &locations( + conf.GetSubConfiguration( + "BackupLocations")); + + // Make sure all the directory records + // are set up + SetupLocations(*mapClientContext, locations); + } + + mpProgressNotifier->NotifyIDMapsSetup(*mapClientContext); + + // Get some ID maps going + SetupIDMapsForSync(); + + // Delete any unused directories? + DeleteUnusedRootDirEntries(*mapClientContext); + +#ifdef ENABLE_VSS + CreateVssBackupComponents(); +#endif + + // Go through the records, syncing them + for(Locations::const_iterator + i(mLocations.begin()); + i != mLocations.end(); ++i) + { + // Set current and new ID map pointers + // in the context + mapClientContext->SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], + mNewIDMaps[(*i)->mIDMapIndex]); + + // Set exclude lists (context doesn't + // take ownership) + mapClientContext->SetExcludeLists( + (*i)->mapExcludeFiles.get(), + (*i)->mapExcludeDirs.get()); + + // Sync the directory + std::string locationPath = (*i)->mPath; +#ifdef ENABLE_VSS + if((*i)->mIsSnapshotCreated) + { + locationPath = (*i)->mSnapshotPath; + } +#endif + + (*i)->mapDirectoryRecord->SyncDirectory(params, + BackupProtocolListDirectory::RootDirectory, + locationPath, std::string("/") + (*i)->mName, **i); + + // Unset exclude lists (just in case) + mapClientContext->SetExcludeLists(0, 0); + } + + // Perform any deletions required -- these are + // delayed until the end to allow renaming to + // happen neatly. + mapClientContext->PerformDeletions(); + +#ifdef ENABLE_VSS + CleanupVssBackupComponents(); +#endif + + // Get the new store marker + mClientStoreMarker = mapClientContext->GetClientStoreMarker(); + mStorageLimitExceeded = mapClientContext->StorageLimitExceeded(); + mReadErrorsOnFilesystemObjects |= + params.mReadErrorsOnFilesystemObjects; + + if(!mStorageLimitExceeded) + { + // The start time of the next run is the end time of this + // run. This is only done if the storage limit wasn't + // exceeded (as things won't have been done properly if + // it was) + mLastSyncTime = syncPeriodEnd; + } + + // Commit the ID Maps + CommitIDMapsAfterSync(); + + // Calculate when the next sync run should be + mNextSyncTime = mCurrentSyncStartTime + + mUpdateStoreInterval + + Random::RandomInt(mUpdateStoreInterval >> + SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); + + // -------------------------------------------------------------------------------------------- + + // We had a successful backup, save the store + // info. If we save successfully, we must + // delete the file next time we start a backup + + mDeleteStoreObjectInfoFile = + SerializeStoreObjectInfo(mLastSyncTime, mNextSyncTime); + + // -------------------------------------------------------------------------------------------- + + return mapClientContext; // releases mapClientContext +} + +#ifdef ENABLE_VSS +bool BackupDaemon::WaitForAsync(IVssAsync *pAsync, + const std::string& description) +{ + BOX_INFO("VSS: waiting for " << description << " to complete"); + HRESULT result; + + do + { + result = pAsync->Wait(1000); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to wait for " << description << + " to complete: " << GetMsgForHresult(result)); + break; + } + + HRESULT result2; + result = pAsync->QueryStatus(&result2, NULL); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query " << description << + " status: " << GetMsgForHresult(result)); + break; + } + + result = result2; + BOX_INFO("VSS: " << description << " status: " << + GetMsgForHresult(result)); + } + while(result == VSS_S_ASYNC_PENDING); + + pAsync->Release(); + + return (result == VSS_S_ASYNC_FINISHED); +} + +#define CALL_MEMBER_FN(object, method) ((object).*(method)) + +bool BackupDaemon::CallAndWaitForAsync(AsyncMethod method, + const std::string& description) +{ + IVssAsync *pAsync; + HRESULT result = CALL_MEMBER_FN(*mpVssBackupComponents, method)(&pAsync); + if(result != S_OK) + { + BOX_ERROR("VSS: " << description << " failed: " << + GetMsgForHresult(result)); + return false; + } + + return WaitForAsync(pAsync, description); +} + +void BackupDaemon::CreateVssBackupComponents() +{ + std::map<char, VSS_ID> volumesIncluded; + + HRESULT result = ::CreateVssBackupComponents(&mpVssBackupComponents); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to create backup components: " << + GetMsgForHresult(result)); + return; + } + + result = mpVssBackupComponents->InitializeForBackup(NULL); + if(result != S_OK) + { + std::string message = GetMsgForHresult(result); + + if (result == VSS_E_UNEXPECTED) + { + message = "Check the Application Log for details, and ensure " + "that the Volume Shadow Copy, COM+ System Application, " + "and Distributed Transaction Coordinator services " + "are running"; + } + + BOX_ERROR("VSS: Failed to initialize for backup: " << message); + return; + } + + result = mpVssBackupComponents->SetContext(VSS_CTX_BACKUP); + if(result == E_NOTIMPL) + { + BOX_INFO("VSS: Failed to set context to VSS_CTX_BACKUP: " + "not implemented, probably Windows XP, ignored."); + } + else if(result != S_OK) + { + BOX_ERROR("VSS: Failed to set context to VSS_CTX_BACKUP: " << + GetMsgForHresult(result)); + return; + } + + result = mpVssBackupComponents->SetBackupState( + false, /* no components for now */ + true, /* might as well ask for a bootable backup */ + VSS_BT_FULL, + false /* what is Partial File Support? */); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to set backup state: " << + GetMsgForHresult(result)); + return; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterMetadata, + "GatherWriterMetadata()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + UINT writerCount; + result = mpVssBackupComponents->GetWriterMetadataCount(&writerCount); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get writer count: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + for(UINT iWriter = 0; iWriter < writerCount; iWriter++) + { + BOX_INFO("VSS: Getting metadata from writer " << iWriter); + VSS_ID writerInstance; + IVssExamineWriterMetadata* pMetadata; + result = mpVssBackupComponents->GetWriterMetadata(iWriter, + &writerInstance, &pMetadata); + if(result != S_OK) + { + BOX_ERROR("Failed to get VSS metadata from writer " << iWriter << + ": " << GetMsgForHresult(result)); + continue; + } + + UINT includeFiles, excludeFiles, numComponents; + result = pMetadata->GetFileCounts(&includeFiles, &excludeFiles, + &numComponents); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata file counts from " + "writer " << iWriter << ": " << + GetMsgForHresult(result)); + pMetadata->Release(); + continue; + } + + for(UINT iComponent = 0; iComponent < numComponents; iComponent++) + { + IVssWMComponent* pComponent; + result = pMetadata->GetComponent(iComponent, &pComponent); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata component " << + iComponent << " from writer " << iWriter << ": " << + GetMsgForHresult(result)); + continue; + } + + PVSSCOMPONENTINFO pComponentInfo; + result = pComponent->GetComponentInfo(&pComponentInfo); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata component " << + iComponent << " info from writer " << iWriter << ": " << + GetMsgForHresult(result)); + pComponent->Release(); + continue; + } + + BOX_TRACE("VSS: writer " << iWriter << " component " << + iComponent << " info:"); + switch(pComponentInfo->type) + { + case VSS_CT_UNDEFINED: BOX_TRACE("VSS: type: undefined"); break; + case VSS_CT_DATABASE: BOX_TRACE("VSS: type: database"); break; + case VSS_CT_FILEGROUP: BOX_TRACE("VSS: type: filegroup"); break; + default: + BOX_WARNING("VSS: type: unknown (" << pComponentInfo->type << ")"); + } + + BOX_TRACE("VSS: logical path: " << + BstrToString(pComponentInfo->bstrLogicalPath)); + BOX_TRACE("VSS: component name: " << + BstrToString(pComponentInfo->bstrComponentName)); + BOX_TRACE("VSS: caption: " << + BstrToString(pComponentInfo->bstrCaption)); + BOX_TRACE("VSS: restore metadata: " << + pComponentInfo->bRestoreMetadata); + BOX_TRACE("VSS: notify on complete: " << + pComponentInfo->bRestoreMetadata); + BOX_TRACE("VSS: selectable: " << + pComponentInfo->bSelectable); + BOX_TRACE("VSS: selectable for restore: " << + pComponentInfo->bSelectableForRestore); + BOX_TRACE("VSS: component flags: " << + BOX_FORMAT_HEX32(pComponentInfo->dwComponentFlags)); + BOX_TRACE("VSS: file count: " << + pComponentInfo->cFileCount); + BOX_TRACE("VSS: databases: " << + pComponentInfo->cDatabases); + BOX_TRACE("VSS: log files: " << + pComponentInfo->cLogFiles); + BOX_TRACE("VSS: dependencies: " << + pComponentInfo->cDependencies); + + pComponent->FreeComponentInfo(pComponentInfo); + pComponent->Release(); + } + + pMetadata->Release(); + } + + VSS_ID snapshotSetId; + result = mpVssBackupComponents->StartSnapshotSet(&snapshotSetId); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to start snapshot set: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + // Add all volumes included as backup locations to the snapshot set + for(Locations::iterator + iLocation = mLocations.begin(); + iLocation != mLocations.end(); + iLocation++) + { + Location& rLocation(**iLocation); + std::string path = rLocation.mPath; + // convert to absolute and remove Unicode prefix + path = ConvertPathToAbsoluteUnicode(path.c_str()).substr(4); + + if(path.length() >= 3 && path[1] == ':' && path[2] == '\\') + { + std::string volumeRoot = path.substr(0, 3); + + std::map<char, VSS_ID>::iterator i = + volumesIncluded.find(path[0]); + + if(i == volumesIncluded.end()) + { + std::wstring volumeRootWide; + volumeRootWide.push_back((WCHAR) path[0]); + volumeRootWide.push_back((WCHAR) ':'); + volumeRootWide.push_back((WCHAR) '\\'); + VSS_ID newVolumeId; + result = mpVssBackupComponents->AddToSnapshotSet( + (VSS_PWSZ)(volumeRootWide.c_str()), GUID_NULL, + &newVolumeId); + if(result == S_OK) + { + BOX_TRACE("VSS: Added volume " << volumeRoot << + " for backup location " << path << + " to snapshot set"); + volumesIncluded[path[0]] = newVolumeId; + rLocation.mSnapshotVolumeId = newVolumeId; + } + else + { + BOX_ERROR("VSS: Failed to add volume " << + volumeRoot << " to snapshot set: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + } + else + { + BOX_TRACE("VSS: Skipping already included volume " << + volumeRoot << " for backup location " << path); + rLocation.mSnapshotVolumeId = i->second; + } + + rLocation.mIsSnapshotCreated = true; + + // If the snapshot path starts with the volume root + // (drive letter), because the path is absolute (as it + // should be), then remove it so that the resulting + // snapshot path can be appended to the snapshot device + // object to make a real path, without a spurious drive + // letter in it. + + if (path.substr(0, volumeRoot.length()) == volumeRoot) + { + path = path.substr(volumeRoot.length()); + } + + rLocation.mSnapshotPath = path; + } + else + { + BOX_WARNING("VSS: Skipping backup location " << path << + " which does not start with a volume specification"); + } + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::PrepareForBackup, + "PrepareForBackup()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::DoSnapshotSet, + "DoSnapshotSet()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterStatus, + "GatherWriterStatus()")) + { + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + result = mpVssBackupComponents->GetWriterStatusCount(&writerCount); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get writer status count: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + for(UINT iWriter = 0; iWriter < writerCount; iWriter++) + { + VSS_ID instance, writer; + BSTR writerNameBstr; + VSS_WRITER_STATE writerState; + HRESULT writerResult; + + result = mpVssBackupComponents->GetWriterStatus(iWriter, + &instance, &writer, &writerNameBstr, &writerState, + &writerResult); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query writer " << iWriter << + " status: " << GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + std::string writerName = BstrToString(writerNameBstr); + ::SysFreeString(writerNameBstr); + + if(writerResult != S_OK) + { + BOX_ERROR("VSS: Writer " << iWriter << " (" << + writerName << ") failed: " << + GetMsgForHresult(writerResult)); + continue; + } + + std::string stateName; + + switch(writerState) + { +#define WRITER_STATE(code) \ + case code: stateName = #code; break; + WRITER_STATE(VSS_WS_UNKNOWN); + WRITER_STATE(VSS_WS_STABLE); + WRITER_STATE(VSS_WS_WAITING_FOR_FREEZE); + WRITER_STATE(VSS_WS_WAITING_FOR_THAW); + WRITER_STATE(VSS_WS_WAITING_FOR_POST_SNAPSHOT); + WRITER_STATE(VSS_WS_WAITING_FOR_BACKUP_COMPLETE); + WRITER_STATE(VSS_WS_FAILED_AT_IDENTIFY); + WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_BACKUP); + WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_SNAPSHOT); + WRITER_STATE(VSS_WS_FAILED_AT_FREEZE); + WRITER_STATE(VSS_WS_FAILED_AT_THAW); + WRITER_STATE(VSS_WS_FAILED_AT_POST_SNAPSHOT); + WRITER_STATE(VSS_WS_FAILED_AT_BACKUP_COMPLETE); + WRITER_STATE(VSS_WS_FAILED_AT_PRE_RESTORE); + WRITER_STATE(VSS_WS_FAILED_AT_POST_RESTORE); + WRITER_STATE(VSS_WS_FAILED_AT_BACKUPSHUTDOWN); +#undef WRITER_STATE + default: + std::ostringstream o; + o << "unknown (" << writerState << ")"; + stateName = o.str(); + } + + BOX_TRACE("VSS: Writer " << iWriter << " (" << + writerName << ") is in state " << stateName); + } + + // lookup new snapshot volume for each location that has a snapshot + for(Locations::iterator + iLocation = mLocations.begin(); + iLocation != mLocations.end(); + iLocation++) + { + Location& rLocation(**iLocation); + if(rLocation.mIsSnapshotCreated) + { + VSS_SNAPSHOT_PROP prop; + result = mpVssBackupComponents->GetSnapshotProperties( + rLocation.mSnapshotVolumeId, &prop); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get snapshot properties " + "for volume " << GuidToString(rLocation.mSnapshotVolumeId) << + " for location " << rLocation.mPath << ": " << + GetMsgForHresult(result)); + rLocation.mIsSnapshotCreated = false; + continue; + } + + rLocation.mSnapshotPath = + WideStringToString(prop.m_pwszSnapshotDeviceObject) + + DIRECTORY_SEPARATOR + rLocation.mSnapshotPath; + VssFreeSnapshotProperties(&prop); + + BOX_INFO("VSS: Location " << rLocation.mPath << " using " + "snapshot path " << rLocation.mSnapshotPath); + } + } + + IVssEnumObject *pEnum; + result = mpVssBackupComponents->Query(GUID_NULL, VSS_OBJECT_NONE, + VSS_OBJECT_SNAPSHOT, &pEnum); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query snapshot list: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + while(result == S_OK) + { + VSS_OBJECT_PROP rgelt; + ULONG count; + result = pEnum->Next(1, &rgelt, &count); + + if(result == S_FALSE) + { + // end of list, break out of the loop + break; + } + else if(result != S_OK) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + GetMsgForHresult(result)); + } + else if(count != 1) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + "Next() returned " << count << " objects instead of 1"); + } + else if(rgelt.Type != VSS_OBJECT_SNAPSHOT) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + "Next() returned a type " << rgelt.Type << " object " + "instead of VSS_OBJECT_SNAPSHOT"); + } + else + { + VSS_SNAPSHOT_PROP *pSnap = &rgelt.Obj.Snap; + BOX_TRACE("VSS: Snapshot ID: " << + GuidToString(pSnap->m_SnapshotId)); + BOX_TRACE("VSS: Snapshot set ID: " << + GuidToString(pSnap->m_SnapshotSetId)); + BOX_TRACE("VSS: Number of volumes: " << + pSnap->m_lSnapshotsCount); + BOX_TRACE("VSS: Snapshot device object: " << + WideStringToString(pSnap->m_pwszSnapshotDeviceObject)); + BOX_TRACE("VSS: Original volume name: " << + WideStringToString(pSnap->m_pwszOriginalVolumeName)); + BOX_TRACE("VSS: Originating machine: " << + WideStringToString(pSnap->m_pwszOriginatingMachine)); + BOX_TRACE("VSS: Service machine: " << + WideStringToString(pSnap->m_pwszServiceMachine)); + BOX_TRACE("VSS: Exposed name: " << + WideStringToString(pSnap->m_pwszExposedName)); + BOX_TRACE("VSS: Exposed path: " << + WideStringToString(pSnap->m_pwszExposedPath)); + BOX_TRACE("VSS: Provider ID: " << + GuidToString(pSnap->m_ProviderId)); + BOX_TRACE("VSS: Snapshot attributes: " << + BOX_FORMAT_HEX32(pSnap->m_lSnapshotAttributes)); + BOX_TRACE("VSS: Snapshot creation time: " << + BOX_FORMAT_HEX32(pSnap->m_tsCreationTimestamp)); + + std::string status; + switch(pSnap->m_eStatus) + { + case VSS_SS_UNKNOWN: status = "Unknown (error)"; break; + case VSS_SS_PREPARING: status = "Preparing"; break; + case VSS_SS_PROCESSING_PREPARE: status = "Preparing (processing)"; break; + case VSS_SS_PREPARED: status = "Prepared"; break; + case VSS_SS_PROCESSING_PRECOMMIT: status = "Precommitting"; break; + case VSS_SS_PRECOMMITTED: status = "Precommitted"; break; + case VSS_SS_PROCESSING_COMMIT: status = "Commiting"; break; + case VSS_SS_COMMITTED: status = "Committed"; break; + case VSS_SS_PROCESSING_POSTCOMMIT: status = "Postcommitting"; break; + case VSS_SS_PROCESSING_PREFINALCOMMIT: status = "Pre final committing"; break; + case VSS_SS_PREFINALCOMMITTED: status = "Pre final committed"; break; + case VSS_SS_PROCESSING_POSTFINALCOMMIT: status = "Post final committing"; break; + case VSS_SS_CREATED: status = "Created"; break; + case VSS_SS_ABORTED: status = "Aborted"; break; + case VSS_SS_DELETED: status = "Deleted"; break; + case VSS_SS_POSTCOMMITTED: status = "Postcommitted"; break; + default: + std::ostringstream buf; + buf << "Unknown code: " << pSnap->m_eStatus; + status = buf.str(); + } + + BOX_TRACE("VSS: Snapshot status: " << status); + VssFreeSnapshotProperties(pSnap); + } + } + + pEnum->Release(); + +CreateVssBackupComponents_cleanup_WriterStatus: + result = mpVssBackupComponents->FreeWriterStatus(); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to free writer status: " << + GetMsgForHresult(result)); + } + +CreateVssBackupComponents_cleanup_WriterMetadata: + result = mpVssBackupComponents->FreeWriterMetadata(); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to free writer metadata: " << + GetMsgForHresult(result)); + } +} + +void BackupDaemon::CleanupVssBackupComponents() +{ + if(mpVssBackupComponents == NULL) + { + return; + } + + CallAndWaitForAsync(&IVssBackupComponents::BackupComplete, + "BackupComplete()"); + + mpVssBackupComponents->Release(); + mpVssBackupComponents = NULL; +} +#endif + +void BackupDaemon::OnBackupStart() +{ + ResetLogFile(); + + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_start"); + + // Reset statistics on uploads + BackupStoreFile::ResetStats(); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(true /* start */); + + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupStart); + + // Setup timer for polling the command socket + mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, + "CommandSocketPollTimer")); + + // Set state and log start + SetState(State_Connected); + BOX_NOTICE("Beginning scan of local files"); +} + +void BackupDaemon::OnBackupFinish() +{ + try + { + // Log + BOX_NOTICE("Finished scan of local files"); + + // Log the stats + BOX_NOTICE("File statistics: total file size uploaded " + << BackupStoreFile::msStats.mBytesInEncodedFiles + << ", bytes already on server " + << BackupStoreFile::msStats.mBytesAlreadyOnServer + << ", encoded size " + << BackupStoreFile::msStats.mTotalFileStreamSize + << ", " << mNumFilesUploaded << " files uploaded, " + << mNumDirsCreated << " dirs created"); + + // Reset statistics again + BackupStoreFile::ResetStats(); + + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupFinish); + + // Stop the timer for polling the command socket, + // to prevent needless alarms while sleeping. + mapCommandSocketPollTimer.reset(); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(false /* finish */); + + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_finish"); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to perform backup finish actions: " << e.what()); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::UseScriptToSeeIfSyncAllowed() +// Purpose: Private. Use a script to see if the sync should be +// allowed now (if configured). Returns -1 if it's +// allowed, time in seconds to wait otherwise. +// Created: 21/6/04 +// +// -------------------------------------------------------------------------- +int BackupDaemon::UseScriptToSeeIfSyncAllowed() +{ + const Configuration &conf(GetConfiguration()); + + // Got a script to run? + if(!conf.KeyExists("SyncAllowScript")) + { + // No. Do sync. + return -1; + } + + // If there's no result, try again in five minutes + int waitInSeconds = (60*5); + + std::string script(conf.GetKeyValue("SyncAllowScript") + + " \"" + GetConfigFileName() + "\""); + + // Run it? + pid_t pid = 0; + try + { + std::auto_ptr<IOStream> pscript(LocalProcessStream(script, + pid)); + + // Read in the result + IOStreamGetLine getLine(*pscript); + std::string line; + if(getLine.GetLine(line, true, 30000)) // 30 seconds should be enough + { + waitInSeconds = BackupDaemon::ParseSyncAllowScriptOutput(script, line); + } + else + { + BOX_ERROR("SyncAllowScript output nothing within " + "30 seconds, waiting 5 minutes to try again" + " (" << script << ")"); + } + } + catch(std::exception &e) + { + BOX_ERROR("Internal error running SyncAllowScript: " + << e.what() << " (" << script << ")"); + } + catch(...) + { + // Ignore any exceptions + // Log that something bad happened + BOX_ERROR("Unknown error running SyncAllowScript (" << + script << ")"); + } + + // Wait and then cleanup child process, if any + if(pid != 0) + { + int status = 0; + ::waitpid(pid, &status, 0); + } + + return waitInSeconds; +} + +int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, + const std::string& output) +{ + int waitInSeconds = (60*5); + std::istringstream iss(output); + + std::string delay; + iss >> delay; + + if(delay == "") + { + BOX_ERROR("SyncAllowScript output an empty line, sleeping for " + << waitInSeconds << " seconds (" << script << ")"); + return waitInSeconds; + } + + // Got a string, interpret + if(delay == "now") + { + // Script says do it now. Obey. + waitInSeconds = -1; + + BOX_NOTICE("SyncAllowScript requested a backup now " + "(" << script << ")"); + } + else + { + try + { + // How many seconds to wait? + waitInSeconds = BoxConvert::Convert<int32_t, const std::string&>(delay); + } + catch(ConversionException &e) + { + BOX_ERROR("SyncAllowScript output an invalid " + "number: '" << output << "' (" << + script << ")"); + throw; + } + + BOX_NOTICE("SyncAllowScript requested a delay of " << + waitInSeconds << " seconds (" << script << ")"); + } + + if(iss.eof()) + { + // No bandwidth limit requested + mMaxBandwidthFromSyncAllowScript = 0; + BOX_NOTICE("SyncAllowScript did not set a maximum bandwidth " + "(" << script << ")"); + } + else + { + std::string maxBandwidth; + iss >> maxBandwidth; + + try + { + // How many seconds to wait? + mMaxBandwidthFromSyncAllowScript = + BoxConvert::Convert<int32_t, const std::string&>(maxBandwidth); + } + catch(ConversionException &e) + { + BOX_ERROR("Invalid maximum bandwidth from " + "SyncAllowScript: '" << + output << "' (" << script << ")"); + throw; + } + + BOX_NOTICE("SyncAllowScript set maximum bandwidth to " << + mMaxBandwidthFromSyncAllowScript << " kB/s (" << + script << ")"); + } + + return waitInSeconds; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::RunBackgroundTask() +// Purpose: Checks for connections or commands on the command +// socket and handles them with minimal delay. Polled +// during lengthy operations such as file uploads. +// Created: 07/04/14 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum) +{ + BOX_TRACE("BackupDaemon::RunBackgroundTask: state = " << state << + ", progress = " << progress << "/" << maximum); + + if(!mapCommandSocketPollTimer.get()) + { + return true; // no background task + } + + if(mapCommandSocketPollTimer->HasExpired()) + { + mapCommandSocketPollTimer->Reset(COMMAND_SOCKET_POLL_INTERVAL); + } + else + { + // Do no more work right now + return true; + } + + if(mapCommandSocketInfo.get()) + { + BOX_TRACE("BackupDaemon::RunBackgroundTask: polling command socket"); + + bool sync_flag_out, sync_is_forced_out; + + WaitOnCommandSocket(0, // RequiredDelay + sync_flag_out, sync_is_forced_out); + + if(sync_flag_out) + { + BOX_WARNING("Ignoring request to sync while " + "already syncing."); + } + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::WaitOnCommandSocket(box_time_t, bool &, bool &) +// Purpose: Waits on a the command socket for a time of UP TO +// the required time but may be much less, and handles +// a command if necessary. +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut) +{ + DoSyncFlagOut = false; + SyncIsForcedOut = false; + + ASSERT(mapCommandSocketInfo.get()); + if(!mapCommandSocketInfo.get()) + { + // failure case isn't too bad + ::sleep(1); + return; + } + + BOX_TRACE("Wait on command socket, delay = " << + BOX_FORMAT_MICROSECONDS(RequiredDelay)); + + try + { + // Timeout value for connections and things + int timeout = ((int)BoxTimeToMilliSeconds(RequiredDelay)) + 1; + // Handle bad boundary cases + if(timeout <= 0) timeout = 1; + if(timeout == INFTIM) timeout = 100000; + + // Wait for socket connection, or handle a command? + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + // There should be no GetLine, as it would be holding onto a + // pointer to a dead mpConnectedSocket. + ASSERT(!mapCommandSocketInfo->mapGetLine.get()); + + // No connection, listen for a new one + mapCommandSocketInfo->mpConnectedSocket.reset( + mapCommandSocketInfo->mListeningSocket.Accept(timeout).release()); + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + // If a connection didn't arrive, there was a timeout, which means we've + // waited long enough and it's time to go. + return; + } + else + { +#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET + bool uidOK = true; + BOX_WARNING("On this platform, no security check can be made on the credentials of peers connecting to the command socket. (bbackupctl)"); +#else + // Security check -- does the process connecting to this socket have + // the same UID as this process? + bool uidOK = false; + // BLOCK + { + uid_t remoteEUID = 0xffff; + gid_t remoteEGID = 0xffff; + if(mapCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID)) + { + // Credentials are available -- check UID + if(remoteEUID == ::getuid()) + { + // Acceptable + uidOK = true; + } + } + } +#endif // PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET + + // Is this an acceptable connection? + if(!uidOK) + { + // Dump the connection + BOX_ERROR("Incoming command connection from peer had different user ID than this process, or security check could not be completed."); + mapCommandSocketInfo->mpConnectedSocket.reset(); + return; + } + else + { + // Log + BOX_INFO("Connection from command socket"); + + // Send a header line summarising the configuration and current state + const Configuration &conf(GetConfiguration()); + std::ostringstream hello; + hello << "bbackupd: " << + (conf.GetKeyValueBool("AutomaticBackup") ? 1 : 0) + << " " << + conf.GetKeyValueInt("UpdateStoreInterval") + << " " << + conf.GetKeyValueInt("MinimumFileAge") + << " " << + conf.GetKeyValueInt("MaxUploadWait") + << "\nstate " << mState << "\n"; + mapCommandSocketInfo->mpConnectedSocket->Write( + hello.str(), timeout); + + // Set the timeout to something very small, so we don't wait too long on waiting + // for any incoming data + timeout = 10; // milliseconds + } + } + + mapCommandSocketInfo->mapGetLine.reset( + new IOStreamGetLine( + *(mapCommandSocketInfo->mpConnectedSocket.get()))); + } + + // So there must be a connection now. + ASSERT(mapCommandSocketInfo->mpConnectedSocket.get() != 0); + ASSERT(mapCommandSocketInfo->mapGetLine.get() != 0); + + // Ping the remote side, to provide errors which will mean the socket gets closed + mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5, + timeout); + + // Wait for a command or something on the socket + std::string command; + while(mapCommandSocketInfo->mapGetLine.get() != 0 + && !mapCommandSocketInfo->mapGetLine->IsEOF() + && mapCommandSocketInfo->mapGetLine->GetLine(command, false /* no preprocessing */, timeout)) + { + BOX_TRACE("Receiving command '" << command + << "' over command socket"); + + bool sendOK = false; + bool sendResponse = true; + + // Command to process! + if(command == "quit" || command == "") + { + // Close the socket. + CloseCommandConnection(); + sendResponse = false; + } + else if(command == "sync") + { + // Sync now! + DoSyncFlagOut = true; + SyncIsForcedOut = false; + sendOK = true; + } + else if(command == "force-sync") + { + // Sync now (forced -- overrides any SyncAllowScript) + DoSyncFlagOut = true; + SyncIsForcedOut = true; + sendOK = true; + } + else if(command == "reload") + { + // Reload the configuration + SetReloadConfigWanted(); + sendOK = true; + } + else if(command == "terminate") + { + // Terminate the daemon cleanly + SetTerminateWanted(); + sendOK = true; + } + + // Send a response back? + if(sendResponse) + { + std::string response = sendOK ? "ok\n" : "error\n"; + mapCommandSocketInfo->mpConnectedSocket->Write( + response, timeout); + } + + // Set timeout to something very small, so this just checks for data which is waiting + timeout = 1; + } + + // Close on EOF? + if(mapCommandSocketInfo->mapGetLine.get() != 0 && + mapCommandSocketInfo->mapGetLine->IsEOF()) + { + CloseCommandConnection(); + } + } + catch(ConnectionException &ce) + { + BOX_NOTICE("Failed to write to command socket: " << ce.what()); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } + catch(std::exception &e) + { + BOX_ERROR("Failed to write to command socket: " << + e.what()); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } + catch(...) + { + BOX_ERROR("Failed to write to command socket: unknown error"); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CloseCommandConnection() +// Purpose: Close the command connection, ignoring any errors +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::CloseCommandConnection() +{ + try + { + BOX_TRACE("Closing command connection"); + mapCommandSocketInfo->mapGetLine.reset(); + mapCommandSocketInfo->mpConnectedSocket.reset(); + } + catch(std::exception &e) + { + BOX_ERROR("Internal error while closing command " + "socket: " << e.what()); + } + catch(...) + { + // Ignore any errors + } +} + + +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemon.cpp +// Purpose: Send a start or finish sync message to the command socket, if it's connected. +// +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SendSyncStartOrFinish(bool SendStart) +{ + // The bbackupctl program can't rely on a state change, because it + // may never change if the server doesn't need to be contacted. + + if(mapCommandSocketInfo.get() && + mapCommandSocketInfo->mpConnectedSocket.get() != 0) + { + std::string message = SendStart ? "start-sync" : "finish-sync"; + try + { + message += "\n"; + mapCommandSocketInfo->mpConnectedSocket->Write(message, + 1); // short timeout, it's overlapped + } + catch(std::exception &e) + { + BOX_ERROR("Internal error while sending to " + "command socket client: " << e.what()); + CloseCommandConnection(); + } + catch(...) + { + CloseCommandConnection(); + } + } +} + + + + +#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_NMTONNAME) + // string comparison ordering for when mount points are handled + // by code, rather than the OS. + typedef struct + { + bool operator()(const std::string &s1, const std::string &s2) + { + if(s1.size() == s2.size()) + { + // Equal size, sort according to natural sort order + return s1 < s2; + } + else + { + // Make sure longer strings go first + return s1.size() > s2.size(); + } + } + } mntLenCompare; +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetupLocations(BackupClientContext &, const Configuration &) +// Purpose: Makes sure that the list of directories records is correctly set up +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf) +{ + // Going to need a copy of the root directory. Get a connection, + // and fetch it. + BackupProtocolCallable& connection(rClientContext.GetConnection()); + + // Ask server for a list of everything in the root directory, + // which is a directory itself + std::auto_ptr<BackupProtocolSuccess> dirreply( + connection.QueryListDirectory( + BackupProtocolListDirectory::RootDirectory, + // only directories + BackupProtocolListDirectory::Flags_Dir, + // exclude old/deleted stuff + BackupProtocolListDirectory::Flags_Deleted | + BackupProtocolListDirectory::Flags_OldVersion, + false /* no attributes */)); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(connection.ReceiveStream()); + dir.ReadFromStream(*dirstream, connection.GetTimeout()); + + // Map of mount names to ID map index + std::map<std::string, int> mounts; + int numIDMaps = 0; + +#ifdef HAVE_MOUNTS +#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_MNTONNAME) + // Linux and others can't tell you where a directory is mounted. So we + // have to read the mount entries from /etc/mtab! Bizarre that the OS + // itself can't tell you, but there you go. + std::set<std::string, mntLenCompare> mountPoints; + // BLOCK + FILE *mountPointsFile = 0; + +#ifdef HAVE_STRUCT_MNTENT_MNT_DIR + // Open mounts file + mountPointsFile = ::setmntent("/proc/mounts", "r"); + if(mountPointsFile == 0) + { + mountPointsFile = ::setmntent("/etc/mtab", "r"); + } + if(mountPointsFile == 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + + try + { + // Read all the entries, and put them in the set + struct mntent *entry = 0; + while((entry = ::getmntent(mountPointsFile)) != 0) + { + BOX_TRACE("Found mount point at " << entry->mnt_dir); + mountPoints.insert(std::string(entry->mnt_dir)); + } + + // Close mounts file + ::endmntent(mountPointsFile); + } + catch(...) + { + ::endmntent(mountPointsFile); + throw; + } +#else // ! HAVE_STRUCT_MNTENT_MNT_DIR + // Open mounts file + mountPointsFile = ::fopen("/etc/mnttab", "r"); + if(mountPointsFile == 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + + try + { + // Read all the entries, and put them in the set + struct mnttab entry; + while(getmntent(mountPointsFile, &entry) == 0) + { + BOX_TRACE("Found mount point at " << entry.mnt_mountp); + mountPoints.insert(std::string(entry.mnt_mountp)); + } + + // Close mounts file + ::fclose(mountPointsFile); + } + catch(...) + { + ::fclose(mountPointsFile); + throw; + } +#endif // HAVE_STRUCT_MNTENT_MNT_DIR + // Check sorting and that things are as we expect + ASSERT(mountPoints.size() > 0); +#ifndef BOX_RELEASE_BUILD + { + std::set<std::string, mntLenCompare>::reverse_iterator i(mountPoints.rbegin()); + ASSERT(*i == "/"); + } +#endif // n BOX_RELEASE_BUILD +#endif // n HAVE_STRUCT_STATFS_F_MNTONNAME || n HAVE_STRUCT_STATVFS_F_MNTONNAME +#endif // HAVE_MOUNTS + + // Then... go through each of the entries in the configuration, + // making sure there's a directory created for it. + std::vector<std::string> locNames = + rLocationsConf.GetSubConfigurationNames(); + + // We only want completely configured locations to be in the list + // when this function exits, so move them all to a temporary list. + // Entries matching a properly configured location will be moved + // back to mLocations. Anything left in this list after the loop + // finishes will be deleted. + Locations tmpLocations = mLocations; + mLocations.clear(); + + // The ID map list will be repopulated automatically by this loop + mIDMapMounts.clear(); + + for(std::vector<std::string>::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + Location* pLoc = NULL; + + // Try to find and reuse an existing Location object + for(Locations::const_iterator + i = tmpLocations.begin(); + i != tmpLocations.end(); i++) + { + if ((*i)->mName == *pLocName) + { + BOX_TRACE("Location already configured: " << *pLocName); + pLoc = *i; + break; + } + } + + const Configuration& rConfig( + rLocationsConf.GetSubConfiguration(*pLocName)); + std::auto_ptr<Location> apLoc; + + try + { + if(pLoc == NULL) + { + // Create a record for it + BOX_TRACE("New location: " << *pLocName); + pLoc = new Location; + + // ensure deletion if setup fails + apLoc.reset(pLoc); + + // Setup names in the location record + pLoc->mName = *pLocName; + pLoc->mPath = rConfig.GetKeyValue("Path"); + } + + // Read the exclude lists from the Configuration + pLoc->mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rConfig)); + pLoc->mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rConfig)); + + // Does this exist on the server? + // Remove from dir object early, so that if we fail + // to stat the local directory, we still don't + // consider to remote one for deletion. + BackupStoreDirectory::Iterator iter(dir); + BackupStoreFilenameClear dirname(pLoc->mName); // generate the filename + BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname); + int64_t oid = 0; + if(en != 0) + { + oid = en->GetObjectID(); + + // Delete the entry from the directory, so we get a list of + // unused root directories at the end of this. + dir.DeleteEntry(oid); + } + + // Do a fsstat on the pathname to find out which mount it's on + { + +#if defined HAVE_STRUCT_STATFS_F_MNTONNAME || defined HAVE_STRUCT_STATVFS_F_MNTONNAME || defined WIN32 + + // BSD style statfs -- includes mount point, which is nice. +#ifdef HAVE_STRUCT_STATVFS_F_MNTONNAME + struct statvfs s; + if(::statvfs(pLoc->mPath.c_str(), &s) != 0) +#else // HAVE_STRUCT_STATVFS_F_MNTONNAME + struct statfs s; + if(::statfs(pLoc->mPath.c_str(), &s) != 0) +#endif // HAVE_STRUCT_STATVFS_F_MNTONNAME + { + THROW_SYS_ERROR("Failed to stat path " + "'" << pLoc->mPath << "' " + "for location " + "'" << pLoc->mName << "'", + CommonException, OSFileError); + } + + // Where the filesystem is mounted + std::string mountName(s.f_mntonname); + +#else // !HAVE_STRUCT_STATFS_F_MNTONNAME && !WIN32 + + // Warn in logs if the directory isn't absolute + if(pLoc->mPath[0] != '/') + { + BOX_WARNING("Location path '" + << pLoc->mPath + << "' is not absolute"); + } + // Go through the mount points found, and find a suitable one + std::string mountName("/"); + { + std::set<std::string, mntLenCompare>::const_iterator i(mountPoints.begin()); + BOX_TRACE(mountPoints.size() + << " potential mount points"); + for(; i != mountPoints.end(); ++i) + { + // Compare first n characters with the filename + // If it matches, the file belongs in that mount point + // (sorting order ensures this) + BOX_TRACE("checking against mount point " << *i); + if(::strncmp(i->c_str(), pLoc->mPath.c_str(), i->size()) == 0) + { + // Match + mountName = *i; + break; + } + } + BOX_TRACE("mount point chosen for " + << pLoc->mPath << " is " + << mountName); + } + +#endif + + // Got it? + std::map<std::string, int>::iterator f(mounts.find(mountName)); + if(f != mounts.end()) + { + // Yes -- store the index + pLoc->mIDMapIndex = f->second; + } + else + { + // No -- new index + pLoc->mIDMapIndex = numIDMaps; + mounts[mountName] = numIDMaps; + + // Store the mount name + mIDMapMounts.push_back(mountName); + + // Increment number of maps + ++numIDMaps; + } + } + + // Does this exist on the server? + if(en == 0) + { + // Doesn't exist, so it has to be created on the server. Let's go! + // First, get the directory's attributes and modification time + box_time_t attrModTime = 0; + BackupClientFileAttributes attr; + try + { + attr.ReadAttributes(pLoc->mPath.c_str(), + true /* directories have zero mod times */, + 0 /* not interested in mod time */, + &attrModTime /* get the attribute modification time */); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to get attributes " + "for path '" << pLoc->mPath + << "', skipping location '" << + pLoc->mName << "'"); + throw; + } + + // Execute create directory command + try + { + std::auto_ptr<IOStream> attrStream( + new MemBlockStream(attr)); + std::auto_ptr<BackupProtocolSuccess> + dirCreate(connection.QueryCreateDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, // containing directory + attrModTime, dirname, attrStream)); + + // Object ID for later creation + oid = dirCreate->GetObjectID(); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to create remote " + "directory '/" << pLoc->mName << + "', skipping location '" << + pLoc->mName << "'"); + throw; + } + + } + + // Create and store the directory object for the root of this location + ASSERT(oid != 0); + if(pLoc->mapDirectoryRecord.get() == NULL) + { + pLoc->mapDirectoryRecord.reset( + new BackupClientDirectoryRecord(oid, *pLocName)); + } + + // Remove it from the temporary list to avoid deletion + tmpLocations.remove(pLoc); + + // Push it back on the vector of locations + mLocations.push_back(pLoc); + + if(apLoc.get() != NULL) + { + // Don't delete it now! + apLoc.release(); + } + } + catch (std::exception &e) + { + BOX_ERROR("Failed to configure location '" + << pLoc->mName << "' path '" + << pLoc->mPath << "': " << e.what() << + ": please check for previous errors"); + mReadErrorsOnFilesystemObjects = true; + } + catch(...) + { + BOX_ERROR("Failed to configure location '" + << pLoc->mName << "' path '" + << pLoc->mPath << "': please check for " + "previous errors"); + mReadErrorsOnFilesystemObjects = true; + } + } + + // Now remove any leftovers + for(BackupDaemon::Locations::iterator + i = tmpLocations.begin(); + i != tmpLocations.end(); i++) + { + BOX_INFO("Removing obsolete location from memory: " << + (*i)->mName); + delete *i; + } + + tmpLocations.clear(); + + // Any entries in the root directory which need deleting? + if(dir.GetNumberOfEntries() > 0 && + mDeleteRedundantLocationsAfter == 0) + { + BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " + "in root directory found, but will not delete because " + "DeleteRedundantLocationsAfter = 0"); + } + else if(dir.GetNumberOfEntries() > 0) + { + box_time_t now = GetCurrentBoxTime(); + + // This should reset the timer if the list of unused + // locations changes, but it will not if the number of + // unused locations does not change, but the locations + // do change, e.g. one mysteriously appears and another + // mysteriously appears. (FIXME) + if (dir.GetNumberOfEntries() != mUnusedRootDirEntries.size() || + mDeleteUnusedRootDirEntriesAfter == 0) + { + mDeleteUnusedRootDirEntriesAfter = now + + SecondsToBoxTime(mDeleteRedundantLocationsAfter); + } + + int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter + - now); + + BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " + "in root directory found, will delete from store " + "after " << secs << " seconds."); + + // Store directories in list of things to delete + mUnusedRootDirEntries.clear(); + BackupStoreDirectory::Iterator iter(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = iter.Next()) != 0) + { + // Add name to list + BackupStoreFilenameClear clear(en->GetName()); + const std::string &name(clear.GetClearFilename()); + mUnusedRootDirEntries.push_back( + std::pair<int64_t,std::string> + (en->GetObjectID(), name)); + // Log this + BOX_INFO("Unused location in root: " << name); + } + ASSERT(mUnusedRootDirEntries.size() > 0); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetupIDMapsForSync() +// Purpose: Sets up ID maps for the sync process -- make sure they're all there +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetupIDMapsForSync() +{ + // Make sure we have some blank, empty ID maps + DeleteIDMapVector(mNewIDMaps); + FillIDMapVector(mNewIDMaps, true /* new maps */); + DeleteIDMapVector(mCurrentIDMaps); + FillIDMapVector(mCurrentIDMaps, false /* new maps */); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &) +// Purpose: Fills the vector with the right number of empty ID maps +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps) +{ + ASSERT(rVector.size() == 0); + rVector.reserve(mIDMapMounts.size()); + + for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) + { + // Create the object + BackupClientInodeToIDMap *pmap = new BackupClientInodeToIDMap(); + try + { + // Get the base filename of this map + std::string filename; + MakeMapBaseName(l, filename); + + // If it's a new one, add a suffix + if(NewMaps) + { + filename += ".n"; + } + + // The new map file should not exist yet. If there's + // one left over from a previous failed run, it's not + // useful to us because we never read from it and will + // overwrite the entries of all files that still + // exist, so we should just delete it and start afresh. + if(NewMaps && FileExists(filename.c_str())) + { + BOX_NOTICE("Found an incomplete ID map " + "database, deleting it to start " + "afresh: " << filename); + if(unlink(filename.c_str()) != 0) + { + BOX_LOG_NATIVE_ERROR(BOX_FILE_MESSAGE( + filename, "Failed to delete " + "incomplete ID map database")); + } + } + + // If it's not a new map, it may not exist in which case an empty map should be created + if(!NewMaps && !FileExists(filename.c_str())) + { + pmap->OpenEmpty(); + } + else + { + // Open the map + pmap->Open(filename.c_str(), !NewMaps /* read only */, NewMaps /* create new */); + } + + // Store on vector + rVector.push_back(pmap); + } + catch(...) + { + delete pmap; + throw; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteCorruptBerkelyDbFiles() +// Purpose: Delete the Berkely db files from disc after they have been corrupted. +// Created: 14/9/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteCorruptBerkelyDbFiles() +{ + for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) + { + // Get the base filename of this map + std::string filename; + MakeMapBaseName(l, filename); + + // Delete the file + BOX_TRACE("Deleting " << filename); + ::unlink(filename.c_str()); + + // Add a suffix for the new map + filename += ".n"; + + // Delete that too + BOX_TRACE("Deleting " << filename); + ::unlink(filename.c_str()); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: MakeMapBaseName(unsigned int, std::string &) +// Purpose: Makes the base name for a inode map +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const +{ + // Get the directory for the maps + const Configuration &config(GetConfiguration()); + std::string dir(config.GetKeyValue("DataDirectory")); + + // Make a leafname + std::string leaf(mIDMapMounts[MountNumber]); + for(unsigned int z = 0; z < leaf.size(); ++z) + { + if(leaf[z] == DIRECTORY_SEPARATOR_ASCHAR) + { + leaf[z] = '_'; + } + } + + // Build the final filename + rNameOut = dir + DIRECTORY_SEPARATOR "mnt" + leaf; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CommitIDMapsAfterSync() +// Purpose: Commits the new ID maps, so the 'new' maps are now the 'current' maps. +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::CommitIDMapsAfterSync() +{ + // Get rid of the maps in memory (leaving them on disc of course) + DeleteIDMapVector(mCurrentIDMaps); + DeleteIDMapVector(mNewIDMaps); + + // Then move the old maps into the new places + for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) + { + std::string target; + MakeMapBaseName(l, target); + std::string newmap(target + ".n"); + + // Try to rename +#ifdef WIN32 + // win32 rename doesn't overwrite existing files + ::remove(target.c_str()); +#endif + if(::rename(newmap.c_str(), target.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("Failed to rename ID map: " << + newmap << " to " << target); + THROW_EXCEPTION(CommonException, OSFileError) + } + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &) +// Purpose: Deletes the contents of a vector of ID maps +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector) +{ + while(!rVector.empty()) + { + // Pop off list + BackupClientInodeToIDMap *toDel = rVector.back(); + rVector.pop_back(); + + // Close and delete + delete toDel; + } + ASSERT(rVector.size() == 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::FindLocationPathName(const std::string &, std::string &) const +// Purpose: Tries to find the path of the root of a backup location. Returns true (and path in rPathOut) +// if it can be found, false otherwise. +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const +{ + // Search for the location + for(Locations::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i) + { + if((*i)->mName == rLocationName) + { + rPathOut = (*i)->mPath; + return true; + } + } + + // Didn't find it + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetState(int) +// Purpose: Record current action of daemon, and update process title to reflect this +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetState(int State) +{ + // Two little checks + if(State == mState) return; + if(State < 0) return; + + // Update + mState = State; + + // Set process title + const static char *stateText[] = {"idle", "connected", "error -- waiting for retry", "over limit on server -- not backing up"}; + SetProcessTitle(stateText[State]); + + // If there's a command socket connected, then inform it -- disconnecting from the + // command socket if there's an error + + std::ostringstream msg; + msg << "state " << State << "\n"; + + if(!mapCommandSocketInfo.get()) + { + return; + } + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + return; + } + + // Something connected to the command socket, tell it about the new state + try + { + mapCommandSocketInfo->mpConnectedSocket->Write(msg.str(), + 1); // very short timeout, it's overlapped anyway + } + catch(ConnectionException &ce) + { + BOX_NOTICE("Failed to write state to command socket: " << + ce.what()); + CloseCommandConnection(); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to write state to command socket: " << + e.what()); + CloseCommandConnection(); + } + catch(...) + { + BOX_ERROR("Failed to write state to command socket: " + "unknown error"); + CloseCommandConnection(); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::TouchFileInWorkingDir(const char *) +// Purpose: Make sure a zero length file of the name exists in the working directory. +// Use for marking times of events in the filesystem. +// Created: 21/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::TouchFileInWorkingDir(const char *Filename) +{ + // Filename + const Configuration &config(GetConfiguration()); + std::string fn(config.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); + fn += Filename; + + // Open and close it to update the timestamp + try + { + FileStream touch(fn, O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to write to timestamp file: " << fn << ": " << + e.what()); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::NotifySysadmin(int) +// Purpose: Run the script to tell the sysadmin about events +// which need attention. +// Created: 25/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::NotifySysadmin(SysadminNotifier::EventCode Event) +{ + static const char *sEventNames[] = + { + "store-full", + "read-error", + "backup-error", + "backup-start", + "backup-finish", + "backup-ok", + 0 + }; + + // BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames)); + // BOX_TRACE("sizeof(*sEventNames) == " << sizeof(*sEventNames)); + // BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX); + ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == SysadminNotifier::MAX + 1); + + if(Event < 0 || Event >= SysadminNotifier::MAX) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + BadNotifySysadminEventCode, "NotifySysadmin() called " + "for unknown event code " << Event); + } + + BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << + sEventNames[Event]); + + if(!GetConfiguration().KeyExists("NotifyAlways") || + !GetConfiguration().GetKeyValueBool("NotifyAlways")) + { + // Don't send lots of repeated messages + // Note: backup-start and backup-finish will always be + // logged, because mLastNotifiedEvent is never set to + // these values and therefore they are never "duplicates". + if(mLastNotifiedEvent == Event) + { + if(Event == SysadminNotifier::BackupOK) + { + BOX_INFO("Suppressing duplicate notification " + "about " << sEventNames[Event]); + } + else + { + BOX_WARNING("Suppressing duplicate notification " + "about " << sEventNames[Event]); + } + return; + } + } + + // Is there a notification script? + const Configuration &conf(GetConfiguration()); + if(!conf.KeyExists("NotifyScript")) + { + // Log, and then return + if(Event != SysadminNotifier::BackupStart && + Event != SysadminNotifier::BackupFinish) + { + BOX_INFO("Not notifying administrator about event " + << sEventNames[Event] << ", set NotifyScript " + "to do this in future"); + } + return; + } + + // Script to run + std::string script(conf.GetKeyValue("NotifyScript") + " " + + sEventNames[Event] + " \"" + GetConfigFileName() + "\""); + + // Log what we're about to do + BOX_INFO("About to notify administrator about event " + << sEventNames[Event] << ", running script '" << script << "'"); + + // Then do it + int returnCode = ::system(script.c_str()); + if(returnCode != 0) + { + BOX_WARNING("Notify script returned error code: " << + returnCode << " (" << script << ")"); + } + else if(Event != SysadminNotifier::BackupStart && + Event != SysadminNotifier::BackupFinish) + { + mLastNotifiedEvent = Event; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &) +// Purpose: Deletes any unused entries in the root directory, if they're scheduled to be deleted. +// Created: 13/5/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) +{ + if(mUnusedRootDirEntries.empty()) + { + BOX_INFO("Not deleting unused entries - none in list"); + return; + } + + if(mDeleteUnusedRootDirEntriesAfter == 0) + { + BOX_INFO("Not deleting unused entries - " + "zero delete time (bad)"); + return; + } + + // Check time + box_time_t now = GetCurrentBoxTime(); + if(now < mDeleteUnusedRootDirEntriesAfter) + { + int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter + - now); + BOX_INFO("Not deleting unused entries - too early (" + << secs << " seconds remaining)"); + return; + } + + // Entries to delete, and it's the right time to do so... + BOX_NOTICE("Deleting unused locations from store root..."); + BackupProtocolCallable &connection(rContext.GetConnection()); + for(std::vector<std::pair<int64_t,std::string> >::iterator + i(mUnusedRootDirEntries.begin()); + i != mUnusedRootDirEntries.end(); ++i) + { + connection.QueryDeleteDirectory(i->first); + rContext.GetProgressNotifier().NotifyFileDeleted( + i->first, i->second); + } + + // Reset state + mDeleteUnusedRootDirEntriesAfter = 0; + mUnusedRootDirEntries.clear(); +} + +// -------------------------------------------------------------------------- + +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mNumEntries; + int64_t mObjectID; // this object ID + int64_t mContainerID; // ID of container + uint64_t mAttributesModTime; + int32_t mOptionsPresent; // bit mask of optional sections / features present + +} loc_StreamFormat; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CommandSocketInfo::CommandSocketInfo() +// Purpose: Constructor +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +BackupDaemon::CommandSocketInfo::CommandSocketInfo() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CommandSocketInfo::~CommandSocketInfo() +// Purpose: Destructor +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +BackupDaemon::CommandSocketInfo::~CommandSocketInfo() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SerializeStoreObjectInfo( +// box_time_t theLastSyncTime, +// box_time_t theNextSyncTime) +// Purpose: Serializes remote directory and file information +// into a stream of bytes, using an Archive +// abstraction. +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- + +static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; +static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; +static const int STOREOBJECTINFO_VERSION = 2; + +bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime, + box_time_t theNextSyncTime) const +{ + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + return false; + } + + std::string StoreObjectInfoFile = + GetConfiguration().GetKeyValue("StoreObjectInfoFile"); + + if(StoreObjectInfoFile.size() <= 0) + { + return false; + } + + bool created = false; + + try + { + FileStream aFile(StoreObjectInfoFile.c_str(), + O_WRONLY | O_CREAT | O_TRUNC); + created = true; + + Archive anArchive(aFile, 0); + + anArchive.Write(STOREOBJECTINFO_MAGIC_ID_VALUE); + anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); + anArchive.Write(STOREOBJECTINFO_VERSION); + anArchive.Write(GetLoadedConfigModifiedTime()); + anArchive.Write(mClientStoreMarker); + anArchive.Write(theLastSyncTime); + anArchive.Write(theNextSyncTime); + + // + // + // + int64_t iCount = mLocations.size(); + anArchive.Write(iCount); + + for(Locations::const_iterator i = mLocations.begin(); + i != mLocations.end(); i++) + { + ASSERT(*i); + (*i)->Serialize(anArchive); + } + + // + // + // + iCount = mIDMapMounts.size(); + anArchive.Write(iCount); + + for(int v = 0; v < iCount; v++) + anArchive.Write(mIDMapMounts[v]); + + // + // + // + iCount = mUnusedRootDirEntries.size(); + anArchive.Write(iCount); + + for(int v = 0; v < iCount; v++) + { + anArchive.Write(mUnusedRootDirEntries[v].first); + anArchive.Write(mUnusedRootDirEntries[v].second); + } + + if (iCount > 0) + { + anArchive.Write(mDeleteUnusedRootDirEntriesAfter); + } + + // + // + // + aFile.Close(); + BOX_INFO("Saved store object info file version " << + STOREOBJECTINFO_VERSION << " (" << + StoreObjectInfoFile << ")"); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to write StoreObjectInfoFile: " << + StoreObjectInfoFile << ": " << e.what()); + } + catch(...) + { + BOX_ERROR("Failed to write StoreObjectInfoFile: " << + StoreObjectInfoFile << ": unknown error"); + } + + return created; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeserializeStoreObjectInfo( +// box_time_t & theLastSyncTime, +// box_time_t & theNextSyncTime) +// Purpose: Deserializes remote directory and file information +// from a stream of bytes, using an Archive +// abstraction. +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, + box_time_t & theNextSyncTime) +{ + // + // + // + DeleteAllLocations(); + + // + // + // + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + BOX_NOTICE("Store object info file is not enabled. Will " + "download directory listings from store."); + return false; + } + + std::string StoreObjectInfoFile = + GetConfiguration().GetKeyValue("StoreObjectInfoFile"); + + if(StoreObjectInfoFile.size() <= 0) + { + return false; + } + + int64_t fileSize; + if (!FileExists(StoreObjectInfoFile, &fileSize) || fileSize == 0) + { + BOX_NOTICE(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file does not exist or is empty")); + } + else + { + try + { + FileStream aFile(StoreObjectInfoFile, O_RDONLY); + Archive anArchive(aFile, 0); + + // + // see if the content looks like a valid serialised archive + // + int iMagicValue = 0; + anArchive.Read(iMagicValue); + + if(iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is not a valid " + "or compatible serialised archive")); + return false; + } + + // + // get a bit optimistic and read in a string identifier + // + std::string strMagicValue; + anArchive.Read(strMagicValue); + + if(strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is not a valid " + "or compatible serialised archive")); + return false; + } + + // + // check if we are loading some future format + // version by mistake + // + int iVersion = 0; + anArchive.Read(iVersion); + + if(iVersion != STOREOBJECTINFO_VERSION) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file version " << + iVersion << " is not supported")); + return false; + } + + // + // check if this state file is even valid + // for the loaded bbackupd.conf file + // + box_time_t lastKnownConfigModTime; + anArchive.Read(lastKnownConfigModTime); + + if(lastKnownConfigModTime != GetLoadedConfigModifiedTime()) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is older than " + "configuration file")); + return false; + } + + // + // this is it, go at it + // + anArchive.Read(mClientStoreMarker); + anArchive.Read(theLastSyncTime); + anArchive.Read(theNextSyncTime); + + // + // + // + int64_t iCount = 0; + anArchive.Read(iCount); + + for(int v = 0; v < iCount; v++) + { + Location* pLocation = new Location; + if(!pLocation) + { + throw std::bad_alloc(); + } + + pLocation->Deserialize(anArchive); + mLocations.push_back(pLocation); + } + + // + // + // + iCount = 0; + anArchive.Read(iCount); + + for(int v = 0; v < iCount; v++) + { + std::string strItem; + anArchive.Read(strItem); + + mIDMapMounts.push_back(strItem); + } + + // + // + // + iCount = 0; + anArchive.Read(iCount); + + for(int v = 0; v < iCount; v++) + { + int64_t anId; + anArchive.Read(anId); + + std::string aName; + anArchive.Read(aName); + + mUnusedRootDirEntries.push_back(std::pair<int64_t, std::string>(anId, aName)); + } + + if (iCount > 0) + anArchive.Read(mDeleteUnusedRootDirEntriesAfter); + + // + // + // + aFile.Close(); + + BOX_INFO(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Loaded store object info file version " << iVersion)); + return true; + } + catch(std::exception &e) + { + BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Internal error reading store object info " + "file: " << e.what())); + } + catch(...) + { + BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Internal error reading store object info " + "file: unknown error")); + } + } + + BOX_NOTICE("No usable cache, will download directory listings from " + "server."); + + DeleteAllLocations(); + + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; + theLastSyncTime = 0; + theNextSyncTime = 0; + + return false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteStoreObjectInfo() +// Purpose: Deletes the serialised state file, to prevent us +// from using it again if a backup is interrupted. +// +// Created: 2006/02/12 +// +// -------------------------------------------------------------------------- + +bool BackupDaemon::DeleteStoreObjectInfo() const +{ + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + return false; + } + + std::string storeObjectInfoFile(GetConfiguration().GetKeyValue("StoreObjectInfoFile")); + + // Check to see if the file exists + if(!FileExists(storeObjectInfoFile.c_str())) + { + // File doesn't exist -- so can't be deleted. But something + // isn't quite right, so log a message + BOX_WARNING("StoreObjectInfoFile did not exist when it " + "was supposed to: " << storeObjectInfoFile); + + // Return true to stop things going around in a loop + return true; + } + + // Actually delete it + if(::unlink(storeObjectInfoFile.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("Failed to delete the old " + "StoreObjectInfoFile: " << storeObjectInfoFile); + return false; + } + + return true; +} diff --git a/lib/bbackupd/BackupDaemon.h b/lib/bbackupd/BackupDaemon.h new file mode 100644 index 00000000..f9b8ba31 --- /dev/null +++ b/lib/bbackupd/BackupDaemon.h @@ -0,0 +1,539 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemon.h +// Purpose: Backup daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPDAEMON__H +#define BACKUPDAEMON__H + +#include <vector> +#include <string> +#include <memory> + +#include "BackupClientContext.h" +#include "BackupClientDirectoryRecord.h" +#include "BoxTime.h" +#include "Daemon.h" +#include "Logging.h" +#include "Socket.h" +#include "SocketListen.h" +#include "SocketStream.h" +#include "TLSContext.h" + +#include "autogen_BackupProtocol.h" +#include "autogen_BackupStoreException.h" + +#ifdef WIN32 + #include "WinNamedPipeListener.h" + #include "WinNamedPipeStream.h" +#endif + +#ifdef ENABLE_VSS +# include <comdef.h> +# include <Vss.h> +# include <VsWriter.h> +# include <VsBackup.h> +#endif + +#define COMMAND_SOCKET_POLL_INTERVAL 1000 + +class BackupClientDirectoryRecord; +class BackupClientContext; +class Configuration; +class BackupClientInodeToIDMap; +class ExcludeList; +class IOStreamGetLine; +class Archive; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupDaemon +// Purpose: Backup daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class BackupDaemon : public Daemon, public ProgressNotifier, public LocationResolver, +public RunStatusProvider, public SysadminNotifier, public BackgroundTask +{ +public: + BackupDaemon(); + ~BackupDaemon(); + +private: + // methods below do partial (specialized) serialization of + // client state only + bool SerializeStoreObjectInfo(box_time_t theLastSyncTime, + box_time_t theNextSyncTime) const; + bool DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, + box_time_t & theNextSyncTime); + bool DeleteStoreObjectInfo() const; + BackupDaemon(const BackupDaemon &); + +public: + #ifdef WIN32 + // add command-line options to handle Windows services + std::string GetOptionString(); + int ProcessOption(signed int option); + int Main(const std::string &rConfigFileName); + + // This shouldn't be here, but apparently gcc on + // Windows has no idea about inherited methods... + virtual int Main(const char *DefaultConfigFile, int argc, + const char *argv[]) + { + return Daemon::Main(DefaultConfigFile, argc, argv); + } + #endif + + void Run(); + virtual const char *DaemonName() const; + virtual std::string DaemonBanner() const; + virtual void Usage(); + const ConfigurationVerify *GetConfigVerify() const; + + bool FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const; + + enum + { + // Add stuff to this, make sure the textual equivalents in SetState() are changed too. + State_Initialising = -1, + State_Idle = 0, + State_Connected = 1, + State_Error = 2, + State_StorageLimitExceeded = 3 + }; + + int GetState() {return mState;} + static std::string GetStateName(int state) + { + std::string stateName; + + #define STATE(x) case BackupDaemon::State_ ## x: stateName = #x; break; + switch (state) + { + STATE(Initialising); + STATE(Idle); + STATE(Connected); + STATE(Error); + STATE(StorageLimitExceeded); + default: + stateName = "unknown"; + } + #undef STATE + + return stateName; + } + + // Allow other classes to call this too + void NotifySysadmin(SysadminNotifier::EventCode Event); + +private: + void Run2(); + +public: + void InitCrypto(); + std::auto_ptr<BackupClientContext> RunSyncNowWithExceptionHandling(); + std::auto_ptr<BackupClientContext> RunSyncNow(); + void ResetCachedState(); + void OnBackupStart(); + void OnBackupFinish(); + // TouchFileInWorkingDir is only here for use by Boxi. + // This does NOT constitute an API! + void TouchFileInWorkingDir(const char *Filename); + +protected: + virtual std::auto_ptr<BackupClientContext> GetNewContext + ( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode + ); + +private: + void DeleteAllLocations(); + void SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf); + + void DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector); + void DeleteAllIDMaps() + { + DeleteIDMapVector(mCurrentIDMaps); + DeleteIDMapVector(mNewIDMaps); + } + void FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps); + + void SetupIDMapsForSync(); + void CommitIDMapsAfterSync(); + void DeleteCorruptBerkelyDbFiles(); + + void MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const; + + void SetState(int State); + + void WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut); + void CloseCommandConnection(); + void SendSyncStartOrFinish(bool SendStart); + + void DeleteUnusedRootDirEntries(BackupClientContext &rContext); + + // For warning user about potential security hole + virtual void SetupInInitialProcess(); + + int UseScriptToSeeIfSyncAllowed(); + +public: + int ParseSyncAllowScriptOutput(const std::string& script, + const std::string& output); + typedef std::list<Location *> Locations; + Locations GetLocations() { return mLocations; } + +private: + int mState; // what the daemon is currently doing + + Locations mLocations; + + std::vector<std::string> mIDMapMounts; + std::vector<BackupClientInodeToIDMap *> mCurrentIDMaps; + std::vector<BackupClientInodeToIDMap *> mNewIDMaps; + + int mDeleteRedundantLocationsAfter; + + // For the command socket + class CommandSocketInfo + { + public: + CommandSocketInfo(); + ~CommandSocketInfo(); + private: + CommandSocketInfo(const CommandSocketInfo &); // no copying + CommandSocketInfo &operator=(const CommandSocketInfo &); + public: +#ifdef WIN32 + WinNamedPipeListener<1 /* listen backlog */> mListeningSocket; + std::auto_ptr<WinNamedPipeStream> mpConnectedSocket; +#else + SocketListen<SocketStream, 1 /* listen backlog */> mListeningSocket; + std::auto_ptr<SocketStream> mpConnectedSocket; +#endif + std::auto_ptr<IOStreamGetLine> mapGetLine; + }; + + // Using a socket? + std::auto_ptr<CommandSocketInfo> mapCommandSocketInfo; + + // Stop notifications being repeated. + SysadminNotifier::EventCode mLastNotifiedEvent; + + // Unused entries in the root directory wait a while before being deleted + box_time_t mDeleteUnusedRootDirEntriesAfter; // time to delete them + std::vector<std::pair<int64_t,std::string> > mUnusedRootDirEntries; + + int64_t mClientStoreMarker; + bool mStorageLimitExceeded; + bool mReadErrorsOnFilesystemObjects; + box_time_t mLastSyncTime, mNextSyncTime; + box_time_t mCurrentSyncStartTime, mUpdateStoreInterval, + mBackupErrorDelay; + TLSContext mTlsContext; + bool mDeleteStoreObjectInfoFile; + bool mDoSyncForcedByPreviousSyncError; + int64_t mNumFilesUploaded, mNumDirsCreated; + int mMaxBandwidthFromSyncAllowScript; + +public: + int GetMaxBandwidthFromSyncAllowScript() { return mMaxBandwidthFromSyncAllowScript; } + bool StopRun() { return this->Daemon::StopRun(); } + bool StorageLimitExceeded() { return mStorageLimitExceeded; } + +private: + bool mLogAllFileAccess; + +public: + ProgressNotifier* GetProgressNotifier() { return mpProgressNotifier; } + LocationResolver* GetLocationResolver() { return mpLocationResolver; } + RunStatusProvider* GetRunStatusProvider() { return mpRunStatusProvider; } + SysadminNotifier* GetSysadminNotifier() { return mpSysadminNotifier; } + void SetProgressNotifier (ProgressNotifier* p) { mpProgressNotifier = p; } + void SetLocationResolver (LocationResolver* p) { mpLocationResolver = p; } + void SetRunStatusProvider(RunStatusProvider* p) { mpRunStatusProvider = p; } + void SetSysadminNotifier (SysadminNotifier* p) { mpSysadminNotifier = p; } + virtual bool RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum); + +private: + ProgressNotifier* mpProgressNotifier; + LocationResolver* mpLocationResolver; + RunStatusProvider* mpRunStatusProvider; + SysadminNotifier* mpSysadminNotifier; + std::auto_ptr<Timer> mapCommandSocketPollTimer; + std::auto_ptr<BackupClientContext> mapClientContext; + + /* ProgressNotifier implementation */ +public: + virtual void NotifyIDMapsSetup(BackupClientContext& rContext) { } + + virtual void NotifyScanDirectory( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Scanning directory: " << rLocalPath); + } + + if (!RunBackgroundTask(BackgroundTask::Scanning_Dirs, 0, 0)) + { + THROW_EXCEPTION(BackupStoreException, + CancelledByBackgroundTask); + } + } + virtual void NotifyDirStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to access directory: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyFileStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to access file: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyDirListFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to list directory: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyMountPointSkipped( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + #ifdef WIN32 + BOX_WARNING("Ignored directory: " << rLocalPath << + ": is an NTFS junction/reparse point; create " + "a new location if you want to back it up"); + #else + BOX_WARNING("Ignored directory: " << rLocalPath << + ": is a mount point; create a new location " + "if you want to back it up"); + #endif + } + virtual void NotifyFileExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Skipping excluded file: " << rLocalPath); + } + } + virtual void NotifyDirExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Skipping excluded directory: " << rLocalPath); + } + } + virtual void NotifyUnsupportedFileType( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Ignoring file of unknown type: " << rLocalPath); + } + virtual void NotifyFileReadFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Error reading file: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyFileModifiedInFuture( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Some files have modification times excessively " + "in the future. Check clock synchronisation. " + "Example file (only one shown): " << rLocalPath); + } + virtual void NotifyFileSkippedServerFull( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Skipped file: server is full: " << rLocalPath); + } + virtual void NotifyFileUploadException( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const BoxException& rException) + { + if (rException.GetType() == CommonException::ExceptionType && + rException.GetSubType() == CommonException::AccessDenied) + { + BOX_ERROR("Failed to upload file: " << rLocalPath + << ": Access denied"); + } + else + { + BOX_ERROR("Failed to upload file: " << rLocalPath + << ": caught exception: " << rException.what() + << " (" << rException.GetType() + << "/" << rException.GetSubType() << ")"); + } + } + virtual void NotifyFileUploadServerError( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int type, int subtype) + { + BOX_ERROR("Failed to upload file: " << rLocalPath << + ": server error: " << + BackupProtocolError::GetMessage(subtype)); + } + virtual void NotifyFileUploading( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploading complete file: " << rLocalPath); + } + } + virtual void NotifyFileUploadingPatch( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t EstimatedBytesToUpload) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploading patch to file: " << rLocalPath << + ", estimated upload size = " << + EstimatedBytesToUpload); + } + } + virtual void NotifyFileUploadingAttributes( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploading new file attributes: " << + rLocalPath); + } + } + virtual void NotifyFileUploaded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploaded file: " << rLocalPath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + "): total size = " << FileSize << ", " + "uploaded size = " << UploadedSize); + } + mNumFilesUploaded++; + } + virtual void NotifyFileSynchronised( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) + { + if (mLogAllFileAccess) + { + BOX_INFO("Synchronised file: " << rLocalPath); + } + } + virtual void NotifyDirectoryCreated( + int64_t ObjectID, + const std::string& rLocalPath, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Created directory: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + mNumDirsCreated++; + } + virtual void NotifyDirectoryDeleted( + int64_t ObjectID, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Deleted directory: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + } + virtual void NotifyFileDeleted( + int64_t ObjectID, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Deleted file: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", " << (length - offset) << " remain, eta " << + BoxTimeToSeconds(finish - elapsed) << "s"); + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", " << (length - offset) << " remain"); + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", unknown bytes remaining"); + } + +#ifdef WIN32 + private: + bool mInstallService, mRemoveService, mRunAsService; + std::string mServiceName; +#endif + +#ifdef ENABLE_VSS + IVssBackupComponents* mpVssBackupComponents; + void CreateVssBackupComponents(); + bool WaitForAsync(IVssAsync *pAsync, const std::string& description); + typedef HRESULT (__stdcall IVssBackupComponents::*AsyncMethod)(IVssAsync**); + bool CallAndWaitForAsync(AsyncMethod method, + const std::string& description); + void CleanupVssBackupComponents(); +#endif +}; + +#endif // BACKUPDAEMON__H diff --git a/lib/bbackupd/BackupDaemonInterface.h b/lib/bbackupd/BackupDaemonInterface.h new file mode 100644 index 00000000..d5c47c85 --- /dev/null +++ b/lib/bbackupd/BackupDaemonInterface.h @@ -0,0 +1,166 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemonInterface.h +// Purpose: Interfaces for managing a BackupDaemon +// Created: 2008/12/30 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPDAEMONINTERFACE__H +#define BACKUPDAEMONINTERFACE__H + +#include <string> + +#include "BoxTime.h" + +class Archive; +class BackupClientContext; +class BackupDaemon; + +// -------------------------------------------------------------------------- +// +// Class +// Name: SysadminNotifier +// Purpose: Provides a NotifySysadmin() method to send mail to the sysadmin +// Created: 2005/11/15 +// +// -------------------------------------------------------------------------- +class SysadminNotifier +{ + public: + virtual ~SysadminNotifier() { } + + typedef enum + { + StoreFull = 0, + ReadError, + BackupError, + BackupStart, + BackupFinish, + BackupOK, + MAX + // When adding notifications, remember to add + // strings to NotifySysadmin() + } + EventCode; + + virtual void NotifySysadmin(EventCode Event) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: ProgressNotifier +// Purpose: Provides methods for the backup library to inform the user +// interface about its progress with the backup +// Created: 2005/11/20 +// +// -------------------------------------------------------------------------- + +class BackupClientContext; +class BackupClientDirectoryRecord; + +class ProgressNotifier +{ + public: + virtual ~ProgressNotifier() { } + virtual void NotifyIDMapsSetup(BackupClientContext& rContext) = 0; + virtual void NotifyScanDirectory( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyDirStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyFileStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyDirListFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyMountPointSkipped( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyDirExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyUnsupportedFileType( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileReadFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyFileModifiedInFuture( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileSkippedServerFull( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploadException( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const BoxException& rException) = 0; + virtual void NotifyFileUploadServerError( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int type, int subtype) = 0; + virtual void NotifyFileUploading( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploadingPatch( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t EstimatedBytesToUpload) = 0; + virtual void NotifyFileUploadingAttributes( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploaded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) = 0; + virtual void NotifyFileSynchronised( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) = 0; + virtual void NotifyDirectoryCreated( + int64_t ObjectID, + const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirectoryDeleted( + int64_t ObjectID, + const std::string& rRemotePath) = 0; + virtual void NotifyFileDeleted( + int64_t ObjectID, + const std::string& rRemotePath) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: LocationResolver +// Purpose: Interface for classes that can resolve locations to paths, +// like BackupDaemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class LocationResolver +{ +public: + virtual ~LocationResolver() { } + virtual bool FindLocationPathName(const std::string &rLocationName, + std::string &rPathOut) const = 0; +}; + +#endif // BACKUPDAEMONINTERFACE__H diff --git a/lib/bbackupd/Win32BackupService.cpp b/lib/bbackupd/Win32BackupService.cpp new file mode 100644 index 00000000..6d027abf --- /dev/null +++ b/lib/bbackupd/Win32BackupService.cpp @@ -0,0 +1,48 @@ +// Win32 service functions for Box Backup, by Nick Knight + +#ifdef WIN32 + +#include "Box.h" +#include "BackupDaemon.h" +#include "MainHelper.h" +#include "BoxPortsAndFiles.h" +#include "BackupStoreException.h" + +#include "MemLeakFindOn.h" + +#include "Win32BackupService.h" + +Win32BackupService* gpDaemonService = NULL; +extern HANDLE gStopServiceEvent; +extern DWORD gServiceReturnCode; + +unsigned int WINAPI RunService(LPVOID lpParameter) +{ + DWORD retVal = gpDaemonService->WinService((const char*) lpParameter); + gServiceReturnCode = retVal; + SetEvent(gStopServiceEvent); + return retVal; +} + +void TerminateService(void) +{ + gpDaemonService->SetTerminateWanted(); +} + +DWORD Win32BackupService::WinService(const char* pConfigFileName) +{ + DWORD ret; + + if (pConfigFileName != NULL) + { + ret = this->Main(pConfigFileName); + } + else + { + ret = this->Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE); + } + + return ret; +} + +#endif // WIN32 diff --git a/lib/bbackupd/Win32BackupService.h b/lib/bbackupd/Win32BackupService.h new file mode 100644 index 00000000..e7f077f2 --- /dev/null +++ b/lib/bbackupd/Win32BackupService.h @@ -0,0 +1,21 @@ +// Box Backup service daemon implementation by Nick Knight + +#ifndef WIN32BACKUPSERVICE_H +#define WIN32BACKUPSERVICE_H + +#ifdef WIN32 + +class Configuration; +class ConfigurationVerify; +class BackupDaemon; + +class Win32BackupService : public BackupDaemon +{ +public: + DWORD WinService(const char* pConfigFileName); +}; + +#endif // WIN32 + +#endif // WIN32BACKUPSERVICE_H + diff --git a/lib/bbackupd/Win32ServiceFunctions.cpp b/lib/bbackupd/Win32ServiceFunctions.cpp new file mode 100644 index 00000000..2df914a7 --- /dev/null +++ b/lib/bbackupd/Win32ServiceFunctions.cpp @@ -0,0 +1,384 @@ +//*************************************************************** +// From the book "Win32 System Services: The Heart of Windows 98 +// and Windows 2000" +// by Marshall Brain +// Published by Prentice Hall +// Copyright 1995 Prentice Hall. +// +// This code implements the Windows API Service interface +// for the Box Backup for Windows native port. +// Adapted for Box Backup by Nick Knight. +//*************************************************************** + +#ifdef WIN32 + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif +#ifdef HAVE_PROCESS_H + #include <process.h> +#endif + +extern void TerminateService(void); +extern unsigned int WINAPI RunService(LPVOID lpParameter); + +// Global variables + +TCHAR* gServiceName = TEXT("Box Backup Service"); +SERVICE_STATUS gServiceStatus; +SERVICE_STATUS_HANDLE gServiceStatusHandle = 0; +HANDLE gStopServiceEvent = 0; +DWORD gServiceReturnCode = 0; + +#define SERVICE_NAME "boxbackup" + +void ShowMessage(char *s) +{ + MessageBox(0, s, "Box Backup Message", + MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); +} + +void ErrorHandler(char *s, DWORD err) +{ + char buf[256]; + memset(buf, 0, sizeof(buf)); + _snprintf(buf, sizeof(buf)-1, "%s: %s", s, + GetErrorMessage(err).c_str()); + BOX_ERROR(buf); + MessageBox(0, buf, "Error", + MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); + ExitProcess(err); +} + +void WINAPI ServiceControlHandler( DWORD controlCode ) +{ + switch ( controlCode ) + { + case SERVICE_CONTROL_INTERROGATE: + break; + + case SERVICE_CONTROL_SHUTDOWN: + case SERVICE_CONTROL_STOP: + Beep(1000,100); + TerminateService(); + gServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + SetEvent(gStopServiceEvent); + return; + + case SERVICE_CONTROL_PAUSE: + break; + + case SERVICE_CONTROL_CONTINUE: + break; + + default: + if ( controlCode >= 128 && controlCode <= 255 ) + // user defined control code + break; + else + // unrecognised control code + break; + } + + SetServiceStatus( gServiceStatusHandle, &gServiceStatus ); +} + +// ServiceMain is called when the SCM wants to +// start the service. When it returns, the service +// has stopped. It therefore waits on an event +// just before the end of the function, and +// that event gets set when it is time to stop. +// It also returns on any error because the +// service cannot start if there is an eror. + +static char* spConfigFileName; + +VOID ServiceMain(DWORD argc, LPTSTR *argv) +{ + // initialise service status + gServiceStatus.dwServiceType = SERVICE_WIN32; + gServiceStatus.dwCurrentState = SERVICE_STOPPED; + gServiceStatus.dwControlsAccepted = 0; + gServiceStatus.dwWin32ExitCode = NO_ERROR; + gServiceStatus.dwServiceSpecificExitCode = NO_ERROR; + gServiceStatus.dwCheckPoint = 0; + gServiceStatus.dwWaitHint = 0; + + gServiceStatusHandle = RegisterServiceCtrlHandler(gServiceName, + ServiceControlHandler); + + if (gServiceStatusHandle) + { + // service is starting + gServiceStatus.dwCurrentState = SERVICE_START_PENDING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + // do initialisation here + gStopServiceEvent = CreateEvent(0, TRUE, FALSE, 0); + if (!gStopServiceEvent) + { + gServiceStatus.dwControlsAccepted &= + ~(SERVICE_ACCEPT_STOP | + SERVICE_ACCEPT_SHUTDOWN); + gServiceStatus.dwCurrentState = SERVICE_STOPPED; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + return; + } + + HANDLE ourThread = (HANDLE)_beginthreadex( + NULL, + 0, + RunService, + spConfigFileName, + CREATE_SUSPENDED, + NULL); + + SetThreadPriority(ourThread, THREAD_PRIORITY_LOWEST); + ResumeThread(ourThread); + + // we are now running so tell the SCM + gServiceStatus.dwControlsAccepted |= + (SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); + gServiceStatus.dwCurrentState = SERVICE_RUNNING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + // do cleanup here + WaitForSingleObject(gStopServiceEvent, INFINITE); + CloseHandle(gStopServiceEvent); + gStopServiceEvent = 0; + + // service was stopped + gServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + // service is now stopped + gServiceStatus.dwControlsAccepted &= + ~(SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); + + gServiceStatus.dwCurrentState = SERVICE_STOPPED; + + if (gServiceReturnCode != 0) + { + gServiceStatus.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR; + gServiceStatus.dwServiceSpecificExitCode = gServiceReturnCode; + } + + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + } +} + +int OurService(const char* pConfigFileName) +{ + spConfigFileName = strdup(pConfigFileName); + + SERVICE_TABLE_ENTRY serviceTable[] = + { + { SERVICE_NAME, (LPSERVICE_MAIN_FUNCTION) ServiceMain }, + { NULL, NULL } + }; + BOOL success; + + // Register with the SCM + success = StartServiceCtrlDispatcher(serviceTable); + + free(spConfigFileName); + spConfigFileName = NULL; + + if (!success) + { + ErrorHandler("Failed to start service. Did you start " + "Box Backup from the Service Control Manager? " + "(StartServiceCtrlDispatcher)", GetLastError()); + return 1; + } + + return 0; +} + +int InstallService(const char* pConfigFileName, const std::string& rServiceName) +{ + if (pConfigFileName != NULL) + { + EMU_STRUCT_STAT st; + + if (emu_stat(pConfigFileName, &st) != 0) + { + BOX_LOG_SYS_ERROR("Failed to open configuration file " + "'" << pConfigFileName << "'"); + return 1; + } + + if (!(st.st_mode & S_IFREG)) + { + + BOX_ERROR("Failed to open configuration file '" << + pConfigFileName << "': not a file"); + return 1; + } + } + + SC_HANDLE scm = OpenSCManager(0, 0, SC_MANAGER_CREATE_SERVICE); + + if (!scm) + { + BOX_ERROR("Failed to open service control manager: " << + GetErrorMessage(GetLastError())); + return 1; + } + + char cmd[MAX_PATH]; + GetModuleFileName(NULL, cmd, sizeof(cmd)-1); + cmd[sizeof(cmd)-1] = 0; + + std::string cmdWithArgs(cmd); + cmdWithArgs += " -s -S \"" + rServiceName + "\""; + + if (pConfigFileName != NULL) + { + cmdWithArgs += " \""; + cmdWithArgs += pConfigFileName; + cmdWithArgs += "\""; + } + + std::string serviceDesc = "Box Backup (" + rServiceName + ")"; + + SC_HANDLE newService = CreateService( + scm, + rServiceName.c_str(), + serviceDesc.c_str(), + SERVICE_ALL_ACCESS, + SERVICE_WIN32_OWN_PROCESS, + SERVICE_AUTO_START, + SERVICE_ERROR_NORMAL, + cmdWithArgs.c_str(), + 0,0,0,0,0); + + DWORD err = GetLastError(); + CloseServiceHandle(scm); + + if (!newService) + { + switch (err) + { + case ERROR_SERVICE_EXISTS: + { + BOX_ERROR("Failed to create Box Backup " + "service: it already exists"); + } + break; + + case ERROR_SERVICE_MARKED_FOR_DELETE: + { + BOX_ERROR("Failed to create Box Backup " + "service: it is waiting to be deleted"); + } + break; + + case ERROR_DUPLICATE_SERVICE_NAME: + { + BOX_ERROR("Failed to create Box Backup " + "service: a service with this name " + "already exists"); + } + break; + + default: + { + BOX_ERROR("Failed to create Box Backup " + "service: error " << + GetErrorMessage(GetLastError())); + } + } + + return 1; + } + + BOX_INFO("Created Box Backup service"); + + SERVICE_DESCRIPTION desc; + desc.lpDescription = "Backs up your data files over the Internet"; + + if (!ChangeServiceConfig2(newService, SERVICE_CONFIG_DESCRIPTION, + &desc)) + { + BOX_WARNING("Failed to set description for Box Backup " + "service: " << GetErrorMessage(GetLastError())); + } + + CloseServiceHandle(newService); + + return 0; +} + +int RemoveService(const std::string& rServiceName) +{ + SC_HANDLE scm = OpenSCManager(0,0,SC_MANAGER_CREATE_SERVICE); + + if (!scm) + { + BOX_ERROR("Failed to open service control manager: " << + GetErrorMessage(GetLastError())); + return 1; + } + + SC_HANDLE service = OpenService(scm, rServiceName.c_str(), + SERVICE_ALL_ACCESS|DELETE); + DWORD err = GetLastError(); + CloseServiceHandle(scm); + + if (!service) + { + if (err == ERROR_SERVICE_DOES_NOT_EXIST || + err == ERROR_IO_PENDING) + // hello microsoft? anyone home? + { + BOX_ERROR("Failed to open Box Backup service: " + "not installed or not found"); + } + else + { + BOX_ERROR("Failed to open Box Backup service: " << + GetErrorMessage(err)); + } + return 1; + } + + SERVICE_STATUS status; + if (!ControlService(service, SERVICE_CONTROL_STOP, &status)) + { + err = GetLastError(); + if (err != ERROR_SERVICE_NOT_ACTIVE) + { + BOX_WARNING("Failed to stop Box Backup service: " << + GetErrorMessage(err)); + } + } + + BOOL deleted = DeleteService(service); + err = GetLastError(); + CloseServiceHandle(service); + + if (deleted) + { + BOX_INFO("Box Backup service deleted"); + return 0; + } + else if (err == ERROR_SERVICE_MARKED_FOR_DELETE) + { + BOX_ERROR("Failed to remove Box Backup service: " + "it is already being deleted"); + } + else + { + BOX_ERROR("Failed to remove Box Backup service: " << + GetErrorMessage(err)); + } + + return 1; +} + +#endif // WIN32 diff --git a/lib/bbackupd/Win32ServiceFunctions.h b/lib/bbackupd/Win32ServiceFunctions.h new file mode 100644 index 00000000..e04c368f --- /dev/null +++ b/lib/bbackupd/Win32ServiceFunctions.h @@ -0,0 +1,19 @@ +//*************************************************************** +// From the book "Win32 System Services: The Heart of Windows 98 +// and Windows 2000" +// by Marshall Brain +// Published by Prentice Hall +// Copyright 1995 Prentice Hall. +// +// This code implements the Windows API Service interface +// for the Box Backup for Windows native port. +//*************************************************************** + +#ifndef WIN32SERVICEFUNCTIONS_H +#define WIN32SERVICEFUNCTIONS_H + +int RemoveService (const std::string& rServiceName); +int InstallService (const char* pConfigFilePath, const std::string& rServiceName); +int OurService (const char* pConfigFileName); + +#endif diff --git a/lib/bbackupquery/BackupQueries.cpp b/lib/bbackupquery/BackupQueries.cpp new file mode 100644 index 00000000..bcb1827e --- /dev/null +++ b/lib/bbackupquery/BackupQueries.cpp @@ -0,0 +1,2410 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupQueries.cpp +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <stdio.h> +#include <errno.h> +#include <stdlib.h> +#include <limits.h> +#include <sys/types.h> +#include <sys/stat.h> + +#ifdef HAVE_DIRENT_H + #include <dirent.h> +#endif + +#include <algorithm> +#include <cstring> +#include <limits> +#include <iostream> +#include <ostream> +#include <set> + +#include "BackupClientFileAttributes.h" +#include "BackupClientMakeExcludeList.h" +#include "BackupClientRestore.h" +#include "BackupQueries.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFilenameClear.h" +#include "BoxTimeToText.h" +#include "CommonException.h" +#include "Configuration.h" +#include "ExcludeList.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "IOStream.h" +#include "Logging.h" +#include "PathUtils.h" +#include "SelfFlushingStream.h" +#include "Utils.h" +#include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" + +#include "MemLeakFindOn.h" + +// min() and max() macros from stdlib.h break numeric_limits<>::min(), etc. +#undef min +#undef max + +#define COMPARE_RETURN_SAME 1 +#define COMPARE_RETURN_DIFFERENT 2 +#define COMPARE_RETURN_ERROR 3 +#define COMMAND_RETURN_ERROR 4 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::BackupQueries() +// Purpose: Constructor +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +BackupQueries::BackupQueries(BackupProtocolCallable &rConnection, + const Configuration &rConfiguration, bool readWrite) + : mReadWrite(readWrite), + mrConnection(rConnection), + mrConfiguration(rConfiguration), + mQuitNow(false), + mRunningAsRoot(false), + mWarnedAboutOwnerAttributes(false), + mReturnCode(0) // default return code +{ + #ifdef WIN32 + mRunningAsRoot = TRUE; + #else + mRunningAsRoot = (::geteuid() == 0); + #endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::~BackupQueries() +// Purpose: Destructor +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +BackupQueries::~BackupQueries() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::DoCommand(const char *, bool) +// Purpose: Perform a command +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::DoCommand(ParsedCommand& rCommand) +{ + // Check... + + if(rCommand.mFailed) + { + BOX_ERROR("Parse failed: unknown command '" << + rCommand.mCmdElements[0] << "' or failed to convert " + "encoding of arguments"); + return; + } + + if(rCommand.mCmdElements.size() < 1) + { + // blank command + return; + } + + if(rCommand.pSpec->type == Command_sh && + rCommand.mCmdElements.size() == 2) + { + // Yes, run shell command + int result = ::system(rCommand.mCmdElements[1].c_str()); + if(result != 0) + { + BOX_WARNING("System command returned error code " << + result); + SetReturnCode(ReturnCode::Command_Error); + } + return; + } + + if(rCommand.pSpec->type == Command_Unknown) + { + // No such command + BOX_ERROR("Unrecognised command: " << rCommand.mCmdElements[0]); + return; + } + + // Arguments + std::vector<std::string> args(rCommand.mCmdElements.begin() + 1, + rCommand.mCmdElements.end()); + + // Set up options + bool opts[256]; + for(int o = 0; o < 256; ++o) opts[o] = false; + // BLOCK + { + // options + const char *c = rCommand.mOptions.c_str(); + while(*c != 0) + { + // Valid option? + if(::strchr(rCommand.pSpec->opts, *c) == NULL) + { + BOX_ERROR("Invalid option '" << *c << "' for " + "command " << rCommand.pSpec->name); + return; + } + opts[(int)*c] = true; + ++c; + } + } + + if(rCommand.pSpec->type != Command_Quit) + { + // If not a quit command, set the return code to zero + SetReturnCode(ReturnCode::Command_OK); + } + + // Handle command + switch(rCommand.pSpec->type) + { + case Command_Quit: + mQuitNow = true; + break; + + case Command_List: + CommandList(args, opts); + break; + + case Command_pwd: + { + // Simple implementation, so do it here + BOX_NOTICE(GetCurrentDirectoryName() << " (" << + BOX_FORMAT_OBJECTID(GetCurrentDirectoryID()) << + ")"); + } + break; + + case Command_cd: + CommandChangeDir(args, opts); + break; + + case Command_lcd: + CommandChangeLocalDir(args); + break; + + case Command_sh: + BOX_ERROR("The command to run must be specified as an argument."); + break; + + case Command_GetObject: + CommandGetObject(args, opts); + break; + + case Command_Get: + CommandGet(args, opts); + break; + + case Command_Compare: + CommandCompare(args, opts); + break; + + case Command_Restore: + CommandRestore(args, opts); + break; + + case Command_Usage: + CommandUsage(opts); + break; + + case Command_Help: + CommandHelp(args); + break; + + case Command_Undelete: + CommandUndelete(args, opts); + break; + + case Command_Delete: + CommandDelete(args, opts); + break; + + default: + BOX_ERROR("Unknown command: " << rCommand.mCmdElements[0]); + break; + } +} + +#define LIST_OPTION_TIMES_ATTRIBS 'a' +#define LIST_OPTION_SORT_NO_DIRS_FIRST 'D' +#define LIST_OPTION_NOFLAGS 'F' +#define LIST_OPTION_DISPLAY_HASH 'h' +#define LIST_OPTION_SORT_ID 'i' +#define LIST_OPTION_NOOBJECTID 'I' +#define LIST_OPTION_SORT_REVERSE 'r' +#define LIST_OPTION_RECURSIVE 'R' +#define LIST_OPTION_SIZEINBLOCKS 's' +#define LIST_OPTION_SORT_SIZE 'S' +#define LIST_OPTION_TIMES_LOCAL 't' +#define LIST_OPTION_TIMES_UTC 'T' +#define LIST_OPTION_SORT_NONE 'U' + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandList(const std::vector<std::string> &, const bool *) +// Purpose: List directories (optionally recursive) +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandList(const std::vector<std::string> &args, const bool *opts) +{ + // default to using the current directory + int64_t rootDir = GetCurrentDirectoryID(); + + // name of base directory + std::string listRoot; // blank + + // Got a directory in the arguments? + if(args.size() > 0) + { +#ifdef WIN32 + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) + return; +#else + const std::string& storeDirEncoded(args[0]); +#endif + + // Attempt to find the directory + rootDir = FindDirectoryObjectID(storeDirEncoded, + opts[LIST_OPTION_ALLOWOLD], + opts[LIST_OPTION_ALLOWDELETED]); + + if(rootDir == 0) + { + BOX_ERROR("Directory '" << args[0] << "' not found " + "on store."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + } + + // List it + List(rootDir, listRoot, opts, true /* first level to list */); +} + +static std::string GetTimeString(BackupStoreDirectory::Entry& en, + bool useLocalTime, bool showAttrModificationTimes) +{ + std::ostringstream out; + box_time_t originalTime, newAttributesTime; + + // there is no attribute modification time in the directory + // entry, unfortunately, so we can't display it. + originalTime = en.GetModificationTime(); + out << BoxTimeToISO8601String(originalTime, useLocalTime); + + if(en.HasAttributes()) + { + const StreamableMemBlock &storeAttr(en.GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + + box_time_t NewModificationTime, NewAttrModificationTime; + attr.GetModificationTimes(&NewModificationTime, + &NewAttrModificationTime); + + if (showAttrModificationTimes) + { + newAttributesTime = NewAttrModificationTime; + } + else + { + newAttributesTime = NewModificationTime; + } + + if (newAttributesTime == originalTime) + { + out << "*"; + } + else + { + out << "~" << BoxTimeToISO8601String(newAttributesTime, + useLocalTime); + } + } + else + { + out << " "; + } + + return out.str(); +} + +/* We need a way to pass options to sort functions for sorting. The algorithm + * doesn't seem to provide a way to do this, so I'm using a global variable. + * Which is not thread safe, but we don't currently use threads so that should + * be OK. Do not use threads without checking! + */ +const bool *gThreadUnsafeOptions; + +int DirsFirst(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + if (a->IsDir() && !b->IsDir()) + { + return -1; // a < b + } + else if (!a->IsDir() && b->IsDir()) + { + return 1; // b > a + } + else + { + return 0; // continue comparison + } +} + +#define MAYBE_DIRS_FIRST(a, b) \ + if (!gThreadUnsafeOptions[LIST_OPTION_SORT_NO_DIRS_FIRST]) \ + { \ + int result = DirsFirst(a, b); \ + if (result < 0) return true; /* a < b */ \ + else if (result > 0) return false; /* a > b */ \ + /* else: fall through */ \ + } + +#define MAYBE_REVERSE(result) \ + (result != gThreadUnsafeOptions[LIST_OPTION_SORT_REVERSE]) +// result is false, opts[reverse] is false => return false +// result is false, opts[reverse] is true => return true +// result is true, opts[reverse] is false => return true +// result is true, opts[reverse] is true => return false +// this is logical XOR, for which the boolean operator is !=. + +bool SortById(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + bool result = (a->GetObjectID() < b->GetObjectID()); + return MAYBE_REVERSE(result); +} + +bool SortBySize(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + bool result = (a->GetSizeInBlocks() < b->GetSizeInBlocks()); + return MAYBE_REVERSE(result); +} + +bool SortByName(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + BackupStoreFilenameClear afc(a->GetName()); + BackupStoreFilenameClear bfc(b->GetName()); + std::string an = afc.GetClearFilename(); + std::string bn = bfc.GetClearFilename(); + bool result = (an < bn); + return MAYBE_REVERSE(result); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::List(int64_t, const std::string &, const bool *, bool) +// Purpose: Do the actual listing of directories and files +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::List(int64_t DirID, const std::string &rListRoot, + const bool *opts, bool FirstLevel, std::ostream* pOut) +{ +#ifdef WIN32 + DWORD n_chars; + HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE); +#endif + + // Generate exclude flags + int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + if(!opts[LIST_OPTION_ALLOWOLD]) excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + if(!opts[LIST_OPTION_ALLOWDELETED]) excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; + + // Do communication + try + { + mrConnection.QueryListDirectory( + DirID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // both files and directories + excludeFlags, + true /* want attributes */); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to list directory: " << e.what()); + SetReturnCode(ReturnCode::Command_Error); + return; + } + catch (...) + { + BOX_ERROR("Failed to list directory: unknown error"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Store entry pointers in a std::vector for sorting + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + std::vector<BackupStoreDirectory::Entry*> sorted_entries; + while((en = i.Next()) != 0) + { + sorted_entries.push_back(en); + } + + // Typedef to avoid mind-bending while dealing with pointers to functions. + typedef bool (EntryComparator_t)(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b); + // Default is no comparator, i.e. no sorting. + EntryComparator_t* pComparator = NULL; + + if (opts[LIST_OPTION_SORT_ID]) + { + pComparator = &SortById; + } + else if (opts[LIST_OPTION_SORT_SIZE]) + { + pComparator = &SortBySize; + } + else if (opts[LIST_OPTION_SORT_NONE]) + { + // do nothing + } + else // sort by name + { + pComparator = &SortByName; + } + + if (pComparator != NULL) + { + gThreadUnsafeOptions = opts; + sort(sorted_entries.begin(), sorted_entries.end(), + pComparator); + gThreadUnsafeOptions = NULL; + } + + for (std::vector<BackupStoreDirectory::Entry*>::const_iterator + i = sorted_entries.begin(); + i != sorted_entries.end(); i++) + { + en = *i; + std::ostringstream buf; + + // Display this entry + BackupStoreFilenameClear clear(en->GetName()); + + // Object ID? + if(!opts[LIST_OPTION_NOOBJECTID]) + { + // add object ID to line + buf << std::hex << std::internal << std::setw(8) << + std::setfill('0') << en->GetObjectID() << + std::dec << " "; + } + + // Flags? + if(!opts[LIST_OPTION_NOFLAGS]) + { + static const char *flags = BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES; + char displayflags[16]; + // make sure f is big enough + ASSERT(sizeof(displayflags) >= sizeof(BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES) + 3); + // Insert flags + char *f = displayflags; + const char *t = flags; + int16_t en_flags = en->GetFlags(); + while(*t != 0) + { + *f = ((en_flags&1) == 0)?'-':*t; + en_flags >>= 1; + f++; + t++; + } + // attributes flags + *(f++) = (en->HasAttributes())?'a':'-'; + + // terminate + *(f++) = ' '; + *(f++) = '\0'; + buf << displayflags; + + if(en_flags != 0) + { + buf << "[ERROR: Entry has additional flags set] "; + } + } + + if(opts[LIST_OPTION_TIMES_UTC]) + { + // Show UTC times... + buf << GetTimeString(*en, false, + opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; + } + + if(opts[LIST_OPTION_TIMES_LOCAL]) + { + // Show local times... + buf << GetTimeString(*en, true, + opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; + } + + if(opts[LIST_OPTION_DISPLAY_HASH]) + { + buf << std::hex << std::internal << std::setw(16) << + std::setfill('0') << en->GetAttributesHash() << + std::dec; + } + + if(opts[LIST_OPTION_SIZEINBLOCKS]) + { + buf << std::internal << std::setw(5) << + std::setfill('0') << en->GetSizeInBlocks() << + " "; + } + + // add name + if(!FirstLevel) + { +#ifdef WIN32 + std::string listRootDecoded; + if(!ConvertUtf8ToConsole(rListRoot.c_str(), + listRootDecoded)) return; + listRootDecoded += "/"; + buf << listRootDecoded; + WriteConsole(hOut, listRootDecoded.c_str(), + strlen(listRootDecoded.c_str()), &n_chars, NULL); +#else + buf << rListRoot << "/"; +#endif + } + + std::string fileName; + try + { + fileName = clear.GetClearFilename(); + } + catch(CipherException &e) + { + fileName = "<decrypt failed>"; + } + +#ifdef WIN32 + std::string fileNameUtf8 = fileName; + if(!ConvertUtf8ToConsole(fileNameUtf8, fileName)) + { + fileName = fileNameUtf8 + " [convert encoding failed]"; + } +#endif + + buf << fileName; + + if(!en->GetName().IsEncrypted()) + { + buf << " [FILENAME NOT ENCRYPTED]"; + } + + buf << std::endl; + + if(pOut) + { + (*pOut) << buf.str(); + } + else + { +#ifdef WIN32 + std::string line = buf.str(); + if (!WriteConsole(hOut, line.c_str(), line.size(), + &n_chars, NULL)) + { + // WriteConsole failed, try standard method + std::cout << buf.str(); + } +#else + std::cout << buf.str(); +#endif + } + + // Directory? + if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) + { + // Recurse? + if(opts[LIST_OPTION_RECURSIVE]) + { + std::string subroot(rListRoot); + if(!FirstLevel) subroot += '/'; + subroot += clear.GetClearFilename(); + List(en->GetObjectID(), subroot, opts, + false /* not the first level to list */, + pOut); + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::FindDirectoryObjectID(const +// std::string &) +// Purpose: Find the object ID of a directory on the store, +// or return 0 for not found. If pStack != 0, the +// object is set to the stack of directories. +// Will start from the current directory stack. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, + bool AllowOldVersion, bool AllowDeletedDirs, + std::vector<std::pair<std::string, int64_t> > *pStack) +{ + // Split up string into elements + std::vector<std::string> dirElements; + SplitString(rDirName, '/', dirElements); + + // Start from current stack, or root, whichever is required + std::vector<std::pair<std::string, int64_t> > stack; + int64_t dirID = BackupProtocolListDirectory::RootDirectory; + if(rDirName.size() > 0 && rDirName[0] == '/') + { + // Root, do nothing + } + else + { + // Copy existing stack + stack = mDirStack; + if(stack.size() > 0) + { + dirID = stack[stack.size() - 1].second; + } + } + + // Generate exclude flags + int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + if(!AllowOldVersion) excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + if(!AllowDeletedDirs) excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; + + // Read directories + for(unsigned int e = 0; e < dirElements.size(); ++e) + { + if(dirElements[e].size() > 0) + { + if(dirElements[e] == ".") + { + // Ignore. + } + else if(dirElements[e] == "..") + { + // Up one! + if(stack.size() > 0) + { + // Remove top element + stack.pop_back(); + + // New dir ID + dirID = (stack.size() > 0)?(stack[stack.size() - 1].second):BackupProtocolListDirectory::RootDirectory; + } + else + { + // At root anyway + dirID = BackupProtocolListDirectory::RootDirectory; + } + } + else + { + // Not blank element. Read current directory. + std::auto_ptr<BackupProtocolSuccess> dirreply(mrConnection.QueryListDirectory( + dirID, + BackupProtocolListDirectory::Flags_Dir, // just directories + excludeFlags, + true /* want attributes */)); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Then... find the directory within it + BackupStoreDirectory::Iterator i(dir); + BackupStoreFilenameClear dirname(dirElements[e]); + BackupStoreDirectory::Entry *en = i.FindMatchingClearName(dirname); + if(en == 0) + { + // Not found + return 0; + } + + // Object ID for next round of searching + dirID = en->GetObjectID(); + + // Push onto stack + stack.push_back(std::pair<std::string, int64_t>(dirElements[e], dirID)); + } + } + } + + // If required, copy the new stack to the caller + if(pStack) + { + *pStack = stack; + } + + return dirID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::GetCurrentDirectoryID() +// Purpose: Returns the ID of the current directory +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::GetCurrentDirectoryID() +{ + // Special case for root + if(mDirStack.size() == 0) + { + return BackupProtocolListDirectory::RootDirectory; + } + + // Otherwise, get from the last entry on the stack + return mDirStack[mDirStack.size() - 1].second; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::GetCurrentDirectoryName() +// Purpose: Gets the name of the current directory +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +std::string BackupQueries::GetCurrentDirectoryName() +{ + // Special case for root + if(mDirStack.size() == 0) + { + return std::string("/"); + } + + // Build path + std::string r; + for(unsigned int l = 0; l < mDirStack.size(); ++l) + { + r += "/"; +#ifdef WIN32 + std::string dirName; + if(!ConvertUtf8ToConsole(mDirStack[l].first.c_str(), dirName)) + return "error"; + r += dirName; +#else + r += mDirStack[l].first; +#endif + } + + return r; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandChangeDir(const std::vector<std::string> &) +// Purpose: Change directory command +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandChangeDir(const std::vector<std::string> &args, const bool *opts) +{ + if(args.size() != 1 || args[0].size() == 0) + { + BOX_ERROR("Incorrect usage. cd [-o] [-d] <directory>"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + +#ifdef WIN32 + std::string dirName; + if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) return; +#else + const std::string& dirName(args[0]); +#endif + + std::vector<std::pair<std::string, int64_t> > newStack; + int64_t id = FindDirectoryObjectID(dirName, opts['o'], opts['d'], + &newStack); + + if(id == 0) + { + BOX_ERROR("Directory '" << args[0] << "' not found."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Store new stack + mDirStack = newStack; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandChangeLocalDir(const std::vector<std::string> &) +// Purpose: Change local directory command +// Created: 2003/10/11 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandChangeLocalDir(const std::vector<std::string> &args) +{ + if(args.size() != 1 || args[0].size() == 0) + { + BOX_ERROR("Incorrect usage. lcd <local-directory>"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Try changing directory +#ifdef WIN32 + std::string dirName; + if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) + { + BOX_ERROR("Failed to convert path from console encoding."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + int result = ::chdir(dirName.c_str()); +#else + int result = ::chdir(args[0].c_str()); +#endif + if(result != 0) + { + if(errno == ENOENT || errno == ENOTDIR) + { + BOX_ERROR("Directory '" << args[0] << "' does not exist."); + } + else + { + BOX_LOG_SYS_ERROR("Failed to change to directory " + "'" << args[0] << "'"); + } + + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Report current dir + char wd[PATH_MAX]; + if(::getcwd(wd, PATH_MAX) == 0) + { + BOX_LOG_SYS_ERROR("Error getting current directory"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + +#ifdef WIN32 + if(!ConvertUtf8ToConsole(wd, dirName)) + { + BOX_ERROR("Failed to convert new path from console encoding."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + BOX_INFO("Local current directory is now '" << dirName << "'."); +#else + BOX_INFO("Local current directory is now '" << wd << "'."); +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandGetObject(const std::vector<std::string> &, const bool *) +// Purpose: Gets an object without any translation. +// Created: 2003/10/11 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandGetObject(const std::vector<std::string> &args, const bool *opts) +{ + // Check args + if(args.size() != 2) + { + BOX_ERROR("Incorrect usage. getobject <object-id> " + "<local-filename>"); + return; + } + + int64_t id = ::strtoll(args[0].c_str(), 0, 16); + if(id == std::numeric_limits<long long>::min() || id == std::numeric_limits<long long>::max() || id == 0) + { + BOX_ERROR("Not a valid object ID (specified in hex): " << + args[0]); + return; + } + + // Does file exist? + EMU_STRUCT_STAT st; + if(EMU_STAT(args[1].c_str(), &st) == 0 || errno != ENOENT) + { + BOX_ERROR("The local file '" << args[1] << " already exists."); + return; + } + + // Open file + FileStream out(args[1].c_str(), O_WRONLY | O_CREAT | O_EXCL); + + // Request that object + try + { + // Request object + std::auto_ptr<BackupProtocolSuccess> getobj(mrConnection.QueryGetObject(id)); + + // Stream that object out to the file + std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); + objectStream->CopyStreamTo(out); + + BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(id) << + " fetched successfully."); + } + catch(ConnectionException &e) + { + if(mrConnection.GetLastErrorType() == BackupProtocolError::Err_DoesNotExist) + { + BOX_ERROR("Object ID " << BOX_FORMAT_OBJECTID(id) << + " does not exist on store."); + ::unlink(args[1].c_str()); + } + else + { + BOX_ERROR("Error occured fetching object."); + ::unlink(args[1].c_str()); + } + } + catch(...) + { + ::unlink(args[1].c_str()); + BOX_ERROR("Error occured fetching object."); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::FindFileID(const std::string& +// rNameOrIdString, const bool *options, +// int64_t *pDirIdOut, std::string* pFileNameOut) +// Purpose: Locate a file on the store (either by name or by +// object ID, depending on opts['i'], where name can +// include a path) and return the file ID, placing the +// directory ID in *pDirIdOut and the filename part +// of the path in *pFileNameOut (if not NULL). +// Created: 2008-09-12 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::FindFileID(const std::string& rNameOrIdString, + const bool *opts, int64_t *pDirIdOut, std::string* pFileNameOut, + int16_t flagsInclude, int16_t flagsExclude, int16_t* pFlagsOut) +{ + // Find object ID somehow + int64_t fileId; + int64_t dirId = GetCurrentDirectoryID(); + std::string fileName = rNameOrIdString; + + if(!opts['i']) + { + // does this remote filename include a path? + std::string::size_type index = fileName.rfind('/'); + if(index != std::string::npos) + { + std::string dirName(fileName.substr(0, index)); + fileName = fileName.substr(index + 1); + + dirId = FindDirectoryObjectID(dirName); + if(dirId == 0) + { + BOX_ERROR("Directory '" << dirName << + "' not found."); + return 0; + } + } + } + + BackupStoreFilenameClear fn(fileName); + + // Need to look it up in the current directory + mrConnection.QueryListDirectory( + dirId, flagsInclude, flagsExclude, + true /* do want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + BackupStoreDirectory::Entry *en; + + if(opts['i']) + { + // Specified as ID. + fileId = ::strtoll(rNameOrIdString.c_str(), 0, 16); + if(fileId == std::numeric_limits<long long>::min() || + fileId == std::numeric_limits<long long>::max() || + fileId == 0) + { + BOX_ERROR("Not a valid object ID (specified in hex): " + << rNameOrIdString); + return 0; + } + + // Check that the item is actually in the directory + en = dir.FindEntryByID(fileId); + if(en == 0) + { + BOX_ERROR("File ID " << + BOX_FORMAT_OBJECTID(fileId) << + " not found in current directory on store.\n" + "(You can only access files by ID from the " + "current directory.)"); + return 0; + } + } + else + { + // Specified by name, find the object in the directory to get the ID + BackupStoreDirectory::Iterator i(dir); + en = i.FindMatchingClearName(fn); + if(en == 0) + { + BOX_ERROR("Filename '" << rNameOrIdString << "' " + "not found in current directory on store.\n" + "(Subdirectories in path not searched.)"); + return 0; + } + + fileId = en->GetObjectID(); + } + + *pDirIdOut = dirId; + + if(pFlagsOut) + { + *pFlagsOut = en->GetFlags(); + } + + if(pFileNameOut) + { + BackupStoreFilenameClear entryName(en->GetName()); + *pFileNameOut = entryName.GetClearFilename(); + } + + return fileId; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandGet(const std::vector<std::string> &, const bool *) +// Purpose: Command to get a file from the store +// Created: 2003/10/12 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) +{ + // At least one argument? + // Check args + if(args.size() < 1 || (opts['i'] && args.size() != 2) || args.size() > 2) + { + BOX_ERROR("Incorrect usage.\n" + "get <remote-filename> [<local-filename>] or\n" + "get -i <object-id> <local-filename>"); + return; + } + + // Find object ID somehow + int64_t fileId, dirId; + std::string localName; + +#ifdef WIN32 + for (std::vector<std::string>::iterator + i = args.begin(); i != args.end(); i++) + { + std::string out; + if(!ConvertConsoleToUtf8(i->c_str(), out)) + { + BOX_ERROR("Failed to convert encoding."); + return; + } + *i = out; + } +#endif + + int16_t flagsExclude; + + if(opts['i']) + { + // can retrieve anything by ID + flagsExclude = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + } + else + { + // only current versions by name + flagsExclude = + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted; + } + + + fileId = FindFileID(args[0], opts, &dirId, &localName, + BackupProtocolListDirectory::Flags_File, // just files + flagsExclude, NULL /* don't care about flags found */); + + if (fileId == 0) + { + // error already reported + return; + } + + if(opts['i']) + { + // Specified as ID. Must have a local name in the arguments + // (check at beginning of function ensures this) + localName = args[1]; + } + else + { + // Specified by name. Local name already set by FindFileID, + // but may be overridden by user supplying a second argument. + if(args.size() == 2) + { + localName = args[1]; + } + } + + // Does local file already exist? (don't want to overwrite) + EMU_STRUCT_STAT st; + if(EMU_STAT(localName.c_str(), &st) == 0 || errno != ENOENT) + { + BOX_ERROR("The local file " << localName << " already exists, " + "will not overwrite it."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Request it from the store + try + { + // Request object + mrConnection.QueryGetFile(dirId, fileId); + + // Stream containing encoded file + std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); + + // Decode it + BackupStoreFile::DecodeFile(*objectStream, localName.c_str(), mrConnection.GetTimeout()); + + // Done. + BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(fileId) << + " fetched successfully."); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to fetch file: " << + e.what()); + ::unlink(localName.c_str()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to fetch file: " << + e.what()); + ::unlink(localName.c_str()); + } + catch(...) + { + BOX_ERROR("Failed to fetch file: unknown error"); + ::unlink(localName.c_str()); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CompareParams::CompareParams() +// Purpose: Constructor +// Created: 29/1/04 +// +// -------------------------------------------------------------------------- +BackupQueries::CompareParams::CompareParams(bool QuickCompare, + bool IgnoreExcludes, bool IgnoreAttributes, + box_time_t LatestFileUploadTime) +: BoxBackupCompareParams(QuickCompare, IgnoreExcludes, IgnoreAttributes, + LatestFileUploadTime), + mDifferences(0), + mDifferencesExplainedByModTime(0), + mUncheckedFiles(0), + mExcludedDirs(0), + mExcludedFiles(0) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandCompare(const std::vector<std::string> &, const bool *) +// Purpose: Command to compare data on the store with local data +// Created: 2003/10/12 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandCompare(const std::vector<std::string> &args, const bool *opts) +{ + box_time_t LatestFileUploadTime = GetCurrentBoxTime(); + + // Try and work out the time before which all files should be on the server + { + std::string syncTimeFilename(mrConfiguration.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); + syncTimeFilename += "last_sync_start"; + // Stat it to get file time + EMU_STRUCT_STAT st; + if(EMU_STAT(syncTimeFilename.c_str(), &st) == 0) + { + // Files modified after this time shouldn't be on the server, so report errors slightly differently + LatestFileUploadTime = FileModificationTime(st) - + SecondsToBoxTime(mrConfiguration.GetKeyValueInt("MinimumFileAge")); + } + else + { + BOX_WARNING("Failed to determine the time of the last " + "synchronisation -- checks not performed."); + } + } + + // Parameters, including count of differences + BackupQueries::CompareParams params(opts['q'], // quick compare? + opts['E'], // ignore excludes + opts['A'], // ignore attributes + LatestFileUploadTime); + + params.mQuietCompare = opts['Q']; + + // Quick compare? + if(params.QuickCompare()) + { + BOX_WARNING("Quick compare used -- file attributes are not " + "checked."); + } + + if(!opts['l'] && opts['a'] && args.size() == 0) + { + // Compare all locations + const Configuration &rLocations( + mrConfiguration.GetSubConfiguration("BackupLocations")); + std::vector<std::string> locNames = + rLocations.GetSubConfigurationNames(); + for(std::vector<std::string>::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + CompareLocation(*pLocName, params); + } + } + else if(opts['l'] && !opts['a'] && args.size() == 1) + { + // Compare one location + CompareLocation(args[0], params); + } + else if(!opts['l'] && !opts['a'] && args.size() == 2) + { + // Compare directory to directory + + // Can't be bothered to do all the hard work to work out which location it's on, and hence which exclude list + if(!params.IgnoreExcludes()) + { + BOX_ERROR("Cannot use excludes on directory to directory comparison -- use -E flag to specify ignored excludes."); + return; + } + else + { + // Do compare + Compare(args[0], args[1], params); + } + } + else + { + BOX_ERROR("Incorrect usage.\ncompare -a\n or compare -l <location-name>\n or compare <store-dir-name> <local-dir-name>"); + return; + } + + if (!params.mQuietCompare) + { + BOX_INFO("[ " << + params.mDifferencesExplainedByModTime << " (of " << + params.mDifferences << ") differences probably " + "due to file modifications after the last upload ]"); + } + + BOX_INFO("Differences: " << params.mDifferences << " (" << + params.mExcludedDirs << " dirs excluded, " << + params.mExcludedFiles << " files excluded, " << + params.mUncheckedFiles << " files not checked)"); + + // Set return code? + if(opts['c']) + { + if (params.mUncheckedFiles != 0) + { + SetReturnCode(ReturnCode::Compare_Error); + } + else if (params.mDifferences != 0) + { + SetReturnCode(ReturnCode::Compare_Different); + } + else + { + SetReturnCode(ReturnCode::Compare_Same); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CompareLocation(const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a location +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::CompareLocation(const std::string &rLocation, + BoxBackupCompareParams &rParams) +{ + // Find the location's sub configuration + const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations")); + if(!locations.SubConfigurationExists(rLocation.c_str())) + { + BOX_ERROR("Location " << rLocation << " does not exist."); + return; + } + const Configuration &loc(locations.GetSubConfiguration(rLocation.c_str())); + + #ifdef WIN32 + { + std::string path = loc.GetKeyValue("Path"); + if (path.size() > 0 && path[path.size()-1] == + DIRECTORY_SEPARATOR_ASCHAR) + { + BOX_WARNING("Location '" << rLocation << "' path ends " + "with '" DIRECTORY_SEPARATOR "', " + "compare may fail!"); + } + } + #endif + + // Generate the exclude lists + if(!rParams.IgnoreExcludes()) + { + rParams.LoadExcludeLists(loc); + } + + // Then get it compared + Compare(std::string("/") + rLocation, loc.GetKeyValue("Path"), rParams); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::Compare(const std::string &, +// const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a store directory against a local directory +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::Compare(const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams) +{ +#ifdef WIN32 + std::string localDirEncoded; + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(rLocalDir.c_str(), localDirEncoded)) return; + if(!ConvertConsoleToUtf8(rStoreDir.c_str(), storeDirEncoded)) return; +#else + const std::string& localDirEncoded(rLocalDir); + const std::string& storeDirEncoded(rStoreDir); +#endif + + // Get the directory ID of the directory -- only use current data + int64_t dirID = FindDirectoryObjectID(storeDirEncoded); + + // Found? + if(dirID == 0) + { + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalDir.c_str(), &st) == 0) + { + if(FileAttrModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyRemoteFileMissing(localDirEncoded, + storeDirEncoded, modifiedAfterLastSync); + return; + } + + // Go! + Compare(dirID, storeDirEncoded, localDirEncoded, rParams); +} + +void BackupQueries::CompareOneFile(int64_t DirID, + BackupStoreDirectory::Entry *pEntry, + const std::string& rLocalPath, + const std::string& rStorePath, + BoxBackupCompareParams &rParams) +{ + int64_t fileId = pEntry->GetObjectID(); + int64_t fileSize = 0; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalPath.c_str(), &st) == 0) + { + fileSize = st.st_size; + } + + try + { + // Files the same flag? + bool equal = true; + + // File modified after last sync flag + bool modifiedAfterLastSync = false; + + bool hasDifferentAttribs = false; + + bool alreadyReported = false; + + if(rParams.QuickCompare()) + { + // Compare file -- fetch it + mrConnection.QueryGetBlockIndexByID(fileId); + + // Stream containing block index + std::auto_ptr<IOStream> blockIndexStream(mrConnection.ReceiveStream()); + + // Compare + equal = BackupStoreFile::CompareFileContentsAgainstBlockIndex( + rLocalPath.c_str(), *blockIndexStream, + mrConnection.GetTimeout()); + } + else + { + // Compare file -- fetch it + mrConnection.QueryGetFile(DirID, pEntry->GetObjectID()); + + // Stream containing encoded file + std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); + + // Decode it + std::auto_ptr<BackupStoreFile::DecodedStream> fileOnServerStream; + + // Got additional attributes? + if(pEntry->HasAttributes()) + { + // Use these attributes + const StreamableMemBlock &storeAttr(pEntry->GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + fileOnServerStream.reset( + BackupStoreFile::DecodeFileStream( + *objectStream, + mrConnection.GetTimeout(), + &attr).release()); + } + else + { + // Use attributes stored in file + fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout()).release()); + } + + // Should always be something in the auto_ptr, it's how the interface is defined. But be paranoid. + if(!fileOnServerStream.get()) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Compare attributes + BackupClientFileAttributes localAttr; + box_time_t fileModTime = 0; + localAttr.ReadAttributes(rLocalPath.c_str(), false /* don't zero mod times */, &fileModTime); + modifiedAfterLastSync = (fileModTime > rParams.LatestFileUploadTime()); + bool ignoreAttrModTime = true; + + #ifdef WIN32 + // attr mod time is really + // creation time, so check it + ignoreAttrModTime = false; + #endif + + if(!rParams.IgnoreAttributes() && + #ifdef PLATFORM_DISABLE_SYMLINK_ATTRIB_COMPARE + !fileOnServerStream->IsSymLink() && + #endif + !localAttr.Compare(fileOnServerStream->GetAttributes(), + ignoreAttrModTime, + fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */)) + { + hasDifferentAttribs = true; + } + + // Compare contents, if it's a regular file not a link + // Remember, we MUST read the entire stream from the server. + SelfFlushingStream flushObject(*objectStream); + + if(!fileOnServerStream->IsSymLink()) + { + SelfFlushingStream flushFile(*fileOnServerStream); + // Open the local file + std::auto_ptr<FileStream> apLocalFile; + + try + { + apLocalFile.reset(new FileStream(rLocalPath.c_str())); + } + catch(std::exception &e) + { + rParams.NotifyLocalFileReadFailed(rLocalPath, + rStorePath, fileSize, e); + alreadyReported = true; + } + catch(...) + { + rParams.NotifyLocalFileReadFailed(rLocalPath, + rStorePath, fileSize); + alreadyReported = true; + } + + if(apLocalFile.get()) + { + equal = apLocalFile->CompareWith(*fileOnServerStream, + mrConnection.GetTimeout()); + } + } + } + + rParams.NotifyFileCompared(rLocalPath, rStorePath, fileSize, + hasDifferentAttribs, !equal, modifiedAfterLastSync, + pEntry->HasAttributes()); + } + catch(BoxException &e) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize, + e); + } + catch(std::exception &e) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize, + e); + } + catch(...) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::Compare(int64_t, const std::string &, +// const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a store directory against a local directory +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams) +{ + rParams.NotifyDirComparing(rLocalDir, rStoreDir); + + // Get info on the local directory + EMU_STRUCT_STAT st; + if(EMU_LSTAT(rLocalDir.c_str(), &st) != 0) + { + // What kind of error? + if(errno == ENOTDIR || errno == ENOENT) + { + rParams.NotifyLocalDirMissing(rLocalDir, rStoreDir); + } + else + { + rParams.NotifyLocalDirAccessFailed(rLocalDir, rStoreDir); + } + return; + } + + // Get the directory listing from the store + mrConnection.QueryListDirectory( + DirID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // get everything + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted, + // except for old versions and deleted files + true /* want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Test out the attributes + if(!dir.HasAttributes()) + { + rParams.NotifyStoreDirMissingAttributes(rLocalDir, rStoreDir); + } + else + { + // Fetch the attributes + const StreamableMemBlock &storeAttr(dir.GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + + // Get attributes of local directory + BackupClientFileAttributes localAttr; + localAttr.ReadAttributes(rLocalDir.c_str(), + true /* directories have zero mod times */); + + if(attr.Compare(localAttr, true, true /* ignore modification times */)) + { + rParams.NotifyDirCompared(rLocalDir, rStoreDir, + false, false /* actually we didn't check :) */); + } + else + { + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalDir.c_str(), &st) == 0) + { + if(FileAttrModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyDirCompared(rLocalDir, rStoreDir, + true, modifiedAfterLastSync); + } + } + + // Open the local directory + DIR *dirhandle = ::opendir(rLocalDir.c_str()); + if(dirhandle == 0) + { + rParams.NotifyLocalDirAccessFailed(rLocalDir, rStoreDir); + return; + } + + try + { + // Read the files and directories into sets + std::set<std::string> localFiles; + std::set<std::string> localDirs; + struct dirent *localDirEn = 0; + while((localDirEn = readdir(dirhandle)) != 0) + { + // Not . and ..! + if(localDirEn->d_name[0] == '.' && + (localDirEn->d_name[1] == '\0' || (localDirEn->d_name[1] == '.' && localDirEn->d_name[2] == '\0'))) + { + // ignore, it's . or .. + +#ifdef HAVE_VALID_DIRENT_D_TYPE + if (localDirEn->d_type != DT_DIR) + { + BOX_ERROR("d_type does not really " + "work on your platform. " + "Reconfigure Box!"); + return; + } +#endif + + continue; + } + + std::string localDirPath(MakeFullPath(rLocalDir, + localDirEn->d_name)); + std::string storeDirPath(rStoreDir + "/" + + localDirEn->d_name); + +#ifndef HAVE_VALID_DIRENT_D_TYPE + EMU_STRUCT_STAT st; + if(EMU_LSTAT(localDirPath.c_str(), &st) != 0) + { + // Check whether dir is excluded before trying + // to stat it, to fix problems with .gvfs + // directories that are not readable by root + // causing compare to crash: + // http://lists.boxbackup.org/pipermail/boxbackup/2010-January/000013.html + if(rParams.IsExcludedDir(localDirPath)) + { + rParams.NotifyExcludedDir(localDirPath, + storeDirPath); + continue; + } + else + { + THROW_EXCEPTION_MESSAGE(CommonException, + OSFileError, localDirPath); + } + } + + // Entry -- file or dir? + if(S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) + { + // File or symbolic link + localFiles.insert(std::string(localDirEn->d_name)); + } + else if(S_ISDIR(st.st_mode)) + { + // Directory + localDirs.insert(std::string(localDirEn->d_name)); + } +#else + // Entry -- file or dir? + if(localDirEn->d_type == DT_REG || localDirEn->d_type == DT_LNK) + { + // File or symbolic link + localFiles.insert(std::string(localDirEn->d_name)); + } + else if(localDirEn->d_type == DT_DIR) + { + // Directory + localDirs.insert(std::string(localDirEn->d_name)); + } +#endif + } + // Close directory + if(::closedir(dirhandle) != 0) + { + BOX_LOG_SYS_ERROR("Failed to close local directory " + "'" << rLocalDir << "'"); + } + dirhandle = 0; + + // Do the same for the store directories + std::set<std::pair<std::string, BackupStoreDirectory::Entry *> > storeFiles; + std::set<std::pair<std::string, BackupStoreDirectory::Entry *> > storeDirs; + + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *storeDirEn = 0; + while((storeDirEn = i.Next()) != 0) + { + // Decrypt filename + BackupStoreFilenameClear name(storeDirEn->GetName()); + + // What is it? + if((storeDirEn->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == BackupStoreDirectory::Entry::Flags_File) + { + // File + storeFiles.insert(std::pair<std::string, BackupStoreDirectory::Entry *>(name.GetClearFilename(), storeDirEn)); + } + else + { + // Dir + storeDirs.insert(std::pair<std::string, BackupStoreDirectory::Entry *>(name.GetClearFilename(), storeDirEn)); + } + } + +#ifdef _MSC_VER + typedef std::set<std::string>::iterator string_set_iter_t; +#else + typedef std::set<std::string>::const_iterator string_set_iter_t; +#endif + + // Now compare files. + for(std::set<std::pair<std::string, BackupStoreDirectory::Entry *> >::const_iterator i = storeFiles.begin(); i != storeFiles.end(); ++i) + { + const std::string& fileName(i->first); + + std::string localPath(MakeFullPath(rLocalDir, fileName)); + std::string storePath(rStoreDir + "/" + fileName); + + rParams.NotifyFileComparing(localPath, storePath); + + // Does the file exist locally? + string_set_iter_t local(localFiles.find(fileName)); + if(local == localFiles.end()) + { + // Not found -- report + rParams.NotifyLocalFileMissing(localPath, + storePath); + } + else + { + CompareOneFile(DirID, i->second, localPath, + storePath, rParams); + + // Remove from set so that we know it's been compared + localFiles.erase(local); + } + } + + // Report any files which exist locally, but not on the store + for(string_set_iter_t i = localFiles.begin(); i != localFiles.end(); ++i) + { + std::string localPath(MakeFullPath(rLocalDir, *i)); + std::string storePath(rStoreDir + "/" + *i); + + // Should this be ignored (ie is excluded)? + if(!rParams.IsExcludedFile(localPath)) + { + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(localPath.c_str(), &st) == 0) + { + if(FileModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyRemoteFileMissing(localPath, + storePath, modifiedAfterLastSync); + } + else + { + rParams.NotifyExcludedFile(localPath, + storePath); + } + } + + // Finished with the files, clear the sets to reduce memory usage slightly + localFiles.clear(); + storeFiles.clear(); + + // Now do the directories, recursively to check subdirectories + for(std::set<std::pair<std::string, BackupStoreDirectory::Entry *> >::const_iterator i = storeDirs.begin(); i != storeDirs.end(); ++i) + { + std::string localPath(MakeFullPath(rLocalDir, i->first)); + std::string storePath(rStoreDir + "/" + i->first); + + // Does the directory exist locally? + string_set_iter_t local(localDirs.find(i->first)); + if(local == localDirs.end() && + rParams.IsExcludedDir(localPath)) + { + rParams.NotifyExcludedFileNotDeleted(localPath, + storePath); + } + else if(local == localDirs.end()) + { + // Not found -- report + rParams.NotifyLocalFileMissing(localPath, + storePath); + } + else if(rParams.IsExcludedDir(localPath)) + { + // don't recurse into excluded directories + } + else + { + // Compare directory + Compare(i->second->GetObjectID(), + storePath, localPath, rParams); + + // Remove from set so that we know it's been compared + localDirs.erase(local); + } + } + + // Report any directories which exist locally, but not on the store + for(std::set<std::string>::const_iterator + i = localDirs.begin(); + i != localDirs.end(); ++i) + { + std::string localPath(MakeFullPath(rLocalDir, *i)); + std::string storePath(rStoreDir + "/" + *i); + + // Should this be ignored (ie is excluded)? + if(!rParams.IsExcludedDir(localPath)) + { + bool modifiedAfterLastSync = false; + + // Check the dir modification time + EMU_STRUCT_STAT st; + if(EMU_STAT(localPath.c_str(), &st) == 0 && + FileModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + + rParams.NotifyRemoteFileMissing(localPath, + storePath, modifiedAfterLastSync); + } + else + { + rParams.NotifyExcludedDir(localPath, storePath); + } + } + } + catch(...) + { + if(dirhandle != 0) + { + ::closedir(dirhandle); + } + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandRestore(const std::vector<std::string> &, const bool *) +// Purpose: Restore a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandRestore(const std::vector<std::string> &args, const bool *opts) +{ + // Check arguments + if(args.size() < 1 || args.size() > 2) + { + BOX_ERROR("Incorrect usage. restore [-drif] <remote-name> " + "[<local-name>]"); + return; + } + + // Restoring deleted things? + bool restoreDeleted = opts['d']; + + std::string storeDirEncoded; + + // Get directory ID + int64_t dirID = 0; + if(opts['i']) + { + // Specified as ID. + dirID = ::strtoll(args[0].c_str(), 0, 16); + if(dirID == std::numeric_limits<long long>::min() || dirID == std::numeric_limits<long long>::max() || dirID == 0) + { + BOX_ERROR("Not a valid object ID (specified in hex): " + << args[0]); + return; + } + std::ostringstream oss; + oss << BOX_FORMAT_OBJECTID(args[0]); + storeDirEncoded = oss.str(); + } + else + { +#ifdef WIN32 + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) + return; +#else + storeDirEncoded = args[0]; +#endif + + // Look up directory ID + dirID = FindDirectoryObjectID(storeDirEncoded, + false /* no old versions */, + restoreDeleted /* find deleted dirs */); + } + + // Allowable? + if(dirID == 0) + { + BOX_ERROR("Directory '" << args[0] << "' not found on server"); + return; + } + + if(dirID == BackupProtocolListDirectory::RootDirectory) + { + BOX_ERROR("Cannot restore the root directory -- restore locations individually."); + return; + } + + std::string localName; + + if(args.size() == 2) + { + #ifdef WIN32 + if(!ConvertConsoleToUtf8(args[1].c_str(), localName)) + { + return; + } + #else + localName = args[1]; + #endif + } + else + { + localName = args[0]; + } + + // Go and restore... + int result; + + try + { + // At TRACE level, we print a line for each file and + // directory, so we don't need dots. + + result = BackupClientRestore(mrConnection, dirID, + storeDirEncoded.c_str(), localName.c_str(), + true /* print progress dots */, restoreDeleted, + false /* don't undelete after restore! */, + opts['r'] /* resume? */, + opts['f'] /* force continue after errors */); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to restore: " << e.what()); + SetReturnCode(ReturnCode::Command_Error); + return; + } + catch(...) + { + BOX_ERROR("Failed to restore: unknown exception"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + switch(result) + { + case Restore_Complete: + BOX_INFO("Restore complete."); + break; + + case Restore_CompleteWithErrors: + BOX_WARNING("Restore complete, but some files could not be " + "restored."); + break; + + case Restore_ResumePossible: + BOX_ERROR("Resume possible -- repeat command with -r flag " + "to resume."); + SetReturnCode(ReturnCode::Command_Error); + break; + + case Restore_TargetExists: + BOX_ERROR("The target directory exists. You cannot restore " + "over an existing directory."); + SetReturnCode(ReturnCode::Command_Error); + break; + + case Restore_TargetPathNotFound: + BOX_ERROR("The target directory path does not exist.\n" + "To restore to a directory whose parent " + "does not exist, create the parent first."); + SetReturnCode(ReturnCode::Command_Error); + break; + + case Restore_UnknownError: + BOX_ERROR("Unknown error during restore."); + SetReturnCode(ReturnCode::Command_Error); + break; + + default: + BOX_ERROR("Unknown restore result " << result << "."); + SetReturnCode(ReturnCode::Command_Error); + break; + } +} + + + +// These are autogenerated by a script. +extern const char *help_commands[]; +extern const char *help_text[]; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandHelp(const std::vector<std::string> &args) +// Purpose: Display help on commands +// Created: 15/2/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandHelp(const std::vector<std::string> &args) +{ + if(args.size() == 0) + { + // Display a list of all commands + printf("Available commands are:\n"); + for(int c = 0; help_commands[c] != 0; ++c) + { + printf(" %s\n", help_commands[c]); + } + printf("Type \"help <command>\" for more information on a command.\n\n"); + } + else + { + // Display help on a particular command + int c; + for(c = 0; help_commands[c] != 0; ++c) + { + if(::strcmp(help_commands[c], args[0].c_str()) == 0) + { + // Found the command, print help + printf("\n%s\n", help_text[c]); + break; + } + } + if(help_commands[c] == 0) + { + printf("No help found for command '%s'\n", args[0].c_str()); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUsage() +// Purpose: Display storage space used on server +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUsage(const bool *opts) +{ + bool MachineReadable = opts['m']; + + // Request full details from the server + std::auto_ptr<BackupProtocolAccountUsage> usage(mrConnection.QueryGetAccountUsage()); + + // Display each entry in turn + int64_t hardLimit = usage->GetBlocksHardLimit(); + int32_t blockSize = usage->GetBlockSize(); + CommandUsageDisplayEntry("Used", usage->GetBlocksUsed(), hardLimit, + blockSize, MachineReadable); + CommandUsageDisplayEntry("Old files", usage->GetBlocksInOldFiles(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Deleted files", usage->GetBlocksInDeletedFiles(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Directories", usage->GetBlocksInDirectories(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Soft limit", usage->GetBlocksSoftLimit(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Hard limit", hardLimit, hardLimit, blockSize, + MachineReadable); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUsageDisplayEntry(const char *, +// int64_t, int64_t, int32_t, bool) +// Purpose: Display an entry in the usage table +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUsageDisplayEntry(const char *Name, int64_t Size, +int64_t HardLimit, int32_t BlockSize, bool MachineReadable) +{ + std::cout << FormatUsageLineStart(Name, MachineReadable) << + FormatUsageBar(Size, Size * BlockSize, HardLimit * BlockSize, + MachineReadable) << std::endl; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUndelete(const std::vector<std::string> &, const bool *) +// Purpose: Undelete a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUndelete(const std::vector<std::string> &args, const bool *opts) +{ + if (!mReadWrite) + { + BOX_ERROR("This command requires a read-write connection. " + "Please reconnect with the -w option."); + return; + } + + // Check arguments + if(args.size() != 1) + { + BOX_ERROR("Incorrect usage. undelete <name> or undelete -i <object-id>"); + return; + } + +#ifdef WIN32 + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) return; +#else + const std::string& storeDirEncoded(args[0]); +#endif + + // Find object ID somehow + int64_t fileId, parentId; + std::string fileName; + int16_t flagsOut; + + fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, + /* include files and directories */ + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + /* include old and deleted files */ + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + &flagsOut); + + if (fileId == 0) + { + // error already reported + return; + } + + // Undelete it on the store + try + { + // Undelete object + if(flagsOut & BackupProtocolListDirectory::Flags_File) + { + mrConnection.QueryUndeleteFile(parentId, fileId); + } + else + { + mrConnection.QueryUndeleteDirectory(fileId); + } + } + catch (BoxException &e) + { + BOX_ERROR("Failed to undelete object: " << + e.what()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to undelete object: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Failed to undelete object: unknown error"); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandDelete(const +// std::vector<std::string> &, const bool *) +// Purpose: Deletes a file +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandDelete(const std::vector<std::string> &args, + const bool *opts) +{ + if (!mReadWrite) + { + BOX_ERROR("This command requires a read-write connection. " + "Please reconnect with the -w option."); + return; + } + + // Check arguments + if(args.size() != 1) + { + BOX_ERROR("Incorrect usage. delete <name>"); + return; + } + +#ifdef WIN32 + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) return; +#else + const std::string& storeDirEncoded(args[0]); +#endif + + // Find object ID somehow + int64_t fileId, parentId; + std::string fileName; + int16_t flagsOut; + + fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, + /* include files and directories */ + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + /* exclude old and deleted files */ + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted, + &flagsOut); + + if (fileId == 0) + { + // error already reported + return; + } + + BackupStoreFilenameClear fn(fileName); + + // Delete it on the store + try + { + // Delete object + if(flagsOut & BackupProtocolListDirectory::Flags_File) + { + mrConnection.QueryDeleteFile(parentId, fn); + } + else + { + mrConnection.QueryDeleteDirectory(fileId); + } + } + catch (BoxException &e) + { + BOX_ERROR("Failed to delete object: " << + e.what()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to delete object: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Failed to delete object: unknown error"); + } +} diff --git a/lib/bbackupquery/BackupQueries.h b/lib/bbackupquery/BackupQueries.h new file mode 100644 index 00000000..96df34f5 --- /dev/null +++ b/lib/bbackupquery/BackupQueries.h @@ -0,0 +1,440 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupQueries.h +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPQUERIES__H +#define BACKUPQUERIES__H + +#include <iostream> +#include <string> +#include <vector> + +#include "BoxTime.h" +#include "BoxBackupCompareParams.h" +#include "BackupStoreDirectory.h" + +class BackupProtocolCallable; +class Configuration; +class ExcludeList; + +typedef enum +{ + Command_Unknown = 0, + Command_Quit, + Command_List, + Command_pwd, + Command_cd, + Command_lcd, + Command_sh, + Command_GetObject, + Command_Get, + Command_Compare, + Command_Restore, + Command_Help, + Command_Usage, + Command_Undelete, + Command_Delete, +} +CommandType; + +struct QueryCommandSpecification; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupQueries +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +class BackupQueries +{ +public: + BackupQueries(BackupProtocolCallable &rConnection, + const Configuration &rConfiguration, + bool readWrite); + ~BackupQueries(); +private: + BackupQueries(const BackupQueries &); +public: + struct ParsedCommand + { + std::vector<std::string> mCmdElements; + std::string mOptions; + std::string mCompleteCommand; + bool mInOptions, mFailed; + QueryCommandSpecification* pSpec; + // mArgCount is the same as mCmdElements.size() for a complete + // command, but if the command line ends in a space, + // e.g. during readline parsing, it can be one greater, + // to indicate that we should complete the next item instead. + size_t mCompleteArgCount; + ParsedCommand(const std::string& Command, + bool isFromCommandLine); + bool IsEmpty() { return mCmdElements.empty(); } + bool IsFailed() { return mFailed; } + }; + + void DoCommand(ParsedCommand& rCommand); + + // Ready to stop? + bool Stop() {return mQuitNow;} + + // Return code? + int GetReturnCode() {return mReturnCode;} + + void List(int64_t DirID, const std::string &rListRoot, const bool *opts, + bool FirstLevel, std::ostream* pOut = NULL); + void CommandList(const std::vector<std::string> &args, const bool *opts); + + // Commands + void CommandChangeDir(const std::vector<std::string> &args, const bool *opts); + void CommandChangeLocalDir(const std::vector<std::string> &args); + void CommandGetObject(const std::vector<std::string> &args, const bool *opts); + void CommandGet(std::vector<std::string> args, const bool *opts); + void CommandCompare(const std::vector<std::string> &args, const bool *opts); + void CommandRestore(const std::vector<std::string> &args, const bool *opts); + void CommandUndelete(const std::vector<std::string> &args, const bool *opts); + void CommandDelete(const std::vector<std::string> &args, + const bool *opts); + void CommandUsage(const bool *opts); + void CommandUsageDisplayEntry(const char *Name, int64_t Size, + int64_t HardLimit, int32_t BlockSize, bool MachineReadable); + void CommandHelp(const std::vector<std::string> &args); + + class CompareParams : public BoxBackupCompareParams + { + public: + CompareParams(bool QuickCompare, bool IgnoreExcludes, + bool IgnoreAttributes, box_time_t LatestFileUploadTime); + + bool mQuietCompare; + int mDifferences; + int mDifferencesExplainedByModTime; + int mUncheckedFiles; + int mExcludedDirs; + int mExcludedFiles; + + std::string ConvertForConsole(const std::string& rUtf8String) + { + #ifdef WIN32 + std::string output; + + if(!ConvertUtf8ToConsole(rUtf8String.c_str(), output)) + { + BOX_WARNING("Character set conversion failed " + "on string: " << rUtf8String); + return rUtf8String; + } + + return output; + #else + return rUtf8String; + #endif + } + + virtual void NotifyLocalDirMissing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Local directory '" << + ConvertForConsole(rLocalPath) << "' " + "does not exist, but remote directory does."); + mDifferences ++; + } + + virtual void NotifyLocalDirAccessFailed( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_LOG_SYS_WARNING("Failed to access local directory " + "'" << ConvertForConsole(rLocalPath) << "'"); + mUncheckedFiles ++; + } + + virtual void NotifyStoreDirMissingAttributes( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Store directory '" << + ConvertForConsole(rRemotePath) << "' " + "doesn't have attributes."); + } + + virtual void NotifyRemoteFileMissing( + const std::string& rLocalPath, + const std::string& rRemotePath, + bool modifiedAfterLastSync) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "exists, but remote file '" << + ConvertForConsole(rRemotePath) << "' " + "does not."); + mDifferences ++; + + if(modifiedAfterLastSync) + { + mDifferencesExplainedByModTime ++; + BOX_INFO("(the file above was modified after " + "the last sync time -- might be " + "reason for difference)"); + } + } + + virtual void NotifyLocalFileMissing( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Remote file '" << + ConvertForConsole(rRemotePath) << "' " + "exists, but local file '" << + ConvertForConsole(rLocalPath) << "' does not."); + mDifferences ++; + } + + virtual void NotifyExcludedFileNotDeleted( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "is excluded, but remote file '" << + ConvertForConsole(rRemotePath) << "' " + "still exists."); + mDifferences ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + BoxException& rException) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath) << "': " << + rException.what() << " (" << + rException.GetType() << "/" << + rException.GetSubType() << ")"); + mUncheckedFiles ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath) << "': " << + rException.what()); + mUncheckedFiles ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath)); + mUncheckedFiles ++; + } + + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) + { + BOX_ERROR("Failed to read local file '" << + ConvertForConsole(rLocalPath) << "': " << + rException.what()); + mUncheckedFiles ++; + } + + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) + { + BOX_ERROR("Failed to read local file '" << + ConvertForConsole(rLocalPath)); + mUncheckedFiles ++; + } + + virtual void NotifyExcludedFile(const std::string& rLocalPath, + const std::string& rRemotePath) + { + mExcludedFiles ++; + } + + virtual void NotifyExcludedDir(const std::string& rLocalPath, + const std::string& rRemotePath) + { + mExcludedDirs ++; + } + + virtual void NotifyDirComparing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_INFO("Comparing directory: " << rLocalPath); + } + + virtual void NotifyDirCompared( + const std::string& rLocalPath, + const std::string& rRemotePath, + bool HasDifferentAttributes, + bool modifiedAfterLastSync) + { + if(HasDifferentAttributes) + { + BOX_WARNING("Local directory '" << + ConvertForConsole(rLocalPath) << "' " + "has different attributes to " + "store directory '" << + ConvertForConsole(rRemotePath) << "'."); + mDifferences ++; + + if(modifiedAfterLastSync) + { + mDifferencesExplainedByModTime ++; + BOX_INFO("(the directory above was " + "modified after the last sync " + "time -- might be reason for " + "difference)"); + } + } + } + + virtual void NotifyFileComparing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_TRACE("Comparing file: " << rLocalPath); + } + + virtual void NotifyFileCompared(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + bool HasDifferentAttributes, bool HasDifferentContents, + bool ModifiedAfterLastSync, bool NewAttributesApplied) + { + int NewDifferences = 0; + + if(HasDifferentAttributes) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "has different attributes to " + "store file '" << + ConvertForConsole(rRemotePath) << "'."); + NewDifferences ++; + } + + if(HasDifferentContents) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "has different contents to " + "store file '" << + ConvertForConsole(rRemotePath) << "'."); + NewDifferences ++; + } + + if(HasDifferentAttributes || HasDifferentContents) + { + if(ModifiedAfterLastSync) + { + mDifferencesExplainedByModTime += + NewDifferences; + BOX_INFO("(the file above was modified " + "after the last sync time -- " + "might be reason for difference)"); + } + else if(NewAttributesApplied) + { + BOX_INFO("(the file above has had new " + "attributes applied)\n"); + } + } + + mDifferences += NewDifferences; + } + }; + void CompareLocation(const std::string &rLocation, + BoxBackupCompareParams &rParams); + void Compare(const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams); + void Compare(int64_t DirID, const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams); + void CompareOneFile(int64_t DirID, BackupStoreDirectory::Entry *pEntry, + const std::string& rLocalPath, const std::string& rStorePath, + BoxBackupCompareParams &rParams); + +public: + + class ReturnCode + { + public: + typedef enum { + Command_OK = 0, + Compare_Same = 1, + Compare_Different, + Compare_Error, + Command_Error, + } Type; + }; + + // Were private, but needed by completion functions: + int64_t GetCurrentDirectoryID(); + int64_t FindDirectoryObjectID(const std::string &rDirName, + bool AllowOldVersion = false, bool AllowDeletedDirs = false, + std::vector<std::pair<std::string, int64_t> > *pStack = 0); + +private: + + // Utility functions + int64_t FindFileID(const std::string& rNameOrIdString, + const bool *opts, int64_t *pDirIdOut, + std::string* pFileNameOut, int16_t flagsInclude, + int16_t flagsExclude, int16_t* pFlagsOut); + std::string GetCurrentDirectoryName(); + void SetReturnCode(int code) {mReturnCode = code;} + +private: + bool mReadWrite; + BackupProtocolCallable &mrConnection; + const Configuration &mrConfiguration; + bool mQuitNow; + std::vector<std::pair<std::string, int64_t> > mDirStack; + bool mRunningAsRoot; + bool mWarnedAboutOwnerAttributes; + int mReturnCode; +}; + +typedef std::vector<std::string> (*CompletionHandler) + (BackupQueries::ParsedCommand& rCommand, const std::string& prefix, + BackupProtocolCallable& rProtocol, const Configuration& rConfig, + BackupQueries& rQueries); + +std::vector<std::string> CompleteCommand(BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolCallable& rProtocol, + const Configuration& rConfig, BackupQueries& rQueries); +std::vector<std::string> CompleteOptions(BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolCallable& rProtocol, + const Configuration& rConfig, BackupQueries& rQueries); + +#define MAX_COMPLETION_HANDLERS 4 + +struct QueryCommandSpecification +{ + const char* name; + const char* opts; + CommandType type; + CompletionHandler complete[MAX_COMPLETION_HANDLERS]; +}; + +// Data about commands +extern QueryCommandSpecification commands[]; + +extern const char *alias[]; +extern const int aliasIs[]; + +#define LIST_OPTION_ALLOWOLD 'o' +#define LIST_OPTION_ALLOWDELETED 'd' + +#endif // BACKUPQUERIES__H + diff --git a/lib/bbackupquery/BoxBackupCompareParams.h b/lib/bbackupquery/BoxBackupCompareParams.h new file mode 100644 index 00000000..655df947 --- /dev/null +++ b/lib/bbackupquery/BoxBackupCompareParams.h @@ -0,0 +1,112 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxBackupCompareParams.h +// Purpose: Parameters and notifiers for a compare operation +// Created: 2008/12/30 +// +// -------------------------------------------------------------------------- + +#ifndef BOXBACKUPCOMPAREPARAMS__H +#define BOXBACKUPCOMPAREPARAMS__H + +#include <memory> +#include <string> + +#include "BoxTime.h" +#include "ExcludeList.h" +#include "BackupClientMakeExcludeList.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: BoxBackupCompareParams +// Purpose: Parameters and notifiers for a compare operation +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +class BoxBackupCompareParams +{ +private: + std::auto_ptr<const ExcludeList> mapExcludeFiles, mapExcludeDirs; + bool mQuickCompare; + bool mIgnoreExcludes; + bool mIgnoreAttributes; + box_time_t mLatestFileUploadTime; + +public: + BoxBackupCompareParams(bool QuickCompare, bool IgnoreExcludes, + bool IgnoreAttributes, box_time_t LatestFileUploadTime) + : mQuickCompare(QuickCompare), + mIgnoreExcludes(IgnoreExcludes), + mIgnoreAttributes(IgnoreAttributes), + mLatestFileUploadTime(LatestFileUploadTime) + { } + + virtual ~BoxBackupCompareParams() { } + + bool QuickCompare() { return mQuickCompare; } + bool IgnoreExcludes() { return mIgnoreExcludes; } + bool IgnoreAttributes() { return mIgnoreAttributes; } + box_time_t LatestFileUploadTime() { return mLatestFileUploadTime; } + + void LoadExcludeLists(const Configuration& rLoc) + { + mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rLoc)); + mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rLoc)); + } + bool IsExcludedFile(const std::string& rLocalPath) + { + if (!mapExcludeFiles.get()) return false; + return mapExcludeFiles->IsExcluded(rLocalPath); + } + bool IsExcludedDir(const std::string& rLocalPath) + { + if (!mapExcludeDirs.get()) return false; + return mapExcludeDirs->IsExcluded(rLocalPath); + } + + virtual void NotifyLocalDirMissing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyLocalDirAccessFailed(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyStoreDirMissingAttributes(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyRemoteFileMissing(const std::string& rLocalPath, + const std::string& rRemotePath, + bool modifiedAfterLastSync) = 0; + virtual void NotifyLocalFileMissing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyExcludedFileNotDeleted(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + BoxException& rException) = 0; + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) = 0; + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) = 0; + virtual void NotifyExcludedFile(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyExcludedDir(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirComparing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirCompared(const std::string& rLocalPath, + const std::string& rRemotePath, bool HasDifferentAttributes, + bool modifiedAfterLastSync) = 0; + virtual void NotifyFileComparing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyFileCompared(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + bool HasDifferentAttributes, bool HasDifferentContents, + bool modifiedAfterLastSync, bool newAttributesApplied) = 0; +}; + +#endif // BOXBACKUPCOMPAREPARAMS__H diff --git a/lib/bbackupquery/CommandCompletion.cpp b/lib/bbackupquery/CommandCompletion.cpp new file mode 100644 index 00000000..761fc97e --- /dev/null +++ b/lib/bbackupquery/CommandCompletion.cpp @@ -0,0 +1,604 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CommandCompletion.cpp +// Purpose: Parts of BackupQueries that depend on readline +// Created: 2011/01/21 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_LIBREADLINE + #ifdef HAVE_READLINE_READLINE_H + #include <readline/readline.h> + #elif defined(HAVE_EDITLINE_READLINE_H) + #include <editline/readline.h> + #elif defined(HAVE_READLINE_H) + #include <readline.h> + #endif +#endif + +#ifdef HAVE_READLINE_HISTORY + #ifdef HAVE_READLINE_HISTORY_H + #include <readline/history.h> + #elif defined(HAVE_HISTORY_H) + #include <history.h> + #endif +#endif + +#include <cstring> + +#include "BackupQueries.h" +#include "Configuration.h" + +#include "autogen_BackupProtocol.h" + +#include "MemLeakFindOn.h" + +#define COMPARE_RETURN_SAME 1 +#define COMPARE_RETURN_DIFFERENT 2 +#define COMPARE_RETURN_ERROR 3 +#define COMMAND_RETURN_ERROR 4 + +#define COMPLETION_FUNCTION(name, code) \ +std::vector<std::string> Complete ## name( \ + BackupQueries::ParsedCommand& rCommand, \ + const std::string& prefix, \ + BackupProtocolCallable& rProtocol, const Configuration& rConfig, \ + BackupQueries& rQueries) \ +{ \ + std::vector<std::string> completions; \ + \ + try \ + { \ + code \ + } \ + catch(std::exception &e) \ + { \ + BOX_TRACE("Failed to complete " << prefix << ": " << e.what()); \ + } \ + catch(...) \ + { \ + BOX_TRACE("Failed to complete " << prefix << ": " \ + "unknown error"); \ + } \ + \ + return completions; \ +} + +#define DELEGATE_COMPLETION(name) \ + completions = Complete ## name(rCommand, prefix, rProtocol, rConfig, \ + rQueries); + +COMPLETION_FUNCTION(None,) + +#ifdef HAVE_RL_FILENAME_COMPLETION_FUNCTION + #define RL_FILENAME_COMPLETION_FUNCTION rl_filename_completion_function + #define HAVE_A_FILENAME_COMPLETION_FUNCTION 1 +#elif defined HAVE_FILENAME_COMPLETION_FUNCTION + #define RL_FILENAME_COMPLETION_FUNCTION filename_completion_function + #define HAVE_A_FILENAME_COMPLETION_FUNCTION 1 +#endif + +#ifdef HAVE_A_FILENAME_COMPLETION_FUNCTION +COMPLETION_FUNCTION(Default, + int i = 0; + + while (const char *match = RL_FILENAME_COMPLETION_FUNCTION(prefix.c_str(), i)) + { + completions.push_back(match); + ++i; + } +) +#else // !HAVE_A_FILENAME_COMPLETION_FUNCTION +COMPLETION_FUNCTION(Default,) +#endif // HAVE_A_FILENAME_COMPLETION_FUNCTION + +COMPLETION_FUNCTION(Command, + int len = prefix.length(); + + for(int i = 0; commands[i].name != NULL; i++) + { + if(::strncmp(commands[i].name, prefix.c_str(), len) == 0) + { + completions.push_back(commands[i].name); + } + } +) + +void CompleteOptionsInternal(const std::string& prefix, + BackupQueries::ParsedCommand& rCommand, + std::vector<std::string>& completions) +{ + std::string availableOptions = rCommand.pSpec->opts; + + for(std::string::iterator + opt = availableOptions.begin(); + opt != availableOptions.end(); opt++) + { + if(rCommand.mOptions.find(*opt) == std::string::npos) + { + if(prefix == "") + { + // complete with possible option strings + completions.push_back(std::string("-") + *opt); + } + else + { + // complete with possible additional options + completions.push_back(prefix + *opt); + } + } + } +} + +COMPLETION_FUNCTION(Options, + CompleteOptionsInternal(prefix, rCommand, completions); +) + +std::string EncodeFileName(const std::string &rUnEncodedName) +{ +#ifdef WIN32 + std::string encodedName; + if(!ConvertConsoleToUtf8(rUnEncodedName, encodedName)) + { + return std::string(); + } + return encodedName; +#else + return rUnEncodedName; +#endif +} + +int16_t GetExcludeFlags(BackupQueries::ParsedCommand& rCommand) +{ + int16_t excludeFlags = 0; + + if (rCommand.mOptions.find(LIST_OPTION_ALLOWOLD) == std::string::npos) + { + excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + } + + if (rCommand.mOptions.find(LIST_OPTION_ALLOWDELETED) == std::string::npos) + { + excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; + } + + return excludeFlags; +} + +std::vector<std::string> CompleteRemoteFileOrDirectory( + BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolCallable& rProtocol, + BackupQueries& rQueries, int16_t includeFlags) +{ + std::vector<std::string> completions; + + // default to using the current directory + int64_t listDirId = rQueries.GetCurrentDirectoryID(); + std::string searchPrefix; + std::string listDir = prefix; + + if(rCommand.mCompleteArgCount == rCommand.mCmdElements.size()) + { + // completing an empty name, from the current directory + // nothing to change + } + else + { + // completing a partially-completed subdirectory name + searchPrefix = prefix; + listDir = ""; + + // do we need to list a subdirectory to complete? + size_t lastSlash = searchPrefix.rfind('/'); + if(lastSlash == std::string::npos) + { + // no slashes, so the whole name is the prefix + // nothing to change + } + else + { + // listing a partially-completed subdirectory name + listDir = searchPrefix.substr(0, lastSlash); + + listDirId = rQueries.FindDirectoryObjectID(listDir, + rCommand.mOptions.find(LIST_OPTION_ALLOWOLD) + != std::string::npos, + rCommand.mOptions.find(LIST_OPTION_ALLOWDELETED) + != std::string::npos); + + if(listDirId == 0) + { + // no matches for subdir to list, + // return empty-handed. + return completions; + } + + // matched, and updated listDir and listDirId already + searchPrefix = searchPrefix.substr(lastSlash + 1); + } + } + + // Always include directories, because they contain files. + // We will append a slash later for each directory if we're + // actually looking for files. + // + // If we're looking for directories, then only list directories. + + bool completeFiles = includeFlags & + BackupProtocolListDirectory::Flags_File; + bool completeDirs = includeFlags & + BackupProtocolListDirectory::Flags_Dir; + int16_t listFlags = 0; + + if(completeFiles) + { + listFlags = BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING; + } + else if(completeDirs) + { + listFlags = BackupProtocolListDirectory::Flags_Dir; + } + + rProtocol.QueryListDirectory(listDirId, + listFlags, GetExcludeFlags(rCommand), + false /* no attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream()); + dir.ReadFromStream(*dirstream, rProtocol.GetTimeout()); + + // Then... display everything + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + BackupStoreFilenameClear clear(en->GetName()); + std::string name = clear.GetClearFilename().c_str(); + if(name.compare(0, searchPrefix.length(), searchPrefix) == 0) + { + bool dir_added = false; + + if(en->IsDir() && + (includeFlags & BackupProtocolListDirectory::Flags_Dir) == 0) + { + // Was looking for a file, but this is a + // directory, so append a slash to the name + name += "/"; + } + + #ifdef HAVE_LIBREADLINE + if(strchr(name.c_str(), ' ')) + { + int n_quote = 0; + + for(int k = strlen(rl_line_buffer); k >= 0; k--) + { + if (rl_line_buffer[k] == '\"') { + ++n_quote; + } + } + + dir_added = false; + + if (!(n_quote % 2)) + { + name = "\"" + (listDir == "" ? name : listDir + "/" + name); + dir_added = true; + } + + name = name + "\""; + } + #endif + + if(listDir == "" || dir_added) + { + completions.push_back(name); + } + else + { + completions.push_back(listDir + "/" + name); + } + } + } + + return completions; +} + +COMPLETION_FUNCTION(RemoteDir, + completions = CompleteRemoteFileOrDirectory(rCommand, prefix, + rProtocol, rQueries, + BackupProtocolListDirectory::Flags_Dir); +) + +COMPLETION_FUNCTION(RemoteFile, + completions = CompleteRemoteFileOrDirectory(rCommand, prefix, + rProtocol, rQueries, + BackupProtocolListDirectory::Flags_File); +) + +COMPLETION_FUNCTION(LocalDir, + DELEGATE_COMPLETION(Default); +) + +COMPLETION_FUNCTION(LocalFile, + DELEGATE_COMPLETION(Default); +) + +COMPLETION_FUNCTION(LocationName, + const Configuration &locations(rConfig.GetSubConfiguration( + "BackupLocations")); + + std::vector<std::string> locNames = + locations.GetSubConfigurationNames(); + + for(std::vector<std::string>::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + if(pLocName->compare(0, pLocName->length(), prefix) == 0) + { + completions.push_back(*pLocName); + } + } +) + +COMPLETION_FUNCTION(RemoteFileIdInCurrentDir, + int64_t listDirId = rQueries.GetCurrentDirectoryID(); + int16_t excludeFlags = GetExcludeFlags(rCommand); + + rProtocol.QueryListDirectory( + listDirId, + BackupProtocolListDirectory::Flags_File, + excludeFlags, false /* no attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream()); + dir.ReadFromStream(*dirstream, rProtocol.GetTimeout()); + + // Then... compare each item + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + std::ostringstream hexId; + hexId << std::hex << en->GetObjectID(); + if(hexId.str().compare(0, prefix.length(), prefix) == 0) + { + completions.push_back(hexId.str()); + } + } +) + +// TODO implement completion of hex IDs up to the maximum according to Usage +COMPLETION_FUNCTION(RemoteId,) + +COMPLETION_FUNCTION(GetFileOrId, + if(rCommand.mOptions.find('i') != std::string::npos) + { + DELEGATE_COMPLETION(RemoteFileIdInCurrentDir); + } + else + { + DELEGATE_COMPLETION(RemoteFile); + } +) + +COMPLETION_FUNCTION(CompareLocationOrRemoteDir, + if(rCommand.mOptions.find('l') != std::string::npos) + { + DELEGATE_COMPLETION(LocationName); + } + else + { + DELEGATE_COMPLETION(RemoteDir); + } +) + +COMPLETION_FUNCTION(CompareNoneOrLocalDir, + if(rCommand.mOptions.find('l') != std::string::npos) + { + // no completions + DELEGATE_COMPLETION(None); + } + else + { + DELEGATE_COMPLETION(LocalDir); + } +) + +COMPLETION_FUNCTION(RestoreRemoteDirOrId, + if(rCommand.mOptions.find('i') != std::string::npos) + { + DELEGATE_COMPLETION(RemoteId); + } + else + { + DELEGATE_COMPLETION(RemoteDir); + } +) + +// Data about commands +QueryCommandSpecification commands[] = +{ + { "quit", "", Command_Quit, {} }, + { "exit", "", Command_Quit, {} }, + { "list", "adDFhiIorRsStTU", Command_List, {CompleteRemoteDir} }, + { "pwd", "", Command_pwd, {} }, + { "cd", "od", Command_cd, {CompleteRemoteDir} }, + { "lcd", "", Command_lcd, {CompleteLocalDir} }, + { "sh", "", Command_sh, {CompleteDefault} }, + { "getobject", "", Command_GetObject, + {CompleteRemoteId, CompleteLocalDir} }, + { "get", "i", Command_Get, + {CompleteGetFileOrId, CompleteLocalDir} }, + { "compare", "alcqAEQ", Command_Compare, + {CompleteCompareLocationOrRemoteDir, CompleteCompareNoneOrLocalDir} }, + { "restore", "drif", Command_Restore, + {CompleteRestoreRemoteDirOrId, CompleteLocalDir} }, + { "help", "", Command_Help, {} }, + { "usage", "m", Command_Usage, {} }, + { "undelete", "i", Command_Undelete, + {CompleteGetFileOrId} }, + { "delete", "i", Command_Delete, {CompleteGetFileOrId} }, + { NULL, NULL, Command_Unknown, {} } +}; + +const char *alias[] = {"ls", 0}; +const int aliasIs[] = {Command_List, 0}; + +BackupQueries::ParsedCommand::ParsedCommand(const std::string& Command, + bool isFromCommandLine) +: mInOptions(false), + mFailed(false), + pSpec(NULL), + mCompleteArgCount(0) +{ + mCompleteCommand = Command; + + // is the command a shell command? + if(Command[0] == 's' && Command[1] == 'h' && Command[2] == ' ' && Command[3] != '\0') + { + // Yes, run shell command + for(int i = 0; commands[i].type != Command_Unknown; i++) + { + if(commands[i].type == Command_sh) + { + pSpec = &(commands[i]); + break; + } + } + + mCmdElements[0] = "sh"; + mCmdElements[1] = Command.c_str() + 3; + return; + } + + // split command into components + bool inQuoted = false; + mInOptions = false; + + std::string currentArg; + for (std::string::const_iterator c = Command.begin(); + c != Command.end(); c++) + { + // Terminating char? + if(*c == ((inQuoted)?'"':' ')) + { + if(!currentArg.empty()) + { + mCmdElements.push_back(currentArg); + + // Because we just found a space, and the last + // word was not options (otherwise currentArg + // would be empty), we've received a complete + // command or non-option argument. + mCompleteArgCount++; + } + + currentArg.resize(0); + inQuoted = false; + mInOptions = false; + } + // Start of quoted parameter? + else if(currentArg.empty() && *c == '"') + { + inQuoted = true; + } + // Start of options? You can't have options if there's no + // command before them, so treat the options as a command (which + // doesn't exist, so it will fail to parse) in that case. + else if(currentArg.empty() && *c == '-' && !mCmdElements.empty()) + { + mInOptions = true; + } + else if(mInOptions) + { + // Option char + mOptions += *c; + } + else + { + // Normal string char, part of current arg + currentArg += *c; + } + } + + if(!currentArg.empty()) + { + mCmdElements.push_back(currentArg); + } + + // If there are no commands then there's nothing to do except return + if(mCmdElements.empty()) + { + return; + } + + // Work out which command it is... + int cmd = 0; + while(commands[cmd].name != 0 && + mCmdElements[0] != commands[cmd].name) + { + cmd++; + } + + if(commands[cmd].name == 0) + { + // Check for aliases + int a; + for(a = 0; alias[a] != 0; ++a) + { + if(mCmdElements[0] == alias[a]) + { + // Found an alias + cmd = aliasIs[a]; + break; + } + } + } + + if(commands[cmd].name == 0) + { + mFailed = true; + return; + } + + pSpec = &(commands[cmd]); + + #ifdef WIN32 + if(isFromCommandLine) + { + std::string converted; + + if(!ConvertEncoding(mCompleteCommand, CP_ACP, converted, + GetConsoleCP())) + { + BOX_ERROR("Failed to convert encoding"); + mFailed = true; + } + + mCompleteCommand = converted; + + for(std::vector<std::string>::iterator + i = mCmdElements.begin(); + i != mCmdElements.end(); i++) + { + if(!ConvertEncoding(*i, CP_ACP, converted, + GetConsoleCP())) + { + BOX_ERROR("Failed to convert encoding"); + mFailed = true; + } + + *i = converted; + } + } + #endif +} + diff --git a/lib/bbackupquery/Documentation.txt b/lib/bbackupquery/Documentation.txt new file mode 100644 index 00000000..b16a6f7c --- /dev/null +++ b/lib/bbackupquery/Documentation.txt @@ -0,0 +1,194 @@ + +bbackupquery utility -- examine store, compare files, restore, etc. + +This file has markers for automatic help generation script -- '>' marks a start of a command/help topic, +and '<' marks the end of a section. + +Command line: +============= + +> bbackupquery [-q] [-c configfile] [commands ...] + + -q -- quiet, no information prompts + -c -- specify another bbackupd configuation file + +The commands following the options are executed, then (if there was no quit +command) an interactive mode is entered. + +If a command contains a space, enclose it in quotes. Example + + bbackupquery "list testdir1" quit + +to list the contents of testdir1, and then exit without interactive mode. +< + +Commands: +========= + +All directory names relative to a "current" directory, or from root if they +start with '/'. The initial directory is always the root directory. + + +> ls [options] [directory-name] + + List contents of current directory, or specified directory. + + -R -- recursively list all files + -d -- list deleted files/directories + -o -- list old versions of files/directories + -I -- don't display object ID + -F -- don't display flags + -t -- show file modification time in local time + (and attr mod time if has the object has attributes, ~ separated) + -T -- show file modification time in GMT + -a -- show updated attribute instead of file modification time + -s -- show file size in blocks used on server + (only very approximate indication of size locally) + -h -- show file attributes hash + -D -- sort directories together with files (not dirs first) + -i -- sort by object ID (the old default) + -S -- sort by object size in blocks + -U -- don't sort the results (new default is to sort by name) + +list can be used as an alias. +< + +> list + + Alias for 'ls'. Type 'help ls' for options. +< + +> cd [options] <directory-name> + + Change directory + + -d -- consider deleted directories for traversal + -o -- consider old versions of directories for traversal + (this option should never be useful in a correctly formed store) +< + +> pwd + + Print current directory, always root relative. +< + +> lcd <local-directory-name> + + Change local directory. + + Type "sh ls" to list the contents. +< + +> sh <shell command> + + All of the parameters after the "sh" are run as a shell command. + + For example, to list the contents of the location directory, type "sh ls" +< + +> get <object-filename> [<local-filename>] +get -i <object-id> <local-filename> + + Gets a file from the store. Object is specified as the filename within + the current directory, and local filename is optional. Ignores old and + deleted files when searching the directory for the file to retrieve. + + To get an old or deleted file, use the -i option and select the object + as a hex object ID (first column in listing). The local filename must + be specified. +< + +> compare -a +compare -l <location-name> +compare <store-dir-name> <local-dir-name> + + Compares the (current) data on the store with the data on the disc. + All the data will be downloaded -- this is potentially a very long + operation. + + -a -- compare all locations + -l -- compare one backup location as specified in the configuration file. + -c -- set return code + -q -- quick compare. Only checks file contents against checksums, + doesn't do a full download + -A -- ignore attribute differences + -E -- ignore exclusion settings + + Comparing with the root directory is an error, use -a option instead. + + If -c is set, then the return code (if quit is the next command) will be + 1 Comparison was exact + 2 Differences were found + 3 An error occured + This can be used for automated tests. +< + +> restore [-drif] <directory-name> [<local-directory-name>] + + Restores a directory to the local disc. The local directory specified + must not exist (unless a previous restore is being restarted). If the + local directory is omitted, the default is to restore to the same + directory name and path, relative to the current local directory, + as set with the "lcd" command. + + The root cannot be restored -- restore locations individually. + + -d -- restore a deleted directory or deleted files inside + -r -- resume an interrupted restoration + -i -- directory name is actually an ID + -f -- force restore to continue if errors are encountered + + If a restore operation is interrupted for any reason, it can be restarted + using the -r switch. Restore progress information is saved in a file at + regular intervals during the restore operation to allow restarts. +< + +> getobject <object-id> <local-filename> + + Gets the object specified by the object id (in hex) and stores the raw + contents in the local file specified. + + This is only useful for debugging as it does not decode files from the + stored format, which is encrypted and compressed. +< + +> usage [-m] + + Show space used on the server for this account. + + -m -- display the output in machine-readable form + + Used: Total amount of space used on the server. + Old files: Space used by old files + Deleted files: Space used by deleted files + Directories: Space used by the directory structure. + + When Used exceeds the soft limit, the server will start to remove old and + deleted files until the usage drops below the soft limit. + + After a while, you would expect to see the usage stay at just below the + soft limit. You only need more space if the space used by old and deleted + files is near zero. +< + +> undelete <directory-name> +undelete -i <object-id> + + Removes the deleted flag from the specified directory name (in the + current directory) or hex object ID. Be careful not to use this + command where a directory already exists with the same name which is + not marked as deleted. +< + +> delete <file-name> + + Sets the deleted flag on the specified file name (in the current + directory, or with a relative path). +< + +> quit + + End session and exit. +< + + diff --git a/lib/bbackupquery/Makefile.extra b/lib/bbackupquery/Makefile.extra new file mode 100644 index 00000000..5d37c09f --- /dev/null +++ b/lib/bbackupquery/Makefile.extra @@ -0,0 +1,6 @@ + +# AUTOGEN SEEDING +autogen_Documentation.cpp: makedocumentation.pl Documentation.txt + $(_PERL) makedocumentation.pl + + diff --git a/lib/bbackupquery/makedocumentation.pl.in b/lib/bbackupquery/makedocumentation.pl.in new file mode 100755 index 00000000..503ac9c8 --- /dev/null +++ b/lib/bbackupquery/makedocumentation.pl.in @@ -0,0 +1,75 @@ +#!@PERL@ +use strict; + +print "Creating built-in documentation for bbackupquery...\n"; + +open DOC, "Documentation.txt" or die "Can't open Documentation.txt file: $!"; +my $section; +my %help; +my @in_order; + +while(<DOC>) +{ + if(m/\A>\s+(\w+)/) + { + $section = $1; + m/\A>\s+(.+)\Z/; + $help{$section} = $1."\n"; + push @in_order,$section; + } + elsif(m/\A</) + { + $section = ''; + } + elsif($section ne '') + { + $help{$section} .= $_; + } +} + +close DOC; + +open OUT,">autogen_Documentation.cpp" or die "Can't open output file for writing"; + +print OUT <<__E; +// +// Automatically generated file, do not edit. +// + +#include "Box.h" + +#include "MemLeakFindOn.h" + +const char *help_commands[] = +{ +__E + +for(@in_order) +{ + print OUT qq:\t"$_",\n:; +} + +print OUT <<__E; + 0 +}; + +const char *help_text[] = +{ +__E + +for(@in_order) +{ + my $t = $help{$_}; + $t =~ s/\t/ /g; + $t =~ s/\n/\\n/g; + $t =~ s/"/\\"/g; + print OUT qq:\t"$t",\n:; +} + +print OUT <<__E; + 0 +}; + +__E + +close OUT; diff --git a/lib/bbstored/BBStoreDHousekeeping.cpp b/lib/bbstored/BBStoreDHousekeeping.cpp new file mode 100644 index 00000000..86d6409c --- /dev/null +++ b/lib/bbstored/BBStoreDHousekeeping.cpp @@ -0,0 +1,261 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BBStoreDHousekeeping.cpp +// Purpose: Implementation of housekeeping functions for bbstored +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> + +#include "BackupStoreDaemon.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreAccounts.h" +#include "HousekeepStoreAccount.h" +#include "BoxTime.h" +#include "Configuration.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::HousekeepingProcess() +// Purpose: Do housekeeping +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreDaemon::HousekeepingInit() +{ + + mLastHousekeepingRun = 0; +} + +void BackupStoreDaemon::HousekeepingProcess() +{ + HousekeepingInit(); + + // Get the time between housekeeping runs + const Configuration &rconfig(GetConfiguration()); + int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping")); + + while(!StopRun()) + { + RunHousekeepingIfNeeded(); + + // Stop early? + if(StopRun()) + { + break; + } + + // Calculate how long should wait before doing the next + // housekeeping run + int64_t timeNow = GetCurrentBoxTime(); + time_t secondsToGo = BoxTimeToSeconds( + (mLastHousekeepingRun + housekeepingInterval) - + timeNow); + if(secondsToGo < 1) secondsToGo = 1; + if(secondsToGo > 60) secondsToGo = 60; + int32_t millisecondsToGo = ((int)secondsToGo) * 1000; + + // Check to see if there's any message pending + CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); + } +} + +void BackupStoreDaemon::RunHousekeepingIfNeeded() +{ + // Get the time between housekeeping runs + const Configuration &rconfig(GetConfiguration()); + int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping")); + + // Time now + int64_t timeNow = GetCurrentBoxTime(); + + // Do housekeeping if the time interval has elapsed since the last check + if((timeNow - mLastHousekeepingRun) < housekeepingInterval) + { + BOX_TRACE("No need for housekeeping, " << + BoxTimeToSeconds(timeNow - mLastHousekeepingRun) << + " seconds since last run is less than " << + BoxTimeToSeconds(housekeepingInterval)); + return; + } + else + { + BOX_TRACE("Running housekeeping now, because " << + BoxTimeToSeconds(timeNow - mLastHousekeepingRun) << + " seconds since last run is more than " << + BoxTimeToSeconds(housekeepingInterval)); + } + + // Store the time + mLastHousekeepingRun = timeNow; + BOX_INFO("Starting housekeeping"); + + // Get the list of accounts + std::vector<int32_t> accounts; + if(mpAccountDatabase) + { + mpAccountDatabase->GetAllAccountIDs(accounts); + } + + SetProcessTitle("housekeeping, active"); + + // Check them all + for(std::vector<int32_t>::const_iterator i = accounts.begin(); i != accounts.end(); ++i) + { + try + { + std::string rootDir; + int discSet = 0; + + { + // Tag log output to identify account + std::ostringstream tag; + tag << "hk/" << BOX_FORMAT_ACCOUNT(*i); + Logging::Tagger tagWithClientID(tag.str()); + + // Get the account root + mpAccounts->GetAccountRoot(*i, rootDir, discSet); + + // Reset tagging as HousekeepStoreAccount will + // do that itself, to avoid duplicate tagging. + // Happens automatically when tagWithClientID + // goes out of scope. + } + + // Do housekeeping on this account + HousekeepStoreAccount housekeeping(*i, rootDir, + discSet, this); + housekeeping.DoHousekeeping(); + } + catch(BoxException &e) + { + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(*i) << " threw exception, " + "aborting run for this account: " << + e.what() << " (" << + e.GetType() << "/" << e.GetSubType() << ")"); + } + catch(std::exception &e) + { + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(*i) << " threw exception, " + "aborting run for this account: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(*i) << " threw exception, " + "aborting run for this account: " + "unknown exception"); + } + + int64_t timeNow = GetCurrentBoxTime(); + time_t secondsToGo = BoxTimeToSeconds( + (mLastHousekeepingRun + housekeepingInterval) - + timeNow); + if(secondsToGo < 1) secondsToGo = 1; + if(secondsToGo > 60) secondsToGo = 60; + int32_t millisecondsToGo = ((int)secondsToGo) * 1000; + + // Check to see if there's any message pending + CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); + + // Stop early? + if(StopRun()) + { + break; + } + } + + BOX_INFO("Finished housekeeping"); + + // Placed here for accuracy, if StopRun() is true, for example. + SetProcessTitle("housekeeping, idle"); +} + +void BackupStoreDaemon::OnIdle() +{ + if (!IsSingleProcess()) + { + return; + } + + if (!mHousekeepingInited) + { + HousekeepingInit(); + mHousekeepingInited = true; + } + + RunHousekeepingIfNeeded(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::CheckForInterProcessMsg(int, int) +// Purpose: Process a message, returning true if the housekeeping process +// should abort for the specified account. +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitTime) +{ + if(!mInterProcessCommsSocket.IsOpened()) + { + return false; + } + + // First, check to see if it's EOF -- this means something has gone wrong, and the housekeeping should terminate. + if(mInterProcessComms.IsEOF()) + { + SetTerminateWanted(); + return true; + } + + // Get a line, and process the message + std::string line; + if(mInterProcessComms.GetLine(line, false /* no pre-processing */, MaximumWaitTime)) + { + BOX_TRACE("Housekeeping received command '" << line << + "' over interprocess comms"); + + int account = 0; + + if(line == "h") + { + // HUP signal received by main process + SetReloadConfigWanted(); + return true; + } + else if(line == "t") + { + // Terminate signal received by main process + SetTerminateWanted(); + return true; + } + else if(sscanf(line.c_str(), "r%x", &account) == 1) + { + // Main process is trying to lock an account -- are we processing it? + if(account == AccountNum) + { + // Yes! -- need to stop now so when it retries to get the lock, it will succeed + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(AccountNum) << + "giving way to client connection"); + return true; + } + } + } + + return false; +} + + diff --git a/lib/bbstored/BackupStoreDaemon.cpp b/lib/bbstored/BackupStoreDaemon.cpp new file mode 100644 index 00000000..8fddf125 --- /dev/null +++ b/lib/bbstored/BackupStoreDaemon.cpp @@ -0,0 +1,377 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDaemon.cpp +// Purpose: Backup store daemon +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> +#include <stdio.h> +#include <signal.h> + +#ifdef HAVE_SYSLOG_H + #include <syslog.h> +#endif + +#include "BackupStoreContext.h" +#include "BackupStoreDaemon.h" +#include "BackupStoreConfigVerify.h" +#include "autogen_BackupProtocol.h" +#include "RaidFileController.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreAccounts.h" +#include "BannerText.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::BackupStoreDaemon() +// Purpose: Constructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreDaemon::BackupStoreDaemon() + : mpAccountDatabase(0), + mpAccounts(0), + mExtendedLogging(false), + mHaveForkedHousekeeping(false), + mIsHousekeepingProcess(false), + mHousekeepingInited(false), + mInterProcessComms(mInterProcessCommsSocket), + mpTestHook(NULL) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::~BackupStoreDaemon() +// Purpose: Destructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreDaemon::~BackupStoreDaemon() +{ + // Must delete this one before the database ... + if(mpAccounts != 0) + { + delete mpAccounts; + mpAccounts = 0; + } + // ... as the mpAccounts object has a reference to it + if(mpAccountDatabase != 0) + { + delete mpAccountDatabase; + mpAccountDatabase = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::DaemonName() +// Purpose: Name of daemon +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +const char *BackupStoreDaemon::DaemonName() const +{ + return "bbstored"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::DaemonBanner() +// Purpose: Daemon banner +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +std::string BackupStoreDaemon::DaemonBanner() const +{ + return BANNER_TEXT("Backup Store Server"); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::GetConfigVerify() +// Purpose: Configuration definition +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify *BackupStoreDaemon::GetConfigVerify() const +{ + return &BackupConfigFileVerify; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::SetupInInitialProcess() +// Purpose: Setup before we fork -- get raid file controller going +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +void BackupStoreDaemon::SetupInInitialProcess() +{ + const Configuration &config(GetConfiguration()); + + // Initialise the raid files controller + RaidFileController &rcontroller = RaidFileController::GetController(); + + std::string raidFileConfig; + + #ifdef WIN32 + if (!config.KeyExists("RaidFileConf")) + { + raidFileConfig = BOX_GET_DEFAULT_RAIDFILE_CONFIG_FILE; + } + else + { + raidFileConfig = config.GetKeyValue("RaidFileConf"); + } + #else + raidFileConfig = config.GetKeyValue("RaidFileConf"); + #endif + + rcontroller.Initialise(raidFileConfig); + + // Load the account database + std::auto_ptr<BackupStoreAccountDatabase> pdb(BackupStoreAccountDatabase::Read(config.GetKeyValue("AccountDatabase").c_str())); + mpAccountDatabase = pdb.release(); + + // Create a accounts object + mpAccounts = new BackupStoreAccounts(*mpAccountDatabase); + + // Ready to go! +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::Run() +// Purpose: Run shim for the store daemon -- read some config details +// Created: 2003/10/24 +// +// -------------------------------------------------------------------------- +void BackupStoreDaemon::Run() +{ + // Get extended logging flag + mExtendedLogging = false; + const Configuration &config(GetConfiguration()); + mExtendedLogging = config.GetKeyValueBool("ExtendedLogging"); + + // Fork off housekeeping daemon -- must only do this the first + // time Run() is called. Housekeeping runs synchronously on Win32 + // because IsSingleProcess() is always true + +#ifndef WIN32 + if(!IsSingleProcess() && !mHaveForkedHousekeeping) + { + // Open a socket pair for communication + int sv[2] = {-1,-1}; + if(::socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, sv) != 0) + { + THROW_EXCEPTION(ServerException, SocketPairFailed) + } + int whichSocket = 0; + + // Fork + switch(::fork()) + { + case -1: + { + // Error + THROW_EXCEPTION(ServerException, ServerForkError) + } + break; + case 0: + { + // In child process + mIsHousekeepingProcess = true; + SetProcessTitle("housekeeping, idle"); + whichSocket = 1; + // Change the log name + ::openlog("bbstored/hk", LOG_PID, LOG_LOCAL6); + // Log that housekeeping started + BOX_INFO("Housekeeping process started"); + // Ignore term and hup + // Parent will handle these and alert the + // child via the socket, don't want to + // randomly die! + ::signal(SIGHUP, SIG_IGN); + ::signal(SIGTERM, SIG_IGN); + } + break; + default: + { + // Parent process + whichSocket = 0; + } + break; + } + + // Mark that this has been, so -HUP doesn't try and do this again + mHaveForkedHousekeeping = true; + + // Attach the comms thing to the right socket, and close the other one + mInterProcessCommsSocket.Attach(sv[whichSocket]); + + if(::close(sv[(whichSocket == 0)?1:0]) != 0) + { + THROW_EXCEPTION(ServerException, SocketCloseError) + } + } +#endif // WIN32 + + if(mIsHousekeepingProcess) + { + // Housekeeping process -- do other stuff + HousekeepingProcess(); + } + else + { + // In server process -- use the base class to do the magic + ServerTLS<BOX_PORT_BBSTORED>::Run(); + + if (!mInterProcessCommsSocket.IsOpened()) + { + return; + } + + // Why did it stop? Tell the housekeeping process to do the same + if(IsReloadConfigWanted()) + { + mInterProcessCommsSocket.Write("h\n", 2); + } + + if(IsTerminateWanted()) + { + mInterProcessCommsSocket.Write("t\n", 2); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::Connection(SocketStreamTLS &) +// Purpose: Handles a connection, by catching exceptions and +// delegating to Connection2 +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +void BackupStoreDaemon::Connection(std::auto_ptr<SocketStreamTLS> apStream) +{ + try + { + Connection2(apStream); + } + catch(BoxException &e) + { + BOX_ERROR("Error in child process, terminating connection: " << + e.what() << " (" << e.GetType() << "/" << + e.GetSubType() << ")"); + } + catch(std::exception &e) + { + BOX_ERROR("Error in child process, terminating connection: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Error in child process, terminating connection: " << + "unknown exception"); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::Connection2(SocketStreamTLS &) +// Purpose: Handles a connection from bbackupd +// Created: 2006/11/12 +// +// -------------------------------------------------------------------------- +void BackupStoreDaemon::Connection2(std::auto_ptr<SocketStreamTLS> apStream) +{ + // Get the common name from the certificate + std::string clientCommonName(apStream->GetPeerCommonName()); + + // Log the name + BOX_INFO("Client certificate CN: " << clientCommonName); + + // Check it + int32_t id; + if(::sscanf(clientCommonName.c_str(), "BACKUP-%x", &id) != 1) + { + // Bad! Disconnect immediately + BOX_WARNING("Failed login: invalid client common name: " << + clientCommonName); + return; + } + + // Make ps listings clearer + std::ostringstream tag; + tag << "client=" << BOX_FORMAT_ACCOUNT(id); + SetProcessTitle(tag.str().c_str()); + Logging::Tagger tagWithClientID(tag.str()); + + // Create a context, using this ID + BackupStoreContext context(id, this, GetConnectionDetails()); + + if (mpTestHook) + { + context.SetTestHook(*mpTestHook); + } + + // See if the client has an account? + if(mpAccounts && mpAccounts->AccountExists(id)) + { + std::string root; + int discSet; + mpAccounts->GetAccountRoot(id, root, discSet); + context.SetClientHasAccount(root, discSet); + } + + // Handle a connection with the backup protocol + std::auto_ptr<SocketStream> apPlainStream(apStream); + BackupProtocolServer server(apPlainStream); + server.SetLogToSysLog(mExtendedLogging); + server.SetTimeout(BACKUP_STORE_TIMEOUT); + try + { + server.DoServer(context); + } + catch(...) + { + LogConnectionStats(id, context.GetAccountName(), server); + throw; + } + LogConnectionStats(id, context.GetAccountName(), server); + context.CleanUp(); +} + +void BackupStoreDaemon::LogConnectionStats(uint32_t accountId, + const std::string& accountName, const BackupProtocolServer &server) +{ + // Log the amount of data transferred + BOX_NOTICE("Connection statistics for " << + BOX_FORMAT_ACCOUNT(accountId) << " " + "(name=" << accountName << "):" + " IN=" << server.GetBytesRead() << + " OUT=" << server.GetBytesWritten() << + " NET_IN=" << (server.GetBytesRead() - server.GetBytesWritten()) << + " TOTAL=" << (server.GetBytesRead() + server.GetBytesWritten())); +} diff --git a/lib/bbstored/BackupStoreDaemon.h b/lib/bbstored/BackupStoreDaemon.h new file mode 100644 index 00000000..a2dab5e5 --- /dev/null +++ b/lib/bbstored/BackupStoreDaemon.h @@ -0,0 +1,101 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreDaemon.h +// Purpose: Backup store daemon +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREDAEMON__H +#define BACKUPSTOREDAEMON__H + +#include "ServerTLS.h" +#include "BoxPortsAndFiles.h" +#include "BackupConstants.h" +#include "BackupStoreContext.h" +#include "HousekeepStoreAccount.h" +#include "IOStreamGetLine.h" + +class BackupStoreAccounts; +class BackupStoreAccountDatabase; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreDaemon +// Purpose: Backup store daemon implementation +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +class BackupStoreDaemon : public ServerTLS<BOX_PORT_BBSTORED>, + HousekeepingInterface, HousekeepingCallback +{ +public: + BackupStoreDaemon(); + ~BackupStoreDaemon(); +private: + BackupStoreDaemon(const BackupStoreDaemon &rToCopy); +public: + + // For BackupStoreContext to communicate with housekeeping process + void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) + { +#ifndef WIN32 + mInterProcessCommsSocket.Write(Msg, MsgLen); +#endif + } + +protected: + + virtual void SetupInInitialProcess(); + + virtual void Run(); + + virtual void Connection(std::auto_ptr<SocketStreamTLS> apStream); + void Connection2(std::auto_ptr<SocketStreamTLS> apStream); + + virtual const char *DaemonName() const; + virtual std::string DaemonBanner() const; + + const ConfigurationVerify *GetConfigVerify() const; + + // Housekeeping functions + void HousekeepingProcess(); + + void LogConnectionStats(uint32_t accountId, + const std::string& accountName, const BackupProtocolServer &server); + +public: + // HousekeepingInterface implementation + virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0); + void RunHousekeepingIfNeeded(); + +private: + BackupStoreAccountDatabase *mpAccountDatabase; + BackupStoreAccounts *mpAccounts; + bool mExtendedLogging; + bool mHaveForkedHousekeeping; + bool mIsHousekeepingProcess; + bool mHousekeepingInited; + + SocketStream mInterProcessCommsSocket; + IOStreamGetLine mInterProcessComms; + + virtual void OnIdle(); + void HousekeepingInit(); + int64_t mLastHousekeepingRun; + +public: + void SetTestHook(BackupStoreContext::TestHook& rTestHook) + { + mpTestHook = &rTestHook; + } + +private: + BackupStoreContext::TestHook* mpTestHook; +}; + + +#endif // BACKUPSTOREDAEMON__H + diff --git a/lib/common/Archive.h b/lib/common/Archive.h index 6d5ce88b..2b27b303 100644 --- a/lib/common/Archive.h +++ b/lib/common/Archive.h @@ -26,7 +26,7 @@ class Archive { public: Archive(IOStream &Stream, int Timeout) - : mrStream(Stream) + : mrStream(Stream) { mTimeout = Timeout; } @@ -38,6 +38,7 @@ public: ~Archive() { } + // // // @@ -46,21 +47,29 @@ public: Write((int) Item); } void WriteExact(uint32_t Item) { Write((int)Item); } + // TODO FIXME: use of "int" here is dangerous and deprecated. It can lead to + // incompatible serialisation on non-32-bit machines. Passing anything other + // than one of the specifically supported fixed size types should be forbidden. void Write(int Item) { int32_t privItem = htonl(Item); - mrStream.Write(&privItem, sizeof(privItem)); + mrStream.Write(&privItem, sizeof(privItem), mTimeout); } void Write(int64_t Item) { int64_t privItem = box_hton64(Item); - mrStream.Write(&privItem, sizeof(privItem)); + mrStream.Write(&privItem, sizeof(privItem), mTimeout); + } + void WriteInt16(uint16_t Item) + { + uint16_t privItem = htons(Item); + mrStream.Write(&privItem, sizeof(privItem), mTimeout); } void WriteExact(uint64_t Item) { Write(Item); } void Write(uint64_t Item) { uint64_t privItem = box_hton64(Item); - mrStream.Write(&privItem, sizeof(privItem)); + mrStream.Write(&privItem, sizeof(privItem), mTimeout); } void Write(uint8_t Item) { @@ -71,7 +80,7 @@ public: { int size = Item.size(); Write(size); - mrStream.Write(Item.c_str(), size); + mrStream.Write(Item.c_str(), size, mTimeout); } // // @@ -100,17 +109,29 @@ public: void Read(int &rItemOut) { int32_t privItem; - if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) + if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), + 0 /* not interested in bytes read if this fails */, + mTimeout)) { - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead); } rItemOut = ntohl(privItem); } + void ReadFullBuffer(void* Buffer, size_t Size) + { + if(!mrStream.ReadFullBuffer(Buffer, Size, + 0 /* not interested in bytes read if this fails */, + mTimeout)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead); + } + } void ReadIfPresent(int &rItemOut, int ValueIfNotPresent) { int32_t privItem; int bytesRead; - if(mrStream.ReadFullBuffer(&privItem, sizeof(privItem), &bytesRead)) + if(mrStream.ReadFullBuffer(&privItem, sizeof(privItem), + &bytesRead, mTimeout)) { rItemOut = ntohl(privItem); } @@ -122,48 +143,70 @@ public: else { // bad number of remaining bytes - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead); } } void Read(int64_t &rItemOut) { int64_t privItem; - if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) - { - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) - } + ReadFullBuffer(&privItem, sizeof(privItem)); rItemOut = box_ntoh64(privItem); } void ReadExact(uint64_t &rItemOut) { Read(rItemOut); } void Read(uint64_t &rItemOut) { uint64_t privItem; - if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) - { - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) - } + ReadFullBuffer(&privItem, sizeof(privItem)); rItemOut = box_ntoh64(privItem); } + void ReadInt16(uint16_t &rItemOut) + { + uint16_t privItem; + ReadFullBuffer(&privItem, sizeof(privItem)); + rItemOut = ntohs(privItem); + } void Read(uint8_t &rItemOut) { int privItem; Read(privItem); rItemOut = privItem; } + void ReadIfPresent(std::string &rItemOut, const std::string& ValueIfNotPresent) + { + ReadString(rItemOut, &ValueIfNotPresent); + } void Read(std::string &rItemOut) { + ReadString(rItemOut, NULL); + } +private: + void ReadString(std::string &rItemOut, const std::string* pValueIfNotPresent) + { int size; - Read(size); + int bytesRead; + if(!mrStream.ReadFullBuffer(&size, sizeof(size), &bytesRead, mTimeout)) + { + if(bytesRead == 0 && pValueIfNotPresent != NULL) + { + // item is simply not present + rItemOut = *pValueIfNotPresent; + return; + } + else + { + // bad number of remaining bytes + THROW_EXCEPTION(CommonException, + ArchiveBlockIncompleteRead) + } + } + size = ntohl(size); // Assume most strings are relatively small char buf[256]; if(size < (int) sizeof(buf)) { // Fetch rest of pPayload, relying on the Protocol to error on stupidly large sizes for us - if(!mrStream.ReadFullBuffer(buf, size, 0 /* not interested in bytes read if this fails */, mTimeout)) - { - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) - } + ReadFullBuffer(buf, size); // assign to this string, storing the header and the extra payload rItemOut.assign(buf, size); } @@ -174,10 +217,7 @@ public: char *ppayload = dataB; // Fetch rest of pPayload, relying on the Protocol to error on stupidly large sizes for us - if(!mrStream.ReadFullBuffer(ppayload, size, 0 /* not interested in bytes read if this fails */, mTimeout)) - { - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) - } + ReadFullBuffer(ppayload, size); // assign to this string, storing the header and the extra pPayload rItemOut.assign(ppayload, size); } diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h index f0772c9c..9ca0c11c 100644 --- a/lib/common/BannerText.h +++ b/lib/common/BannerText.h @@ -16,7 +16,7 @@ #define BANNER_TEXT(UtilityName) \ "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers and " \ - "contributors 2003-2011" + "contributors 2003-2014" #endif // BANNERTEXT__H diff --git a/lib/common/Box.h b/lib/common/Box.h index 316f4364..8ce2a625 100644 --- a/lib/common/Box.h +++ b/lib/common/Box.h @@ -116,16 +116,8 @@ { \ if((!HideExceptionMessageGuard::ExceptionsHidden() \ && !HideSpecificExceptionGuard::IsHidden( \ - type::ExceptionType, type::subtype)) \ - || Logging::Guard::IsGuardingFrom(Log::EVERYTHING)) \ + type::ExceptionType, type::subtype))) \ { \ - std::auto_ptr<Logging::Guard> guard; \ - \ - if(Logging::Guard::IsGuardingFrom(Log::EVERYTHING)) \ - { \ - guard.reset(new Logging::Guard(Log::EVERYTHING)); \ - } \ - \ OPTIONAL_DO_BACKTRACE \ BOX_WARNING("Exception thrown: " \ #type "(" #subtype ") " \ @@ -140,16 +132,8 @@ _box_throw_line << message; \ if((!HideExceptionMessageGuard::ExceptionsHidden() \ && !HideSpecificExceptionGuard::IsHidden( \ - type::ExceptionType, type::subtype)) \ - || Logging::Guard::IsGuardingFrom(Log::EVERYTHING)) \ + type::ExceptionType, type::subtype))) \ { \ - std::auto_ptr<Logging::Guard> guard; \ - \ - if(Logging::Guard::IsGuardingFrom(Log::EVERYTHING)) \ - { \ - guard.reset(new Logging::Guard(Log::EVERYTHING)); \ - } \ - \ OPTIONAL_DO_BACKTRACE \ BOX_WARNING("Exception thrown: " \ #type "(" #subtype ") (" << \ diff --git a/lib/common/BoxConfig-MSVC.h b/lib/common/BoxConfig-MSVC.h index eeb25d2e..2ec2edd7 100644 --- a/lib/common/BoxConfig-MSVC.h +++ b/lib/common/BoxConfig-MSVC.h @@ -386,9 +386,6 @@ /* Define to empty if `const' does not conform to ANSI C. */ /* #undef const */ -/* Define to `int' if <sys/types.h> doesn't define. */ -#define gid_t int - /* Define to `int' if <sys/types.h> does not define. */ /* #undef mode_t */ @@ -400,6 +397,3 @@ /* Define to `unsigned' if <sys/types.h> does not define. */ /* #undef size_t */ - -/* Define to `int' if <sys/types.h> doesn't define. */ -#define uid_t int diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h index 53a967e8..f7c74bfc 100644 --- a/lib/common/BoxPlatform.h +++ b/lib/common/BoxPlatform.h @@ -21,11 +21,13 @@ #define PLATFORM_DEV_NULL "/dev/null" -#ifdef _MSC_VER -#include "BoxConfig-MSVC.h" -#define NEED_BOX_VERSION_H +#if defined BOX_CMAKE +# include "BoxConfig.cmake.h" +#elif defined _MSC_VER +# include "BoxConfig-MSVC.h" +# define NEED_BOX_VERSION_H #else -#include "BoxConfig.h" +# include "BoxConfig.h" #endif #ifdef WIN32 @@ -40,9 +42,12 @@ #endif #endif +#include "emu.h" + #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> #endif + #ifdef HAVE_INTTYPES_H #include <inttypes.h> #else @@ -93,66 +98,19 @@ #endif // Handle differing xattr APIs -#ifdef HAVE_SYS_XATTR_H - #if !defined(HAVE_LLISTXATTR) && defined(HAVE_LISTXATTR) && HAVE_DECL_XATTR_NOFOLLOW - #define llistxattr(a,b,c) listxattr(a,b,c,XATTR_NOFOLLOW) - #endif - #if !defined(HAVE_LGETXATTR) && defined(HAVE_GETXATTR) && HAVE_DECL_XATTR_NOFOLLOW - #define lgetxattr(a,b,c,d) getxattr(a,b,c,d,0,XATTR_NOFOLLOW) - #endif - #if !defined(HAVE_LSETXATTR) && defined(HAVE_SETXATTR) && HAVE_DECL_XATTR_NOFOLLOW - #define lsetxattr(a,b,c,d,e) setxattr(a,b,c,d,0,(e)|XATTR_NOFOLLOW) - #endif +#if !defined(HAVE_LLISTXATTR) && defined(HAVE_LISTXATTR) && HAVE_DECL_XATTR_NOFOLLOW + #define llistxattr(a,b,c) listxattr(a,b,c,XATTR_NOFOLLOW) + #define HAVE_LLISTXATTR #endif -#if defined WIN32 && !defined __MINGW32__ - typedef __int8 int8_t; - typedef __int16 int16_t; - typedef __int32 int32_t; - typedef __int64 int64_t; - - typedef unsigned __int8 u_int8_t; - typedef unsigned __int16 u_int16_t; - typedef unsigned __int32 u_int32_t; - typedef unsigned __int64 u_int64_t; - - #define HAVE_U_INT8_T - #define HAVE_U_INT16_T - #define HAVE_U_INT32_T - #define HAVE_U_INT64_T -#endif // WIN32 && !__MINGW32__ - -// Define missing types -#ifndef HAVE_UINT8_T - typedef u_int8_t uint8_t; +#if !defined(HAVE_LGETXATTR) && defined(HAVE_GETXATTR) && HAVE_DECL_XATTR_NOFOLLOW + #define lgetxattr(a,b,c,d) getxattr(a,b,c,d,0,XATTR_NOFOLLOW) + #define HAVE_LGETXATTR #endif -#ifndef HAVE_UINT16_T - typedef u_int16_t uint16_t; -#endif - -#ifndef HAVE_UINT32_T - typedef u_int32_t uint32_t; -#endif - -#ifndef HAVE_UINT64_T - typedef u_int64_t uint64_t; -#endif - -#ifndef HAVE_U_INT8_T - typedef uint8_t u_int8_t; -#endif - -#ifndef HAVE_U_INT16_T - typedef uint16_t u_int16_t; -#endif - -#ifndef HAVE_U_INT32_T - typedef uint32_t u_int32_t; -#endif - -#ifndef HAVE_U_INT64_T - typedef uint64_t u_int64_t; +#if !defined(HAVE_LSETXATTR) && defined(HAVE_SETXATTR) && HAVE_DECL_XATTR_NOFOLLOW + #define lsetxattr(a,b,c,d,e) setxattr(a,b,c,d,0,(e)|XATTR_NOFOLLOW) + #define HAVE_LSETXATTR #endif #if !HAVE_DECL_INFTIM @@ -173,7 +131,7 @@ #endif #ifdef WIN32 - typedef u_int64_t InodeRefType; + typedef uint64_t InodeRefType; #else typedef ino_t InodeRefType; #endif @@ -182,8 +140,6 @@ #define WIN32_LEAN_AND_MEAN #endif -#include "emu.h" - #ifdef WIN32 #define INVALID_FILE INVALID_HANDLE_VALUE typedef HANDLE tOSFileHandle; diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp index f62b1c35..77daae6d 100644 --- a/lib/common/BoxTime.cpp +++ b/lib/common/BoxTime.cpp @@ -35,21 +35,30 @@ // -------------------------------------------------------------------------- box_time_t GetCurrentBoxTime() { - #ifdef HAVE_GETTIMEOFDAY - struct timeval tv; - if (gettimeofday(&tv, NULL) != 0) - { - BOX_LOG_SYS_ERROR("Failed to gettimeofday(), " - "dropping precision"); - } - else - { - box_time_t timeNow = (tv.tv_sec * MICRO_SEC_IN_SEC_LL) - + tv.tv_usec; - return timeNow; - } - #endif - +#ifdef HAVE_GETTIMEOFDAY + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) + { + BOX_LOG_SYS_ERROR("Failed to gettimeofday(), " + "dropping precision"); + } + else + { + box_time_t time_now = (tv.tv_sec * MICRO_SEC_IN_SEC_LL) + tv.tv_usec; + return time_now; + } +#elif WIN32 + // There's no Win32 API function that returns the current time as a UNIX timestamp with + // sub-second precision. So we use time(0) and add the fractional part from + // GetSystemTime() in the hope that the difference between these two (if any) is a whole + // number of seconds. + box_time_t time_now = SecondsToBoxTime(time(0)); + SYSTEMTIME system_time; + GetSystemTime(&system_time); + time_now += MilliSecondsToBoxTime(system_time.wMilliseconds); + return time_now; +#endif + return SecondsToBoxTime(time(0)); } @@ -83,7 +92,7 @@ std::string FormatTime(box_time_t time, bool includeDate, bool showMicros) if (showMicros) { - buf << "." << std::setw(6) << micros; + buf << "." << std::setw(3) << (int)(micros / 1000); } } else @@ -108,8 +117,7 @@ void ShortSleep(box_time_t duration, bool logDuration) { if(logDuration) { - BOX_TRACE("Sleeping for " << BoxTimeToMicroSeconds(duration) << - " microseconds"); + BOX_TRACE("Sleeping for " << BOX_FORMAT_MICROSECONDS(duration)); } #ifdef WIN32 @@ -118,7 +126,9 @@ void ShortSleep(box_time_t duration, bool logDuration) struct timespec ts; memset(&ts, 0, sizeof(ts)); ts.tv_sec = duration / MICRO_SEC_IN_SEC; - ts.tv_nsec = duration % MICRO_SEC_IN_SEC; + ts.tv_nsec = (duration % MICRO_SEC_IN_SEC) * 1000; + + box_time_t start_time = GetCurrentBoxTime(); while (nanosleep(&ts, &ts) == -1 && errno == EINTR) { @@ -140,6 +150,10 @@ void ShortSleep(box_time_t duration, bool logDuration) BOX_TRACE("nanosleep interrupted with " << remain_ns << " nanosecs remaining, sleeping again"); } + + box_time_t sleep_time = GetCurrentBoxTime() - start_time; + BOX_TRACE("Actually slept for " << BOX_FORMAT_MICROSECONDS(sleep_time) << + ", was aiming for " << BOX_FORMAT_MICROSECONDS(duration)); #endif } diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h index 3108d809..6afaada3 100644 --- a/lib/common/BoxTime.h +++ b/lib/common/BoxTime.h @@ -10,8 +10,8 @@ #ifndef BOXTIME__H #define BOXTIME__H -// Time is presented as an unsigned 64 bit integer, in microseconds -typedef int64_t box_time_t; +// Time is presented as a signed 64 bit integer, in microseconds +typedef int64_t box_time_t; #define NANO_SEC_IN_SEC (1000000000LL) #define NANO_SEC_IN_USEC (1000) diff --git a/lib/common/BufferedStream.cpp b/lib/common/BufferedStream.cpp index b58253f3..847cf66c 100644 --- a/lib/common/BufferedStream.cpp +++ b/lib/common/BufferedStream.cpp @@ -96,7 +96,7 @@ IOStream::pos_type BufferedStream::BytesLeftToRead() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void BufferedStream::Write(const void *pBuffer, int NBytes) +void BufferedStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, NotSupported); } @@ -189,7 +189,9 @@ void BufferedStream::Close() // -------------------------------------------------------------------------- bool BufferedStream::StreamDataLeft() { - return mrSource.StreamDataLeft(); + // Return true if either the source has data left to read, or we have + // buffered data still to be read. + return mrSource.StreamDataLeft() || (mBufferPosition < mBufferSize); } // -------------------------------------------------------------------------- diff --git a/lib/common/BufferedStream.h b/lib/common/BufferedStream.h index 079c482a..3984aceb 100644 --- a/lib/common/BufferedStream.h +++ b/lib/common/BufferedStream.h @@ -25,7 +25,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); @@ -33,6 +34,10 @@ public: virtual bool StreamDataLeft(); virtual bool StreamClosed(); + virtual std::string ToString() const + { + return std::string("Buffered ") + mrSource.ToString(); + } private: BufferedStream(const BufferedStream &rToCopy) : mrSource(rToCopy.mrSource) { /* do not call */ } diff --git a/lib/common/BufferedWriteStream.cpp b/lib/common/BufferedWriteStream.cpp index 797be00d..8fbabe9b 100644 --- a/lib/common/BufferedWriteStream.cpp +++ b/lib/common/BufferedWriteStream.cpp @@ -64,7 +64,7 @@ IOStream::pos_type BufferedWriteStream::BytesLeftToRead() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void BufferedWriteStream::Write(const void *pBuffer, int NBytes) +void BufferedWriteStream::Write(const void *pBuffer, int NBytes, int Timeout) { int numBytesRemain = NBytes; diff --git a/lib/common/BufferedWriteStream.h b/lib/common/BufferedWriteStream.h index 7a1c8c17..5f6d5f19 100644 --- a/lib/common/BufferedWriteStream.h +++ b/lib/common/BufferedWriteStream.h @@ -25,7 +25,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Flush(int Timeout = IOStream::TimeOutInfinite); diff --git a/lib/common/CollectInBufferStream.cpp b/lib/common/CollectInBufferStream.cpp index 90e2e7bc..47b271f0 100644 --- a/lib/common/CollectInBufferStream.cpp +++ b/lib/common/CollectInBufferStream.cpp @@ -98,7 +98,7 @@ IOStream::pos_type CollectInBufferStream::BytesLeftToRead() // Created: 2003/08/26 // // -------------------------------------------------------------------------- -void CollectInBufferStream::Write(const void *pBuffer, int NBytes) +void CollectInBufferStream::Write(const void *pBuffer, int NBytes, int Timeout) { if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } diff --git a/lib/common/CollectInBufferStream.h b/lib/common/CollectInBufferStream.h index d73af8db..297d2851 100644 --- a/lib/common/CollectInBufferStream.h +++ b/lib/common/CollectInBufferStream.h @@ -26,24 +26,31 @@ class CollectInBufferStream : public IOStream public: CollectInBufferStream(); ~CollectInBufferStream(); -private: - // No copying - CollectInBufferStream(const CollectInBufferStream &); - CollectInBufferStream(const IOStream &); -public: + + // Move constructor: + CollectInBufferStream(CollectInBufferStream& rOther) + : mBuffer(rOther.mBuffer.Release()), + mBufferSize(rOther.mBufferSize), + mBytesInBuffer(rOther.mBytesInBuffer), + mReadPosition(rOther.mReadPosition), + mInWritePhase(rOther.mInWritePhase) + { + rOther.Reset(); + } virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual bool StreamDataLeft(); virtual bool StreamClosed(); void SetForReading(); - + void Reset(); - + void *GetBuffer() const; int GetSize() const; bool IsSetForReading() const {return !mInWritePhase;} diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt index 05da2709..610ba1a8 100644 --- a/lib/common/CommonException.txt +++ b/lib/common/CommonException.txt @@ -55,3 +55,5 @@ DatabaseRecordAlreadyExists 47 The database already contains a record with this DatabaseRecordBadSize 48 The database contains a record with an invalid size DatabaseIterateFailed 49 Failed to iterate over the database keys ReferenceNotFound 50 The database does not contain an expected reference +TimersNotInitialised 51 The timer framework should have been ready at this point +InvalidConfiguration 52 Some required values are missing or incorrect in the configuration file. diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp index f49f3c6e..8ce8d389 100644 --- a/lib/common/Configuration.cpp +++ b/lib/common/Configuration.cpp @@ -34,6 +34,8 @@ inline bool iw(int c) static const char *sValueBooleanStrings[] = {"yes", "true", "no", "false", 0}; static const bool sValueBooleanValue[] = {true, true, false, false}; +const ConfigurationCategory ConfigurationVerify::VERIFY_ERROR("VerifyError"); + ConfigurationVerifyKey::ConfigurationVerifyKey ( std::string name, @@ -212,8 +214,8 @@ std::auto_ptr<Configuration> Configuration::LoadAndVerify( if(!rErrorMsg.empty()) { // An error occured, return now - BOX_ERROR("Error in Configuration::LoadInto: " << - rErrorMsg); + BOX_LOG_CATEGORY(Log::ERROR, ConfigurationVerify::VERIFY_ERROR, + "Error in Configuration::LoadInto: " << rErrorMsg); return std::auto_ptr<Configuration>(0); } @@ -222,8 +224,11 @@ std::auto_ptr<Configuration> Configuration::LoadAndVerify( { if(!apConfig->Verify(*pVerify, std::string(), rErrorMsg)) { - BOX_ERROR("Error verifying configuration: " << - rErrorMsg); + BOX_LOG_CATEGORY(Log::ERROR, + ConfigurationVerify::VERIFY_ERROR, + "Error verifying configuration: " << + rErrorMsg.substr(0, rErrorMsg.size() > 0 + ? rErrorMsg.size() - 1 : 0)); return std::auto_ptr<Configuration>(0); } } @@ -425,7 +430,8 @@ const std::string &Configuration::GetKeyValue(const std::string& rKeyName) const if(i == mKeys.end()) { - BOX_ERROR("Missing configuration key: " << rKeyName); + BOX_LOG_CATEGORY(Log::ERROR, ConfigurationVerify::VERIFY_ERROR, + "Missing configuration key: " << rKeyName); THROW_EXCEPTION(CommonException, ConfigNoKey) } else diff --git a/lib/common/Configuration.h b/lib/common/Configuration.h index 4828b315..e6498e80 100644 --- a/lib/common/Configuration.h +++ b/lib/common/Configuration.h @@ -27,6 +27,14 @@ enum ConfigTest_IsBool = 32 }; +class ConfigurationCategory : public Log::Category +{ + public: + ConfigurationCategory(const std::string& name) + : Log::Category(std::string("Configuration/") + name) + { } +}; + class ConfigurationVerifyKey { public: @@ -75,6 +83,7 @@ public: const ConfigurationVerifyKey *mpKeys; int Tests; void *TestFunction; // set to zero for now, will implement later + static const ConfigurationCategory VERIFY_ERROR; }; class FdGetLine; diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp index 0b123675..58a82c0e 100644 --- a/lib/common/DebugMemLeakFinder.cpp +++ b/lib/common/DebugMemLeakFinder.cpp @@ -15,14 +15,19 @@ #undef realloc #undef free -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - +#include <limits.h> #include <signal.h> #include <stdio.h> #include <string.h> +#ifdef HAVE_PROCESS_H +# include <process.h> +#endif + +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif + #include <cstdlib> // for std::atexit #include <map> #include <set> @@ -155,7 +160,17 @@ void *memleakfinder_malloc(size_t size, const char *file, int line) InternalAllocGuard guard; void *b = std::malloc(size); - if(!memleakfinder_global_enable) return b; + + if(!memleakfinder_global_enable) + { + // We may not be tracking this allocation, but if + // someone realloc()s the buffer later then it will + // trigger an untracked buffer warning, which we don't + // want to see either. + memleakfinder_notaleak(b); + return b; + } + if(!memleakfinder_initialised) return b; memleakfinder_malloc_add_block(b, size, file, line); @@ -176,25 +191,58 @@ void *memleakfinder_calloc(size_t blocks, size_t size, const char *file, int lin void *memleakfinder_realloc(void *ptr, size_t size) { + if(!ptr) + { + return memleakfinder_malloc(size, "realloc", 0); + } + + if(!size) + { + memleakfinder_free(ptr); + return NULL; + } + InternalAllocGuard guard; + ASSERT(ptr != NULL); + if(!ptr) return NULL; // defensive + if(!memleakfinder_global_enable || !memleakfinder_initialised) { - return std::realloc(ptr, size); + ptr = std::realloc(ptr, size); + if(!memleakfinder_global_enable) + { + // We may not be tracking this allocation, but if + // someone realloc()s the buffer later then it will + // trigger an untracked buffer warning, which we don't + // want to see either. + memleakfinder_notaleak(ptr); + } + return ptr; } // Check it's been allocated std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); - if(ptr && i == sMallocBlocks.end()) + std::set<void *>::iterator j(sNotLeaks.find(ptr)); + + if(i == sMallocBlocks.end() && j == sNotLeaks.end()) { BOX_WARNING("Block " << ptr << " realloc()ated, but not " "in list. Error? Or allocated in startup static " "objects?"); } + if(j != sNotLeaks.end()) + { + // It's in the list of not-leaks, so don't warn about it, + // but it's being reallocated, so remove it from the list too, + // in case it's reassigned, and add the new block below. + sNotLeaks.erase(j); + } + void *b = std::realloc(ptr, size); - if(ptr && i!=sMallocBlocks.end()) + if(i != sMallocBlocks.end()) { // Worked? if(b != 0) @@ -230,10 +278,19 @@ void memleakfinder_free(void *ptr) { // Check it's been allocated std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); + std::set<void *>::iterator j(sNotLeaks.find(ptr)); + if(i != sMallocBlocks.end()) { sMallocBlocks.erase(i); } + else if(j != sNotLeaks.end()) + { + // It's in the list of not-leaks, so don't warn + // about it, but it's being freed, so remove it + // from the list too, in case it's reassigned. + sNotLeaks.erase(j); + } else { BOX_WARNING("Block " << ptr << " freed, but not " @@ -284,7 +341,8 @@ void memleakfinder_notaleak(void *ptr) ASSERT(!sTrackingDataDestroyed); memleakfinder_notaleak_insert_pre(); - if(memleakfinder_global_enable && memleakfinder_initialised) + + if(memleakfinder_initialised) { sNotLeaks.insert(ptr); } @@ -294,7 +352,9 @@ void memleakfinder_notaleak(void *ptr) sizeof(sNotLeaksPre)/sizeof(*sNotLeaksPre) ) sNotLeaksPre[sNotLeaksPreNum++] = ptr; } -/* { + + /* + { std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); if(i != sMallocBlocks.end()) sMallocBlocks.erase(i); } @@ -531,9 +591,14 @@ extern "C" void memleakfinder_atexit() memleakfinder_reportleaks_appendfile(atexit_filename, atexit_markertext); } -void memleakfinder_setup_exit_report(const char *filename, const char *markertext) +void memleakfinder_setup_exit_report(const std::string& filename, + const char *markertext) { - ::strncpy(atexit_filename, filename, sizeof(atexit_filename)-1); + char buffer[PATH_MAX]; + std::string abs_filename = std::string(getcwd(buffer, sizeof(buffer))) + + DIRECTORY_SEPARATOR + filename; + ::strncpy(atexit_filename, abs_filename.c_str(), + sizeof(atexit_filename)-1); ::strncpy(atexit_markertext, markertext, sizeof(atexit_markertext)-1); atexit_filename[sizeof(atexit_filename)-1] = 0; atexit_markertext[sizeof(atexit_markertext)-1] = 0; @@ -544,9 +609,6 @@ void memleakfinder_setup_exit_report(const char *filename, const char *markertex } } - - - void add_object_block(void *block, size_t size, const char *file, int line, bool array) { InternalAllocGuard guard; @@ -557,6 +619,10 @@ void add_object_block(void *block, size_t size, const char *file, int line, bool if(block != 0) { + std::map<void *, ObjectInfo>::iterator j(sObjectBlocks.find(block)); + // The same block should not already be tracked! + ASSERT(j == sObjectBlocks.end()); + ObjectInfo i; i.size = size; i.file = file; @@ -637,7 +703,7 @@ void *operator new(size_t size) } */ -void *operator new[](size_t size) +void *operator new[](size_t size) throw (std::bad_alloc) { return internal_new(size, "standard libraries", 0); } diff --git a/lib/common/EventWatchFilesystemObject.cpp b/lib/common/EventWatchFilesystemObject.cpp deleted file mode 100644 index 43533fc8..00000000 --- a/lib/common/EventWatchFilesystemObject.cpp +++ /dev/null @@ -1,112 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: EventWatchFilesystemObject.cpp -// Purpose: WaitForEvent compatible object for watching directories -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <errno.h> -#include <fcntl.h> - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - -#include "EventWatchFilesystemObject.h" -#include "autogen_CommonException.h" -#include "Logging.h" - -#include "MemLeakFindOn.h" - - -// -------------------------------------------------------------------------- -// -// Function -// Name: EventWatchFilesystemObject::EventWatchFilesystemObject -// (const char *) -// Purpose: Constructor -- opens the file object -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- -EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) -#ifdef HAVE_KQUEUE - : mDescriptor(::open(Filename, O_RDONLY /*O_EVTONLY*/, 0)) -#endif -{ -#ifdef HAVE_KQUEUE - if(mDescriptor == -1) - { - BOX_LOG_SYS_ERROR("EventWatchFilesystemObject: " - "Failed to open file '" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileOpenError) - } -#else - THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform) -#endif -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: EventWatchFilesystemObject::~EventWatchFilesystemObject() -// Purpose: Destructor -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- -EventWatchFilesystemObject::~EventWatchFilesystemObject() -{ - if(mDescriptor != -1) - { - ::close(mDescriptor); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: EventWatchFilesystemObject::EventWatchFilesystemObject -// (const EventWatchFilesystemObject &) -// Purpose: Copy constructor -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- -EventWatchFilesystemObject::EventWatchFilesystemObject( - const EventWatchFilesystemObject &rToCopy) - : mDescriptor(::dup(rToCopy.mDescriptor)) -{ - if(mDescriptor == -1) - { - THROW_EXCEPTION(CommonException, OSFileError) - } -} - - -#ifdef HAVE_KQUEUE -// -------------------------------------------------------------------------- -// -// Function -// Name: EventWatchFilesystemObject::FillInKEvent(struct kevent &, int) -// Purpose: For WaitForEvent -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- -void EventWatchFilesystemObject::FillInKEvent(struct kevent &rEvent, - int Flags) const -{ - EV_SET(&rEvent, mDescriptor, EVFILT_VNODE, EV_CLEAR, - NOTE_DELETE | NOTE_WRITE, 0, (void*)this); -} -#else -void EventWatchFilesystemObject::FillInPoll(int &fd, short &events, - int Flags) const -{ - THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform) -} -#endif - diff --git a/lib/common/EventWatchFilesystemObject.h b/lib/common/EventWatchFilesystemObject.h deleted file mode 100644 index f9175a49..00000000 --- a/lib/common/EventWatchFilesystemObject.h +++ /dev/null @@ -1,48 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: EventWatchFilesystemObject.h -// Purpose: WaitForEvent compatible object for watching directories -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- - -#ifndef EVENTWATCHFILESYSTEMOBJECT__H -#define EVENTWATCHFILESYSTEMOBJECT__H - -#ifdef HAVE_KQUEUE - #include <sys/event.h> -#endif - - -// -------------------------------------------------------------------------- -// -// Class -// Name: EventWatchFilesystemObject -// Purpose: WaitForEvent compatible object for watching files and directories -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- -class EventWatchFilesystemObject -{ -public: - EventWatchFilesystemObject(const char *Filename); - ~EventWatchFilesystemObject(); - EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy); -private: - // Assignment not allowed - EventWatchFilesystemObject &operator=(const EventWatchFilesystemObject &); -public: - -#ifdef HAVE_KQUEUE - void FillInKEvent(struct kevent &rEvent, int Flags = 0) const; -#else - void FillInPoll(int &fd, short &events, int Flags = 0) const; -#endif - -private: - int mDescriptor; -}; - -#endif // EventWatchFilesystemObject__H - diff --git a/lib/common/ExcludeList.cpp b/lib/common/ExcludeList.cpp index 213c4f8e..3f9f69ee 100644 --- a/lib/common/ExcludeList.cpp +++ b/lib/common/ExcludeList.cpp @@ -10,9 +10,9 @@ #include "Box.h" #ifdef HAVE_REGEX_SUPPORT - #ifdef HAVE_PCREPOSIX_H + #if defined HAVE_PCREPOSIX_H #include <pcreposix.h> - #else + #elif defined HAVE_REGEX_H #include <regex.h> #endif #define EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED @@ -139,9 +139,10 @@ void ExcludeList::AddDefiniteEntries(const std::string &rEntries) if (entry.size() > 0 && entry[entry.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR) { - BOX_WARNING("Exclude entry ends in path " - "separator, will never match: " - << entry); + BOX_LOG_CATEGORY(Log::WARNING, + ConfigurationVerify::VERIFY_ERROR, + "Exclude entry ends in path separator, " + "will never match: " << entry); } mDefinite.insert(entry); @@ -198,9 +199,9 @@ void ExcludeList::AddRegexEntries(const std::string &rEntries) { char buf[1024]; regerror(errcode, pregex, buf, sizeof(buf)); - BOX_ERROR("Invalid regular expression: " << + THROW_EXCEPTION_MESSAGE(CommonException, BadRegularExpression, + "Invalid regular expression: " << entry << ": " << buf); - THROW_EXCEPTION(CommonException, BadRegularExpression) } // Store in list of regular expressions diff --git a/lib/common/FileModificationTime.cpp b/lib/common/FileModificationTime.cpp index 06fc7887..50f1fb62 100644 --- a/lib/common/FileModificationTime.cpp +++ b/lib/common/FileModificationTime.cpp @@ -18,11 +18,14 @@ box_time_t FileModificationTime(const EMU_STRUCT_STAT &st) { -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC - box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); -#else +#if defined HAVE_STRUCT_STAT_ST_ATIM + box_time_t datamodified = (((int64_t)st.st_mtim.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_mtim.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIMESPEC box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#else + box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); #endif return datamodified; @@ -31,7 +34,10 @@ box_time_t FileModificationTime(const EMU_STRUCT_STAT &st) box_time_t FileAttrModificationTime(const EMU_STRUCT_STAT &st) { box_time_t statusmodified = -#ifdef HAVE_STRUCT_STAT_ST_MTIMESPEC +#if defined HAVE_STRUCT_STAT_ST_ATIM + (((int64_t)st.st_ctim.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctim.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIMESPEC (((int64_t)st.st_ctimespec.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); #elif defined HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp index fc0319da..51752f85 100644 --- a/lib/common/FileStream.cpp +++ b/lib/common/FileStream.cpp @@ -67,22 +67,29 @@ void FileStream::AfterOpen() { MEMLEAKFINDER_NOT_A_LEAK(this); - #ifdef WIN32 - BOX_LOG_WIN_WARNING_NUMBER("Failed to open file: " << - mFileName, winerrno); - #else - BOX_LOG_SYS_WARNING("Failed to open file: " << - mFileName); - #endif - +#ifdef WIN32 + if(errno == EACCES) + { + THROW_WIN_FILE_ERRNO("Failed to open file", mFileName, + winerrno, CommonException, AccessDenied); + } + else + { + THROW_WIN_FILE_ERRNO("Failed to open file", mFileName, + winerrno, CommonException, OSFileOpenError); + } +#else if(errno == EACCES) { - THROW_EXCEPTION(CommonException, AccessDenied) + THROW_SYS_FILE_ERROR("Failed to open file", mFileName, + CommonException, AccessDenied); } else { - THROW_EXCEPTION(CommonException, OSFileOpenError) + THROW_SYS_FILE_ERROR("Failed to open file", mFileName, + CommonException, OSFileOpenError); } +#endif } } @@ -244,9 +251,9 @@ IOStream::pos_type FileStream::BytesLeftToRead() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void FileStream::Write(const void *pBuffer, int NBytes) +void FileStream::Write(const void *pBuffer, int NBytes, int Timeout) { - if(mOSFileHandle == INVALID_FILE) + if(mOSFileHandle == INVALID_FILE) { THROW_EXCEPTION(CommonException, FileClosed) } diff --git a/lib/common/FileStream.h b/lib/common/FileStream.h index 9101a968..1426d8a2 100644 --- a/lib/common/FileStream.h +++ b/lib/common/FileStream.h @@ -23,7 +23,7 @@ class FileStream : public IOStream { public: - FileStream(const std::string& rFilename, + FileStream(const std::string& rFilename, int flags = (O_RDONLY | O_BINARY), int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); @@ -40,7 +40,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); @@ -49,6 +50,11 @@ public: virtual bool StreamClosed(); bool CompareWith(IOStream& rOther, int Timeout = IOStream::TimeOutInfinite); + std::string ToString() const + { + return std::string("local file ") + mFileName; + } + const std::string GetFileName() const { return mFileName; } private: tOSFileHandle mOSFileHandle; diff --git a/lib/common/Guards.h b/lib/common/Guards.h index cd2e4628..46b6d2bd 100644 --- a/lib/common/Guards.h +++ b/lib/common/Guards.h @@ -37,9 +37,8 @@ public: { if(mOSFileHandle < 0) { - BOX_LOG_SYS_ERROR("FileHandleGuard: failed to open " - "file '" << rFilename << "'"); - THROW_EXCEPTION(CommonException, OSFileOpenError) + THROW_SYS_FILE_ERROR("Failed to open file", rFilename, + CommonException, OSFileOpenError); } } @@ -78,7 +77,17 @@ class MemoryBlockGuard { public: MemoryBlockGuard(int BlockSize) - : mpBlock(::malloc(BlockSize)) + : mpBlock(::malloc(BlockSize)), + mBlockSize(BlockSize) + { + if(mpBlock == 0) + { + throw std::bad_alloc(); + } + } + + MemoryBlockGuard(void *pBlock) + : mpBlock(pBlock) { if(mpBlock == 0) { @@ -110,9 +119,21 @@ public: } mpBlock = ptrn; } + + void* Release() + { + void* pBlock = mpBlock; + mpBlock = ::malloc(mBlockSize); + if(mpBlock == 0) + { + throw std::bad_alloc(); + } + return pBlock; + } private: void *mpBlock; + int mBlockSize; }; #include "MemLeakFindOff.h" diff --git a/lib/common/IOStream.cpp b/lib/common/IOStream.cpp index fc9d0bc3..3e126d3f 100644 --- a/lib/common/IOStream.cpp +++ b/lib/common/IOStream.cpp @@ -127,8 +127,8 @@ int IOStream::ConvertSeekTypeToOSWhence(int SeekType) // Function // Name: IOStream::ReadFullBuffer(void *, int, int) // Purpose: Reads bytes into buffer, returning whether or not it managed to -// get all the bytes required. Exception and abort use of stream -// if this returns false. +// get all the bytes required. Exception and abort use of stream +// if this returns false. // Created: 2003/08/26 // // -------------------------------------------------------------------------- @@ -165,7 +165,7 @@ bool IOStream::ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int T // Created: 2003/08/26 // // -------------------------------------------------------------------------- -void IOStream::WriteAllBuffered() +void IOStream::WriteAllBuffered(int Timeout) { } @@ -245,7 +245,30 @@ void IOStream::Flush(int Timeout) } } -void IOStream::Write(const char *pBuffer) +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::Write +// Purpose: Convenience method for writing a C++ string to a +// protocol buffer. +// +// -------------------------------------------------------------------------- +void IOStream::Write(const std::string& rBuffer, int Timeout) +{ + Write(rBuffer.c_str(), rBuffer.size(), Timeout); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::ToString() +// Purpose: Returns a string which describes this stream. Useful +// when reporting exceptions about a stream of unknown +// origin, for example in BackupStoreDirectory(). +// Created: 2014/04/28 +// +// -------------------------------------------------------------------------- +std::string IOStream::ToString() const { - Write(pBuffer, strlen(pBuffer)); + return "unknown IOStream"; } diff --git a/lib/common/IOStream.h b/lib/common/IOStream.h index 0b1cedd3..df7216c3 100644 --- a/lib/common/IOStream.h +++ b/lib/common/IOStream.h @@ -47,27 +47,27 @@ public: typedef int64_t pos_type; virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite) = 0; virtual pos_type BytesLeftToRead(); // may return IOStream::SizeOfStreamUnknown (and will for most stream types) - virtual void Write(const void *pBuffer, int NBytes) = 0; - virtual void Write(const char *pBuffer); - virtual void WriteAllBuffered(); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) = 0; + virtual void Write(const std::string& rBuffer, + int Timeout = IOStream::TimeOutInfinite); + virtual void WriteAllBuffered(int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual void Close(); - + // Has all data that can be read been read? virtual bool StreamDataLeft() = 0; // Has the stream been closed (writing not possible) virtual bool StreamClosed() = 0; - + // Utility functions bool ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout = IOStream::TimeOutInfinite); bool CopyStreamTo(IOStream &rCopyTo, int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024); void Flush(int Timeout = IOStream::TimeOutInfinite); - + static int ConvertSeekTypeToOSWhence(int SeekType); + virtual std::string ToString() const; }; - #endif // IOSTREAM__H - - diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h index c4289073..1b537031 100644 --- a/lib/common/IOStreamGetLine.h +++ b/lib/common/IOStreamGetLine.h @@ -33,16 +33,22 @@ private: public: bool GetLine(std::string &rOutput, bool Preprocess = false, int Timeout = IOStream::TimeOutInfinite); - + std::string GetLine() + { + std::string output; + GetLine(output); + return output; + } + // Call to detach, setting file pointer correctly to last bit read. // Only works for lseek-able file descriptors. void DetachFile(); - + virtual bool IsStreamDataLeft() { return mrStream.StreamDataLeft(); } - + // For doing interesting stuff with the remaining data... // Be careful with this! const void *GetBufferedData() const {return mBuffer + mBufferBegin;} @@ -52,7 +58,7 @@ public: protected: int ReadMore(int Timeout = IOStream::TimeOutInfinite); - + private: IOStream &mrStream; }; diff --git a/lib/common/InvisibleTempFileStream.cpp b/lib/common/InvisibleTempFileStream.cpp index abfcb5f6..1a9d6d5a 100644 --- a/lib/common/InvisibleTempFileStream.cpp +++ b/lib/common/InvisibleTempFileStream.cpp @@ -22,7 +22,8 @@ // Created: 2006/10/13 // // -------------------------------------------------------------------------- -InvisibleTempFileStream::InvisibleTempFileStream(const char *Filename, int flags, int mode) +InvisibleTempFileStream::InvisibleTempFileStream(const std::string& Filename, + int flags, int mode) #ifdef WIN32 : FileStream(Filename, flags | O_TEMPORARY, mode) #else @@ -30,7 +31,7 @@ InvisibleTempFileStream::InvisibleTempFileStream(const char *Filename, int flags #endif { #ifndef WIN32 - if(unlink(Filename) != 0) + if(unlink(Filename.c_str()) != 0) { MEMLEAKFINDER_NOT_A_LEAK(this); THROW_EXCEPTION(CommonException, OSFileOpenError) diff --git a/lib/common/InvisibleTempFileStream.h b/lib/common/InvisibleTempFileStream.h index a77d05e2..bb6c3954 100644 --- a/lib/common/InvisibleTempFileStream.h +++ b/lib/common/InvisibleTempFileStream.h @@ -16,7 +16,7 @@ class InvisibleTempFileStream : public FileStream { public: - InvisibleTempFileStream(const char *Filename, + InvisibleTempFileStream(const std::string& Filename, #ifdef WIN32 int flags = (O_RDONLY | O_BINARY), #else diff --git a/lib/common/Logging.cpp b/lib/common/Logging.cpp index 7ce0dd3f..0928a4d4 100644 --- a/lib/common/Logging.cpp +++ b/lib/common/Logging.cpp @@ -13,19 +13,19 @@ #include <time.h> #include <string.h> // for stderror -// c.f. http://bugs.debian.org/512510 -#include <cstdio> +#ifdef HAVE_PROCESS_H +# include <process.h> +#endif #ifdef HAVE_SYSLOG_H - #include <syslog.h> +# include <syslog.h> #endif + #ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif -#ifdef WIN32 - #include <process.h> +# include <unistd.h> #endif +#include <cstdio> #include <cstring> #include <iomanip> @@ -42,16 +42,14 @@ std::vector<Logger*> Logging::sLoggers; std::string Logging::sContext; Console* Logging::spConsole = NULL; Syslog* Logging::spSyslog = NULL; -Log::Level Logging::sGlobalLevel = Log::EVERYTHING; -Logging Logging::sGlobalLogging; //automatic initialisation +Logging Logging::sGlobalLogging; // automatic initialisation std::string Logging::sProgramName; +const Log::Category Logging::UNCATEGORISED("Uncategorised"); +std::auto_ptr<HideFileGuard> Logging::sapHideFileGuard; HideSpecificExceptionGuard::SuppressedExceptions_t HideSpecificExceptionGuard::sSuppressedExceptions; -int Logging::Guard::sGuardCount = 0; -Log::Level Logging::Guard::sOriginalLevel = Log::INVALID; - Logging::Logging() { ASSERT(!spConsole); @@ -139,14 +137,10 @@ void Logging::Remove(Logger* pOldLogger) } } -void Logging::Log(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage) +void Logging::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { - if (level > sGlobalLevel) - { - return; - } - std::string newMessage; if (sContextSet) @@ -154,12 +148,13 @@ void Logging::Log(Log::Level level, const std::string& rFile, newMessage += "[" + sContext + "] "; } - newMessage += rMessage; + newMessage += message; for (std::vector<Logger*>::iterator i = sLoggers.begin(); i != sLoggers.end(); i++) { - bool result = (*i)->Log(level, rFile, line, newMessage); + bool result = (*i)->Log(level, file, line, function, category, + newMessage); if (!result) { return; @@ -167,19 +162,15 @@ void Logging::Log(Log::Level level, const std::string& rFile, } } -void Logging::LogToSyslog(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage) +void Logging::LogToSyslog(Log::Level level, const std::string& rFile, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (!sLogToSyslog) { return; } - if (level > sGlobalLevel) - { - return; - } - std::string newMessage; if (sContextSet) @@ -187,9 +178,9 @@ void Logging::LogToSyslog(Log::Level level, const std::string& rFile, newMessage += "[" + sContext + "] "; } - newMessage += rMessage; + newMessage += message; - spSyslog->Log(level, rFile, line, newMessage); + spSyslog->Log(level, rFile, line, function, category, newMessage); } void Logging::SetContext(std::string context) @@ -255,8 +246,7 @@ Logger::~Logger() bool Logger::IsEnabled(Log::Level level) { - return Logging::IsEnabled(level) && - (int)mCurrentLevel >= (int)level; + return (int)mCurrentLevel >= (int)level; } bool Console::sShowTime = false; @@ -290,8 +280,9 @@ void Console::SetShowPID(bool enabled) sShowPID = enabled; } -bool Console::Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) +bool Console::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (level > GetLevel()) { @@ -299,12 +290,6 @@ bool Console::Log(Log::Level level, const std::string& rFile, } FILE* target = stdout; - - if (level <= Log::WARNING) - { - target = stderr; - } - std::ostringstream buf; if (sShowTime) @@ -354,7 +339,7 @@ bool Console::Log(Log::Level level, const std::string& rFile, buf << "TRACE: "; } - buf << rMessage; + buf << message; #ifdef WIN32 std::string output = buf.str(); @@ -376,8 +361,9 @@ bool Console::Log(Log::Level level, const std::string& rFile, return true; } -bool Syslog::Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) +bool Syslog::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (level > GetLevel()) { @@ -418,7 +404,7 @@ bool Syslog::Log(Log::Level level, const std::string& rFile, msg = "NOTICE: "; } - msg += rMessage; + msg += message; syslog(syslogLevel, "%s", msg.c_str()); @@ -432,6 +418,11 @@ Syslog::Syslog() : mFacility(LOG_LOCAL6) Syslog::~Syslog() { + Shutdown(); +} + +void Syslog::Shutdown() +{ ::closelog(); } @@ -467,8 +458,9 @@ int Syslog::GetNamedFacility(const std::string& rFacility) return LOG_LOCAL6; } -bool FileLogger::Log(Log::Level Level, const std::string& rFile, - int line, std::string& rMessage) +bool FileLogger::Log(Log::Level Level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (mLogFile.StreamClosed()) { @@ -515,7 +507,7 @@ bool FileLogger::Log(Log::Level Level, const std::string& rFile, buf << "[TRACE] "; } - buf << rMessage << "\n"; + buf << message << "\n"; std::string output = buf.str(); #ifdef WIN32 @@ -564,3 +556,211 @@ bool HideSpecificExceptionGuard::IsHidden(int type, int subtype) return false; } +// -------------------------------------------------------------------------- +// +// Function +// Name: Logging::OptionParser::GetOptionString() +// Purpose: Returns the valid Getopt command-line options +// that Logging::OptionParser::ProcessOption will handle. +// Created: 2014/04/09 +// +// -------------------------------------------------------------------------- +std::string Logging::OptionParser::GetOptionString() +{ + return "L:NPqQt:TUvVW:"; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Logging::OptionParser::ProcessOption(signed int option) +// Purpose: Processes the supplied option (equivalent to the +// return code from getopt()). Return zero if the +// option was handled successfully, or nonzero to +// abort the program with that return value. +// Created: 2007/09/18 +// +// -------------------------------------------------------------------------- +int Logging::OptionParser::ProcessOption(signed int option) +{ + switch(option) + { + case 'L': + { + if(sapHideFileGuard.get()) + { + sapHideFileGuard->Add(optarg); + } + else + { + sapHideFileGuard.reset(new HideFileGuard( + optarg, true)); // HideAllButSelected + } + } + break; + + case 'N': + { + mTruncateLogFile = true; + } + break; + + case 'P': + { + Console::SetShowPID(true); + } + break; + + case 'q': + { + if(mCurrentLevel == Log::NOTHING) + { + BOX_FATAL("Too many '-q': " + "Cannot reduce logging " + "level any more"); + return 2; + } + mCurrentLevel--; + } + break; + + case 'Q': + { + mCurrentLevel = Log::NOTHING; + } + break; + + case 't': + { + Logging::SetProgramName(optarg); + Console::SetShowTag(true); + } + break; + + case 'T': + { + Console::SetShowTime(true); + } + break; + + case 'U': + { + Console::SetShowTime(true); + Console::SetShowTimeMicros(true); + } + break; + + case 'v': + { + if(mCurrentLevel == Log::EVERYTHING) + { + BOX_FATAL("Too many '-v': " + "Cannot increase logging " + "level any more"); + return 2; + } + mCurrentLevel++; + } + break; + + case 'V': + { + mCurrentLevel = Log::EVERYTHING; + } + break; + + case 'W': + { + mCurrentLevel = Logging::GetNamedLevel(optarg); + if (mCurrentLevel == Log::INVALID) + { + BOX_FATAL("Invalid logging level: " << optarg); + return 2; + } + } + break; + + case '?': + { + BOX_FATAL("Unknown option on command line: " + << "'" << (char)optopt << "'"); + return 2; + } + break; + + default: + { + BOX_FATAL("Unknown error in getopt: returned " + << "'" << option << "'"); + return 1; + } + } + + // If we didn't explicitly return an error code above, then the option + // was fine, so return 0 to continue processing. + return 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Logging::OptionParser::GetUsageString() +// Purpose: Returns a string suitable for displaying as part +// of a program's command-line usage help message, +// describing the logging options. +// Created: 2014/04/09 +// +// -------------------------------------------------------------------------- +std::string Logging::OptionParser::GetUsageString() +{ + return + " -L <file> Filter out log messages except from specified file, can repeat\n" + " (for example, -L " __FILE__ ")\n" + " -N Truncate log file at startup and on backup start\n" + " -P Show process ID (PID) in console output\n" + " -q Run more quietly, reduce verbosity level by one, can repeat\n" + " -Q Run at minimum verbosity, log nothing to console and system\n" + " -t <tag> Tag console output with specified marker\n" + " -T Timestamp console output\n" + " -U Timestamp console output with microseconds\n" + " -v Run more verbosely, increase verbosity level by one, can repeat\n" + " -V Run at maximum verbosity, log everything to console and system\n" + " -W <level> Set verbosity to error/warning/notice/info/trace/everything\n"; +} + +bool HideCategoryGuard::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) +{ + std::list<Log::Category>::iterator i = std::find(mCategories.begin(), + mCategories.end(), category); + // Return false if category is in our list, to suppress further + // logging (thus, return true if it's not in our list, i.e. we + // found nothing, to allow it). + return (i == mCategories.end()); +} + +bool HideFileGuard::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) +{ + std::list<std::string>::iterator i = std::find(mFileNames.begin(), + mFileNames.end(), file); + bool allow_log_message; + if(mHideAllButSelected) + { + // Return true if filename is in our list, to allow further + // logging (thus, return false if it's not in our list, i.e. we + // found nothing, to suppress it). + allow_log_message = (i != mFileNames.end()); + } + else + { + // Return false if filename is in our list, to suppress further + // logging (thus, return true if it's not in our list, i.e. we + // found nothing, to allow it). + allow_log_message = (i == mFileNames.end()); + } + return allow_log_message; +} + diff --git a/lib/common/Logging.h b/lib/common/Logging.h index 1074b7c3..3dc3e69c 100644 --- a/lib/common/Logging.h +++ b/lib/common/Logging.h @@ -12,9 +12,11 @@ #include <assert.h> +#include <algorithm> #include <cerrno> #include <cstring> #include <iomanip> +#include <list> #include <sstream> #include <vector> @@ -24,14 +26,24 @@ { \ std::ostringstream _box_log_line; \ _box_log_line << stuff; \ - Logging::Log(level, __FILE__, __LINE__, _box_log_line.str()); \ + Logging::Log(level, __FILE__, __LINE__, __FUNCTION__, \ + Logging::UNCATEGORISED, _box_log_line.str()); \ +} + +#define BOX_LOG_CATEGORY(level, category, stuff) \ +{ \ + std::ostringstream _box_log_line; \ + _box_log_line << stuff; \ + Logging::Log(level, __FILE__, __LINE__, __FUNCTION__, \ + category, _box_log_line.str()); \ } #define BOX_SYSLOG(level, stuff) \ { \ std::ostringstream _box_log_line; \ _box_log_line << stuff; \ - Logging::LogToSyslog(level, __FILE__, __LINE__, _box_log_line.str()); \ + Logging::LogToSyslog(level, __FILE__, __LINE__, __FUNCTION__, \ + Logging::UNCATEGORISED, _box_log_line.str()); \ } #define BOX_FATAL(stuff) BOX_LOG(Log::FATAL, stuff) @@ -39,9 +51,7 @@ #define BOX_WARNING(stuff) BOX_LOG(Log::WARNING, stuff) #define BOX_NOTICE(stuff) BOX_LOG(Log::NOTICE, stuff) #define BOX_INFO(stuff) BOX_LOG(Log::INFO, stuff) -#define BOX_TRACE(stuff) \ - if (Logging::IsEnabled(Log::TRACE)) \ - { BOX_LOG(Log::TRACE, stuff) } +#define BOX_TRACE(stuff) BOX_LOG(Log::TRACE, stuff) #define BOX_SYS_ERRNO_MESSAGE(error_number, stuff) \ stuff << ": " << std::strerror(error_number) << \ @@ -85,18 +95,20 @@ BOX_FILE_MESSAGE(filename, message)) #ifdef WIN32 + #define BOX_WIN_ERRNO_MESSAGE(error_number, stuff) \ + stuff << ": " << GetErrorMessage(error_number) + #define BOX_NATIVE_ERRNO_MESSAGE(error_number, stuff) \ + BOX_WIN_ERRNO_MESSAGE(error_number, stuff) #define BOX_LOG_WIN_ERROR(stuff) \ - BOX_ERROR(stuff << ": " << GetErrorMessage(GetLastError())) + BOX_ERROR(BOX_WIN_ERRNO_MESSAGE(GetLastError(), stuff)) #define BOX_LOG_WIN_WARNING(stuff) \ - BOX_WARNING(stuff << ": " << GetErrorMessage(GetLastError())) + BOX_WARNING(BOX_WIN_ERRNO_MESSAGE(GetLastError(), stuff)) #define BOX_LOG_WIN_ERROR_NUMBER(stuff, number) \ - BOX_ERROR(stuff << ": " << GetErrorMessage(number)) + BOX_ERROR(BOX_WIN_ERRNO_MESSAGE(number, stuff)) #define BOX_LOG_WIN_WARNING_NUMBER(stuff, number) \ - BOX_WARNING(stuff << ": " << GetErrorMessage(number)) + BOX_WARNING(BOX_WIN_ERRNO_MESSAGE(number, stuff)) #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_WIN_ERROR(stuff) #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_WIN_WARNING(stuff) - #define BOX_WIN_ERRNO_MESSAGE(error_number, stuff) \ - stuff << ": " << GetErrorMessage(error_number) #define THROW_WIN_ERROR_NUMBER(message, error_number, exception, subtype) \ THROW_EXCEPTION_MESSAGE(exception, subtype, \ BOX_WIN_ERRNO_MESSAGE(error_number, message)) @@ -106,19 +118,33 @@ #define THROW_WIN_FILE_ERROR(message, filename, exception, subtype) \ THROW_WIN_FILE_ERRNO(message, filename, GetLastError(), \ exception, subtype) + #define EMU_ERRNO winerrno + #define THROW_EMU_ERROR(message, exception, subtype) \ + THROW_EXCEPTION_MESSAGE(exception, subtype, \ + BOX_NATIVE_ERRNO_MESSAGE(EMU_ERRNO, message)) #else + #define BOX_NATIVE_ERRNO_MESSAGE(error_number, stuff) \ + BOX_SYS_ERRNO_MESSAGE(error_number, stuff) #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_SYS_ERROR(stuff) #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_SYS_WARNING(stuff) + #define EMU_ERRNO errno + #define THROW_EMU_ERROR(message, exception, subtype) \ + THROW_EXCEPTION_MESSAGE(exception, subtype, \ + BOX_SYS_ERRNO_MESSAGE(EMU_ERRNO, message)) #endif +#define THROW_EMU_FILE_ERROR(message, filename, exception, subtype) \ + THROW_EMU_ERROR(BOX_FILE_MESSAGE(filename, message), \ + exception, subtype) + #ifdef WIN32 -# define BOX_LOG_SOCKET_ERROR(_type, _name, _port, stuff) \ - BOX_LOG_WIN_ERROR_NUMBER(stuff << " (type " << _type << ", name " << \ - _name << ", port " << _port << ")", WSAGetLastError()) +# define BOX_SOCKET_ERROR_MESSAGE(_type, _name, _port, stuff) \ + BOX_WIN_ERRNO_MESSAGE(WSAGetLastError(), stuff << " (type " << _type << \ + ", name " << _name << ", port " << _port << ")") #else -# define BOX_LOG_SOCKET_ERROR(_type, _name, _port, stuff) \ - BOX_LOG_NATIVE_ERROR(stuff << " (type " << _type << ", name " << \ - _name << ", port " << _port << ")") +# define BOX_SOCKET_ERROR_MESSAGE(_type, _name, _port, stuff) \ + BOX_SYS_ERROR_MESSAGE(stuff << " (type " << _type << ", name " << _name << \ + ", port " << _port << ")") #endif #define BOX_FORMAT_HEX32(number) \ @@ -141,14 +167,16 @@ #define BOX_FORMAT_TIMESPEC(timespec) \ timespec.tv_sec << \ + "." << \ std::setw(6) << \ + std::setfill('0') << \ timespec.tv_usec #define BOX_FORMAT_MICROSECONDS(t) \ (int)((t) / 1000000) << "." << \ - std::setw(6) << \ + std::setw(3) << \ std::setfill('0') << \ - (int)((t) % 1000000) << " seconds" + (int)((t % 1000000) / 1000) << " seconds" #undef ERROR @@ -166,6 +194,18 @@ namespace Log EVERYTHING, INVALID = -1 }; + + class Category { + private: + std::string mName; + + public: + Category(const std::string& name) + : mName(name) + { } + const std::string& ToString() { return mName; } + bool operator==(const Category& other) { return mName == other.mName; } + }; } // -------------------------------------------------------------------------- @@ -187,8 +227,9 @@ class Logger Logger(Log::Level level); virtual ~Logger(); - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) = 0; + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) = 0; void Filter(Log::Level level) { @@ -201,19 +242,23 @@ class Logger virtual void SetProgramName(const std::string& rProgramName) = 0; - class Guard + class LevelGuard { private: Logger& mLogger; Log::Level mOldLevel; public: - Guard(Logger& Logger) + LevelGuard(Logger& Logger, Log::Level newLevel = Log::INVALID) : mLogger(Logger) { mOldLevel = Logger.GetLevel(); + if (newLevel != Log::INVALID) + { + Logger.Filter(newLevel); + } } - ~Guard() + ~LevelGuard() { mLogger.Filter(mOldLevel); } @@ -239,8 +284,9 @@ class Console : public Logger static std::string sTag; public: - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage); + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); virtual const char* GetType() { return "Console"; } virtual void SetProgramName(const std::string& rProgramName); @@ -248,6 +294,33 @@ class Console : public Logger static void SetShowTime(bool enabled); static void SetShowTimeMicros(bool enabled); static void SetShowPID(bool enabled); + static bool GetShowTag() { return sShowTag; } + + class SettingsGuard + { + private: + bool mShowTag; + bool mShowTime; + bool mShowTimeMicros; + bool mShowPID; + std::string mTag; + public: + SettingsGuard() + : mShowTag(Console::sShowTag), + mShowTime(Console::sShowTime), + mShowTimeMicros(Console::sShowTimeMicros), + mShowPID(Console::sShowPID), + mTag(Console::sTag) + { } + ~SettingsGuard() + { + Console::SetShowTag(mShowTag); + Console::SetShowTime(mShowTime); + Console::SetShowTimeMicros(mShowTimeMicros); + Console::SetShowPID(mShowPID); + Console::sTag = mTag; + } + }; }; // -------------------------------------------------------------------------- @@ -269,17 +342,80 @@ class Syslog : public Logger Syslog(); virtual ~Syslog(); - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage); + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); virtual const char* GetType() { return "Syslog"; } virtual void SetProgramName(const std::string& rProgramName); virtual void SetFacility(int facility); + virtual void Shutdown(); static int GetNamedFacility(const std::string& rFacility); }; // -------------------------------------------------------------------------- // // Class +// Name: Capture +// Purpose: Keeps log messages for analysis in tests. +// Created: 2014/03/08 +// +// -------------------------------------------------------------------------- + +class Capture : public Logger +{ + public: + struct Message + { + Message(const Log::Category& category) + : mCategory(category) { } + Log::Level level; + std::string file; + int line; + std::string function; + Log::Category mCategory; + std::string message; + }; + + private: + std::vector<Message> mMessages; + + public: + virtual ~Capture() { } + + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) + { + Message msg(category); + msg.level = level; + msg.file = file; + msg.line = line; + msg.function = function; + msg.message = message; + mMessages.push_back(msg); + return true; + } + virtual const char* GetType() { return "Capture"; } + virtual void SetProgramName(const std::string& rProgramName) { } + const std::vector<Message>& GetMessages() const { return mMessages; } + std::string GetString() const + { + std::ostringstream oss; + for (std::vector<Message>::const_iterator i = mMessages.begin(); + i != mMessages.end(); i++) + { + oss << i->message << "\n"; + } + return oss.str(); + } +}; + +// Forward declaration +class HideFileGuard; + +// -------------------------------------------------------------------------- +// +// Class // Name: Logging // Purpose: Static logging helper, keeps track of enabled loggers // and distributes log messages to them. @@ -296,10 +432,10 @@ class Logging static bool sContextSet; static Console* spConsole; static Syslog* spSyslog; - static Log::Level sGlobalLevel; static Logging sGlobalLogging; static std::string sProgramName; - + static std::auto_ptr<HideFileGuard> sapHideFileGuard; + public: Logging (); ~Logging(); @@ -309,55 +445,35 @@ class Logging static void FilterConsole (Log::Level level); static void Add (Logger* pNewLogger); static void Remove (Logger* pOldLogger); - static void Log(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage); - static void LogToSyslog(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage); + static void Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); + static void LogToSyslog(Log::Level level, const std::string& rFile, int line, + const std::string& function, const Log::Category& category, + const std::string& message); static void SetContext(std::string context); static void ClearContext(); - static void SetGlobalLevel(Log::Level level) { sGlobalLevel = level; } - static Log::Level GetGlobalLevel() { return sGlobalLevel; } static Log::Level GetNamedLevel(const std::string& rName); - static bool IsEnabled(Log::Level level) - { - return (int)sGlobalLevel >= (int)level; - } static void SetProgramName(const std::string& rProgramName); static std::string GetProgramName() { return sProgramName; } static void SetFacility(int facility); static Console& GetConsole() { return *spConsole; } static Syslog& GetSyslog() { return *spSyslog; } - class Guard + class ShowTagOnConsole { private: - Log::Level mOldLevel; - static int sGuardCount; - static Log::Level sOriginalLevel; - + bool mOldShowTag; + public: - Guard(Log::Level newLevel) - { - mOldLevel = Logging::GetGlobalLevel(); - if(sGuardCount == 0) - { - sOriginalLevel = mOldLevel; - } - sGuardCount++; - Logging::SetGlobalLevel(newLevel); - } - ~Guard() + ShowTagOnConsole() + : mOldShowTag(Console::GetShowTag()) { - sGuardCount--; - Logging::SetGlobalLevel(mOldLevel); + Console::SetShowTag(true); } - - static bool IsActive() { return (sGuardCount > 0); } - static Log::Level GetOriginalLevel() { return sOriginalLevel; } - static bool IsGuardingFrom(Log::Level originalLevel) + ~ShowTagOnConsole() { - return IsActive() && - (int)sOriginalLevel >= (int)originalLevel; + Console::SetShowTag(mOldShowTag); } }; @@ -365,16 +481,19 @@ class Logging { private: std::string mOldTag; + bool mReplace; public: Tagger() - : mOldTag(Logging::GetProgramName()) + : mOldTag(Logging::GetProgramName()), + mReplace(false) { } - Tagger(const std::string& rTempTag) - : mOldTag(Logging::GetProgramName()) + Tagger(const std::string& rTempTag, bool replace = false) + : mOldTag(Logging::GetProgramName()), + mReplace(replace) { - Logging::SetProgramName(mOldTag + " " + rTempTag); + Change(rTempTag); } ~Tagger() { @@ -383,9 +502,73 @@ class Logging void Change(const std::string& newTempTag) { - Logging::SetProgramName(mOldTag + " " + newTempTag); + if(mReplace || mOldTag.empty()) + { + Logging::SetProgramName(newTempTag); + } + else + { + Logging::SetProgramName(mOldTag + " " + newTempTag); + } + } + }; + + class TempLoggerGuard + { + private: + Logger* mpLogger; + + public: + TempLoggerGuard(Logger* pLogger) + : mpLogger(pLogger) + { + Logging::Add(mpLogger); + } + ~TempLoggerGuard() + { + Logging::Remove(mpLogger); + } + }; + + // Process global options + static std::string GetOptionString(); + static int ProcessOption(signed int option); + static std::string GetUsageString(); + + // -------------------------------------------------------------------------- + // + // Class + // Name: Logging::OptionParser + // Purpose: Process command-line options, some global, some local + // Created: 2014/04/09 + // + // -------------------------------------------------------------------------- + class OptionParser + { + public: + OptionParser(Log::Level InitialLevel = + #ifdef BOX_RELEASE_BUILD + Log::NOTICE + #else + Log::INFO + #endif + ) + : mCurrentLevel(InitialLevel), + mTruncateLogFile(false) + { } + + static std::string GetOptionString(); + int ProcessOption(signed int option); + static std::string GetUsageString(); + int mCurrentLevel; // need an int to do math with + bool mTruncateLogFile; + Log::Level GetCurrentLevel() + { + return (Log::Level) mCurrentLevel; } }; + + static const Log::Category UNCATEGORISED; }; class FileLogger : public Logger @@ -396,13 +579,14 @@ class FileLogger : public Logger : mLogFile("") { /* do not call */ } public: - FileLogger(const std::string& rFileName, Log::Level Level) + FileLogger(const std::string& rFileName, Log::Level Level, bool append) : Logger(Level), - mLogFile(rFileName, O_WRONLY | O_CREAT | O_APPEND) + mLogFile(rFileName, O_WRONLY | O_CREAT | (append ? O_APPEND : O_TRUNC)) { } - virtual bool Log(Log::Level Level, const std::string& rFile, - int Line, std::string& rMessage); + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); virtual const char* GetType() { return "FileLogger"; } virtual void SetProgramName(const std::string& rProgramName) { } @@ -451,6 +635,64 @@ class HideSpecificExceptionGuard static bool IsHidden(int type, int subtype); }; +class HideCategoryGuard : public Logger +{ + private: + std::list<Log::Category> mCategories; + HideCategoryGuard(const HideCategoryGuard& other); // no copying + HideCategoryGuard& operator=(const HideCategoryGuard& other); // no assignment + + public: + HideCategoryGuard(const Log::Category& category) + { + mCategories.push_back(category); + Logging::Add(this); + } + ~HideCategoryGuard() + { + Logging::Remove(this); + } + void Add(const Log::Category& category) + { + mCategories.push_back(category); + } + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); + virtual const char* GetType() { return "HideCategoryGuard"; } + virtual void SetProgramName(const std::string& rProgramName) { } +}; + +class HideFileGuard : public Logger +{ + private: + std::list<std::string> mFileNames; + HideFileGuard(const HideFileGuard& other); // no copying + HideFileGuard& operator=(const HideFileGuard& other); // no assignment + bool mHideAllButSelected; + + public: + HideFileGuard(const std::string& rFileName, bool HideAllButSelected = false) + : mHideAllButSelected(HideAllButSelected) + { + mFileNames.push_back(rFileName); + Logging::Add(this); + } + ~HideFileGuard() + { + Logging::Remove(this); + } + void Add(const std::string& rFileName) + { + mFileNames.push_back(rFileName); + } + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); + virtual const char* GetType() { return "HideFileGuard"; } + virtual void SetProgramName(const std::string& rProgramName) { } +}; + std::string PrintEscapedBinaryData(const std::string& rInput); #endif // LOGGING__H diff --git a/lib/common/MainHelper.h b/lib/common/MainHelper.h index 3c6e9ff0..0303090e 100644 --- a/lib/common/MainHelper.h +++ b/lib/common/MainHelper.h @@ -19,18 +19,21 @@ #include "BoxException.h" #include "Logging.h" -#define MAINHELPER_START \ - if(argc == 2 && ::strcmp(argv[1], "--version") == 0) \ - { printf(BOX_VERSION "\n"); return 0; } \ +#define MAINHELPER_START \ + if(argc == 2 && ::strcmp(argv[1], "--version") == 0) \ + { printf(BOX_VERSION "\n"); return 0; } \ MEMLEAKFINDER_INIT \ - MEMLEAKFINDER_START \ + MEMLEAKFINDER_START \ try { -#define MAINHELPER_END \ - } catch(std::exception &e) { \ +#define MAINHELPER_END \ + } catch(BoxException &e) { \ + BOX_FATAL(e.what() << ": " << e.GetMessage()); \ + return 1; \ + } catch(std::exception &e) { \ BOX_FATAL(e.what()); \ - return 1; \ - } catch(...) { \ + return 1; \ + } catch(...) { \ BOX_FATAL("UNKNOWN"); \ return 1; \ } diff --git a/lib/common/MemBlockStream.cpp b/lib/common/MemBlockStream.cpp index 3a43a304..f49ac96f 100644 --- a/lib/common/MemBlockStream.cpp +++ b/lib/common/MemBlockStream.cpp @@ -52,6 +52,26 @@ MemBlockStream::MemBlockStream(const void *pBuffer, int Size) // -------------------------------------------------------------------------- // // Function +// Name: MemBlockStream::MemBlockStream(const std::string& rMessage) +// Purpose: Convenience constructor for sending a simple string. +// Copies the string, so you can pass a temporary in. +// Created: 2014/01/20 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream(const std::string& rMessage) +: mReadPosition(0) +{ + mTempBuffer.Write(rMessage.c_str(), rMessage.size()); + mTempBuffer.SetForReading(); + mpBuffer = (const char *)(mTempBuffer.GetBuffer()); + mBytesInBuffer = rMessage.size(); + ASSERT(mpBuffer != 0); + ASSERT(mBytesInBuffer >= 0); +} + +// -------------------------------------------------------------------------- +// +// Function // Name: MemBlockStream::MemBlockStream(const StreamableMemBlock &) // Purpose: Constructor (doesn't copy block, careful with lifetimes) // Created: 2003/09/05 @@ -159,7 +179,7 @@ IOStream::pos_type MemBlockStream::BytesLeftToRead() // Created: 2003/09/05 // // -------------------------------------------------------------------------- -void MemBlockStream::Write(const void *pBuffer, int NBytes) +void MemBlockStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, MemBlockStreamNotSupported) } diff --git a/lib/common/MemBlockStream.h b/lib/common/MemBlockStream.h index 5234525b..1ba4b0a6 100644 --- a/lib/common/MemBlockStream.h +++ b/lib/common/MemBlockStream.h @@ -10,10 +10,10 @@ #ifndef MEMBLOCKSTREAM__H #define MEMBLOCKSTREAM__H +#include "CollectInBufferStream.h" #include "IOStream.h" class StreamableMemBlock; -class CollectInBufferStream; // -------------------------------------------------------------------------- // @@ -29,6 +29,7 @@ class MemBlockStream : public IOStream public: MemBlockStream(); MemBlockStream(const void *pBuffer, int Size); + MemBlockStream(const std::string& rMessage); MemBlockStream(const StreamableMemBlock &rBlock); MemBlockStream(const CollectInBufferStream &rBuffer); MemBlockStream(const MemBlockStream &rToCopy); @@ -37,7 +38,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual bool StreamDataLeft(); @@ -46,6 +48,9 @@ public: virtual int GetSize() const { return mBytesInBuffer; } private: + // Use mTempBuffer when we need to hold a copy of the memory block, + // and free it ourselves when done. + CollectInBufferStream mTempBuffer; const char *mpBuffer; int mBytesInBuffer; int mReadPosition; diff --git a/lib/common/MemLeakFindOn.h b/lib/common/MemLeakFindOn.h index c20fe25a..f1113184 100644 --- a/lib/common/MemLeakFindOn.h +++ b/lib/common/MemLeakFindOn.h @@ -15,6 +15,7 @@ #ifndef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__) + #define calloc(X, Y) memleakfinder_calloc(X, Y, __FILE__, __LINE__) #define realloc memleakfinder_realloc #define free memleakfinder_free #define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED diff --git a/lib/common/MemLeakFinder.h b/lib/common/MemLeakFinder.h index 1a2cf90c..07b52e26 100644 --- a/lib/common/MemLeakFinder.h +++ b/lib/common/MemLeakFinder.h @@ -43,7 +43,7 @@ void memleakfinder_reportleaks(); void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext); -void memleakfinder_setup_exit_report(const char *filename, const char *markertext); +void memleakfinder_setup_exit_report(const std::string& filename, const char *markertext); void memleakfinder_startsectionmonitor(); @@ -54,7 +54,8 @@ void memleakfinder_notaleak(void *ptr); void *operator new (size_t size, const char *file, int line); void *operator new[](size_t size, const char *file, int line); -// define the malloc functions now, if required +// Define the malloc functions now, if required. These should match the definitions +// in MemLeakFindOn.h. #ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__) #define calloc(X, Y) memleakfinder_calloc(X, Y, __FILE__, __LINE__) diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp index f96f80b5..8e672ff5 100644 --- a/lib/common/NamedLock.cpp +++ b/lib/common/NamedLock.cpp @@ -21,8 +21,9 @@ #include <sys/file.h> #endif -#include "NamedLock.h" #include "CommonException.h" +#include "NamedLock.h" +#include "Utils.h" #include "MemLeakFindOn.h" @@ -35,7 +36,11 @@ // // -------------------------------------------------------------------------- NamedLock::NamedLock() - : mFileDescriptor(-1) +#ifdef WIN32 +: mFileDescriptor(INVALID_HANDLE_VALUE) +#else +: mFileDescriptor(-1) +#endif { } @@ -49,7 +54,11 @@ NamedLock::NamedLock() // -------------------------------------------------------------------------- NamedLock::~NamedLock() { +#ifdef WIN32 + if(mFileDescriptor != INVALID_HANDLE_VALUE) +#else if(mFileDescriptor != -1) +#endif { ReleaseLock(); } @@ -68,76 +77,151 @@ NamedLock::~NamedLock() bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) { // Check +#ifdef WIN32 + if(mFileDescriptor != INVALID_HANDLE_VALUE) +#else if(mFileDescriptor != -1) +#endif { THROW_EXCEPTION(CommonException, NamedLockAlreadyLockingSomething) } + mFileName = rFilename; + // See if the lock can be got + int flags = O_WRONLY | O_CREAT | O_TRUNC; + #if HAVE_DECL_O_EXLOCK - int fd = ::open(rFilename.c_str(), - O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode); - if(fd != -1) - { - // Got a lock, lovely - mFileDescriptor = fd; - return true; - } - - // Failed. Why? - if(errno != EWOULDBLOCK) - { - // Not the expected error - THROW_EXCEPTION(CommonException, OSFileError) - } + flags |= O_NONBLOCK | O_EXLOCK; + BOX_TRACE("Trying to create lockfile " << rFilename << " using O_EXLOCK"); +#elif defined BOX_OPEN_LOCK + flags |= BOX_OPEN_LOCK; + BOX_TRACE("Trying to create lockfile " << rFilename << " using BOX_OPEN_LOCK"); +#elif !HAVE_DECL_F_SETLK && !defined HAVE_FLOCK + // We have no other way to get a lock, so all we can do is fail if + // the file already exists, and take the risk of stale locks. + flags |= O_EXCL; + BOX_TRACE("Trying to create lockfile " << rFilename << " using O_EXCL"); +#else + BOX_TRACE("Trying to create lockfile " << rFilename << " without special flags"); +#endif - return false; +#ifdef WIN32 + HANDLE fd = openfile(rFilename.c_str(), flags, mode); + if(fd == INVALID_HANDLE_VALUE) #else - int fd = ::open(rFilename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, mode); + int fd = ::open(rFilename.c_str(), flags, mode); if(fd == -1) - { - BOX_WARNING("Failed to open lockfile: " << rFilename); - THROW_EXCEPTION(CommonException, OSFileError) - } - -#ifdef HAVE_FLOCK - if(::flock(fd, LOCK_EX | LOCK_NB) != 0) - { - ::close(fd); +#endif +#if HAVE_DECL_O_EXLOCK + { // if() if(errno == EWOULDBLOCK) { + // Lockfile already exists, and we tried to open it + // exclusively, which means we failed to lock it. + BOX_NOTICE("Failed to lock lockfile with O_EXLOCK: " << rFilename + << ": already locked by another process?"); return false; } else { - THROW_EXCEPTION(CommonException, OSFileError) + THROW_SYS_FILE_ERROR("Failed to open lockfile with O_EXLOCK", + rFilename, CommonException, OSFileError); } } -#elif HAVE_DECL_F_SETLK - struct flock desc; - desc.l_type = F_WRLCK; - desc.l_whence = SEEK_SET; - desc.l_start = 0; - desc.l_len = 0; - if(::fcntl(fd, F_SETLK, &desc) != 0) - { - ::close(fd); - if(errno == EAGAIN) +#else // !HAVE_DECL_O_EXLOCK + { // if() +# if defined BOX_OPEN_LOCK + if(errno == EBUSY) +# else // !BOX_OPEN_LOCK + if(errno == EEXIST && (flags & O_EXCL)) +# endif { + // Lockfile already exists, and we tried to open it + // exclusively, which means we failed to lock it. + BOX_NOTICE("Failed to lock lockfile with O_EXCL: " << rFilename + << ": already locked by another process?"); return false; } else { - THROW_EXCEPTION(CommonException, OSFileError) + THROW_SYS_FILE_ERROR("Failed to open lockfile with O_EXCL", + rFilename, CommonException, OSFileError); } } -#endif + + try + { +# ifdef HAVE_FLOCK + BOX_TRACE("Trying to lock lockfile " << rFilename << " using flock()"); + if(::flock(fd, LOCK_EX | LOCK_NB) != 0) + { + if(errno == EWOULDBLOCK) + { + ::close(fd); + BOX_NOTICE("Failed to lock lockfile with flock(): " << rFilename + << ": already locked by another process"); + return false; + } + else + { + THROW_SYS_FILE_ERROR("Failed to lock lockfile with flock()", + rFilename, CommonException, OSFileError); + } + } +# elif HAVE_DECL_F_SETLK + struct flock desc; + desc.l_type = F_WRLCK; + desc.l_whence = SEEK_SET; + desc.l_start = 0; + desc.l_len = 0; + BOX_TRACE("Trying to lock lockfile " << rFilename << " using fcntl()"); + if(::fcntl(fd, F_SETLK, &desc) != 0) + { + if(errno == EAGAIN) + { + ::close(fd); + BOX_NOTICE("Failed to lock lockfile with fcntl(): " << rFilename + << ": already locked by another process"); + return false; + } + else + { + THROW_SYS_FILE_ERROR("Failed to lock lockfile with fcntl()", + rFilename, CommonException, OSFileError); + } + } +# endif + } + catch(BoxException &e) + { +# ifdef WIN32 + CloseHandle(fd); +# else + ::close(fd); +# endif + BOX_NOTICE("Failed to lock lockfile " << rFilename << ": " << e.what()); + throw; + } +#endif // HAVE_DECL_O_EXLOCK + + if(!FileExists(rFilename)) + { + BOX_ERROR("Locked lockfile " << rFilename << ", but lockfile no longer " + "exists, bailing out"); +# ifdef WIN32 + CloseHandle(fd); +# else + ::close(fd); +# endif + return false; + } // Success mFileDescriptor = fd; + BOX_TRACE("Successfully locked lockfile " << rFilename); return true; -#endif } // -------------------------------------------------------------------------- @@ -151,20 +235,65 @@ bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) void NamedLock::ReleaseLock() { // Got a lock? +#ifdef WIN32 + if(mFileDescriptor == INVALID_HANDLE_VALUE) +#else if(mFileDescriptor == -1) +#endif { THROW_EXCEPTION(CommonException, NamedLockNotHeld) } - + +#ifndef WIN32 + // Delete the file. We need to do this before closing the filehandle, + // if we used flock() or fcntl() to lock it, otherwise someone could + // acquire the lock, release and delete it between us closing (and + // hence releasing) and deleting it, and we'd fail when it came to + // deleting the file. This happens in tests much more often than + // you'd expect! + // + // This doesn't apply on systems using plain lockfile locking, such as + // Windows, and there we need to close the file before deleting it, + // otherwise the system won't let us delete it. + + if(::unlink(mFileName.c_str()) != 0) + { + THROW_EMU_ERROR( + BOX_FILE_MESSAGE(mFileName, "Failed to delete lockfile"), + CommonException, OSFileError); + } +#endif // !WIN32 + // Close the file +# ifdef WIN32 + if(!CloseHandle(mFileDescriptor)) +# else if(::close(mFileDescriptor) != 0) +# endif { - THROW_EXCEPTION(CommonException, OSFileError) + THROW_EMU_ERROR( + BOX_FILE_MESSAGE(mFileName, "Failed to close lockfile"), + CommonException, OSFileError); } - // Mark as unlocked - mFileDescriptor = -1; -} + // Mark as unlocked, so we don't try to close it again if the unlink() fails. +#ifdef WIN32 + mFileDescriptor = INVALID_HANDLE_VALUE; +#else + mFileDescriptor = -1; +#endif +#ifdef WIN32 + // On Windows we need to close the file before deleting it, otherwise + // the system won't let us delete it. + if(::unlink(mFileName.c_str()) != 0) + { + THROW_EMU_ERROR( + BOX_FILE_MESSAGE(mFileName, "Failed to delete lockfile"), + CommonException, OSFileError); + } +#endif // WIN32 + BOX_TRACE("Released lock and deleted lockfile " << mFileName); +} diff --git a/lib/common/NamedLock.h b/lib/common/NamedLock.h index 534115db..a7d0d778 100644 --- a/lib/common/NamedLock.h +++ b/lib/common/NamedLock.h @@ -29,12 +29,21 @@ private: public: bool TryAndGetLock(const std::string& rFilename, int mode = 0755); +# ifdef WIN32 + bool GotLock() {return mFileDescriptor != INVALID_HANDLE_VALUE;} +# else bool GotLock() {return mFileDescriptor != -1;} +# endif void ReleaseLock(); - private: +# ifdef WIN32 + HANDLE mFileDescriptor; +# else int mFileDescriptor; +# endif + + std::string mFileName; }; #endif // NAMEDLOCK__H diff --git a/lib/common/PartialReadStream.cpp b/lib/common/PartialReadStream.cpp index f2f79715..b5f99bb5 100644 --- a/lib/common/PartialReadStream.cpp +++ b/lib/common/PartialReadStream.cpp @@ -104,7 +104,7 @@ IOStream::pos_type PartialReadStream::BytesLeftToRead() // Created: 2003/08/26 // // -------------------------------------------------------------------------- -void PartialReadStream::Write(const void *pBuffer, int NBytes) +void PartialReadStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, CantWriteToPartialReadStream) } diff --git a/lib/common/PartialReadStream.h b/lib/common/PartialReadStream.h index 1b46b0bd..61bdd7d1 100644 --- a/lib/common/PartialReadStream.h +++ b/lib/common/PartialReadStream.h @@ -33,7 +33,8 @@ private: public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); diff --git a/lib/common/RateLimitingStream.h b/lib/common/RateLimitingStream.h index a322b99b..818c90af 100644 --- a/lib/common/RateLimitingStream.h +++ b/lib/common/RateLimitingStream.h @@ -30,9 +30,10 @@ public: int Timeout = IOStream::TimeOutInfinite); // Everything else is delegated to the sink - virtual void Write(const void *pBuffer, int NBytes) + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) { - Write(pBuffer, NBytes); + mrSink.Write(pBuffer, NBytes, Timeout); } virtual pos_type BytesLeftToRead() { diff --git a/lib/common/ReadGatherStream.cpp b/lib/common/ReadGatherStream.cpp index f50e6664..ae252832 100644 --- a/lib/common/ReadGatherStream.cpp +++ b/lib/common/ReadGatherStream.cpp @@ -213,7 +213,7 @@ IOStream::pos_type ReadGatherStream::BytesLeftToRead() // Created: 10/12/03 // // -------------------------------------------------------------------------- -void ReadGatherStream::Write(const void *pBuffer, int NBytes) +void ReadGatherStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, CannotWriteToReadGatherStream); } diff --git a/lib/common/ReadGatherStream.h b/lib/common/ReadGatherStream.h index 613ede3e..9a44480b 100644 --- a/lib/common/ReadGatherStream.h +++ b/lib/common/ReadGatherStream.h @@ -37,7 +37,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); virtual pos_type GetPosition() const; diff --git a/lib/common/ReadLoggingStream.cpp b/lib/common/ReadLoggingStream.cpp index 54c99c95..df493344 100644 --- a/lib/common/ReadLoggingStream.cpp +++ b/lib/common/ReadLoggingStream.cpp @@ -96,7 +96,7 @@ IOStream::pos_type ReadLoggingStream::BytesLeftToRead() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void ReadLoggingStream::Write(const void *pBuffer, int NBytes) +void ReadLoggingStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, NotSupported); } diff --git a/lib/common/ReadLoggingStream.h b/lib/common/ReadLoggingStream.h index b23b542c..bee7e1d6 100644 --- a/lib/common/ReadLoggingStream.h +++ b/lib/common/ReadLoggingStream.h @@ -39,7 +39,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); @@ -48,7 +49,7 @@ public: virtual bool StreamClosed(); private: - ReadLoggingStream(const ReadLoggingStream &rToCopy) + ReadLoggingStream(const ReadLoggingStream &rToCopy) : mrSource(rToCopy.mrSource), mrLogger(rToCopy.mrLogger) { /* do not call */ } }; diff --git a/lib/common/SelfFlushingStream.h b/lib/common/SelfFlushingStream.h index 36e9a4d3..b4efa294 100644 --- a/lib/common/SelfFlushingStream.h +++ b/lib/common/SelfFlushingStream.h @@ -33,6 +33,12 @@ public: ~SelfFlushingStream() { + if(StreamDataLeft()) + { + BOX_WARNING("Not all data was read from stream, " + "discarding the rest"); + } + Flush(); } @@ -50,9 +56,10 @@ public: { return mrSource.BytesLeftToRead(); } - virtual void Write(const void *pBuffer, int NBytes) + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) { - mrSource.Write(pBuffer, NBytes); + mrSource.Write(pBuffer, NBytes, Timeout); } virtual bool StreamDataLeft() { diff --git a/lib/common/StreamableMemBlock.cpp b/lib/common/StreamableMemBlock.cpp index b376f037..9abf78d3 100644 --- a/lib/common/StreamableMemBlock.cpp +++ b/lib/common/StreamableMemBlock.cpp @@ -125,7 +125,9 @@ void StreamableMemBlock::Set(IOStream &rStream, int Timeout) try { // Read in - if(!rStream.ReadFullBuffer(pblock, size, 0 /* not interested in bytes read if this fails */)) + if(!rStream.ReadFullBuffer(pblock, size, + 0 /* not interested in bytes read if this fails */, + Timeout)) { THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) } @@ -252,7 +254,9 @@ void StreamableMemBlock::ReadFromStream(IOStream &rStream, int Timeout) { // Get the size of the block int32_t size_s; - if(!rStream.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + if(!rStream.ReadFullBuffer(&size_s, sizeof(size_s), + 0, /* not interested in bytes read if this fails */ + Timeout)) { THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) } @@ -270,7 +274,9 @@ void StreamableMemBlock::ReadFromStream(IOStream &rStream, int Timeout) try { // Read in - if(!rStream.ReadFullBuffer(pblock, size, 0 /* not interested in bytes read if this fails */)) + if(!rStream.ReadFullBuffer(pblock, size, + 0, /* not interested in bytes read if this fails */ + Timeout)) { THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) } diff --git a/lib/common/Test.cpp b/lib/common/Test.cpp index de87c465..2c51cd61 100644 --- a/lib/common/Test.cpp +++ b/lib/common/Test.cpp @@ -22,7 +22,236 @@ #endif #include "BoxTime.h" +#include "FileStream.h" #include "Test.h" +#include "Utils.h" + +int num_tests_selected = 0; +int num_failures = 0; +int old_failure_count = 0; +int first_fail_line; +std::string original_working_dir; +std::string first_fail_file; +std::string current_test_name; +std::list<std::string> run_only_named_tests; +std::map<std::string, std::string> s_test_status; + +bool setUp(const char* function_name) +{ + current_test_name = function_name; + + if (!run_only_named_tests.empty()) + { + bool run_this_test = false; + + for (std::list<std::string>::iterator + i = run_only_named_tests.begin(); + i != run_only_named_tests.end(); i++) + { + if (*i == current_test_name) + { + run_this_test = true; + break; + } + } + + if (!run_this_test) + { + // not in the list, so don't run it. + return false; + } + } + + printf("\n\n== %s ==\n", function_name); + num_tests_selected++; + old_failure_count = num_failures; + + if (original_working_dir == "") + { + char buf[1024]; + if (getcwd(buf, sizeof(buf)) == NULL) + { + BOX_LOG_SYS_ERROR("getcwd"); + } + original_working_dir = buf; + } + else + { + if (chdir(original_working_dir.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("chdir"); + } + } + +#ifdef _MSC_VER + DIR* pDir = opendir("testfiles"); + if(!pDir) + { + THROW_SYS_FILE_ERROR("Failed to open test temporary directory", + "testfiles", CommonException, Internal); + } + struct dirent* pEntry; + for(pEntry = readdir(pDir); pEntry; pEntry = readdir(pDir)) + { + std::string filename = pEntry->d_name; + if(StartsWith("TestDir", filename) || + StartsWith("0_", filename) || + filename == "accounts.txt" || + StartsWith("file", filename) || + StartsWith("notifyran", filename) || + StartsWith("notifyscript.tag", filename) || + StartsWith("restore", filename) || + filename == "bbackupd-data" || + filename == "syncallowscript.control" || + StartsWith("syncallowscript.notifyran.", filename) || + filename == "test2.downloaded" || + EndsWith("testfile", filename)) + { + std::string filepath = std::string("testfiles\\") + filename; + + int filetype = ObjectExists(filepath); + if(filetype == ObjectExists_File) + { + if(::unlink(filepath.c_str()) != 0) + { + TEST_FAIL_WITH_MESSAGE(BOX_SYS_ERROR_MESSAGE("Failed to delete " + "test fixture file: unlink(\"" << filepath << "\")")); + } + } + else if(filetype == ObjectExists_Dir) + { + std::string cmd = "cmd /c rd /s /q " + filepath; + WCHAR* wide_cmd = ConvertUtf8ToWideString(cmd.c_str()); + if(wide_cmd == NULL) + { + TEST_FAIL_WITH_MESSAGE("Failed to convert string " + "to wide string: " << cmd); + continue; + } + + STARTUPINFOW si; + PROCESS_INFORMATION pi; + + ZeroMemory( &si, sizeof(si) ); + si.cb = sizeof(si); + ZeroMemory( &pi, sizeof(pi) ); + + BOOL result = CreateProcessW( + NULL, // lpApplicationName + wide_cmd, // lpCommandLine + NULL, // lpProcessAttributes + NULL, // lpThreadAttributes + TRUE, // bInheritHandles + 0, // dwCreationFlags + NULL, // lpEnvironment + NULL, // lpCurrentDirectory + &si, // lpStartupInfo + &pi // lpProcessInformation + ); + delete [] wide_cmd; + + if(result == FALSE) + { + TEST_FAIL_WITH_MESSAGE("Failed to delete test " + "fixture file: failed to execute command " + "'" << cmd << "': " << + GetErrorMessage(GetLastError())); + continue; + } + + // Wait until child process exits. + WaitForSingleObject(pi.hProcess, INFINITE); + DWORD exit_code; + result = GetExitCodeProcess(pi.hProcess, &exit_code); + + if(result == FALSE) + { + TEST_FAIL_WITH_MESSAGE("Failed to delete " + "test fixture file: failed to get " + "command exit status: '" << + cmd << "': " << + GetErrorMessage(GetLastError())); + } + else if(exit_code != 0) + { + TEST_FAIL_WITH_MESSAGE("Failed to delete test " + "fixture file: command '" << cmd << "' " + "exited with status " << exit_code); + } + + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + } + else + { + TEST_FAIL_WITH_MESSAGE("Don't know how to delete file " << filepath << + " of type " << filetype); + } + } + } + closedir(pDir); + FileStream touch("testfiles/accounts.txt", O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR); +#else + TEST_THAT_THROWONFAIL(system( + "rm -rf testfiles/TestDir* testfiles/0_0 testfiles/0_1 " + "testfiles/0_2 testfiles/accounts.txt " // testfiles/test* .tgz! + "testfiles/file* testfiles/notifyran testfiles/notifyran.* " + "testfiles/notifyscript.tag* " + "testfiles/restore* testfiles/bbackupd-data " + "testfiles/syncallowscript.control " + "testfiles/syncallowscript.notifyran.* " + "testfiles/test2.downloaded" + ) == 0); + TEST_THAT_THROWONFAIL(system("touch testfiles/accounts.txt") == 0); +#endif + TEST_THAT_THROWONFAIL(mkdir("testfiles/0_0", 0755) == 0); + TEST_THAT_THROWONFAIL(mkdir("testfiles/0_1", 0755) == 0); + TEST_THAT_THROWONFAIL(mkdir("testfiles/0_2", 0755) == 0); + TEST_THAT_THROWONFAIL(mkdir("testfiles/bbackupd-data", 0755) == 0); + + return true; +} + +bool tearDown() +{ + if (num_failures == old_failure_count) + { + BOX_NOTICE(current_test_name << " passed"); + s_test_status[current_test_name] = "passed"; + return true; + } + else + { + BOX_NOTICE(current_test_name << " failed"); \ + s_test_status[current_test_name] = "FAILED"; + return false; + } +} + +bool fail() +{ + num_failures++; + return tearDown(); +} + +int finish_test_suite() +{ + printf("\n"); + printf("Test results:\n"); + + typedef std::map<std::string, std::string>::iterator s_test_status_iterator; + for(s_test_status_iterator i = s_test_status.begin(); + i != s_test_status.end(); i++) + { + BOX_NOTICE("test result: " << i->second << ": " << i->first); + } + + TEST_LINE(num_tests_selected > 0, "No tests matched the patterns " + "specified on the command line"); + + return (num_failures == 0 && num_tests_selected > 0) ? 0 : 1; +} bool TestFileExists(const char *Filename) { @@ -136,7 +365,7 @@ int ReadPidFile(const char *pidFile) if(!TestFileNotEmpty(pidFile)) { TEST_FAIL_WITH_MESSAGE("Server didn't save PID file " - "(perhaps one was already running?)"); + "(perhaps one was already running?)"); return -1; } @@ -145,7 +374,7 @@ int ReadPidFile(const char *pidFile) FILE *f = fopen(pidFile, "r"); if(f == NULL || fscanf(f, "%d", &pid) != 1) { - TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); + TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); return -1; } fclose(f); @@ -155,7 +384,7 @@ int ReadPidFile(const char *pidFile) int LaunchServer(const std::string& rCommandLine, const char *pidFile) { - ::fprintf(stdout, "Starting server: %s\n", rCommandLine.c_str()); + BOX_INFO("Starting server: " << rCommandLine); #ifdef WIN32 @@ -189,14 +418,10 @@ int LaunchServer(const std::string& rCommandLine, const char *pidFile) free(tempCmd); - if (result == 0) - { - DWORD err = GetLastError(); - printf("Launch failed: %s: error %d\n", rCommandLine.c_str(), - (int)err); - TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + TEST_THAT_OR(result != 0, + BOX_LOG_WIN_ERROR("Launch failed: " << rCommandLine); return -1; - } + ); CloseHandle(procInfo.hProcess); CloseHandle(procInfo.hThread); @@ -205,11 +430,10 @@ int LaunchServer(const std::string& rCommandLine, const char *pidFile) #else // !WIN32 - if(RunCommand(rCommandLine) != 0) - { - TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + TEST_THAT_OR(RunCommand(rCommandLine) == 0, + TEST_FAIL_WITH_MESSAGE("Failed to start server: " << rCommandLine); return -1; - } + ) return WaitForServerStartup(pidFile, 0); @@ -230,18 +454,11 @@ int WaitForServerStartup(const char *pidFile, int pidIfKnown) #endif // time for it to start up - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_TRACE("Waiting for server to start"); - } - else - { - ::fprintf(stdout, "Waiting for server to start: "); - } + BOX_TRACE("Waiting for server to start"); for (int i = 0; i < 15; i++) { - if (TestFileNotEmpty(pidFile)) + if (TestFileNotEmpty(pidFile)) { break; } @@ -251,12 +468,6 @@ int WaitForServerStartup(const char *pidFile, int pidIfKnown) break; } - if (Logging::GetGlobalLevel() < Log::TRACE) - { - ::fprintf(stdout, "."); - ::fflush(stdout); - } - ::sleep(1); } @@ -265,42 +476,17 @@ int WaitForServerStartup(const char *pidFile, int pidIfKnown) if (pidIfKnown && !ServerIsAlive(pidIfKnown)) { - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_ERROR("server died!"); - } - else - { - ::fprintf(stdout, " server died!\n"); - } - - TEST_FAIL_WITH_MESSAGE("Server died!"); + TEST_FAIL_WITH_MESSAGE("Server died!"); return -1; } if (!TestFileNotEmpty(pidFile)) { - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_ERROR("timed out!"); - } - else - { - ::fprintf(stdout, " timed out!\n"); - } - - TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); return -1; } - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_TRACE("Server started"); - } - else - { - ::fprintf(stdout, " done.\n"); - } + BOX_TRACE("Server started"); // wait a second for the pid to be written to the file ::sleep(1); @@ -330,12 +516,12 @@ void TestRemoteProcessMemLeaksFunc(const char *filename, // Does the file exist? if(!TestFileExists(filename)) { - if (failures == 0) + if (num_failures == 0) { first_fail_file = file; first_fail_line = line; } - ++failures; + ++num_failures; printf("FAILURE: MemLeak report not available (file %s) " "at %s:%d\n", filename, file, line); } @@ -344,12 +530,12 @@ void TestRemoteProcessMemLeaksFunc(const char *filename, // Is it empty? if(TestGetFileSize(filename) > 0) { - if (failures == 0) + if (num_failures == 0) { first_fail_file = file; first_fail_line = line; } - ++failures; + ++num_failures; printf("FAILURE: Memory leaks found in other process " "(file %s) at %s:%d\n==========\n", filename, file, line); @@ -378,23 +564,29 @@ void force_sync() void wait_for_sync_start() { + BOX_TRACE("Waiting for sync to start..."); TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " "wait-for-sync") == 0); TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + BOX_TRACE("Backup daemon reported that sync has started."); } void wait_for_sync_end() { + BOX_TRACE("Waiting for sync to finish..."); TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " "wait-for-end") == 0); TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + BOX_TRACE("Backup daemon reported that sync has finished."); } void sync_and_wait() { + BOX_TRACE("Starting a sync and waiting for it to finish..."); TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " "sync-and-wait") == 0); TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + BOX_TRACE("Backup daemon reported that sync has finished."); } void terminate_bbackupd(int pid) @@ -419,35 +611,14 @@ void terminate_bbackupd(int pid) // Wait a given number of seconds for something to complete void wait_for_operation(int seconds, const char* message) { - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_TRACE("Waiting " << seconds << " seconds for " << message); - } - else - { - printf("Waiting for %s: ", message); - fflush(stdout); - } + BOX_INFO("Waiting " << seconds << " seconds for " << message); for(int l = 0; l < seconds; ++l) { sleep(1); - if (Logging::GetGlobalLevel() < Log::TRACE) - { - printf("."); - fflush(stdout); - } } - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_TRACE("Finished waiting for " << message); - } - else - { - printf(" done.\n"); - fflush(stdout); - } + BOX_TRACE("Finished waiting for " << message); } void safe_sleep(int seconds) @@ -455,3 +626,15 @@ void safe_sleep(int seconds) ShortSleep(SecondsToBoxTime(seconds), true); } +std::auto_ptr<Configuration> load_config_file(const std::string& config_file, + const ConfigurationVerify& verify) +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify(config_file, &verify, errs)); + TEST_EQUAL_LINE(0, errs.size(), "Failed to load configuration file: " + config_file + + ": " + errs); + TEST_EQUAL_OR(0, errs.size(), config.reset()); + return config; +} + diff --git a/lib/common/Test.h b/lib/common/Test.h index f318c811..4b5cef61 100644 --- a/lib/common/Test.h +++ b/lib/common/Test.h @@ -11,6 +11,10 @@ #define TEST__H #include <cstring> +#include <list> +#include <map> + +#include "Configuration.h" #ifdef WIN32 #define BBACKUPCTL "..\\..\\bin\\bbackupctl\\bbackupctl.exe" @@ -28,30 +32,70 @@ #define TEST_RETURN(actual, expected) TEST_EQUAL((expected << 8), actual); #endif -extern int failures; +extern int num_failures; extern int first_fail_line; +extern int num_tests_selected; +extern int old_failure_count; extern std::string first_fail_file; extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; +extern std::list<std::string> run_only_named_tests; +extern std::string current_test_name; +extern std::map<std::string, std::string> s_test_status; + +//! Simplifies calling setUp() with the current function name in each test. +#define SETUP() \ + if (!setUp(__FUNCTION__)) return true; \ + try \ + { // left open for TEARDOWN() + +#define TEARDOWN() \ + return tearDown(); \ + } \ + catch (BoxException &e) \ + { \ + BOX_NOTICE(__FUNCTION__ << " errored: " << e.what()); \ + num_failures++; \ + tearDown(); \ + s_test_status[__FUNCTION__] = "ERRORED"; \ + return false; \ + } + +//! End the current test. Only use within a test function, because it just returns false! +#define FAIL { \ + std::ostringstream os; \ + os << "failed at " << __FUNCTION__ << ":" << __LINE__; \ + s_test_status[current_test_name] = os.str(); \ + return fail(); \ +} #define TEST_FAIL_WITH_MESSAGE(msg) \ { \ - if (failures == 0) \ + if (num_failures == 0) \ { \ first_fail_file = __FILE__; \ first_fail_line = __LINE__; \ } \ - failures++; \ + num_failures++; \ BOX_ERROR("**** TEST FAILURE: " << msg << " at " << __FILE__ << \ ":" << __LINE__); \ } #define TEST_ABORT_WITH_MESSAGE(msg) {TEST_FAIL_WITH_MESSAGE(msg); return 1;} -#define TEST_THAT(condition) {if(!(condition)) TEST_FAIL_WITH_MESSAGE("Condition [" #condition "] failed")} +#define TEST_THAT_OR(condition, or_command) \ + if(!(condition)) \ + { \ + TEST_FAIL_WITH_MESSAGE("Condition [" #condition "] failed"); \ + or_command; \ + } +#define TEST_THAT(condition) TEST_THAT_OR(condition,) #define TEST_THAT_ABORTONFAIL(condition) {if(!(condition)) TEST_ABORT_WITH_MESSAGE("Condition [" #condition "] failed")} +#define TEST_THAT_THROWONFAIL(condition) \ + TEST_THAT_OR(condition, THROW_EXCEPTION_MESSAGE(CommonException, \ + AssertFailed, "Condition [" #condition "] failed")); // NOTE: The 0- bit is to allow this to work with stuff which has negative constants for flags (eg ConnectionException) -#define TEST_CHECK_THROWS(statement, excepttype, subtype) \ +#define TEST_CHECK_THROWS_AND_OR(statement, excepttype, subtype, and_command, or_command) \ { \ bool didthrow = false; \ HideExceptionMessageGuard hide; \ @@ -69,6 +113,7 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; throw; \ } \ didthrow = true; \ + and_command; \ } \ catch(...) \ { \ @@ -76,12 +121,15 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; } \ if(!didthrow) \ { \ - TEST_FAIL_WITH_MESSAGE("Didn't throw exception " #excepttype "(" #subtype ")") \ + TEST_FAIL_WITH_MESSAGE("Didn't throw exception " #excepttype "(" #subtype ")"); \ + or_command; \ } \ } +#define TEST_CHECK_THROWS(statement, excepttype, subtype) \ + TEST_CHECK_THROWS_AND_OR(statement, excepttype, subtype,,) // utility macro for comparing two strings in a line -#define TEST_EQUAL(_expected, _found) \ +#define TEST_EQUAL_OR(_expected, _found, or_command) \ { \ std::ostringstream _oss1; \ _oss1 << _expected; \ @@ -100,8 +148,11 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; _oss3 << #_found << " != " << #_expected; \ \ TEST_FAIL_WITH_MESSAGE(_oss3.str().c_str()); \ + or_command; \ } \ } +#define TEST_EQUAL(_expected, _found) \ + TEST_EQUAL_OR(_expected, _found,) // utility macro for comparing two strings in a line #define TEST_EQUAL_LINE(_expected, _found, _line) \ @@ -129,7 +180,7 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; } \ } -// utility macro for testing a line +// utility macros for testing a string/output line #define TEST_LINE(_condition, _line) \ TEST_THAT(_condition); \ if (!(_condition)) \ @@ -140,6 +191,28 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; printf("Test failed on <%s>\n", _line_str.c_str()); \ } +#define TEST_LINE_OR(_condition, _line, _or_command) \ + TEST_LINE(_condition, _line); \ + if(!(_condition)) \ + { \ + _or_command; \ + } + +#define TEST_STARTSWITH(expected, actual) \ + TEST_EQUAL_LINE(expected, actual.substr(0, std::string(expected).size()), actual); + +//! Sets up (cleans up) test environment at the start of every test. +bool setUp(const char* function_name); + +//! Checks account for errors and shuts down daemons at end of every test. +bool tearDown(); + +//! Like tearDown() but returns false, because a test failure was detected. +bool fail(); + +//! Report final status of all tests, and return the correct value to test main(). +int finish_test_suite(); + bool TestFileExists(const char *Filename); bool TestDirExists(const char *Filename); @@ -167,5 +240,17 @@ void terminate_bbackupd(int pid); // Wait a given number of seconds for something to complete void wait_for_operation(int seconds, const char* message); void safe_sleep(int seconds); +std::auto_ptr<Configuration> load_config_file(const std::string& config_file, + const ConfigurationVerify& verify); + +#ifndef TEST_EXECUTABLE +# ifdef _MSC_VER + // Our CMakeFiles compile tests to different executable filenames, + // e.g. test_common.exe instead of _test.exe. + #define TEST_EXECUTABLE BOX_MODULE ".exe" +# else + #define TEST_EXECUTABLE "./_test" +# endif +#endif // TEST_EXECUTABLE #endif // TEST__H diff --git a/lib/common/Timer.cpp b/lib/common/Timer.cpp index ad6b5e8d..4f8c989e 100644 --- a/lib/common/Timer.cpp +++ b/lib/common/Timer.cpp @@ -55,8 +55,8 @@ void Timers::Init() sigemptyset(&newact.sa_mask); if (::sigaction(SIGALRM, &newact, &oldact) != 0) { - BOX_ERROR("Failed to install signal handler"); - THROW_EXCEPTION(CommonException, Internal); + THROW_SYS_ERROR("Failed to install signal handler", + CommonException, Internal); } ASSERT(oldact.sa_handler == 0); #endif // WIN32 && !PLATFORM_CYGWIN @@ -72,13 +72,23 @@ void Timers::Init() // Created: 6/11/2006 // // -------------------------------------------------------------------------- -void Timers::Cleanup() +void Timers::Cleanup(bool throw_exception_if_not_initialised) { - ASSERT(spTimers); - if (!spTimers) + if (throw_exception_if_not_initialised) { - BOX_ERROR("Tried to clean up timers when not initialised!"); - return; + ASSERT(spTimers); + if (!spTimers) + { + BOX_ERROR("Tried to clean up timers when not initialised!"); + return; + } + } + else + { + if (!spTimers) + { + return; + } } #if defined WIN32 && ! defined PLATFORM_CYGWIN @@ -87,8 +97,11 @@ void Timers::Cleanup() struct itimerval timeout; memset(&timeout, 0, sizeof(timeout)); - int result = ::setitimer(ITIMER_REAL, &timeout, NULL); - ASSERT(result == 0); + if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) + { + THROW_SYS_ERROR("Failed to set interval timer", + CommonException, Internal); + } struct sigaction newact, oldact; newact.sa_handler = SIG_DFL; @@ -96,8 +109,8 @@ void Timers::Cleanup() sigemptyset(&(newact.sa_mask)); if (::sigaction(SIGALRM, &newact, &oldact) != 0) { - BOX_ERROR("Failed to remove signal handler"); - THROW_EXCEPTION(CommonException, Internal); + THROW_SYS_ERROR("Failed to remove signal handler", + CommonException, Internal); } ASSERT(oldact.sa_handler == Timers::SignalHandler); #endif // WIN32 && !PLATFORM_CYGWIN @@ -118,7 +131,6 @@ void Timers::Cleanup() void Timers::Add(Timer& rTimer) { ASSERT(spTimers); - ASSERT(&rTimer); BOX_TRACE(TIMER_ID_OF(rTimer) " added to global queue, rescheduling"); spTimers->push_back(&rTimer); Reschedule(); @@ -135,8 +147,14 @@ void Timers::Add(Timer& rTimer) // -------------------------------------------------------------------------- void Timers::Remove(Timer& rTimer) { + if(!spTimers) + { + BOX_WARNING(TIMER_ID_OF(rTimer) " was still active after " + "timer subsystem was cleaned up, already removed."); + return; + } + ASSERT(spTimers); - ASSERT(&rTimer); BOX_TRACE(TIMER_ID_OF(rTimer) " removed from global queue, rescheduling"); bool restart = true; @@ -155,7 +173,7 @@ void Timers::Remove(Timer& rTimer) } } } - + Reschedule(); } @@ -185,26 +203,24 @@ void Timers::Reschedule() ASSERT(spTimers); if (spTimers == NULL) { - THROW_EXCEPTION(CommonException, Internal) + THROW_EXCEPTION(CommonException, TimersNotInitialised); } #ifndef WIN32 struct sigaction oldact; if (::sigaction(SIGALRM, NULL, &oldact) != 0) { - BOX_ERROR("Failed to check signal handler"); - THROW_EXCEPTION(CommonException, Internal) + THROW_SYS_ERROR("Failed to check signal handler", + CommonException, Internal); } ASSERT(oldact.sa_handler == Timers::SignalHandler); if (oldact.sa_handler != Timers::SignalHandler) { - BOX_ERROR("Signal handler was " << - (void *)oldact.sa_handler << - ", expected " << - (void *)Timers::SignalHandler); - THROW_EXCEPTION(CommonException, Internal) + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Signal handler was " << (void *)oldact.sa_handler << + ", expected " << (void *)Timers::SignalHandler); } #endif @@ -218,6 +234,8 @@ void Timers::Reschedule() // win32 timers need no management #else box_time_t timeNow = GetCurrentBoxTime(); + int64_t timeToNextEvent; + std::string nameOfNextEvent; // scan for, trigger and remove expired timers. Removal requires // us to restart the scan each time, due to std::vector semantics. @@ -225,6 +243,7 @@ void Timers::Reschedule() while (restart) { restart = false; + timeToNextEvent = 0; for (std::vector<Timer*>::iterator i = spTimers->begin(); i != spTimers->end(); i++) @@ -252,35 +271,14 @@ void Timers::Reschedule() " seconds"); */ } - } - } - - // Now the only remaining timers should all be in the future. - // Scan to find the next one to fire (earliest deadline). - - int64_t timeToNextEvent = 0; - std::string nameOfNextEvent; - - for (std::vector<Timer*>::iterator i = spTimers->begin(); - i != spTimers->end(); i++) - { - Timer& rTimer = **i; - int64_t timeToExpiry = rTimer.GetExpiryTime() - timeNow; - ASSERT(timeToExpiry > 0) - if (timeToExpiry <= 0) - { - timeToExpiry = 1; - } - - if (timeToNextEvent == 0 || timeToNextEvent > timeToExpiry) - { - timeToNextEvent = timeToExpiry; - nameOfNextEvent = rTimer.GetName(); + if (timeToNextEvent == 0 || timeToNextEvent > timeToExpiry) + { + timeToNextEvent = timeToExpiry; + nameOfNextEvent = rTimer.GetName(); + } } } - - ASSERT(timeToNextEvent >= 0); if (timeToNextEvent == 0) { @@ -302,8 +300,8 @@ void Timers::Reschedule() if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) { - BOX_ERROR("Failed to initialise system timer\n"); - THROW_EXCEPTION(CommonException, Internal) + THROW_SYS_ERROR("Failed to initialise system timer", + CommonException, Internal); } #endif } @@ -322,7 +320,6 @@ void Timers::Reschedule() // -------------------------------------------------------------------------- void Timers::SignalHandler(int unused) { - // ASSERT(spTimers); Timers::RequestReschedule(); } diff --git a/lib/common/Timer.h b/lib/common/Timer.h index 09be58fa..17233203 100644 --- a/lib/common/Timer.h +++ b/lib/common/Timer.h @@ -43,7 +43,7 @@ class Timers public: static void Init(); - static void Cleanup(); + static void Cleanup(bool throw_exception_if_not_initialised = true); static void Add (Timer& rTimer); static void Remove(Timer& rTimer); static void RequestReschedule(); diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp index decc80e8..0915f29a 100644 --- a/lib/common/Utils.cpp +++ b/lib/common/Utils.cpp @@ -15,7 +15,7 @@ #include <cstdlib> -#ifdef SHOW_BACKTRACE_ON_EXCEPTION +#ifdef HAVE_EXECINFO_H #include <execinfo.h> #include <stdlib.h> #endif @@ -51,25 +51,40 @@ std::string GetBoxBackupVersion() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput) +void SplitString(std::string String, char SplitOn, std::vector<std::string> &rOutput) { // Split it up. - std::string::size_type b = 0; - std::string::size_type e = 0; - while(e = String.find_first_of(SplitOn, b), e != String.npos) + std::string::size_type begin = 0, end = 0, pos = 0; + + while(end = String.find_first_of(SplitOn, pos), end != String.npos) { - // Get this string - unsigned int len = e - b; - if(len >= 1) + // Is it preceded by the escape character? + if(end > 0 && String[end - 1] == '\\') + { + // Ignore this one, don't change begin, let the next + // match/fallback consume it instead. But remove the + // backslash from the string, and set pos to the + // current position, which no longer contains a + // separator character. + String.erase(end - 1, 1); + pos = end; + } + else { - rOutput.push_back(String.substr(b, len)); + // Extract the substring and move past it. + unsigned int len = end - begin; + if(len >= 1) + { + rOutput.push_back(String.substr(begin, len)); + } + begin = end + 1; + pos = begin; } - b = e + 1; } // Last string - if(b < String.size()) + if(begin < String.size()) { - rOutput.push_back(String.substr(b)); + rOutput.push_back(String.substr(begin)); } /*#ifndef BOX_RELEASE_BUILD BOX_TRACE("Splitting string '" << String << " on " << (char)SplitOn); @@ -80,71 +95,95 @@ void SplitString(const std::string &String, char SplitOn, std::vector<std::strin #endif*/ } -#ifdef SHOW_BACKTRACE_ON_EXCEPTION +bool StartsWith(const std::string& prefix, const std::string& haystack) +{ + return haystack.size() >= prefix.size() && + haystack.substr(0, prefix.size()) == prefix; +} + +bool EndsWith(const std::string& suffix, const std::string& haystack) +{ + return haystack.size() >= suffix.size() && + haystack.substr(haystack.size() - suffix.size()) == suffix; +} + +std::string RemovePrefix(const std::string& prefix, const std::string& haystack) +{ + if(StartsWith(prefix, haystack)) + { + return haystack.substr(prefix.size()); + } + else + { + return ""; + } +} + +std::string RemoveSuffix(const std::string& suffix, const std::string& haystack) +{ + if(EndsWith(suffix, haystack)) + { + return haystack.substr(0, haystack.size() - suffix.size()); + } + else + { + return ""; + } +} + static std::string demangle(const std::string& mangled_name) { + std::string demangled_name = mangled_name; + #ifdef HAVE_CXXABI_H + char buffer[1024]; int status; + size_t length = sizeof(buffer); -#include "MemLeakFindOff.h" char* result = abi::__cxa_demangle(mangled_name.c_str(), - NULL, NULL, &status); -#include "MemLeakFindOn.h" + buffer, &length, &status); - if (result == NULL) + if (status == 0) { - if (status == 0) - { - BOX_WARNING("Demangle failed but no error: " << - mangled_name); - } - else if (status == -1) - { - BOX_WARNING("Demangle failed with " - "memory allocation error: " << - mangled_name); - } - else if (status == -2) - { - // Probably non-C++ name, don't demangle - /* - BOX_WARNING("Demangle failed with " - "with invalid name: " << - mangled_name); - */ - } - else if (status == -3) - { - BOX_WARNING("Demangle failed with " - "with invalid argument: " << - mangled_name); - } - else - { - BOX_WARNING("Demangle failed with " - "with unknown error " << status << - ": " << mangled_name); - } - - return std::string(mangled_name); + demangled_name = result; + } + else if (status == -1) + { + BOX_WARNING("Demangle failed with " + "memory allocation error: " << + mangled_name); + } + else if (status == -2) + { + // Probably non-C++ name, don't demangle + /* + BOX_WARNING("Demangle failed with " + "with invalid name: " << + mangled_name); + */ + } + else if (status == -3) + { + BOX_WARNING("Demangle failed with " + "with invalid argument: " << + mangled_name); } else { - std::string output = result; -#include "MemLeakFindOff.h" - free(result); -#include "MemLeakFindOn.h" - return output; + BOX_WARNING("Demangle failed with " + "with unknown error " << status << + ": " << mangled_name); } - #else // !HAVE_CXXABI_H - return mangled_name; #endif // HAVE_CXXABI_H + + return demangled_name; } void DumpStackBacktrace() { - void *array[10]; - size_t size = backtrace(array, 10); +#ifdef HAVE_EXECINFO_H + void *array[20]; + size_t size = backtrace(array, 20); BOX_TRACE("Obtained " << size << " stack frames."); for(size_t i = 0; i < size; i++) @@ -179,8 +218,10 @@ void DumpStackBacktrace() BOX_TRACE(output.str()); } +#else // !HAVE_EXECINFO_H + BOX_TRACE("Backtrace support was not compiled in"); +#endif // HAVE_EXECINFO_H } -#endif // SHOW_BACKTRACE_ON_EXCEPTION @@ -340,30 +381,3 @@ std::string FormatUsageLineStart(const std::string& rName, return result.str(); } -std::string BoxGetTemporaryDirectoryName() -{ -#ifdef WIN32 - // http://msdn.microsoft.com/library/default.asp? - // url=/library/en-us/fileio/fs/creating_and_using_a_temporary_file.asp - - DWORD dwRetVal; - char lpPathBuffer[1024]; - DWORD dwBufSize = sizeof(lpPathBuffer); - - // Get the temp path. - dwRetVal = GetTempPath(dwBufSize, // length of the buffer - lpPathBuffer); // buffer for path - if (dwRetVal > dwBufSize) - { - THROW_EXCEPTION(CommonException, TempDirPathTooLong) - } - - return std::string(lpPathBuffer); -#elif defined TEMP_DIRECTORY_NAME - return std::string(TEMP_DIRECTORY_NAME); -#else - #error non-static temporary directory names not supported yet -#endif -} - - diff --git a/lib/common/Utils.h b/lib/common/Utils.h index 3134245a..d306ce1c 100644 --- a/lib/common/Utils.h +++ b/lib/common/Utils.h @@ -17,11 +17,13 @@ std::string GetBoxBackupVersion(); -void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput); +void SplitString(std::string String, char SplitOn, std::vector<std::string> &rOutput); +bool StartsWith(const std::string& prefix, const std::string& haystack); +bool EndsWith(const std::string& prefix, const std::string& haystack); +std::string RemovePrefix(const std::string& prefix, const std::string& haystack); +std::string RemoveSuffix(const std::string& suffix, const std::string& haystack); -#ifdef SHOW_BACKTRACE_ON_EXCEPTION - void DumpStackBacktrace(); -#endif +void DumpStackBacktrace(); bool FileExists(const std::string& rFilename, int64_t *pFileSize = 0, bool TreatLinksAsNotExisting = false); diff --git a/lib/common/ZeroStream.cpp b/lib/common/ZeroStream.cpp index d11ed80c..e1342e6f 100644 --- a/lib/common/ZeroStream.cpp +++ b/lib/common/ZeroStream.cpp @@ -76,7 +76,7 @@ IOStream::pos_type ZeroStream::BytesLeftToRead() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void ZeroStream::Write(const void *pBuffer, int NBytes) +void ZeroStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, NotSupported); } diff --git a/lib/common/ZeroStream.h b/lib/common/ZeroStream.h index 0119045b..f91221b0 100644 --- a/lib/common/ZeroStream.h +++ b/lib/common/ZeroStream.h @@ -22,7 +22,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); diff --git a/lib/common/makeexception.pl.in b/lib/common/makeexception.pl.in index b1b3a8ac..bddaa94a 100755 --- a/lib/common/makeexception.pl.in +++ b/lib/common/makeexception.pl.in @@ -73,12 +73,13 @@ class ${class}Exception : public BoxException public: ${class}Exception(unsigned int SubType, const std::string& rMessage = "") - : mSubType(SubType), mMessage(rMessage) + : mSubType(SubType), mMessage(rMessage), + mWhat(GetMessage(SubType) + std::string(rMessage.empty() ? "" : ": ") + rMessage) { } ${class}Exception(const ${class}Exception &rToCopy) - : mSubType(rToCopy.mSubType), mMessage(rToCopy.mMessage) + : mSubType(rToCopy.mSubType), mMessage(rToCopy.mMessage), mWhat(rToCopy.mWhat) { } @@ -113,10 +114,11 @@ print H <<__E; { return mMessage; } - + static const char* GetMessage(int SubType); private: unsigned int mSubType; std::string mMessage; + std::string mWhat; }; #endif // $guardname @@ -133,74 +135,39 @@ print CPP <<__E; #include "MemLeakFindOn.h" -#ifdef EXCEPTION_CODENAMES_EXTENDED - #ifdef EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION -static const char *whats[] = { -__E - -my $last_seen = -1; -for(my $e = 0; $e <= $#exception; $e++) +unsigned int ${class}Exception::GetType() const throw() { - if($exception[$e] ne '') - { - for(my $s = $last_seen + 1; $s < $e; $s++) - { - print CPP "\t\"UNUSED\",\n" - } - my $ext = ($exception_desc[$e] ne '')?" ($exception_desc[$e])":''; - print CPP "\t\"${class} ".$exception[$e].$ext.'"'.(($e==$#exception)?'':',')."\n"; - $last_seen = $e; - } + return ${class}Exception::ExceptionType; } -print CPP <<__E; -}; - #else -static const char *whats[] = { -__E - -$last_seen = -1; -for(my $e = 0; $e <= $#exception; $e++) +unsigned int ${class}Exception::GetSubType() const throw() { - if($exception[$e] ne '') - { - for(my $s = $last_seen + 1; $s < $e; $s++) - { - print CPP "\t\"UNUSED\",\n" - } - print CPP "\t\"${class} ".$exception[$e].'"'.(($e==$#exception)?'':',')."\n"; - $last_seen = $e; - } + return mSubType; } -print CPP <<__E; -}; - #endif -#endif - -unsigned int ${class}Exception::GetType() const throw() +const char * ${class}Exception::what() const throw() { - return ${class}Exception::ExceptionType; + return mWhat.c_str(); } -unsigned int ${class}Exception::GetSubType() const throw() +const char * ${class}Exception::GetMessage(int SubType) { - return mSubType; -} + switch(SubType) + { +__E -const char *${class}Exception::what() const throw() +for(my $e = 0; $e <= $#exception; $e++) { -#ifdef EXCEPTION_CODENAMES_EXTENDED - if(mSubType > (sizeof(whats) / sizeof(whats[0]))) + if($exception[$e] ne '') { - return "${class}"; + print CPP "\t\tcase ".$exception[$e].': return "'.$exception[$e].'";'."\n"; } - return whats[mSubType]; -#else - return "${class}"; -#endif } +print CPP <<__E; + default: return "Unknown"; + } +} __E close H; diff --git a/lib/compress/CompressStream.cpp b/lib/compress/CompressStream.cpp index 9bb73e3d..f7728a21 100644 --- a/lib/compress/CompressStream.cpp +++ b/lib/compress/CompressStream.cpp @@ -177,12 +177,12 @@ int CompressStream::Read(void *pBuffer, int NBytes, int Timeout) // Created: 27/5/04 // // -------------------------------------------------------------------------- -void CompressStream::Write(const void *pBuffer, int NBytes) +void CompressStream::Write(const void *pBuffer, int NBytes, int Timeout) { USE_WRITE_COMPRESSOR if(pCompress == 0) { - mpStream->Write(pBuffer, NBytes); + mpStream->Write(pBuffer, NBytes, Timeout); return; } @@ -207,7 +207,7 @@ void CompressStream::Write(const void *pBuffer, int NBytes) // Created: 27/5/04 // // -------------------------------------------------------------------------- -void CompressStream::WriteAllBuffered() +void CompressStream::WriteAllBuffered(int Timeout) { if(mIsClosed) { @@ -215,7 +215,7 @@ void CompressStream::WriteAllBuffered() } // Just ask compressed data to be written out, but with the sync flag set - WriteCompressedData(true); + WriteCompressedData(true, Timeout); } @@ -238,7 +238,7 @@ void CompressStream::Close() pCompress->FinishInput(); WriteCompressedData(); - // Mark as definately closed + // Mark as definitely closed mIsClosed = true; } } @@ -257,7 +257,7 @@ void CompressStream::Close() // Created: 28/5/04 // // -------------------------------------------------------------------------- -void CompressStream::WriteCompressedData(bool SyncFlush) +void CompressStream::WriteCompressedData(bool SyncFlush, int Timeout) { USE_WRITE_COMPRESSOR if(pCompress == 0) {THROW_EXCEPTION(CompressException, Internal)} @@ -268,7 +268,7 @@ void CompressStream::WriteCompressedData(bool SyncFlush) s = pCompress->Output(mpBuffer, BUFFER_SIZE, SyncFlush); if(s > 0) { - mpStream->Write(mpBuffer, s); + mpStream->Write(mpBuffer, s, Timeout); } } while(s > 0); // Check assumption -- all input has been consumed diff --git a/lib/compress/CompressStream.h b/lib/compress/CompressStream.h index 7959e3dc..7d6b2501 100644 --- a/lib/compress/CompressStream.h +++ b/lib/compress/CompressStream.h @@ -33,8 +33,9 @@ private: public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); - virtual void WriteAllBuffered(); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); + virtual void WriteAllBuffered(int Timeout = IOStream::TimeOutInfinite); virtual void Close(); virtual bool StreamDataLeft(); virtual bool StreamClosed(); @@ -43,7 +44,8 @@ protected: void CheckRead(); void CheckWrite(); void CheckBuffer(); - void WriteCompressedData(bool SyncFlush = false); + void WriteCompressedData(bool SyncFlush = false, + int Timeout = IOStream::TimeOutInfinite); private: IOStream *mpStream; diff --git a/lib/crypto/CipherBlowfish.cpp b/lib/crypto/CipherBlowfish.cpp index e16cc6ed..4c75b1de 100644 --- a/lib/crypto/CipherBlowfish.cpp +++ b/lib/crypto/CipherBlowfish.cpp @@ -206,7 +206,7 @@ void CipherBlowfish::SetupParameters(EVP_CIPHER_CTX *pCipherContext) const } // Set key #ifndef HAVE_OLD_SSL - if(EVP_CipherInit_ex(pCipherContext, NULL, NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1) + if(EVP_CipherInit_ex(pCipherContext, GetCipher(), NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1) #else if(EVP_CipherInit(pCipherContext, NULL, (unsigned char*)mKey.c_str(), (unsigned char*)mInitialisationVector, -1) != 1) #endif diff --git a/lib/crypto/CipherContext.cpp b/lib/crypto/CipherContext.cpp index fd149395..3de88c64 100644 --- a/lib/crypto/CipherContext.cpp +++ b/lib/crypto/CipherContext.cpp @@ -2,7 +2,7 @@ // // File // Name: CipherContext.cpp -// Purpose: Context for symmetric encryption / descryption +// Purpose: Context for symmetric encryption / decryption // Created: 1/12/03 // // -------------------------------------------------------------------------- @@ -50,7 +50,7 @@ CipherContext::~CipherContext() if(mInitialised) { // Clean up - EVP_CIPHER_CTX_cleanup(&ctx); + BOX_OPENSSL_CLEANUP_CTX(ctx); mInitialised = false; } #ifdef HAVE_OLD_SSL @@ -98,7 +98,7 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes // Check for bad usage if(mInitialised) { - THROW_EXCEPTION(CipherException, AlreadyInitialised) + THROW_EXCEPTION(CipherException, AlreadyInitialised); } if(Function != Decrypt && Function != Encrypt) { @@ -109,43 +109,45 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes mFunction = Function; // Initialise the cipher -#ifndef HAVE_OLD_SSL - EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does - - if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL, - (mFunction == Encrypt) ? 1 : 0) != 1) -#else +#ifdef HAVE_OLD_SSL // Use old version of init call if(EVP_CipherInit(&ctx, rDescription.GetCipher(), NULL, NULL, (mFunction == Encrypt) ? 1 : 0) != 1) +#else + BOX_OPENSSL_INIT_CTX(ctx); + + // Don't set key or IV yet, because we will modify the parameters: + if(EVP_CipherInit_ex(BOX_OPENSSL_CTX(ctx), rDescription.GetCipher(), NULL, NULL, NULL, + (mFunction == Encrypt) ? 1 : 0) != 1) #endif { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, "Failed to initialise " << rDescription.GetFullName() - << "cipher: " << LogError("initialising cipher")); + << ": " << LogError("initialising cipher")); } + UsePadding(mPaddingOn); try { mCipherName = rDescription.GetFullName(); #ifndef HAVE_OLD_SSL // Let the description set up everything else - rDescription.SetupParameters(&ctx); + mpDescription = &rDescription; #else // With the old version, a copy needs to be taken first. mpDescription = rDescription.Clone(); // Mark it as not a leak, otherwise static cipher contexts // cause spurious memory leaks to be reported MEMLEAKFINDER_NOT_A_LEAK(mpDescription); - mpDescription->SetupParameters(&ctx); #endif + mpDescription->SetupParameters(BOX_OPENSSL_CTX(ctx)); } catch(...) { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, - "Failed to configure " << mCipherName << " cipher: " << + "Failed to configure " << mCipherName << ": " << LogError("configuring cipher")); - EVP_CIPHER_CTX_cleanup(&ctx); + BOX_OPENSSL_CLEANUP_CTX(ctx); throw; } @@ -166,7 +168,7 @@ void CipherContext::Reset() if(mInitialised) { // Clean up - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_cleanup(BOX_OPENSSL_CTX(ctx)); mInitialised = false; } #ifdef HAVE_OLD_SSL @@ -177,6 +179,7 @@ void CipherContext::Reset() } #endif mWithinTransform = false; + mIV.clear(); } @@ -192,24 +195,22 @@ void CipherContext::Begin() { if(!mInitialised) { - THROW_EXCEPTION(CipherException, NotInitialised) + THROW_EXCEPTION(CipherException, NotInitialised); } - // Warn if in a transformation (not an error, because a context might not have been finalised if an exception occured) if(mWithinTransform) { - BOX_WARNING("CipherContext::Begin called when context " - "flagged as within a transform"); + THROW_EXCEPTION(CipherException, AlreadyInTransform); } - // Initialise the cipher context again - if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) + if(EVP_CipherInit_ex(BOX_OPENSSL_CTX(ctx), NULL, NULL, NULL, + (const unsigned char *)(mIV.size() > 0 ? mIV.c_str() : NULL), + -1) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, - "Failed to reset " << mCipherName << " cipher: " << - LogError("resetting cipher")); + "Failed to set IV for " << mCipherName << ": " << LogError(GetFunction())); } - + // Mark as being within a transform mWithinTransform = true; } @@ -251,18 +252,18 @@ int CipherContext::Transform(void *pOutBuffer, int OutLength, const void *pInBuf } // Check output buffer size - if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx))) + if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)))) { THROW_EXCEPTION(CipherException, OutputBufferTooSmall); } // Do the transform int outLength = OutLength; - if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) + if(EVP_CipherUpdate(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &outLength, + (unsigned char*)pInBuffer, InLength) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure, - "Failed to " << GetFunction() << " (update) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to update " << mCipherName << ": " << LogError(GetFunction())); } return outLength; @@ -300,7 +301,7 @@ int CipherContext::Final(void *pOutBuffer, int OutLength) } // Check output buffer size - if(OutLength < (2 * EVP_CIPHER_CTX_block_size(&ctx))) + if(OutLength < (2 * EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)))) { THROW_EXCEPTION(CipherException, OutputBufferTooSmall); } @@ -308,12 +309,11 @@ int CipherContext::Final(void *pOutBuffer, int OutLength) // Do the transform int outLength = OutLength; #ifndef HAVE_OLD_SSL - if(EVP_CipherFinal(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1) + if(EVP_CipherFinal(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &outLength) != 1) { mWithinTransform = false; THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure, - "Failed to " << GetFunction() << " (final) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to finalise " << mCipherName << ": " << LogError(GetFunction())); } #else OldOpenSSLFinal((unsigned char*)pOutBuffer, outLength); @@ -340,11 +340,11 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) // Old version needs to use a different form, and then set up the cipher again for next time around int outLength = rOutLengthOut; // Have to emulate padding off... - int blockSize = EVP_CIPHER_CTX_block_size(&ctx); + int blockSize = EVP_CIPHER_CTX_block_size(ctx); if(mPaddingOn) { // Just use normal final call - if(EVP_CipherFinal(&ctx, Buffer, &outLength) != 1) + if(EVP_CipherFinal(ctx, Buffer, &outLength) != 1) { THROW_EXCEPTION(CipherException, EVPFinalFailure) } @@ -357,13 +357,13 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) { // NASTY -- fiddling around with internals like this is bad. // But only way to get this working on old versions of OpenSSL. - if(!EVP_EncryptUpdate(&ctx,Buffer,&outLength,ctx.buf,0) + if(!EVP_EncryptUpdate(ctx,Buffer,&outLength,ctx.buf,0) || outLength != blockSize) { THROW_EXCEPTION(CipherException, EVPFinalFailure) } // Clean up - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_free(ctx); } else { @@ -391,12 +391,14 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) } } // Reinitialise the cipher for the next time around - if(EVP_CipherInit(&ctx, mpDescription->GetCipher(), NULL, NULL, + if(EVP_CipherInit_ex(&ctx, mpDescription->GetCipher(), NULL, NULL, + (const unsigned char *)(mIV.size() > 0 ? mIV.c_str() : NULL), (mFunction == Encrypt) ? 1 : 0) != 1) { THROW_EXCEPTION(CipherException, EVPInitFailure) } mpDescription->SetupParameters(&ctx); + UsePadding(mPaddingOn); // Update length for caller rOutLengthOut = outLength; @@ -421,7 +423,7 @@ int CipherContext::InSizeForOutBufferSize(int OutLength) // Strictly speaking, the *2 is unnecessary. However... // Final() is paranoid, and requires two input blocks of space to work. - return OutLength - (EVP_CIPHER_CTX_block_size(&ctx) * 2); + return OutLength - (EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)) * 2); } // -------------------------------------------------------------------------- @@ -442,7 +444,7 @@ int CipherContext::MaxOutSizeForInBufferSize(int InLength) // Final() is paranoid, and requires two input blocks of space to work, and so we need to add // three blocks on to be absolutely sure. - return InLength + (EVP_CIPHER_CTX_block_size(&ctx) * 3); + return InLength + (EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)) * 3); } @@ -456,20 +458,8 @@ int CipherContext::MaxOutSizeForInBufferSize(int InLength) // -------------------------------------------------------------------------- int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength) { - if(!mInitialised) - { - THROW_EXCEPTION(CipherException, NotInitialised) - } - - // Warn if in a transformation - if(mWithinTransform) - { - BOX_WARNING("CipherContext::TransformBlock called when " - "context flagged as within a transform"); - } - // Check output buffer size - if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx))) + if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)))) { // Check if padding is off, in which case the buffer can be smaller if(!mPaddingOn && OutLength <= InLength) @@ -481,40 +471,36 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p THROW_EXCEPTION(CipherException, OutputBufferTooSmall); } } - - // Initialise the cipher context again - if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) - { - THROW_EXCEPTION(CipherException, EVPInitFailure) - } + + Begin(); // Do the entire block - int outLength = 0; + int output_space_used = OutLength; // Update - outLength = OutLength; - if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) + if(EVP_CipherUpdate(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &output_space_used, + (unsigned char*)pInBuffer, InLength) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure, - "Failed to " << GetFunction() << " (update) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to update " << mCipherName << ": " << LogError(GetFunction())); } // Finalise - int outLength2 = OutLength - outLength; -#ifndef HAVE_OLD_SSL - if(EVP_CipherFinal(&ctx, ((unsigned char*)pOutBuffer) + outLength, &outLength2) != 1) + int output_space_remain = OutLength - output_space_used; + +#ifdef HAVE_OLD_SSL + OldOpenSSLFinal(((unsigned char*)pOutBuffer) + output_space_used, output_space_remain); +#else + if(EVP_CipherFinal(BOX_OPENSSL_CTX(ctx), ((unsigned char*)pOutBuffer) + output_space_used, + &output_space_remain) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure, - "Failed to " << GetFunction() << " (final) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to finalise " << mCipherName << ": " << LogError(GetFunction())); } -#else - OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2); #endif - outLength += outLength2; - return outLength; + mWithinTransform = false; + return output_space_used + output_space_remain; } @@ -533,7 +519,7 @@ int CipherContext::GetIVLength() THROW_EXCEPTION(CipherException, NotInitialised) } - return EVP_CIPHER_CTX_iv_length(&ctx); + return EVP_CIPHER_CTX_iv_length(BOX_OPENSSL_CTX(ctx)); } @@ -559,12 +545,14 @@ void CipherContext::SetIV(const void *pIV) "flagged as within a transform"); } + mIV = std::string((const char *)pIV, GetIVLength()); + // Set IV - if(EVP_CipherInit(&ctx, NULL, NULL, (unsigned char *)pIV, -1) != 1) + if(EVP_CipherInit_ex(BOX_OPENSSL_CTX(ctx), NULL, NULL, NULL, + (const unsigned char *)mIV.c_str(), -1) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, - "Failed to " << GetFunction() << " (set IV) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to set IV for " << mCipherName << ": " << LogError(GetFunction())); } #ifdef HAVE_OLD_SSL @@ -601,19 +589,20 @@ const void *CipherContext::SetRandomIV(int &rLengthOut) } // Get length of IV - unsigned int ivLen = EVP_CIPHER_CTX_iv_length(&ctx); - if(ivLen > sizeof(mGeneratedIV)) + uint8_t generated_iv[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH]; + unsigned int ivLen = EVP_CIPHER_CTX_iv_length(BOX_OPENSSL_CTX(ctx)); + if(ivLen > sizeof(generated_iv)) { THROW_EXCEPTION(CipherException, IVSizeImplementationLimitExceeded) } // Generate some random data - Random::Generate(mGeneratedIV, ivLen); - SetIV(mGeneratedIV); + Random::Generate(generated_iv, ivLen); + SetIV(generated_iv); // Return the IV and it's length rLengthOut = ivLen; - return mGeneratedIV; + return mIV.c_str(); } @@ -628,9 +617,11 @@ const void *CipherContext::SetRandomIV(int &rLengthOut) void CipherContext::UsePadding(bool Padding) { #ifndef HAVE_OLD_SSL - if(EVP_CIPHER_CTX_set_padding(&ctx, Padding) != 1) + if(EVP_CIPHER_CTX_set_padding(BOX_OPENSSL_CTX(ctx), Padding) != 1) { - THROW_EXCEPTION(CipherException, EVPSetPaddingFailure) + THROW_EXCEPTION_MESSAGE(CipherException, EVPSetPaddingFailure, + "Failed to set padding for " << mCipherName << ": " << + LogError(GetFunction())); } #endif mPaddingOn = Padding; diff --git a/lib/crypto/CipherContext.h b/lib/crypto/CipherContext.h index 93c889d6..b6e97b4e 100644 --- a/lib/crypto/CipherContext.h +++ b/lib/crypto/CipherContext.h @@ -19,6 +19,22 @@ class CipherDescription; #define CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH 32 +// Macros to allow compatibility with OpenSSL 1.0 and 1.1 APIs. See +// https://github.com/charybdis-ircd/charybdis/blob/release/3.5/libratbox/src/openssl_ratbox.h +// for the gory details. +#if defined(LIBRESSL_VERSION_NUMBER) || (OPENSSL_VERSION_NUMBER >= 0x10100000L) // OpenSSL >= 1.1 +# define BOX_OPENSSL_INIT_CTX(ctx) ctx = EVP_CIPHER_CTX_new(); +# define BOX_OPENSSL_CTX(ctx) ctx +# define BOX_OPENSSL_CLEANUP_CTX(ctx) EVP_CIPHER_CTX_free(ctx) +typedef EVP_CIPHER_CTX* BOX_EVP_CIPHER_CTX; +#else // OpenSSL < 1.1 +# define BOX_OPENSSL_INIT_CTX(ctx) EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does +# define BOX_OPENSSL_CTX(ctx) &ctx +# define BOX_OPENSSL_CLEANUP_CTX(ctx) EVP_CIPHER_CTX_cleanup(&ctx) +typedef EVP_CIPHER_CTX BOX_EVP_CIPHER_CTX; +#endif + + // -------------------------------------------------------------------------- // // Class @@ -74,16 +90,14 @@ public: #endif private: - EVP_CIPHER_CTX ctx; + BOX_EVP_CIPHER_CTX ctx; bool mInitialised; bool mWithinTransform; bool mPaddingOn; - uint8_t mGeneratedIV[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH]; CipherFunction mFunction; std::string mCipherName; -#ifdef HAVE_OLD_SSL - CipherDescription *mpDescription; -#endif + const CipherDescription *mpDescription; + std::string mIV; }; diff --git a/lib/crypto/CipherException.txt b/lib/crypto/CipherException.txt index abdbac87..494ed3cc 100644 --- a/lib/crypto/CipherException.txt +++ b/lib/crypto/CipherException.txt @@ -16,3 +16,4 @@ PseudoRandNotAvailable 12 EVPSetPaddingFailure 13 RandomInitFailed 14 Failed to read from random device LengthRequestedTooLongForRandomHex 15 +AlreadyInTransform 16 Tried to initialise crypto when already in a transform diff --git a/lib/crypto/Random.cpp b/lib/crypto/Random.cpp index 1d6a07f0..c34a6eea 100644 --- a/lib/crypto/Random.cpp +++ b/lib/crypto/Random.cpp @@ -50,7 +50,7 @@ void Random::Initialise() // -------------------------------------------------------------------------- void Random::Generate(void *pOutput, int Length) { - if(RAND_pseudo_bytes((uint8_t*)pOutput, Length) == -1) + if(RAND_bytes((uint8_t*)pOutput, Length) == -1) { THROW_EXCEPTION(CipherException, PseudoRandNotAvailable) } diff --git a/lib/httpserver/HTTPException.txt b/lib/httpserver/HTTPException.txt index 52630cda..c9b3f940 100644 --- a/lib/httpserver/HTTPException.txt +++ b/lib/httpserver/HTTPException.txt @@ -1,16 +1,17 @@ EXCEPTION HTTP 10 -Internal 0 -RequestReadFailed 1 -RequestAlreadyBeenRead 2 -BadRequest 3 -UnknownResponseCodeUsed 4 -NoContentTypeSet 5 -POSTContentTooLong 6 -CannotSetRedirectIfReponseHasData 7 -CannotSetNotFoundIfReponseHasData 8 -NotImplemented 9 -RequestNotInitialised 10 -BadResponse 11 -ResponseReadFailed 12 -NoStreamConfigured 13 +Internal 0 +RequestReadFailed 1 +RequestAlreadyBeenRead 2 +BadRequest 3 +UnknownResponseCodeUsed 4 +NoContentTypeSet 5 +POSTContentTooLong 6 +CannotSetRedirectIfReponseHasData 7 +CannotSetNotFoundIfReponseHasData 8 +NotImplemented 9 +RequestNotInitialised 10 +BadResponse 11 +ResponseReadFailed 12 +NoStreamConfigured 13 +RequestFailedUnexpectedly 14 The request was expected to succeed, but it failed. diff --git a/lib/httpserver/HTTPRequest.cpp b/lib/httpserver/HTTPRequest.cpp index 4c5dc149..a94d96b0 100644 --- a/lib/httpserver/HTTPRequest.cpp +++ b/lib/httpserver/HTTPRequest.cpp @@ -10,10 +10,11 @@ #include "Box.h" #include <string.h> -#include <strings.h> #include <stdlib.h> #include <stdio.h> +#include <sstream> + #include "HTTPRequest.h" #include "HTTPResponse.h" #include "HTTPQueryDecoder.h" @@ -94,6 +95,32 @@ HTTPRequest::~HTTPRequest() } } +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::GetMethodName() +// Purpose: Returns the name of the request's HTTP method verb +// as a string. +// Created: 28/7/15 +// +// -------------------------------------------------------------------------- + +std::string HTTPRequest::GetMethodName() const +{ + switch(mMethod) + { + case Method_UNINITIALISED: return "uninitialised"; + case Method_UNKNOWN: return "unknown"; + case Method_GET: return "GET"; + case Method_HEAD: return "HEAD"; + case Method_POST: return "POST"; + case Method_PUT: return "PUT"; + default: + std::ostringstream oss; + oss << "unknown-" << mMethod; + return oss.str(); + }; +} // -------------------------------------------------------------------------- // @@ -111,7 +138,7 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) // Check caller's logic if(mMethod != Method_UNINITIALISED) { - THROW_EXCEPTION(HTTPException, RequestAlreadyBeenRead) + THROW_EXCEPTION(HTTPException, RequestAlreadyBeenRead); } // Read the first line, which is of a different format to the rest of the lines @@ -126,8 +153,8 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) // Check the method size_t p = 0; // current position in string p = requestLine.find(' '); // end of first word - - if (p == std::string::npos) + + if(p == std::string::npos) { // No terminating space, looks bad p = requestLine.size(); @@ -163,14 +190,14 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) { ++p; } - + // Check there's a URI following... if(requestLinePtr[p] == '\0') { // Didn't get the request line, probably end of connection which had been kept alive return false; } - + // Read the URI, unescaping any %XX hex codes while(requestLinePtr[p] != ' ' && requestLinePtr[p] != '\0') { @@ -201,10 +228,10 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) { code[1] = requestLinePtr[++p]; } - + // Convert into a char code long c = ::strtol(code, NULL, 16); - + // Accept it? if(c > 0 && c <= 255) { @@ -241,28 +268,32 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) int major, minor; if(::sscanf(requestLinePtr + p + 5, "%d.%d", &major, &minor) != 2) { - THROW_EXCEPTION(HTTPException, BadRequest) + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "Unable to parse HTTP version number: " << + requestLinePtr); } - + // Store version mHTTPVersion = (major * HTTPVersion__MajorMultiplier) + minor; } else { // Not good -- wrong string found - THROW_EXCEPTION(HTTPException, BadRequest) + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "Unable to parse HTTP request line: " << + requestLinePtr); } } - + BOX_TRACE("HTTPRequest: method=" << mMethod << ", uri=" << mRequestURI << ", version=" << mHTTPVersion); - + // If HTTP 1.1 or greater, assume keep-alive if(mHTTPVersion >= HTTPVersion_1_1) { mClientKeepAliveRequested = true; } - + // Decode query string? if((mMethod == Method_GET || mMethod == Method_HEAD) && !mQueryString.empty()) { @@ -270,28 +301,28 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) decoder.DecodeChunk(mQueryString.c_str(), mQueryString.size()); decoder.Finish(); } - + // Now parse the headers ParseHeaders(rGetLine, Timeout); - + std::string expected; - if (GetHeader("Expect", &expected)) + if(GetHeader("Expect", &expected)) { - if (expected == "100-continue") + if(expected == "100-continue") { mExpectContinue = true; } } - + // Parse form data? if(mMethod == Method_POST && mContentLength >= 0) { // Too long? Don't allow people to be nasty by sending lots of data if(mContentLength > MAX_CONTENT_SIZE) { - THROW_EXCEPTION(HTTPException, POSTContentTooLong) + THROW_EXCEPTION(HTTPException, POSTContentTooLong); } - + // Some data in the request to follow, parsing it bit by bit HTTPQueryDecoder decoder(mQuery); // Don't forget any data left in the GetLine object @@ -317,7 +348,8 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) if(r == 0) { // Timeout, just error - THROW_EXCEPTION(HTTPException, RequestReadFailed) + THROW_EXCEPTION_MESSAGE(HTTPException, RequestReadFailed, + "Failed to read complete request with the timeout"); } decoder.DecodeChunk(buf, r); bytesToGo -= r; @@ -336,7 +368,7 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) SetForReading(); mpStreamToReadFrom = &(rGetLine.GetUnderlyingStream()); } - + return true; } @@ -346,7 +378,7 @@ void HTTPRequest::ReadContent(IOStream& rStreamToWriteTo) CopyStreamTo(rStreamToWriteTo); IOStream::pos_type bytesCopied = GetSize(); - + while (bytesCopied < mContentLength) { char buffer[1024]; @@ -397,7 +429,8 @@ bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) case HTTPVersion_1_0: rStream.Write("HTTP/1.0"); break; case HTTPVersion_1_1: rStream.Write("HTTP/1.1"); break; default: - THROW_EXCEPTION(HTTPException, NotImplemented); + THROW_EXCEPTION_MESSAGE(HTTPException, NotImplemented, + "Unsupported HTTP version: " << mHTTPVersion); } rStream.Write("\n"); @@ -428,7 +461,8 @@ bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) if (mpCookies) { - THROW_EXCEPTION(HTTPException, NotImplemented); + THROW_EXCEPTION_MESSAGE(HTTPException, NotImplemented, + "Cookie support not implemented yet"); } if (mClientKeepAliveRequested) @@ -445,7 +479,7 @@ bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) { oss << i->first << ": " << i->second << "\n"; } - + if (ExpectContinue) { oss << "Expect: 100-continue\n"; @@ -461,23 +495,22 @@ void HTTPRequest::SendWithStream(IOStream &rStreamToSendTo, int Timeout, IOStream* pStreamToSend, HTTPResponse& rResponse) { IOStream::pos_type size = pStreamToSend->BytesLeftToRead(); - if (size != IOStream::SizeOfStreamUnknown) { mContentLength = size; } - + Send(rStreamToSendTo, Timeout, true); - + rResponse.Receive(rStreamToSendTo, Timeout); if (rResponse.GetResponseCode() != 100) { // bad response, abort now return; } - + pStreamToSend->CopyStreamTo(rStreamToSendTo, Timeout); - + // receive the final response rResponse.Receive(rStreamToSendTo, Timeout); } @@ -502,13 +535,13 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) THROW_EXCEPTION(HTTPException, BadRequest) } - std::string currentLine; + std::string currentLine; if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) { // Timeout THROW_EXCEPTION(HTTPException, RequestReadFailed) } - + // Is this a continuation of the previous line? bool processHeader = haveHeader; if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) @@ -517,7 +550,7 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) processHeader = false; } //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); - + // Parse the header -- this will actually process the header // from the previous run around the loop. if(processHeader) @@ -538,7 +571,7 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) std::string header_name(ToLowerCase(std::string(h, p))); - + if (header_name == "content-length") { // Decode number @@ -556,17 +589,17 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) { // Store host header mHostName = h + dataStart; - + // Is there a port number to split off? std::string::size_type colon = mHostName.find_first_of(':'); if(colon != std::string::npos) { // There's a port in the string... attempt to turn it into an int mHostPort = ::strtol(mHostName.c_str() + colon + 1, 0, 10); - + // Truncate the string to just the hostname mHostName = mHostName.substr(0, colon); - + BOX_TRACE("Host: header, hostname = " << "'" << mHostName << "', host " "port = " << mHostPort); @@ -596,7 +629,7 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) mExtraHeaders.push_back(Header(header_name, h + dataStart)); } - + // Unset have header flag, as it's now been processed haveHeader = false; } @@ -617,7 +650,7 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) { // All done! break; - } + } } } @@ -636,13 +669,13 @@ void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) const char *pos = data; const char *itemStart = pos; std::string name; - + enum { s_NAME, s_VALUE, s_VALUE_QUOTED, s_FIND_NEXT_NAME } state = s_NAME; - do + do { switch(state) { @@ -665,7 +698,7 @@ void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) } } break; - + case s_VALUE: { if(*pos == ';' || *pos == ',' || *pos == '\0') @@ -679,7 +712,7 @@ void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) } } break; - + case s_VALUE_QUOTED: { if(*pos == '"') @@ -693,7 +726,7 @@ void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) } } break; - + case s_FIND_NEXT_NAME: { // Skip over terminators and white space to get to the next name @@ -705,7 +738,7 @@ void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) } } break; - + default: // Ooops THROW_EXCEPTION(HTTPException, Internal) @@ -732,7 +765,7 @@ bool HTTPRequest::GetCookie(const char *CookieName, std::string &rValueOut) cons { return false; } - + // See if it's there CookieJar_t::const_iterator v(mpCookies->find(std::string(CookieName))); if(v != mpCookies->end()) @@ -741,7 +774,7 @@ bool HTTPRequest::GetCookie(const char *CookieName, std::string &rValueOut) cons rValueOut = v->second; return true; } - + return false; } @@ -764,7 +797,7 @@ const std::string &HTTPRequest::GetCookie(const char *CookieName) const { return noCookie; } - + // See if it's there CookieJar_t::const_iterator v(mpCookies->find(std::string(CookieName))); if(v != mpCookies->end()) @@ -772,7 +805,7 @@ const std::string &HTTPRequest::GetCookie(const char *CookieName) const // Return the value return v->second; } - + return noCookie; } diff --git a/lib/httpserver/HTTPRequest.h b/lib/httpserver/HTTPRequest.h index 25effb70..16c4d16c 100644 --- a/lib/httpserver/HTTPRequest.h +++ b/lib/httpserver/HTTPRequest.h @@ -3,7 +3,7 @@ // File // Name: HTTPRequest.h // Purpose: Request object for HTTP connections -// Created: 26/3/04 +// Created: 26/3/2004 // // -------------------------------------------------------------------------- @@ -23,8 +23,12 @@ class IOStreamGetLine; // // Class // Name: HTTPRequest -// Purpose: Request object for HTTP connections -// Created: 26/3/04 +// Purpose: Request object for HTTP connections. Although it +// inherits from CollectInBufferStream, not all of the +// request data is held in memory, only the beginning. +// Use ReadContent() to write it all (including the +// buffered beginning) to another stream, e.g. a file. +// Created: 26/3/2004 // // -------------------------------------------------------------------------- class HTTPRequest : public CollectInBufferStream @@ -77,6 +81,7 @@ public: // // -------------------------------------------------------------------------- enum Method GetMethod() const {return mMethod;} + std::string GetMethodName() const; const std::string &GetRequestURI() const {return mRequestURI;} // Note: the HTTPRequest generates and parses the Host: header diff --git a/lib/httpserver/HTTPResponse.cpp b/lib/httpserver/HTTPResponse.cpp index 1a8c8447..c56f286f 100644 --- a/lib/httpserver/HTTPResponse.cpp +++ b/lib/httpserver/HTTPResponse.cpp @@ -138,8 +138,7 @@ void HTTPResponse::SetContentType(const char *ContentType) // Function // Name: HTTPResponse::Send(IOStream &, bool) // Purpose: Build the response, and send via the stream. -// Optionally omitting the content. -// Created: 26/3/04 +// Created: 26/3/2004 // // -------------------------------------------------------------------------- void HTTPResponse::Send(bool OmitContent) @@ -148,7 +147,7 @@ void HTTPResponse::Send(bool OmitContent) { THROW_EXCEPTION(HTTPException, NoStreamConfigured); } - + if (GetSize() != 0 && mContentType.empty()) { THROW_EXCEPTION(HTTPException, NoContentTypeSet); @@ -184,6 +183,7 @@ void HTTPResponse::Send(bool OmitContent) // static is allowed to be cached for a day header += "\r\nCache-Control: max-age=86400"; } + if(mKeepAlive) { header += "\r\nConnection: keep-alive\r\n\r\n"; @@ -192,12 +192,13 @@ void HTTPResponse::Send(bool OmitContent) { header += "\r\nConnection: close\r\n\r\n"; } + // NOTE: header ends with blank line in all cases - + // Write to stream mpStreamToSendTo->Write(header.c_str(), header.size()); } - + // Send content if(!OmitContent) { @@ -227,16 +228,16 @@ void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) if(rGetLine.IsEOF()) { // Header terminates unexpectedly - THROW_EXCEPTION(HTTPException, BadRequest) + THROW_EXCEPTION(HTTPException, BadRequest) } - std::string currentLine; + std::string currentLine; if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) { // Timeout THROW_EXCEPTION(HTTPException, RequestReadFailed) } - + // Is this a continuation of the previous line? bool processHeader = haveHeader; if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) @@ -245,7 +246,7 @@ void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) processHeader = false; } //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); - + // Parse the header -- this will actually process the header // from the previous run around the loop. if(processHeader) @@ -263,7 +264,7 @@ void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) { ++dataStart; } - + if(p == sizeof("Content-Length")-1 && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0) { @@ -308,7 +309,7 @@ void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) std::string headerName = header.substr(0, p); AddHeader(headerName, h + dataStart); } - + // Unset have header flag, as it's now been processed haveHeader = false; } @@ -329,7 +330,7 @@ void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) { // All done! break; - } + } } } @@ -340,22 +341,22 @@ void HTTPResponse::Receive(IOStream& rStream, int Timeout) if(rGetLine.IsEOF()) { // Connection terminated unexpectedly - THROW_EXCEPTION(HTTPException, BadResponse) + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "HTTP server closed the connection without sending a response"); } - std::string statusLine; + std::string statusLine; if(!rGetLine.GetLine(statusLine, false /* no preprocess */, Timeout)) { // Timeout - THROW_EXCEPTION(HTTPException, ResponseReadFailed) + THROW_EXCEPTION_MESSAGE(HTTPException, ResponseReadFailed, + "Failed to get a response from the HTTP server within the timeout"); } - if (statusLine.substr(0, 7) != "HTTP/1." || - statusLine[8] != ' ') + if (statusLine.substr(0, 7) != "HTTP/1." || statusLine[8] != ' ') { - // Status line terminated unexpectedly - BOX_ERROR("Bad response status line: " << statusLine); - THROW_EXCEPTION(HTTPException, BadResponse) + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "HTTP server sent an invalid HTTP status line: " << statusLine); } if (statusLine[5] == '1' && statusLine[7] == '1') @@ -363,7 +364,7 @@ void HTTPResponse::Receive(IOStream& rStream, int Timeout) // HTTP/1.1 default is to keep alive mKeepAlive = true; } - + // Decode the status code long status = ::strtol(statusLine.substr(9, 3).c_str(), NULL, 10); // returns zero in error case, this is OK @@ -376,7 +377,7 @@ void HTTPResponse::Receive(IOStream& rStream, int Timeout) { return; } - + ParseHeaders(rGetLine, Timeout); // push back whatever bytes we have left @@ -549,7 +550,7 @@ void HTTPResponse::SetAsRedirect(const char *RedirectTo, bool IsLocalURI) if(IsLocalURI) header += msDefaultURIPrefix; header += RedirectTo; mExtraHeaders.push_back(Header("Location", header)); - + // Set up some default content mContentType = "text/html"; #define REDIRECT_HTML_1 "<html><head><title>Redirection</title></head>\n<body><p><a href=\"" @@ -622,7 +623,7 @@ void HTTPResponse::WriteStringDefang(const char *String, unsigned int StringLen) StringLen -= toWrite; String += toWrite; } - + // Is it a bad character next? while(StringLen > 0) { diff --git a/lib/httpserver/HTTPResponse.h b/lib/httpserver/HTTPResponse.h index 04051958..f39825d9 100644 --- a/lib/httpserver/HTTPResponse.h +++ b/lib/httpserver/HTTPResponse.h @@ -63,9 +63,10 @@ public: typedef std::pair<std::string, std::string> Header; void SetResponseCode(int Code); - int GetResponseCode() { return mResponseCode; } + int GetResponseCode() const { return mResponseCode; } void SetContentType(const char *ContentType); const std::string& GetContentType() { return mContentType; } + int64_t GetContentLength() { return mContentLength; } void SetAsRedirect(const char *RedirectTo, bool IsLocalURI = true); void SetAsNotFound(const char *URI); @@ -107,6 +108,7 @@ public: void SetResponseIsDynamicContent(bool IsDynamic) {mResponseIsDynamicContent = IsDynamic;} // Set keep alive control, default is to mark as to be closed void SetKeepAlive(bool KeepAlive) {mKeepAlive = KeepAlive;} + bool IsKeepAlive() {return mKeepAlive;} void SetCookie(const char *Name, const char *Value, const char *Path = "/", int ExpiresAt = 0); @@ -127,7 +129,7 @@ public: }; static const char *ResponseCodeToString(int ResponseCode); - + void WriteStringDefang(const char *String, unsigned int StringLen); void WriteStringDefang(const std::string &rString) {WriteStringDefang(rString.c_str(), rString.size());} @@ -163,9 +165,9 @@ private: bool mKeepAlive; std::string mContentType; std::vector<Header> mExtraHeaders; - int mContentLength; // only used when reading response from stream + int64_t mContentLength; // only used when reading response from stream IOStream* mpStreamToSendTo; // nonzero only when constructed with a stream - + static std::string msDefaultURIPrefix; void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout); diff --git a/lib/httpserver/HTTPServer.cpp b/lib/httpserver/HTTPServer.cpp index be1db687..a2daed99 100644 --- a/lib/httpserver/HTTPServer.cpp +++ b/lib/httpserver/HTTPServer.cpp @@ -27,8 +27,8 @@ // Created: 26/3/04 // // -------------------------------------------------------------------------- -HTTPServer::HTTPServer() - : mTimeout(20000) // default timeout leaves a little while for clients to get the second request in. +HTTPServer::HTTPServer(int Timeout) +: mTimeout(Timeout) { } @@ -86,7 +86,7 @@ const ConfigurationVerify *HTTPServer::GetConfigVerify() const 0 } }; - + static ConfigurationVerifyKey verifyrootkeys[] = { HTTPSERVER_VERIFY_ROOT_KEYS @@ -132,10 +132,10 @@ void HTTPServer::Run() // Created: 26/3/04 // // -------------------------------------------------------------------------- -void HTTPServer::Connection(SocketStream &rStream) +void HTTPServer::Connection(std::auto_ptr<SocketStream> apConn) { // Create a get line object to use - IOStreamGetLine getLine(rStream); + IOStreamGetLine getLine(*apConn); // Notify dervived claases HTTPConnectionOpening(); @@ -150,10 +150,10 @@ void HTTPServer::Connection(SocketStream &rStream) // Didn't get request, connection probably closed. break; } - + // Generate a response - HTTPResponse response(&rStream); - + HTTPResponse response(apConn.get()); + try { Handle(request, response); @@ -169,9 +169,18 @@ void HTTPServer::Connection(SocketStream &rStream) { SendInternalErrorResponse("unknown", response); } - - // Keep alive? - if(request.GetClientKeepAliveRequested()) + + // Keep alive? response.GetSize() works for CollectInBufferStream, but + // when we make HTTPResponse stream the response instead, we'll need to + // figure out whether we can get the response length from the IOStream + // to be streamed, or not. Also, we don't currently support chunked + // encoding, and http://tools.ietf.org/html/rfc7230#section-3.3.1 says + // that "If any transfer coding other than chunked is applied to a + // response payload body, the sender MUST either apply chunked as the + // final transfer coding or terminate the message by closing the + // connection. So for now, keepalive stays off. + if(false && request.GetClientKeepAliveRequested() && + response.GetSize() >= 0) { // Mark the response to the client as supporting keepalive response.SetKeepAlive(true); diff --git a/lib/httpserver/HTTPServer.h b/lib/httpserver/HTTPServer.h index d9f74949..8ac1ff83 100644 --- a/lib/httpserver/HTTPServer.h +++ b/lib/httpserver/HTTPServer.h @@ -27,7 +27,8 @@ class HTTPResponse; class HTTPServer : public ServerStream<SocketStream, 80> { public: - HTTPServer(); + HTTPServer(int Timeout = 60000); + // default timeout leaves 1 minute for clients to get a second request in. ~HTTPServer(); private: // no copying @@ -62,7 +63,7 @@ private: const char *DaemonName() const; const ConfigurationVerify *GetConfigVerify() const; void Run(); - void Connection(SocketStream &rStream); + void Connection(std::auto_ptr<SocketStream> apStream); }; // Root level diff --git a/lib/httpserver/S3Client.cpp b/lib/httpserver/S3Client.cpp index cd5988d5..21814066 100644 --- a/lib/httpserver/S3Client.cpp +++ b/lib/httpserver/S3Client.cpp @@ -34,7 +34,7 @@ // Name: S3Client::GetObject(const std::string& rObjectURI) // Purpose: Retrieve the object with the specified URI (key) // from your S3 bucket. -// Created: 09/01/09 +// Created: 09/01/2009 // // -------------------------------------------------------------------------- @@ -46,12 +46,29 @@ HTTPResponse S3Client::GetObject(const std::string& rObjectURI) // -------------------------------------------------------------------------- // // Function +// Name: S3Client::HeadObject(const std::string& rObjectURI) +// Purpose: Retrieve the metadata for the object with the +// specified URI (key) from your S3 bucket. +// Created: 03/08/2015 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::HeadObject(const std::string& rObjectURI) +{ + return FinishAndSendRequest(HTTPRequest::Method_HEAD, rObjectURI); +} + + +HTTPResponse HeadObject(const std::string& rObjectURI); +// -------------------------------------------------------------------------- +// +// Function // Name: S3Client::PutObject(const std::string& rObjectURI, // IOStream& rStreamToSend, const char* pContentType) // Purpose: Upload the stream to S3, creating or overwriting the // object with the specified URI (key) in your S3 // bucket. -// Created: 09/01/09 +// Created: 09/01/2009 // // -------------------------------------------------------------------------- @@ -77,7 +94,7 @@ HTTPResponse S3Client::PutObject(const std::string& rObjectURI, // connection to the server if necessary, which may // throw a ConnectionException. Returns the HTTP // response returned by S3, which may be a 500 error. -// Created: 09/01/09 +// Created: 09/01/2009 // // -------------------------------------------------------------------------- @@ -113,7 +130,7 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, { request.AddHeader("Content-Type", pStreamContentType); } - + std::string s3suffix = ".s3.amazonaws.com"; std::string bucket; if (mHostName.size() > s3suffix.size()) @@ -126,18 +143,18 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, s3suffix.size()); } } - + std::ostringstream data; data << request.GetVerb() << "\n"; data << "\n"; /* Content-MD5 */ data << request.GetContentType() << "\n"; data << date.str() << "\n"; - + if (! bucket.empty()) { data << "/" << bucket; } - + data << request.GetRequestURI(); std::string data_string = data.str(); @@ -148,7 +165,7 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, (const unsigned char*)data_string.c_str(), data_string.size(), digest_buffer, &digest_size); std::string digest((const char *)digest_buffer, digest_size); - + base64::encoder encoder; std::string auth_code = "AWS " + mAccessKey + ":" + encoder.encode(digest); @@ -200,6 +217,7 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, } else { + BOX_TRACE("S3Client: " << mHostName << " ! " << ce.what()); throw; } } @@ -218,26 +236,57 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, // necessary, which may throw a ConnectionException. // Returns the HTTP response returned by S3, which may // be a 500 error. -// Created: 09/01/09 +// Created: 09/01/2009 // // -------------------------------------------------------------------------- HTTPResponse S3Client::SendRequest(HTTPRequest& rRequest, IOStream* pStreamToSend, const char* pStreamContentType) -{ +{ HTTPResponse response; - + if (pStreamToSend) { - rRequest.SendWithStream(*mapClientSocket, - 30000 /* milliseconds */, + rRequest.SendWithStream(*mapClientSocket, mNetworkTimeout, pStreamToSend, response); } else { - rRequest.Send(*mapClientSocket, 30000 /* milliseconds */); - response.Receive(*mapClientSocket, 30000 /* milliseconds */); + rRequest.Send(*mapClientSocket, mNetworkTimeout); + response.Receive(*mapClientSocket, mNetworkTimeout); + } + + if(!response.IsKeepAlive()) + { + BOX_TRACE("Server will close the connection, closing our end too."); + mapClientSocket.reset(); + } + else + { + BOX_TRACE("Server will keep the connection open for more requests."); } - + return response; -} +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::CheckResponse(HTTPResponse&, +// std::string& message) +// Purpose: Check the status code of an Amazon S3 response, and +// throw an exception with a useful message (including +// the supplied message) if it's not a 200 OK response. +// Created: 26/07/2015 +// +// -------------------------------------------------------------------------- + +void S3Client::CheckResponse(const HTTPResponse& response, const std::string& message) const +{ + if(response.GetResponseCode() != HTTPResponse::Code_OK) + { + THROW_EXCEPTION_MESSAGE(HTTPException, RequestFailedUnexpectedly, + message); + } +} + diff --git a/lib/httpserver/S3Client.h b/lib/httpserver/S3Client.h index 3c4126ac..4cbb4b96 100644 --- a/lib/httpserver/S3Client.h +++ b/lib/httpserver/S3Client.h @@ -36,7 +36,8 @@ class S3Client : mpSimulator(pSimulator), mHostName(rHostName), mAccessKey(rAccessKey), - mSecretKey(rSecretKey) + mSecretKey(rSecretKey), + mNetworkTimeout(30000) { } S3Client(std::string HostName, int Port, const std::string& rAccessKey, @@ -45,12 +46,16 @@ class S3Client mHostName(HostName), mPort(Port), mAccessKey(rAccessKey), - mSecretKey(rSecretKey) + mSecretKey(rSecretKey), + mNetworkTimeout(30000) { } HTTPResponse GetObject(const std::string& rObjectURI); + HTTPResponse HeadObject(const std::string& rObjectURI); HTTPResponse PutObject(const std::string& rObjectURI, IOStream& rStreamToSend, const char* pContentType = NULL); + void CheckResponse(const HTTPResponse& response, const std::string& message) const; + int GetNetworkTimeout() const { return mNetworkTimeout; } private: HTTPServer* mpSimulator; @@ -58,6 +63,7 @@ class S3Client int mPort; std::auto_ptr<SocketStream> mapClientSocket; std::string mAccessKey, mSecretKey; + int mNetworkTimeout; // milliseconds HTTPResponse FinishAndSendRequest(HTTPRequest::Method Method, const std::string& rRequestURI, diff --git a/lib/httpserver/S3Simulator.cpp b/lib/httpserver/S3Simulator.cpp index 4f6bb3e6..df8910d7 100644 --- a/lib/httpserver/S3Simulator.cpp +++ b/lib/httpserver/S3Simulator.cpp @@ -40,12 +40,12 @@ // -------------------------------------------------------------------------- const ConfigurationVerify* S3Simulator::GetConfigVerify() const { - static ConfigurationVerifyKey verifyserverkeys[] = + static ConfigurationVerifyKey verifyserverkeys[] = { HTTPSERVER_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) // no default addresses }; - static ConfigurationVerify verifyserver[] = + static ConfigurationVerify verifyserver[] = { { "Server", @@ -55,8 +55,8 @@ const ConfigurationVerify* S3Simulator::GetConfigVerify() const 0 } }; - - static ConfigurationVerifyKey verifyrootkeys[] = + + static ConfigurationVerifyKey verifyrootkeys[] = { ConfigurationVerifyKey("AccessKey", ConfigTest_Exists), ConfigurationVerifyKey("SecretKey", ConfigTest_Exists), @@ -99,11 +99,11 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) const Configuration& rConfig(GetConfiguration()); std::string access_key = rConfig.GetKeyValue("AccessKey"); std::string secret_key = rConfig.GetKeyValue("SecretKey"); - + std::string md5, date, bucket; rRequest.GetHeader("content-md5", &md5); rRequest.GetHeader("date", &date); - + std::string host = rRequest.GetHostName(); std::string s3suffix = ".s3.amazonaws.com"; if (host.size() > s3suffix.size()) @@ -116,7 +116,7 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) s3suffix.size()); } } - + std::ostringstream data; data << rRequest.GetVerb() << "\n"; data << md5 << "\n"; @@ -127,7 +127,7 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) std::vector<HTTPRequest::Header> headers = rRequest.GetHeaders(); std::sort(headers.begin(), headers.end()); - + for (std::vector<HTTPRequest::Header>::iterator i = headers.begin(); i != headers.end(); i++) { @@ -135,13 +135,13 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) { data << i->first << ":" << i->second << "\n"; } - } - + } + if (! bucket.empty()) { data << "/" << bucket; } - + data << rRequest.GetRequestURI(); std::string data_string = data.str(); @@ -152,17 +152,17 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) (const unsigned char*)data_string.c_str(), data_string.size(), digest_buffer, &digest_size); std::string digest((const char *)digest_buffer, digest_size); - + base64::encoder encoder; std::string expectedAuth = "AWS " + access_key + ":" + encoder.encode(digest); - + if (expectedAuth[expectedAuth.size() - 1] == '\n') { expectedAuth = expectedAuth.substr(0, expectedAuth.size() - 1); } - + std::string actualAuth; if (!rRequest.GetHeader("authorization", &actualAuth) || actualAuth != expectedAuth) @@ -170,7 +170,7 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) rResponse.SetResponseCode(HTTPResponse::Code_Unauthorized); SendInternalErrorResponse("Authentication Failed", rResponse); - } + } else if (rRequest.GetMethod() == HTTPRequest::Method_GET) { HandleGet(rRequest, rResponse); @@ -198,7 +198,7 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) { SendInternalErrorResponse("Unknown exception", rResponse); } - + if (rResponse.GetResponseCode() != 200 && rResponse.GetSize() == 0) { @@ -207,7 +207,10 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) s << rResponse.GetResponseCode(); SendInternalErrorResponse(s.str().c_str(), rResponse); } - + + BOX_NOTICE(rResponse.GetResponseCode() << " " << rRequest.GetMethodName() << " " << + rRequest.GetRequestURI()); + return; } diff --git a/lib/httpserver/S3Simulator.h b/lib/httpserver/S3Simulator.h index f80770ee..eef4f400 100644 --- a/lib/httpserver/S3Simulator.h +++ b/lib/httpserver/S3Simulator.h @@ -27,13 +27,20 @@ class HTTPResponse; class S3Simulator : public HTTPServer { public: - S3Simulator() { } + // Increase timeout to 5 minutes, from HTTPServer default of 1 minute, + // to help with debugging. + S3Simulator() : HTTPServer(300000) { } ~S3Simulator() { } const ConfigurationVerify* GetConfigVerify() const; virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse); virtual void HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse); virtual void HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse); + + virtual const char *DaemonName() const + { + return "s3simulator"; + } }; #endif // S3SIMULATOR__H diff --git a/lib/httpserver/cdecode.cpp b/lib/httpserver/cdecode.cpp index e632f182..11c59d62 100644 --- a/lib/httpserver/cdecode.cpp +++ b/lib/httpserver/cdecode.cpp @@ -12,7 +12,7 @@ extern "C" int base64_decode_value(char value_in) { - static const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; + static signed const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; static const char decoding_size = sizeof(decoding); value_in -= 43; if (value_in < 0 || value_in > decoding_size) return -1; diff --git a/lib/intercept/intercept.cpp b/lib/intercept/intercept.cpp index 7a33b610..72bd8d4e 100644 --- a/lib/intercept/intercept.cpp +++ b/lib/intercept/intercept.cpp @@ -15,7 +15,6 @@ #include <sys/syscall.h> #endif #include <sys/types.h> -#include <unistd.h> #ifdef HAVE_SYS_UIO_H #include <sys/uio.h> @@ -132,7 +131,7 @@ void intercept_setup_error(const char *filename, unsigned int errorafter, int er intercept_delay_ms = 0; } -void intercept_setup_delay(const char *filename, unsigned int delay_after, +void intercept_setup_delay(const char *filename, unsigned int delay_after, int delay_ms, int syscall_to_delay, int num_delays) { BOX_TRACE("Setup for delay: " << filename << @@ -243,6 +242,10 @@ extern "C" int open(const char *path, int flags, ...) #endif // DEFINE_ONLY_OPEN64 { + // Some newer architectures don't have an open() syscall, but use openat() instead. + // In these cases we will need to call sys_openat() instead of sys_open(). + // https://chromium.googlesource.com/linux-syscall-support/ + if(intercept_count > 0) { if(intercept_filename != NULL && @@ -265,6 +268,8 @@ extern "C" int #ifdef PLATFORM_NO_SYSCALL int r = TEST_open(path, flags, mode); +#elif HAVE_DECL_SYS_OPENAT && !HAVE_DECL_SYS_OPEN + int r = syscall(SYS_openat, AT_FDCWD, path, flags, mode); #else int r = syscall(SYS_open, path, flags, mode); #endif @@ -390,7 +395,7 @@ lseek(int fildes, off_t offset, int whence) #else #ifdef HAVE_LSEEK_DUMMY_PARAM off_t r = syscall(SYS_lseek, fildes, 0 /* extra 0 required here! */, offset, whence); - #elif defined(_FILE_OFFSET_BITS) + #elif defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS == 32 // Don't bother trying to call SYS__llseek on 32 bit since it is // fiddly and not needed for the tests off_t r = syscall(SYS_lseek, fildes, (uint32_t)offset, whence); @@ -514,7 +519,7 @@ DIR *opendir(const char *dirname) { if (opendir_real == NULL) { - opendir_real = (opendir_t*)find_function("opendir"); + opendir_real = (opendir_t*)find_function(FUNC_OPENDIR); } if (opendir_real == NULL) @@ -547,7 +552,7 @@ struct dirent *readdir(DIR *dir) if (readdir_real == NULL) { - readdir_real = (readdir_t*)find_function("readdir"); + readdir_real = (readdir_t*)find_function(FUNC_READDIR); } if (readdir_real == NULL) diff --git a/lib/intercept/intercept.h b/lib/intercept/intercept.h index 80a17d3f..4de5f9f2 100644 --- a/lib/intercept/intercept.h +++ b/lib/intercept/intercept.h @@ -13,6 +13,18 @@ #include <dirent.h> +#ifdef __NetBSD__ //__NetBSD_Version__ is defined in sys/param.h +#include <sys/param.h> +#endif + +#if defined __NetBSD_Version__ && __NetBSD_Version__ >= 399000800 //3.99.8 vers. +#define FUNC_OPENDIR "__opendir30" +#define FUNC_READDIR "__readdir30" +#else +#define FUNC_OPENDIR "opendir" +#define FUNC_READDIR "readdir" +#endif + #include <sys/types.h> #include <sys/stat.h> @@ -50,5 +62,14 @@ void intercept_setup_stat_post_hook (lstat_post_hook_t hookfn); void intercept_clear_setup(); +// Some newer architectures don't have an open() syscall, but use openat() instead. +// In these cases we define SYS_open (which is otherwise undefined) to equal SYS_openat +// (which is defined) so that everywhere else we can call intercept_setup_error(SYS_open) +// without caring about the difference. +// https://chromium.googlesource.com/linux-syscall-support/ +#if !HAVE_DECL_SYS_OPEN && HAVE_DECL_SYS_OPENAT +# define SYS_open SYS_openat +#endif + #endif // !PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE #endif // !INTERCEPT_H diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp index bcff54c6..7b755395 100644 --- a/lib/raidfile/RaidFileRead.cpp +++ b/lib/raidfile/RaidFileRead.cpp @@ -44,8 +44,12 @@ #define READ_NUMBER_DISCS_REQUIRED 3 #define READV_MAX_BLOCKS 64 -// We want to use POSIX fstat() for now, not the emulated one -#undef fstat +// We want to use POSIX fstat() for now, not the emulated one, because it's +// difficult to rewrite all this code to use HANDLEs instead of ints. + +const RaidFileReadCategory RaidFileRead::OPEN_IN_RECOVERY("OpenInRecovery"); +const RaidFileReadCategory RaidFileRead::IO_ERROR("IoError"); +const RaidFileReadCategory RaidFileRead::RECOVERING_IO_ERROR("RecoverIoError"); // -------------------------------------------------------------------------- // @@ -548,7 +552,8 @@ void RaidFileRead_Raid::MoveDamagedFileAlertDaemon(int SetNumber, const std::str // -------------------------------------------------------------------------- void RaidFileRead_Raid::AttemptToRecoverFromIOError(bool Stripe1) { - BOX_WARNING("Attempting to recover from I/O error: " << mSetNumber << + BOX_LOG_CATEGORY(Log::WARNING, RaidFileRead::RECOVERING_IO_ERROR, + "Attempting to recover from I/O error: " << mSetNumber << " " << mFilename << ", on stripe " << (Stripe1?1:2)); // Close offending file @@ -740,7 +745,7 @@ int RaidFileRead_Raid::ReadRecovered(void *pBuffer, int NBytes) // Go XORing! unsigned int *b1 = (unsigned int*)mRecoveryBuffer; unsigned int *b2 = (unsigned int *)(mRecoveryBuffer + mBlockSize); - if((mStripe1Handle == -1)) + if(mStripe1Handle == -1) { b1 = b2; b2 = (unsigned int*)mRecoveryBuffer; @@ -861,10 +866,10 @@ void RaidFileRead_Raid::SetPosition(pos_type FilePosition) { if(errno == EIO) { - BOX_ERROR("I/O error when seeking in " << - mSetNumber << " " << mFilename << - " (to " << FilePosition << "), " << - "stripe 1"); + BOX_LOG_CATEGORY(Log::ERROR, RaidFileRead::IO_ERROR, + "I/O error when seeking in set " << mSetNumber << + ": " << mFilename << " (to " << FilePosition << + "), " << "stripe 1"); // Attempt to recover AttemptToRecoverFromIOError(true /* is stripe 1 */); ASSERT(mStripe1Handle == -1); @@ -881,10 +886,10 @@ void RaidFileRead_Raid::SetPosition(pos_type FilePosition) { if(errno == EIO) { - BOX_ERROR("I/O error when seeking in " << - mSetNumber << " " << mFilename << - " (to " << FilePosition << "), " << - "stripe 2"); + BOX_LOG_CATEGORY(Log::ERROR, RaidFileRead::IO_ERROR, + "I/O error when seeking in set " << mSetNumber << + ": " << mFilename << " (to " << FilePosition << + "), " << "stripe 2"); // Attempt to recover AttemptToRecoverFromIOError(false /* is stripe 2 */); ASSERT(mStripe2Handle == -1); @@ -1155,8 +1160,9 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string bool oktotryagain = true; if(stripe1errno == EIO) { - BOX_ERROR("I/O error on opening " << - SetNumber << " " << Filename << + BOX_LOG_CATEGORY(Log::ERROR, + RaidFileRead::RECOVERING_IO_ERROR, "I/O error " + "on opening " << SetNumber << " " << Filename << " stripe 1, trying recovery mode"); RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, true /* is stripe 1 */); @@ -1172,8 +1178,9 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string if(stripe2errno == EIO) { - BOX_ERROR("I/O error on opening " << - SetNumber << " " << Filename << + BOX_LOG_CATEGORY(Log::ERROR, + RaidFileRead::RECOVERING_IO_ERROR, "I/O error " + "on opening " << SetNumber << " " << Filename << " stripe 2, trying recovery mode"); RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, false /* is stripe 2 */); @@ -1196,7 +1203,8 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string if(existance == RaidFileUtil::AsRaidWithMissingReadable) { - BOX_ERROR("Attempting to open RAID file " << SetNumber << + BOX_LOG_CATEGORY(Log::ERROR, RaidFileRead::OPEN_IN_RECOVERY, + "Attempting to open RAID file " << SetNumber << " " << Filename << " in recovery mode (stripe " << ((existingFiles & RaidFileUtil::Stripe1Exists)?1:2) << " present)"); @@ -1498,8 +1506,8 @@ bool RaidFileRead::DirectoryExists(const RaidFileDiscSet &rSet, const std::strin std::string dn(rSet[l] + DIRECTORY_SEPARATOR + rDirName); // check for existence - struct stat st; - if(::stat(dn.c_str(), &st) == 0) + EMU_STRUCT_STAT st; + if(EMU_STAT(dn.c_str(), &st) == 0) { // Directory? if(st.st_mode & S_IFDIR) @@ -1510,7 +1518,10 @@ bool RaidFileRead::DirectoryExists(const RaidFileDiscSet &rSet, const std::strin else { // No. It's a file. Bad! - THROW_EXCEPTION(RaidFileException, UnexpectedFileInDirPlace) + THROW_FILE_ERROR("Expected a directory, " + "found something else", dn, + RaidFileException, + UnexpectedFileInDirPlace); } } else @@ -1519,7 +1530,9 @@ bool RaidFileRead::DirectoryExists(const RaidFileDiscSet &rSet, const std::strin if(errno != ENOENT) { // No. Bad things. - THROW_EXCEPTION(RaidFileException, OSError) + THROW_SYS_FILE_ERROR("Failed to check for " + "existing RaidFile directory", dn, + RaidFileException, OSError); } } } @@ -1617,12 +1630,15 @@ bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirN #ifdef HAVE_VALID_DIRENT_D_TYPE if(DirReadType == DirReadType_FilesOnly && en->d_type == DT_REG) #else - struct stat st; + EMU_STRUCT_STAT st; std::string fullName(dn + DIRECTORY_SEPARATOR + en->d_name); - if(::lstat(fullName.c_str(), &st) != 0) + if(EMU_LSTAT(fullName.c_str(), &st) != 0) { - THROW_EXCEPTION(RaidFileException, OSError) + THROW_SYS_FILE_ERROR("Failed to stat", + fullName, RaidFileException, + OSError); } + if(DirReadType == DirReadType_FilesOnly && (st.st_mode & S_IFDIR) == 0) #endif { @@ -1721,7 +1737,7 @@ bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirN // Created: 2003/08/21 // // -------------------------------------------------------------------------- -void RaidFileRead::Write(const void *pBuffer, int NBytes) +void RaidFileRead::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(RaidFileException, UnsupportedReadWriteOrClose) } @@ -1767,6 +1783,7 @@ IOStream::pos_type RaidFileRead::GetDiscUsageInBlocks() return RaidFileUtil::DiscUsageInBlocks(GetFileSize(), rdiscSet); } - - - +std::string RaidFileRead::ToString() const +{ + return std::string("RaidFile ") + mFilename; +} diff --git a/lib/raidfile/RaidFileRead.h b/lib/raidfile/RaidFileRead.h index 8a04409d..a3c792d0 100644 --- a/lib/raidfile/RaidFileRead.h +++ b/lib/raidfile/RaidFileRead.h @@ -16,9 +16,17 @@ #include <vector> #include "IOStream.h" +#include "Logging.h" class RaidFileDiscSet; +class RaidFileReadCategory : public Log::Category +{ + public: + RaidFileReadCategory(const std::string& name) + : Log::Category(std::string("RaidFileRead/") + name) + { } +}; // -------------------------------------------------------------------------- // @@ -56,14 +64,20 @@ public: static bool ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector<std::string> &rOutput); // Common IOStream interface implementation - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamClosed(); virtual pos_type BytesLeftToRead(); pos_type GetDiscUsageInBlocks(); + std::string ToString() const; typedef int64_t FileSizeType; + static const RaidFileReadCategory OPEN_IN_RECOVERY; + static const RaidFileReadCategory IO_ERROR; + static const RaidFileReadCategory RECOVERING_IO_ERROR; + protected: int mSetNumber; std::string mFilename; diff --git a/lib/raidfile/RaidFileUtil.cpp b/lib/raidfile/RaidFileUtil.cpp index 7c6299ec..48625997 100644 --- a/lib/raidfile/RaidFileUtil.cpp +++ b/lib/raidfile/RaidFileUtil.cpp @@ -14,10 +14,30 @@ #include "RaidFileUtil.h" #include "FileModificationTime.h" -#include "RaidFileRead.h" // for type definition +#include "RaidFileRead.h" // for type definition #include "MemLeakFindOn.h" +int64_t adjust_timestamp(int64_t timestamp, size_t file_size) +{ +#ifndef BOX_RELEASE_BUILD + // Remove the microseconds part of the timestamp, + // to simulate filesystem with 1-second timestamp + // resolution, e.g. MacOS X HFS, old Linuxes. + // Otherwise it's easy to write tests that rely + // on more accurate timestamps, and pass on + // platforms that have them, and fail on others. + timestamp -= (timestamp % MICRO_SEC_IN_SEC); +#endif + + // The resolution of timestamps may be very + // low, e.g. 1 second. So add the size to it + // to give a bit more chance of it changing. + // TODO: Make this better. + timestamp += file_size; + + return timestamp; +} // -------------------------------------------------------------------------- // @@ -39,8 +59,7 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, *pExistingFiles = 0; } - // For stat call, although the results are not examined - struct stat st; + EMU_STRUCT_STAT st; // check various files int startDisc = 0; @@ -50,29 +69,15 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, { *pStartDisc = startDisc; } - if(::stat(writeFile.c_str(), &st) == 0) + if(EMU_STAT(writeFile.c_str(), &st) == 0) { // write file exists, use that // Get unique ID if(pRevisionID != 0) { - #ifdef WIN32 - *pRevisionID = st.st_mtime; - #else - *pRevisionID = FileModificationTime(st); - #endif - -#ifdef BOX_RELEASE_BUILD - // The resolution of timestamps may be very - // low, e.g. 1 second. So add the size to it - // to give a bit more chance of it changing. - // TODO: Make this better. - // Disabled in debug mode, to simulate - // filesystem with 1-second timestamp - // resolution, e.g. MacOS X HFS, Linux. - (*pRevisionID) += st.st_size; -#endif + *pRevisionID = FileModificationTime(st); + *pRevisionID = adjust_timestamp(*pRevisionID, st.st_size); } // return non-raid file @@ -91,7 +96,7 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, for(int f = 0; f < setSize; ++f) { std::string componentFile(RaidFileUtil::MakeRaidComponentName(rDiscSet, rFilename, (f + startDisc) % setSize)); - if(::stat(componentFile.c_str(), &st) == 0) + if(EMU_STAT(componentFile.c_str(), &st) == 0) { // Component file exists, add to count rfCount++; @@ -103,12 +108,7 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, // Revision ID if(pRevisionID != 0) { - #ifdef WIN32 - int64_t rid = st.st_mtime; - #else - int64_t rid = FileModificationTime(st); - #endif - + int64_t rid = FileModificationTime(st); if(rid > revisionID) revisionID = rid; revisionIDplus += st.st_size; } @@ -116,16 +116,8 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, } if(pRevisionID != 0) { + revisionID = adjust_timestamp(revisionID, revisionIDplus); (*pRevisionID) = revisionID; -#ifdef BOX_RELEASE_BUILD - // The resolution of timestamps may be very low, e.g. - // 1 second. So add the size to it to give a bit more - // chance of it changing. - // TODO: Make this better. - // Disabled in debug mode, to simulate filesystem with - // 1-second timestamp resolution, e.g. MacOS X HFS, Linux. - (*pRevisionID) += revisionIDplus; -#endif } // Return a status based on how many parts are available diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp index 82aeef3d..8f95ba65 100644 --- a/lib/raidfile/RaidFileWrite.cpp +++ b/lib/raidfile/RaidFileWrite.cpp @@ -42,8 +42,8 @@ // Must have this number of discs in the set #define TRANSFORM_NUMBER_DISCS_REQUIRED 3 -// we want to use POSIX fstat() for now, not the emulated one -#undef fstat +// We want to use POSIX fstat() for now, not the emulated one, because it's +// difficult to rewrite all this code to use HANDLEs instead of ints. // -------------------------------------------------------------------------- // @@ -243,7 +243,7 @@ void RaidFileWrite::Open(bool AllowOverwrite) // Created: 2003/07/10 // // -------------------------------------------------------------------------- -void RaidFileWrite::Write(const void *pBuffer, int Length) +void RaidFileWrite::Write(const void *pBuffer, int Length, int Timeout) { // open? if(mOSFileHandle == -1) @@ -672,7 +672,9 @@ void RaidFileWrite::TransformToRaidStorage() { \ if (::unlink(file) != 0 && errno != ENOENT) \ { \ - THROW_EXCEPTION(RaidFileException, OSError); \ + THROW_EMU_ERROR("Failed to unlink raidfile " \ + "stripe: " << file, RaidFileException, \ + OSError); \ } \ } CHECK_UNLINK(stripe1Filename.c_str()); diff --git a/lib/raidfile/RaidFileWrite.h b/lib/raidfile/RaidFileWrite.h index e2887167..ab9b399a 100644 --- a/lib/raidfile/RaidFileWrite.h +++ b/lib/raidfile/RaidFileWrite.h @@ -41,13 +41,15 @@ public: RaidFileWrite(int SetNumber, const std::string &Filename, int refcount); ~RaidFileWrite(); + private: RaidFileWrite(const RaidFileWrite &rToCopy); public: // IOStream interface virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); // will exception - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual void Close(); // will discard the file! Use commit instead. @@ -65,8 +67,6 @@ public: static void CreateDirectory(int SetNumber, const std::string &rDirName, bool Recursive = false, int mode = 0777); static void CreateDirectory(const RaidFileDiscSet &rSet, const std::string &rDirName, bool Recursive = false, int mode = 0777); - -private: private: int mSetNumber; diff --git a/lib/server/ConnectionException.txt b/lib/server/ConnectionException.txt index c3429116..7dcaadeb 100644 --- a/lib/server/ConnectionException.txt +++ b/lib/server/ConnectionException.txt @@ -25,3 +25,4 @@ Protocol_HandshakeFailed 48 Protocol_StreamWhenObjExpected 49 Protocol_ObjWhenStreamExpected 50 Protocol_TimeOutWhenSendingStream 52 Probably a network issue between client and server. +Protocol_StreamsNotConsumed 53 The server command handler did not consume all streams that were sent. diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp index 7419f973..d3c8441f 100644 --- a/lib/server/Daemon.cpp +++ b/lib/server/Daemon.cpp @@ -9,23 +9,27 @@ #include "Box.h" -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - #include <errno.h> #include <stdio.h> #include <signal.h> #include <string.h> #include <stdarg.h> +#ifdef HAVE_PROCESS_H +# include <process.h> +#endif + +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif + #ifdef HAVE_BSD_UNISTD_H #include <bsd/unistd.h> #endif #ifdef WIN32 + #include <Strsafe.h> #include <ws2tcpip.h> - #include <process.h> #endif #include "depot.h" @@ -36,12 +40,13 @@ # include "BoxVersion.h" #endif +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "Configuration.h" #include "Daemon.h" #include "FileModificationTime.h" #include "Guards.h" #include "Logging.h" -#include "ServerException.h" #include "UnixUser.h" #include "Utils.h" @@ -106,11 +111,11 @@ Daemon::~Daemon() // -------------------------------------------------------------------------- std::string Daemon::GetOptionString() { - return "c:" + return std::string("c:" #ifndef WIN32 "DF" #endif - "hkKo:O:PqQt:TUvVW:"; + "hkKo:O:") + Logging::OptionParser::GetOptionString(); } void Daemon::Usage() @@ -133,16 +138,7 @@ void Daemon::Usage() " -K Stop writing log messages to console while daemon is running\n" " -o <file> Log to a file, defaults to maximum verbosity\n" " -O <level> Set file log verbosity to error/warning/notice/info/trace/everything\n" - " -P Show process ID (PID) in console output\n" - " -q Run more quietly, reduce verbosity level by one, can repeat\n" - " -Q Run at minimum verbosity, log nothing to console and system\n" - " -t <tag> Tag console output with specified marker\n" - " -T Timestamp console output\n" - " -U Timestamp console output with microseconds\n" - " -v Run more verbosely, increase verbosity level by one, can repeat\n" - " -V Run at maximum verbosity, log everything to console and system\n" - " -W <level> Set verbosity to error/warning/notice/info/trace/everything\n" - ; + << Logging::OptionParser::GetUsageString(); } // -------------------------------------------------------------------------- @@ -218,94 +214,9 @@ int Daemon::ProcessOption(signed int option) } break; - case 'P': - { - Console::SetShowPID(true); - } - break; - - case 'q': - { - if(mLogLevel == Log::NOTHING) - { - BOX_FATAL("Too many '-q': " - "Cannot reduce logging " - "level any more"); - return 2; - } - mLogLevel--; - } - break; - - case 'Q': - { - mLogLevel = Log::NOTHING; - } - break; - - case 't': - { - Logging::SetProgramName(optarg); - Console::SetShowTag(true); - } - break; - - case 'T': - { - Console::SetShowTime(true); - } - break; - - case 'U': - { - Console::SetShowTime(true); - Console::SetShowTimeMicros(true); - } - break; - - case 'v': - { - if(mLogLevel == Log::EVERYTHING) - { - BOX_FATAL("Too many '-v': " - "Cannot increase logging " - "level any more"); - return 2; - } - mLogLevel++; - } - break; - - case 'V': - { - mLogLevel = Log::EVERYTHING; - } - break; - - case 'W': - { - mLogLevel = Logging::GetNamedLevel(optarg); - if (mLogLevel == Log::INVALID) - { - BOX_FATAL("Invalid logging level: " << optarg); - return 2; - } - } - break; - - case '?': - { - BOX_FATAL("Unknown option on command line: " - << "'" << (char)optopt << "'"); - return 2; - } - break; - default: { - BOX_FATAL("Unknown error in getopt: returned " - << "'" << option << "'"); - return 1; + return mLogLevel.ProcessOption(option); } } @@ -351,12 +262,6 @@ int Daemon::Main(const std::string& rDefaultConfigFile, int argc, int Daemon::ProcessOptions(int argc, const char *argv[]) { - #ifdef BOX_RELEASE_BUILD - mLogLevel = Log::NOTICE; - #else - mLogLevel = Log::INFO; - #endif - if (argc == 2 && strcmp(argv[1], "/?") == 0) { Usage(); @@ -368,7 +273,7 @@ int Daemon::ProcessOptions(int argc, const char *argv[]) // reset getopt, just in case anybody used it before. // unfortunately glibc and BSD differ on this point! // http://www.ussg.iu.edu/hypermail/linux/kernel/0305.3/0262.html - #if HAVE_DECL_OPTRESET == 1 || defined WIN32 + #if HAVE_DECL_OPTRESET == 1 || defined BOX_BSD_GETOPT optind = 1; optreset = 1; #elif defined __GLIBC__ @@ -406,13 +311,14 @@ int Daemon::ProcessOptions(int argc, const char *argv[]) return 2; } - Logging::FilterConsole((Log::Level)mLogLevel); - Logging::FilterSyslog ((Log::Level)mLogLevel); + Logging::FilterConsole(mLogLevel.GetCurrentLevel()); + Logging::FilterSyslog (mLogLevel.GetCurrentLevel()); if (mLogFileLevel != Log::INVALID) { mapLogFileLogger.reset( - new FileLogger(mLogFile, mLogFileLevel)); + new FileLogger(mLogFile, mLogFileLevel, + !mLogLevel.mTruncateLogFile)); } return 0; @@ -473,17 +379,17 @@ bool Daemon::Configure(const std::string& rConfigFileName) BOX_ERROR("Failed to load or verify configuration file"); return false; } - + if(!Configure(*apConfig)) { BOX_ERROR("Failed to verify configuration file"); - return false; + return false; } - + // Store configuration mConfigFileName = rConfigFileName; mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); - + return true; } @@ -513,14 +419,14 @@ bool Daemon::Configure(const Configuration& rConfig) BOX_ERROR("Configuration errors: " << errors); return false; } - + // Store configuration mapConfiguration = apConf; - + // Let the derived class have a go at setting up stuff // in the initial process SetupInInitialProcess(); - + return true; } @@ -664,7 +570,7 @@ int Daemon::Main(const std::string &rConfigFileName) // Write PID to file char pid[32]; - int pidsize = sprintf(pid, "%d", (int)getpid()); + int pidsize = snprintf(pid, sizeof(pid), "%d", (int)getpid()); if(::write(pidFile, pid, pidsize) != pidsize) { @@ -676,9 +582,8 @@ int Daemon::Main(const std::string &rConfigFileName) // Set up memory leak reporting #ifdef BOX_MEMORY_LEAK_TESTING { - char filename[256]; - sprintf(filename, "%s.memleaks", DaemonName()); - memleakfinder_setup_exit_report(filename, DaemonName()); + memleakfinder_setup_exit_report(std::string(DaemonName()) + + ".memleaks", DaemonName()); } #endif // BOX_MEMORY_LEAK_TESTING @@ -986,7 +891,9 @@ const Configuration &Daemon::GetConfiguration() const if(mapConfiguration.get() == 0) { // Shouldn't get anywhere near this if a configuration file can't be loaded - THROW_EXCEPTION(ServerException, Internal) + THROW_EXCEPTION_MESSAGE(ServerException, Internal, + "The daemon has not been configured; no config file " + "has been loaded."); } return *mapConfiguration; diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h index 2718c288..b5384918 100644 --- a/lib/server/Daemon.h +++ b/lib/server/Daemon.h @@ -85,7 +85,16 @@ protected: bool IsSingleProcess() { return mSingleProcess; } virtual std::string GetOptionString(); virtual int ProcessOption(signed int option); - + void ResetLogFile() + { + if(mapLogFileLogger.get()) + { + mapLogFileLogger.reset( + new FileLogger(mLogFile, mLogFileLevel, + !mLogLevel.mTruncateLogFile)); + } + } + private: static void SignalHandler(int sigraised); box_time_t GetConfigFileModifiedTime() const; @@ -99,7 +108,7 @@ private: bool mRunInForeground; bool mKeepConsoleOpenAfterFork; bool mHaveConfigFile; - int mLogLevel; // need an int to do math with + Logging::OptionParser mLogLevel; std::string mLogFile; Log::Level mLogFileLevel; std::auto_ptr<FileLogger> mapLogFileLogger; diff --git a/lib/server/Message.h b/lib/server/Message.h index 0d073d49..9f2245ec 100644 --- a/lib/server/Message.h +++ b/lib/server/Message.h @@ -37,10 +37,11 @@ public: // reading and writing with Protocol objects virtual void SetPropertiesFromStreamData(Protocol &rProtocol); - virtual void WritePropertiesToStreamData(Protocol &rProtocol) const; + virtual void WritePropertiesToStreamData(Protocol &rProtocol) const; virtual void LogSysLog(const char *Action) const { } virtual void LogFile(const char *Action, FILE *file) const { } + virtual std::string ToString() const = 0; }; /* diff --git a/lib/server/Protocol.cpp b/lib/server/Protocol.cpp index 382f1c37..0adf9543 100644 --- a/lib/server/Protocol.cpp +++ b/lib/server/Protocol.cpp @@ -17,10 +17,11 @@ #include <new> +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "Protocol.h" #include "ProtocolWire.h" -#include "IOStream.h" -#include "ServerException.h" +#include "SocketStream.h" #include "PartialReadStream.h" #include "ProtocolUncertainStream.h" #include "Logging.h" @@ -44,8 +45,8 @@ // Created: 2003/08/19 // // -------------------------------------------------------------------------- -Protocol::Protocol(IOStream &rStream) -: mrStream(rStream), +Protocol::Protocol(std::auto_ptr<SocketStream> apConn) +: mapConn(apConn), mHandshakeDone(false), mMaxObjectSize(PROTOCOL_DEFAULT_MAXOBJSIZE), mTimeout(PROTOCOL_DEFAULT_TIMEOUT), @@ -103,8 +104,8 @@ void Protocol::Handshake() ::strncpy(hsSend.mIdent, GetProtocolIdentString(), sizeof(hsSend.mIdent)); // Send it - mrStream.Write(&hsSend, sizeof(hsSend)); - mrStream.WriteAllBuffered(); + mapConn->Write(&hsSend, sizeof(hsSend), GetTimeout()); + mapConn->WriteAllBuffered(); // Receive a handshake from the peer PW_Handshake hsReceive; @@ -114,10 +115,10 @@ void Protocol::Handshake() while(bytesToRead > 0) { // Get some data from the stream - int bytesRead = mrStream.Read(readInto, bytesToRead, mTimeout); + int bytesRead = mapConn->Read(readInto, bytesToRead, GetTimeout()); if(bytesRead == 0) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + THROW_EXCEPTION(ConnectionException, Protocol_Timeout) } readInto += bytesRead; bytesToRead -= bytesRead; @@ -127,7 +128,7 @@ void Protocol::Handshake() // Are they the same? if(::memcmp(&hsSend, &hsReceive, sizeof(hsSend)) != 0) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_HandshakeFailed) + THROW_EXCEPTION(ConnectionException, Protocol_HandshakeFailed) } // Mark as done @@ -158,9 +159,10 @@ void Protocol::CheckAndReadHdr(void *hdr) } // Get some data into this header - if(!mrStream.ReadFullBuffer(hdr, sizeof(PW_ObjectHeader), 0 /* not interested in bytes read if this fails */, mTimeout)) + if(!mapConn->ReadFullBuffer(hdr, sizeof(PW_ObjectHeader), + 0 /* not interested in bytes read if this fails */, mTimeout)) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + THROW_EXCEPTION(ConnectionException, Protocol_Timeout) } } @@ -168,8 +170,9 @@ void Protocol::CheckAndReadHdr(void *hdr) // -------------------------------------------------------------------------- // // Function -// Name: Protocol::Recieve() -// Purpose: Recieves an object from the stream, creating it from the factory object type +// Name: Protocol::ReceiveInternal() +// Purpose: Receives an object from the stream, creating it +// from the factory object type // Created: 2003/08/19 // // -------------------------------------------------------------------------- @@ -182,14 +185,14 @@ std::auto_ptr<Message> Protocol::ReceiveInternal() // Hope it's not a stream if(ntohl(objHeader.mObjType) == SPECIAL_STREAM_OBJECT_TYPE) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_StreamWhenObjExpected) + THROW_EXCEPTION(ConnectionException, Protocol_StreamWhenObjExpected) } // Check the object size - u_int32_t objSize = ntohl(objHeader.mObjSize); + uint32_t objSize = ntohl(objHeader.mObjSize); if(objSize < sizeof(objHeader) || objSize > mMaxObjectSize) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_ObjTooBig) + THROW_EXCEPTION(ConnectionException, Protocol_ObjTooBig) } // Create a blank object @@ -199,9 +202,10 @@ std::auto_ptr<Message> Protocol::ReceiveInternal() EnsureBufferAllocated(objSize); // Read data - if(!mrStream.ReadFullBuffer(mpBuffer, objSize - sizeof(objHeader), 0 /* not interested in bytes read if this fails */, mTimeout)) + if(!mapConn->ReadFullBuffer(mpBuffer, objSize - sizeof(objHeader), + 0 /* not interested in bytes read if this fails */, mTimeout)) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + THROW_EXCEPTION(ConnectionException, Protocol_Timeout) } // Setup ready to read out data from the buffer @@ -231,7 +235,7 @@ std::auto_ptr<Message> Protocol::ReceiveInternal() // Exception if not all the data was consumed if(dataLeftOver) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_BadCommandRecieved) + THROW_EXCEPTION(ConnectionException, Protocol_BadCommandRecieved) } return obj; @@ -240,7 +244,7 @@ std::auto_ptr<Message> Protocol::ReceiveInternal() // -------------------------------------------------------------------------- // // Function -// Name: Protocol::Send() +// Name: Protocol::SendInternal() // Purpose: Send an object to the other side of the connection. // Created: 2003/08/19 // @@ -292,8 +296,8 @@ void Protocol::SendInternal(const Message &rObject) pobjHeader->mObjType = htonl(rObject.GetType()); // Write data - mrStream.Write(mpBuffer, writtenSize); - mrStream.WriteAllBuffered(); + mapConn->Write(mpBuffer, writtenSize, GetTimeout()); + mapConn->WriteAllBuffered(); } // -------------------------------------------------------------------------- @@ -346,7 +350,7 @@ void Protocol::EnsureBufferAllocated(int Size) #define READ_CHECK_BYTES_AVAILABLE(bytesRequired) \ if((mReadOffset + (int)(bytesRequired)) > mValidDataSize) \ { \ - THROW_EXCEPTION(ConnectionException, Conn_Protocol_BadCommandRecieved) \ + THROW_EXCEPTION(ConnectionException, Protocol_BadCommandRecieved) \ } // -------------------------------------------------------------------------- @@ -619,7 +623,7 @@ void Protocol::Write(const std::string &rValue) // -------------------------------------------------------------------------- // // Function -// Name: Protocol::ReceieveStream() +// Name: Protocol::ReceiveStream() // Purpose: Receive a stream from the remote side // Created: 2003/08/26 // @@ -633,11 +637,11 @@ std::auto_ptr<IOStream> Protocol::ReceiveStream() // Hope it's not an object if(ntohl(objHeader.mObjType) != SPECIAL_STREAM_OBJECT_TYPE) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_ObjWhenStreamExpected) + THROW_EXCEPTION(ConnectionException, Protocol_ObjWhenStreamExpected) } // Get the stream size - u_int32_t streamSize = ntohl(objHeader.mObjSize); + uint32_t streamSize = ntohl(objHeader.mObjSize); // Inform sub class InformStreamReceiving(streamSize); @@ -647,13 +651,13 @@ std::auto_ptr<IOStream> Protocol::ReceiveStream() { BOX_TRACE("Receiving stream, size uncertain"); return std::auto_ptr<IOStream>( - new ProtocolUncertainStream(mrStream)); + new ProtocolUncertainStream(*mapConn)); } else { BOX_TRACE("Receiving stream, size " << streamSize << " bytes"); return std::auto_ptr<IOStream>( - new PartialReadStream(mrStream, streamSize)); + new PartialReadStream(*mapConn, streamSize)); } } @@ -709,7 +713,7 @@ void Protocol::SendStream(IOStream &rStream) objHeader.mObjType = htonl(SPECIAL_STREAM_OBJECT_TYPE); // Write header - mrStream.Write(&objHeader, sizeof(objHeader)); + mapConn->Write(&objHeader, sizeof(objHeader), GetTimeout()); // Could be sent in one of two ways if(uncertainSize) { @@ -744,7 +748,7 @@ void Protocol::SendStream(IOStream &rStream) // Send final byte to finish the stream BOX_TRACE("Sending end of stream byte"); uint8_t endOfStream = ProtocolStreamHeader_EndOfStream; - mrStream.Write(&endOfStream, 1); + mapConn->Write(&endOfStream, 1, GetTimeout()); BOX_TRACE("Sent end of stream byte"); } catch(...) @@ -759,13 +763,14 @@ void Protocol::SendStream(IOStream &rStream) else { // Fixed size stream, send it all in one go - if(!rStream.CopyStreamTo(mrStream, mTimeout, 4096 /* slightly larger buffer */)) + if(!rStream.CopyStreamTo(*mapConn, GetTimeout(), + 4096 /* slightly larger buffer */)) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_TimeOutWhenSendingStream) + THROW_EXCEPTION(ConnectionException, Protocol_TimeOutWhenSendingStream) } } // Make sure everything is written - mrStream.WriteAllBuffered(); + mapConn->WriteAllBuffered(); } @@ -816,7 +821,7 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) Block[-1] = header; // Write everything out - mrStream.Write(Block - 1, writeSize + 1); + mapConn->Write(Block - 1, writeSize + 1, GetTimeout()); BOX_TRACE("Sent " << (writeSize+1) << " bytes to stream"); // move the remainer to the beginning of the block for the next time round @@ -831,12 +836,12 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) // -------------------------------------------------------------------------- // // Function -// Name: Protocol::InformStreamReceiving(u_int32_t) +// Name: Protocol::InformStreamReceiving(uint32_t) // Purpose: Informs sub classes about streams being received // Created: 2003/10/27 // // -------------------------------------------------------------------------- -void Protocol::InformStreamReceiving(u_int32_t Size) +void Protocol::InformStreamReceiving(uint32_t Size) { if(GetLogToSysLog()) { @@ -863,12 +868,12 @@ void Protocol::InformStreamReceiving(u_int32_t Size) // -------------------------------------------------------------------------- // // Function -// Name: Protocol::InformStreamSending(u_int32_t) +// Name: Protocol::InformStreamSending(uint32_t) // Purpose: Informs sub classes about streams being sent // Created: 2003/10/27 // // -------------------------------------------------------------------------- -void Protocol::InformStreamSending(u_int32_t Size) +void Protocol::InformStreamSending(uint32_t Size) { if(GetLogToSysLog()) { @@ -1177,6 +1182,12 @@ const uint16_t Protocol::sProtocolStreamHeaderLengths[256] = 0 // 255 = special (reserved) }; +int64_t Protocol::GetBytesRead() const +{ + return mapConn->GetBytesRead(); +} - - +int64_t Protocol::GetBytesWritten() const +{ + return mapConn->GetBytesWritten(); +} diff --git a/lib/server/Protocol.h b/lib/server/Protocol.h index 42cb0ff8..fbe6461c 100644 --- a/lib/server/Protocol.h +++ b/lib/server/Protocol.h @@ -19,6 +19,7 @@ #include "Message.h" class IOStream; +class SocketStream; // default timeout is 15 minutes #define PROTOCOL_DEFAULT_TIMEOUT (15*60*1000) @@ -36,7 +37,7 @@ class IOStream; class Protocol { public: - Protocol(IOStream &rStream); + Protocol(std::auto_ptr<SocketStream> apConn); virtual ~Protocol(); private: @@ -66,14 +67,14 @@ public: // Purpose: Sets the timeout for sending and reciving // Created: 2003/08/19 // - // -------------------------------------------------------------------------- + // -------------------------------------------------------------------------- void SetTimeout(int NewTimeout) {mTimeout = NewTimeout;} // -------------------------------------------------------------------------- // // Function - // Name: Protocol::GetTimeout() + // Name: Protocol::GetTimeout() // Purpose: Get current timeout for sending and receiving // Created: 2003/09/06 // @@ -175,6 +176,8 @@ public: FILE *GetLogToFile() { return mLogToFile; } void SetLogToSysLog(bool Log = false) {mLogToSysLog = Log;} void SetLogToFile(FILE *File = 0) {mLogToFile = File;} + int64_t GetBytesRead() const; + int64_t GetBytesWritten() const; protected: virtual std::auto_ptr<Message> MakeMessage(int ObjType) = 0; @@ -183,14 +186,14 @@ protected: void CheckAndReadHdr(void *hdr); // don't use type here to avoid dependency // Will be used for logging - virtual void InformStreamReceiving(u_int32_t Size); - virtual void InformStreamSending(u_int32_t Size); + virtual void InformStreamReceiving(uint32_t Size); + virtual void InformStreamSending(uint32_t Size); private: void EnsureBufferAllocated(int Size); int SendStreamSendBlock(uint8_t *Block, int BytesInBlock); - IOStream &mrStream; + std::auto_ptr<SocketStream> mapConn; bool mHandshakeDone; unsigned int mMaxObjectSize; int mTimeout; @@ -208,4 +211,3 @@ class ProtocolContext }; #endif // PROTOCOL__H - diff --git a/lib/server/ProtocolUncertainStream.cpp b/lib/server/ProtocolUncertainStream.cpp index 84a213a8..aeb15816 100644 --- a/lib/server/ProtocolUncertainStream.cpp +++ b/lib/server/ProtocolUncertainStream.cpp @@ -8,8 +8,9 @@ // -------------------------------------------------------------------------- #include "Box.h" +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "ProtocolUncertainStream.h" -#include "ServerException.h" #include "Protocol.h" #include "MemLeakFindOn.h" @@ -172,7 +173,7 @@ IOStream::pos_type ProtocolUncertainStream::BytesLeftToRead() // Created: 2003/12/05 // // -------------------------------------------------------------------------- -void ProtocolUncertainStream::Write(const void *pBuffer, int NBytes) +void ProtocolUncertainStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(ServerException, CantWriteToProtocolUncertainStream) } diff --git a/lib/server/ProtocolUncertainStream.h b/lib/server/ProtocolUncertainStream.h index 4954cf88..2e97ba6a 100644 --- a/lib/server/ProtocolUncertainStream.h +++ b/lib/server/ProtocolUncertainStream.h @@ -33,7 +33,8 @@ private: public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); diff --git a/lib/server/ProtocolWire.h b/lib/server/ProtocolWire.h index ff62b66e..6dee445b 100644 --- a/lib/server/ProtocolWire.h +++ b/lib/server/ProtocolWire.h @@ -26,8 +26,8 @@ typedef struct typedef struct { - u_int32_t mObjSize; - u_int32_t mObjType; + uint32_t mObjSize; + uint32_t mObjType; } PW_ObjectHeader; #define SPECIAL_STREAM_OBJECT_TYPE 0xffffffff diff --git a/lib/server/SSLLib.cpp b/lib/server/SSLLib.cpp index 004d2d98..1bcadb0d 100644 --- a/lib/server/SSLLib.cpp +++ b/lib/server/SSLLib.cpp @@ -18,9 +18,10 @@ #include <wincrypt.h> #endif +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "CryptoUtils.h" #include "SSLLib.h" -#include "ServerException.h" #include "MemLeakFindOn.h" @@ -79,7 +80,7 @@ void SSLLib::Initialise() BOX_LOG_WIN_ERROR("Failed to release crypto context"); } } -#elif HAVE_RANDOM_DEVICE +#elif defined HAVE_RANDOM_DEVICE if(::RAND_load_file(RANDOM_DEVICE, 1024) != 1024) { THROW_EXCEPTION(ServerException, SSLRandomInitFailed) diff --git a/lib/server/ServerControl.cpp b/lib/server/ServerControl.cpp index b9650cee..f1a718df 100644 --- a/lib/server/ServerControl.cpp +++ b/lib/server/ServerControl.cpp @@ -15,13 +15,14 @@ #include <signal.h> #endif +#include "BoxTime.h" +#include "IOStreamGetLine.h" #include "ServerControl.h" #include "Test.h" #ifdef WIN32 #include "WinNamedPipeStream.h" -#include "IOStreamGetLine.h" #include "BoxPortsAndFiles.h" static std::string sPipeName; @@ -197,18 +198,18 @@ bool KillServer(int pid, bool WaitForProcess) } #endif - for (int i = 0; i < 30; i++) + printf("Waiting for server to die (pid %d): ", pid); + + for (int i = 0; i < 300; i++) { - if (i == 0) + if (i % 10 == 0) { - printf("Waiting for server to die (pid %d): ", pid); + printf("."); + fflush(stdout); } - printf("."); - fflush(stdout); - if (!ServerIsAlive(pid)) break; - ::sleep(1); + ShortSleep(MilliSecondsToBoxTime(100), false); if (!ServerIsAlive(pid)) break; } @@ -226,3 +227,64 @@ bool KillServer(int pid, bool WaitForProcess) return !ServerIsAlive(pid); } +bool KillServer(std::string pid_file, bool WaitForProcess) +{ + FileStream fs(pid_file); + IOStreamGetLine getline(fs); + std::string line = getline.GetLine(); + int pid = atoi(line.c_str()); + bool status = KillServer(pid, WaitForProcess); + TEST_EQUAL_LINE(true, status, std::string("kill(") + pid_file + ")"); + +#ifdef WIN32 + if(WaitForProcess) + { + int unlink_result = unlink(pid_file.c_str()); + TEST_EQUAL_LINE(0, unlink_result, std::string("unlink ") + pid_file); + if(unlink_result != 0) + { + return false; + } + } +#endif + + return status; +} + +int StartDaemon(int current_pid, const std::string& cmd_line, const char* pid_file) +{ + TEST_THAT_OR(current_pid == 0, return 0); + + int new_pid = LaunchServer(cmd_line, pid_file); + TEST_THAT_OR(new_pid != -1 && new_pid != 0, return 0); + + ::sleep(1); + TEST_THAT_OR(ServerIsAlive(new_pid), return 0); + return new_pid; +} + +bool StopDaemon(int current_pid, const std::string& pid_file, + const std::string& memleaks_file, bool wait_for_process) +{ + TEST_THAT_OR(current_pid != 0, return false); + TEST_THAT_OR(ServerIsAlive(current_pid), return false); + TEST_THAT_OR(KillServer(current_pid, wait_for_process), return false); + ::sleep(1); + + TEST_THAT_OR(!ServerIsAlive(current_pid), return false); + + #ifdef WIN32 + int unlink_result = unlink(pid_file.c_str()); + TEST_EQUAL_LINE(0, unlink_result, std::string("unlink ") + pid_file); + if(unlink_result != 0) + { + return false; + } + #else + TestRemoteProcessMemLeaks(memleaks_file.c_str()); + #endif + + return true; +} + + diff --git a/lib/server/ServerControl.h b/lib/server/ServerControl.h index b2e51864..be2464c1 100644 --- a/lib/server/ServerControl.h +++ b/lib/server/ServerControl.h @@ -5,6 +5,10 @@ bool HUPServer(int pid); bool KillServer(int pid, bool WaitForProcess = false); +bool KillServer(std::string pid_file, bool WaitForProcess = false); +int StartDaemon(int current_pid, const std::string& cmd_line, const char* pid_file); +bool StopDaemon(int current_pid, const std::string& pid_file, + const std::string& memleaks_file, bool wait_for_process); #ifdef WIN32 #include "WinNamedPipeStream.h" diff --git a/lib/server/ServerException.h b/lib/server/ServerException.h deleted file mode 100644 index 8851b90a..00000000 --- a/lib/server/ServerException.h +++ /dev/null @@ -1,46 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: ServerException.h -// Purpose: Exception -// Created: 2003/07/08 -// -// -------------------------------------------------------------------------- - -#ifndef SERVEREXCEPTION__H -#define SERVEREXCEPTION__H - -// Compatibility header -#include "autogen_ServerException.h" -#include "autogen_ConnectionException.h" - -// Rename old connection exception names to new names without Conn_ prefix -// This is all because ConnectionException used to be derived from ServerException -// with some funky magic with subtypes. Perhaps a little unreliable, and the -// usefulness of it never really was used. -#define Conn_SocketWriteError SocketWriteError -#define Conn_SocketReadError SocketReadError -#define Conn_SocketNameLookupError SocketNameLookupError -#define Conn_SocketShutdownError SocketShutdownError -#define Conn_SocketConnectError SocketConnectError -#define Conn_TLSHandshakeFailed TLSHandshakeFailed -#define Conn_TLSShutdownFailed TLSShutdownFailed -#define Conn_TLSWriteFailed TLSWriteFailed -#define Conn_TLSReadFailed TLSReadFailed -#define Conn_TLSNoPeerCertificate TLSNoPeerCertificate -#define Conn_TLSPeerCertificateInvalid TLSPeerCertificateInvalid -#define Conn_TLSClosedWhenWriting TLSClosedWhenWriting -#define Conn_TLSHandshakeTimedOut TLSHandshakeTimedOut -#define Conn_Protocol_Timeout Protocol_Timeout -#define Conn_Protocol_ObjTooBig Protocol_ObjTooBig -#define Conn_Protocol_BadCommandRecieved Protocol_BadCommandRecieved -#define Conn_Protocol_UnknownCommandRecieved Protocol_UnknownCommandRecieved -#define Conn_Protocol_TriedToExecuteReplyCommand Protocol_TriedToExecuteReplyCommand -#define Conn_Protocol_UnexpectedReply Protocol_UnexpectedReply -#define Conn_Protocol_HandshakeFailed Protocol_HandshakeFailed -#define Conn_Protocol_StreamWhenObjExpected Protocol_StreamWhenObjExpected -#define Conn_Protocol_ObjWhenStreamExpected Protocol_ObjWhenStreamExpected -#define Conn_Protocol_TimeOutWhenSendingStream Protocol_TimeOutWhenSendingStream - -#endif // SERVEREXCEPTION__H - diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h index a9b56169..3f6eed7e 100644 --- a/lib/server/ServerStream.h +++ b/lib/server/ServerStream.h @@ -17,6 +17,7 @@ #include <sys/wait.h> #endif +#include "autogen_ServerException.h" #include "Daemon.h" #include "SocketListen.h" #include "Utils.h" @@ -286,7 +287,7 @@ public: #endif // The derived class does some server magic with the connection - HandleConnection(*connection); + HandleConnection(connection); // Since rChildExit == true, the forked process will call _exit() on return from this fn return; @@ -305,7 +306,7 @@ public: #endif // !WIN32 // Just handle in this process SetProcessTitle("handling"); - HandleConnection(*connection); + HandleConnection(connection); SetProcessTitle("idle"); #ifndef WIN32 } @@ -344,10 +345,20 @@ public: p = ::waitpid(0 /* any child in process group */, &status, WNOHANG); - if(p == -1 && errno != ECHILD && errno != EINTR) + if(p == -1) { - THROW_EXCEPTION(ServerException, - ServerWaitOnChildError) + if (errno == ECHILD || errno == EINTR) + { + // Nothing actually happened, so there's no reason + // to wait again. + break; + } + else + { + THROW_SYS_ERROR("Failed to wait for daemon child " + "process", ServerException, + ServerWaitOnChildError); + } } else if(p == 0) { @@ -377,12 +388,12 @@ public: } #endif - virtual void HandleConnection(StreamType &rStream) + virtual void HandleConnection(std::auto_ptr<StreamType> apStream) { - Connection(rStream); + Connection(apStream); } - virtual void Connection(StreamType &rStream) = 0; + virtual void Connection(std::auto_ptr<StreamType> apStream) = 0; protected: // For checking code in derived classes -- use if you have an algorithm which diff --git a/lib/server/ServerTLS.h b/lib/server/ServerTLS.h index a74a671e..f748f4b2 100644 --- a/lib/server/ServerTLS.h +++ b/lib/server/ServerTLS.h @@ -52,18 +52,19 @@ public: std::string certFile(serverconf.GetKeyValue("CertificateFile")); std::string keyFile(serverconf.GetKeyValue("PrivateKeyFile")); std::string caFile(serverconf.GetKeyValue("TrustedCAsFile")); - mContext.Initialise(true /* as server */, certFile.c_str(), keyFile.c_str(), caFile.c_str()); + mContext.Initialise(true /* as server */, certFile.c_str(), + keyFile.c_str(), caFile.c_str()); // Then do normal stream server stuff ServerStream<SocketStreamTLS, Port, ListenBacklog, ForkToHandleRequests>::Run2(rChildExit); } - virtual void HandleConnection(SocketStreamTLS &rStream) + virtual void HandleConnection(std::auto_ptr<SocketStreamTLS> apStream) { - rStream.Handshake(mContext, true /* is server */); + apStream->Handshake(mContext, true /* is server */); // this-> in next line required to build under some gcc versions - this->Connection(rStream); + this->Connection(apStream); } private: diff --git a/lib/server/Socket.cpp b/lib/server/Socket.cpp index f2a4996b..c9c1773d 100644 --- a/lib/server/Socket.cpp +++ b/lib/server/Socket.cpp @@ -24,8 +24,9 @@ #include <string.h> #include <stdio.h> +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "Socket.h" -#include "ServerException.h" #include "MemLeakFindOn.h" @@ -69,12 +70,12 @@ void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, } else { - THROW_EXCEPTION(ConnectionException, Conn_SocketNameLookupError); + THROW_EXCEPTION(ConnectionException, SocketNameLookupError); } } else { - THROW_EXCEPTION(ConnectionException, Conn_SocketNameLookupError); + THROW_EXCEPTION(ConnectionException, SocketNameLookupError); } } break; diff --git a/lib/server/SocketListen.h b/lib/server/SocketListen.h index 39c60ba6..39fe7e24 100644 --- a/lib/server/SocketListen.h +++ b/lib/server/SocketListen.h @@ -29,8 +29,9 @@ #include <memory> #include <string> +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "Socket.h" -#include "ServerException.h" #include "MemLeakFindOn.h" @@ -73,7 +74,8 @@ private: // Created: 2003/07/31 // // -------------------------------------------------------------------------- -template<typename SocketType, int ListenBacklog = 128, typename SocketLockingType = _NoSocketLocking, int MaxMultiListenSockets = 16> +template<typename SocketType, int ListenBacklog = 128, + typename SocketLockingType = _NoSocketLocking, int MaxMultiListenSockets = 16> class SocketListen { public: @@ -112,10 +114,9 @@ public: if(::close(mSocketHandle) == -1) #endif { - BOX_LOG_SOCKET_ERROR(mType, mName, mPort, - "Failed to close network socket"); - THROW_EXCEPTION(ServerException, - SocketCloseError) + THROW_EXCEPTION_MESSAGE(ServerException, SocketCloseError, + BOX_SOCKET_ERROR_MESSAGE(mType, mName, mPort, + "Failed to close network socket")); } } mSocketHandle = -1; @@ -152,9 +153,9 @@ public: 0 /* let OS choose protocol */); if(mSocketHandle == -1) { - BOX_LOG_SOCKET_ERROR(Type, Name, Port, - "Failed to create a network socket"); - THROW_EXCEPTION(ServerException, SocketOpenError) + THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, + BOX_SOCKET_ERROR_MESSAGE(Type, Name, Port, + "Failed to create a network socket")); } // Set an option to allow reuse (useful for -HUP situations!) @@ -167,28 +168,28 @@ public: &option, sizeof(option)) == -1) #endif { - BOX_LOG_SOCKET_ERROR(Type, Name, Port, - "Failed to set socket options"); - THROW_EXCEPTION(ServerException, SocketOpenError) + THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, + BOX_SOCKET_ERROR_MESSAGE(Type, Name, Port, + "Failed to set socket options")); } // Bind it to the right port, and start listening if(::bind(mSocketHandle, &addr.sa_generic, addrLen) == -1 || ::listen(mSocketHandle, ListenBacklog) == -1) { - int err_number = errno; - - BOX_LOG_SOCKET_ERROR(Type, Name, Port, - "Failed to bind socket"); - - // Dispose of the socket - ::close(mSocketHandle); - mSocketHandle = -1; - - THROW_SYS_FILE_ERRNO("Failed to bind or listen " - "on socket", Name, err_number, - ServerException, SocketBindError); - } + try + { + THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, + BOX_SOCKET_ERROR_MESSAGE(Type, Name, Port, + "Failed to bind socket to name/port")); + } + catch(ServerException &e) // finally + { + // Dispose of the socket + Close(); + throw; + } + } } // ------------------------------------------------------------------ @@ -248,10 +249,10 @@ public: } else { - BOX_LOG_SOCKET_ERROR(mType, mName, mPort, - "Failed to poll connection"); - THROW_EXCEPTION(ServerException, - SocketPollError) + THROW_EXCEPTION_MESSAGE(ServerException, + SocketPollError, + BOX_SOCKET_ERROR_MESSAGE(mType, mName, + mPort, "Failed to poll connection")); } break; case 0: // timed out @@ -268,9 +269,9 @@ public: // Got socket (or error), unlock (implicit in destruction) if(sock == -1) { - BOX_LOG_SOCKET_ERROR(mType, mName, mPort, - "Failed to accept connection"); - THROW_EXCEPTION(ServerException, SocketAcceptError) + THROW_EXCEPTION_MESSAGE(ServerException, SocketAcceptError, + BOX_SOCKET_ERROR_MESSAGE(mType, mName, + mPort, "Failed to accept connection")); } // Log it diff --git a/lib/server/SocketStream.cpp b/lib/server/SocketStream.cpp index 6ef4b8d1..edb5e5b8 100644 --- a/lib/server/SocketStream.cpp +++ b/lib/server/SocketStream.cpp @@ -25,10 +25,24 @@ #include <ucred.h> #endif +#ifdef HAVE_BSD_UNISTD_H + #include <bsd/unistd.h> +#endif + +#ifdef HAVE_SYS_PARAM_H + #include <sys/param.h> +#endif + +#ifdef HAVE_SYS_UCRED_H + #include <sys/ucred.h> +#endif + +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "SocketStream.h" -#include "ServerException.h" #include "CommonException.h" #include "Socket.h" +#include "Utils.h" #include "MemLeakFindOn.h" @@ -162,25 +176,31 @@ void SocketStream::Open(Socket::Type Type, const std::string& rName, int Port) 0 /* let OS choose protocol */); if(mSocketHandle == INVALID_SOCKET_VALUE) { - BOX_LOG_SOCKET_ERROR(Type, rName, Port, - "Failed to create a network socket"); - THROW_EXCEPTION(ServerException, SocketOpenError) + THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, + BOX_SOCKET_ERROR_MESSAGE(Type, rName, Port, + "Failed to create a network socket")); } // Connect it if(::connect(mSocketHandle, &addr.sa_generic, addrLen) == -1) { // Dispose of the socket - BOX_LOG_SOCKET_ERROR(Type, rName, Port, - "Failed to connect to socket"); + try + { + THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, + BOX_SOCKET_ERROR_MESSAGE(Type, rName, Port, + "Failed to connect to socket")); + } + catch(ServerException &e) + { #ifdef WIN32 - ::closesocket(mSocketHandle); + ::closesocket(mSocketHandle); #else // !WIN32 - ::close(mSocketHandle); + ::close(mSocketHandle); #endif // WIN32 - - mSocketHandle = INVALID_SOCKET_VALUE; - THROW_EXCEPTION(ConnectionException, Conn_SocketConnectError) + mSocketHandle = INVALID_SOCKET_VALUE; + throw; + } } ResetCounters(); @@ -199,7 +219,9 @@ void SocketStream::Open(Socket::Type Type, const std::string& rName, int Port) // -------------------------------------------------------------------------- int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) { - if(mSocketHandle == INVALID_SOCKET_VALUE) + CheckForMissingTimeout(Timeout); + + if(mSocketHandle == INVALID_SOCKET_VALUE) { THROW_EXCEPTION(ServerException, BadSocketHandle) } @@ -210,7 +232,7 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) p.fd = mSocketHandle; p.events = POLLIN; p.revents = 0; - switch(::poll(&p, 1, (Timeout == IOStream::TimeOutInfinite)?INFTIM:Timeout)) + switch(::poll(&p, 1, PollTimeout(Timeout, 0))) { case -1: // error @@ -256,7 +278,7 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) // Other error BOX_LOG_SYS_ERROR("Failed to read from socket"); THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError); + SocketReadError); } } @@ -270,6 +292,41 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) return r; } +bool SocketStream::Poll(short Events, int Timeout) +{ + // Wait for data to send. + struct pollfd p; + p.fd = GetSocketHandle(); + p.events = Events; + p.revents = 0; + + box_time_t start = GetCurrentBoxTime(); + int result; + + do + { + result = ::poll(&p, 1, PollTimeout(Timeout, start)); + } + while(result == -1 && errno == EINTR); + + switch(result) + { + case -1: + // error - Bad! + THROW_SYS_ERROR("Failed to poll socket", ServerException, + SocketPollError); + break; + + case 0: + // Condition not met, timed out + return false; + + default: + // good to go! + return true; + } +} + // -------------------------------------------------------------------------- // // Function @@ -278,20 +335,21 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void SocketStream::Write(const void *pBuffer, int NBytes) +void SocketStream::Write(const void *pBuffer, int NBytes, int Timeout) { - if(mSocketHandle == INVALID_SOCKET_VALUE) + if(mSocketHandle == INVALID_SOCKET_VALUE) { THROW_EXCEPTION(ServerException, BadSocketHandle) } - + // Buffer in byte sized type. ASSERT(sizeof(char) == 1); const char *buffer = (char *)pBuffer; - + // Bytes left to send int bytesLeft = NBytes; - + box_time_t start = GetCurrentBoxTime(); + while(bytesLeft > 0) { // Try to send. @@ -304,41 +362,30 @@ void SocketStream::Write(const void *pBuffer, int NBytes) { // Error. mWriteClosed = true; // assume can't write again - BOX_LOG_SYS_ERROR("Failed to write to socket"); - THROW_EXCEPTION(ConnectionException, - Conn_SocketWriteError); + THROW_SYS_ERROR("Failed to write to socket", + ConnectionException, SocketWriteError); } - + // Knock off bytes sent bytesLeft -= sent; // Move buffer pointer buffer += sent; mBytesWritten += sent; - + // Need to wait until it can send again? if(bytesLeft > 0) { - BOX_TRACE("Waiting to send data on socket " << + BOX_TRACE("Waiting to send data on socket " << mSocketHandle << " (" << bytesLeft << " of " << NBytes << " bytes left)"); - - // Wait for data to send. - struct pollfd p; - p.fd = mSocketHandle; - p.events = POLLOUT; - p.revents = 0; - - if(::poll(&p, 1, 16000 /* 16 seconds */) == -1) + + if(!Poll(POLLOUT, PollTimeout(Timeout, start))) { - // Don't exception if it's just a signal - if(errno != EINTR) - { - BOX_LOG_SYS_ERROR("Failed to poll " - "socket"); - THROW_EXCEPTION(ServerException, - SocketPollError) - } + THROW_EXCEPTION_MESSAGE(ConnectionException, + Protocol_Timeout, "Timed out waiting " + "to send " << bytesLeft << " of " << + NBytes << " bytes"); } } } @@ -354,7 +401,7 @@ void SocketStream::Write(const void *pBuffer, int NBytes) // -------------------------------------------------------------------------- void SocketStream::Close() { - if(mSocketHandle == INVALID_SOCKET_VALUE) + if(mSocketHandle == INVALID_SOCKET_VALUE) { THROW_EXCEPTION(ServerException, BadSocketHandle) } @@ -385,19 +432,19 @@ void SocketStream::Shutdown(bool Read, bool Write) { THROW_EXCEPTION(ServerException, BadSocketHandle) } - + // Do anything? if(!Read && !Write) return; - + int how = SHUT_RDWR; if(Read && !Write) how = SHUT_RD; if(!Read && Write) how = SHUT_WR; - + // Shut it down! if(::shutdown(mSocketHandle, how) == -1) { BOX_LOG_SYS_ERROR("Failed to shutdown socket"); - THROW_EXCEPTION(ConnectionException, Conn_SocketShutdownError) + THROW_EXCEPTION(ConnectionException, SocketShutdownError) } } @@ -478,8 +525,13 @@ bool SocketStream::GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut) if(::getsockopt(mSocketHandle, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) { +#ifdef HAVE_STRUCT_UCRED_UID rUidOut = cred.uid; rGidOut = cred.gid; +#else // HAVE_STRUCT_UCRED_CR_UID + rUidOut = cred.cr_uid; + rGidOut = cred.cr_gid; +#endif return true; } @@ -509,3 +561,11 @@ bool SocketStream::GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut) return false; } +void SocketStream::CheckForMissingTimeout(int Timeout) +{ + if (Timeout == IOStream::TimeOutInfinite) + { + BOX_WARNING("Network operation started with no timeout!"); + DumpStackBacktrace(); + } +} diff --git a/lib/server/SocketStream.h b/lib/server/SocketStream.h index 2fb5e391..fd57af8f 100644 --- a/lib/server/SocketStream.h +++ b/lib/server/SocketStream.h @@ -10,6 +10,13 @@ #ifndef SOCKETSTREAM__H #define SOCKETSTREAM__H +#include <climits> + +#ifdef HAVE_SYS_POLL_H +# include <sys/poll.h> +#endif + +#include "BoxTime.h" #include "IOStream.h" #include "Socket.h" @@ -41,7 +48,16 @@ public: void Attach(int socket); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); + + // Why not inherited from IOStream? Never mind, we want to enforce + // supplying a timeout for network operations anyway. + virtual void Write(const std::string& rBuffer, int Timeout) + { + IOStream::Write(rBuffer, Timeout); + } + virtual void Close(); virtual bool StreamDataLeft(); virtual bool StreamClosed(); @@ -53,6 +69,42 @@ public: protected: void MarkAsReadClosed() {mReadClosed = true;} void MarkAsWriteClosed() {mWriteClosed = true;} + void CheckForMissingTimeout(int Timeout); + + // Converts a timeout in milliseconds (or IOStream::TimeOutInfinite) + // into one that can be passed to poll() (also in milliseconds), also + // compensating for time elapsed since the wait should have started, + // if known. + int PollTimeout(int timeout, box_time_t start_time) + { + if (timeout == IOStream::TimeOutInfinite) + { + return INFTIM; + } + + if (start_time == 0) + { + return timeout; // no adjustment possible + } + + box_time_t end_time = start_time + MilliSecondsToBoxTime(timeout); + box_time_t now = GetCurrentBoxTime(); + box_time_t remaining = end_time - now; + + if (remaining < 0) + { + return 0; // no delay + } + else if (BoxTimeToMilliSeconds(remaining) > INT_MAX) + { + return INT_MAX; + } + else + { + return (int) BoxTimeToMilliSeconds(remaining); + } + } + bool Poll(short Events, int Timeout); private: tOSSocketHandle mSocketHandle; diff --git a/lib/server/SocketStreamTLS.cpp b/lib/server/SocketStreamTLS.cpp index 576b53a2..e6299bfa 100644 --- a/lib/server/SocketStreamTLS.cpp +++ b/lib/server/SocketStreamTLS.cpp @@ -19,9 +19,10 @@ #include <poll.h> #endif +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "BoxTime.h" #include "CryptoUtils.h" -#include "ServerException.h" #include "SocketStreamTLS.h" #include "SSLLib.h" #include "TLSContext.h" @@ -131,7 +132,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) tOSSocketHandle socket = GetSocketHandle(); BIO_set_fd(mpBIO, socket, BIO_NOCLOSE); - + // Then the SSL object mpSSL = ::SSL_new(rContext.GetRawContext()); if(mpSSL == 0) @@ -154,7 +155,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) THROW_EXCEPTION(ServerException, SocketSetNonBlockingFailed) } #endif - + // FIXME: This is less portable than the above. However, it MAY be needed // for cygwin, which has/had bugs with fcntl // @@ -196,7 +197,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) if(WaitWhenRetryRequired(se, TLS_HANDSHAKE_TIMEOUT) == false) { // timed out - THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeTimedOut) + THROW_EXCEPTION(ConnectionException, TLSHandshakeTimedOut) } break; @@ -205,12 +206,12 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) if(IsServer) { CryptoUtils::LogError("accepting connection"); - THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) + THROW_EXCEPTION(ConnectionException, TLSHandshakeFailed) } else { CryptoUtils::LogError("connecting"); - THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) + THROW_EXCEPTION(ConnectionException, TLSHandshakeFailed) } } } @@ -222,23 +223,25 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) // // Function // Name: WaitWhenRetryRequired(int, int) -// Purpose: Waits until the condition required by the TLS layer is met. -// Returns true if the condition is met, false if timed out. +// Purpose: Waits until the condition required by the TLS layer +// is met. Returns true if the condition is met, false +// if timed out. // Created: 2003/08/15 // // -------------------------------------------------------------------------- bool SocketStreamTLS::WaitWhenRetryRequired(int SSLErrorCode, int Timeout) { - struct pollfd p; - p.fd = GetSocketHandle(); + CheckForMissingTimeout(Timeout); + + short events; switch(SSLErrorCode) { case SSL_ERROR_WANT_READ: - p.events = POLLIN; + events = POLLIN; break; case SSL_ERROR_WANT_WRITE: - p.events = POLLOUT; + events = POLLOUT; break; default: @@ -246,45 +249,8 @@ bool SocketStreamTLS::WaitWhenRetryRequired(int SSLErrorCode, int Timeout) THROW_EXCEPTION(ServerException, Internal) break; } - p.revents = 0; - - int64_t start, end; - start = BoxTimeToMilliSeconds(GetCurrentBoxTime()); - end = start + Timeout; - int result; - - do - { - int64_t now = BoxTimeToMilliSeconds(GetCurrentBoxTime()); - int poll_timeout = (int)(end - now); - if (poll_timeout < 0) poll_timeout = 0; - if (Timeout == IOStream::TimeOutInfinite) - { - poll_timeout = INFTIM; - } - result = ::poll(&p, 1, poll_timeout); - } - while(result == -1 && errno == EINTR); - - switch(result) - { - case -1: - // error - Bad! - THROW_EXCEPTION(ServerException, SocketPollError) - break; - - case 0: - // Condition not met, timed out - return false; - break; - - default: - // good to go! - return true; - break; - } - return true; + return Poll(events, Timeout); } // -------------------------------------------------------------------------- @@ -297,6 +263,7 @@ bool SocketStreamTLS::WaitWhenRetryRequired(int SSLErrorCode, int Timeout) // -------------------------------------------------------------------------- int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) { + CheckForMissingTimeout(Timeout); if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} // Make sure zero byte reads work as expected @@ -304,7 +271,7 @@ int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) { return 0; } - + while(true) { int r = ::SSL_read(mpSSL, pBuffer, NBytes); @@ -337,7 +304,7 @@ int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) default: CryptoUtils::LogError("reading"); - THROW_EXCEPTION(ConnectionException, Conn_TLSReadFailed) + THROW_EXCEPTION(ConnectionException, TLSReadFailed) break; } } @@ -351,23 +318,23 @@ int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) // Created: 2003/08/06 // // -------------------------------------------------------------------------- -void SocketStreamTLS::Write(const void *pBuffer, int NBytes) +void SocketStreamTLS::Write(const void *pBuffer, int NBytes, int Timeout) { if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} - + // Make sure zero byte writes work as expected if(NBytes == 0) { return; } - + // from man SSL_write // // SSL_write() will only return with success, when the // complete contents of buf of length num has been written. // // So no worries about partial writes and moving the buffer around - + while(true) { // try the write @@ -385,24 +352,24 @@ void SocketStreamTLS::Write(const void *pBuffer, int NBytes) case SSL_ERROR_ZERO_RETURN: // Connection closed MarkAsWriteClosed(); - THROW_EXCEPTION(ConnectionException, Conn_TLSClosedWhenWriting) + THROW_EXCEPTION(ConnectionException, TLSClosedWhenWriting) break; case SSL_ERROR_WANT_READ: case SSL_ERROR_WANT_WRITE: - // wait for the requried data + // wait for the required data { #ifndef BOX_RELEASE_BUILD - bool conditionmet = + bool conditionmet = #endif - WaitWhenRetryRequired(se, IOStream::TimeOutInfinite); + WaitWhenRetryRequired(se, Timeout); ASSERT(conditionmet); } break; default: CryptoUtils::LogError("writing"); - THROW_EXCEPTION(ConnectionException, Conn_TLSWriteFailed) + THROW_EXCEPTION(ConnectionException, TLSWriteFailed) break; } } @@ -444,7 +411,7 @@ void SocketStreamTLS::Shutdown(bool Read, bool Write) if(::SSL_shutdown(mpSSL) < 0) { CryptoUtils::LogError("shutting down"); - THROW_EXCEPTION(ConnectionException, Conn_TLSShutdownFailed) + THROW_EXCEPTION(ConnectionException, TLSShutdownFailed) } // Don't ask the base class to shutdown -- BIO does this, apparently. @@ -467,15 +434,15 @@ std::string SocketStreamTLS::GetPeerCommonName() if(cert == 0) { ::X509_free(cert); - THROW_EXCEPTION(ConnectionException, Conn_TLSNoPeerCertificate) + THROW_EXCEPTION(ConnectionException, TLSNoPeerCertificate) } - // Subject details - X509_NAME *subject = ::X509_get_subject_name(cert); + // Subject details + X509_NAME *subject = ::X509_get_subject_name(cert); if(subject == 0) { ::X509_free(cert); - THROW_EXCEPTION(ConnectionException, Conn_TLSPeerCertificateInvalid) + THROW_EXCEPTION(ConnectionException, TLSPeerCertificateInvalid) } // Common name @@ -483,7 +450,7 @@ std::string SocketStreamTLS::GetPeerCommonName() if(::X509_NAME_get_text_by_NID(subject, NID_commonName, commonName, sizeof(commonName)) <= 0) { ::X509_free(cert); - THROW_EXCEPTION(ConnectionException, Conn_TLSPeerCertificateInvalid) + THROW_EXCEPTION(ConnectionException, TLSPeerCertificateInvalid) } // Terminate just in case commonName[sizeof(commonName)-1] = '\0'; diff --git a/lib/server/SocketStreamTLS.h b/lib/server/SocketStreamTLS.h index bb40ed10..3fda98c1 100644 --- a/lib/server/SocketStreamTLS.h +++ b/lib/server/SocketStreamTLS.h @@ -43,7 +43,8 @@ public: void Handshake(const TLSContext &rContext, bool IsServer = false); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual void Close(); virtual void Shutdown(bool Read = true, bool Write = true); diff --git a/lib/server/TLSContext.cpp b/lib/server/TLSContext.cpp index 341043e9..1a6d4a53 100644 --- a/lib/server/TLSContext.cpp +++ b/lib/server/TLSContext.cpp @@ -12,8 +12,9 @@ #define TLS_CLASS_IMPLEMENTATION_CPP #include <openssl/ssl.h> +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "CryptoUtils.h" -#include "ServerException.h" #include "SSLLib.h" #include "TLSContext.h" @@ -22,6 +23,17 @@ #define MAX_VERIFICATION_DEPTH 2 #define CIPHER_LIST "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH" +// Macros to allow compatibility with OpenSSL 1.0 and 1.1 APIs. See +// https://github.com/charybdis-ircd/charybdis/blob/release/3.5/libratbox/src/openssl_ratbox.h +// for the gory details. +#if defined(LIBRESSL_VERSION_NUMBER) || (OPENSSL_VERSION_NUMBER >= 0x10100000L) // OpenSSL >= 1.1 +# define BOX_TLS_SERVER_METHOD TLS_server_method +# define BOX_TLS_CLIENT_METHOD TLS_client_method +#else // OpenSSL < 1.1 +# define BOX_TLS_SERVER_METHOD TLSv1_server_method +# define BOX_TLS_CLIENT_METHOD TLSv1_client_method +#endif + // -------------------------------------------------------------------------- // // Function @@ -66,7 +78,7 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c ::SSL_CTX_free(mpContext); } - mpContext = ::SSL_CTX_new(AsServer?TLSv1_server_method():TLSv1_client_method()); + mpContext = ::SSL_CTX_new(AsServer ? BOX_TLS_SERVER_METHOD() : BOX_TLS_CLIENT_METHOD()); if(mpContext == NULL) { THROW_EXCEPTION(ServerException, TLSAllocationFailed) diff --git a/lib/server/TcpNice.cpp b/lib/server/TcpNice.cpp index 20619e49..79e91eeb 100644 --- a/lib/server/TcpNice.cpp +++ b/lib/server/TcpNice.cpp @@ -146,7 +146,7 @@ void NiceSocketStream::Write(const void *pBuffer, int NBytes) int socket = mapSocket->GetSocketHandle(); int rtt = 50; // WAG -# if HAVE_DECL_SOL_TCP && HAVE_DECL_TCP_INFO && HAVE_STRUCT_TCP_INFO_TCPI_RTT +# if HAVE_DECL_SOL_TCP && defined HAVE_STRUCT_TCP_INFO_TCPI_RTT struct tcp_info info; socklen_t optlen = sizeof(info); if(getsockopt(socket, SOL_TCP, TCP_INFO, &info, &optlen) == -1) @@ -154,7 +154,7 @@ void NiceSocketStream::Write(const void *pBuffer, int NBytes) BOX_LOG_SYS_WARNING("getsockopt(" << socket << ", SOL_TCP, " "TCP_INFO) failed"); } - else if(optlen != sizeof(info)) + else if(optlen < sizeof(info)) { BOX_WARNING("getsockopt(" << socket << ", SOL_TCP, " "TCP_INFO) return structure size " << optlen << ", " @@ -164,7 +164,7 @@ void NiceSocketStream::Write(const void *pBuffer, int NBytes) { rtt = info.tcpi_rtt; } -# endif +# endif // HAVE_DECL_SOL_TCP && defined HAVE_STRUCT_TCP_INFO_TCPI_RTT int newWindow = mTcpNice.GetNextWindowSize(mBytesWrittenThisPeriod, elapsed, rtt); diff --git a/lib/server/TcpNice.h b/lib/server/TcpNice.h index e2027749..4381df42 100644 --- a/lib/server/TcpNice.h +++ b/lib/server/TcpNice.h @@ -83,7 +83,7 @@ private: // // -------------------------------------------------------------------------- -class NiceSocketStream : public IOStream +class NiceSocketStream : public SocketStream { private: std::auto_ptr<SocketStream> mapSocket; @@ -166,6 +166,10 @@ public: } virtual void SetEnabled(bool enabled); + off_t GetBytesRead() const { return mapSocket->GetBytesRead(); } + off_t GetBytesWritten() const { return mapSocket->GetBytesWritten(); } + void ResetCounters() { mapSocket->ResetCounters(); } + private: NiceSocketStream(const NiceSocketStream &rToCopy) { /* do not call */ } diff --git a/lib/server/WinNamedPipeListener.h b/lib/server/WinNamedPipeListener.h index 26e76e3d..956a7b5a 100644 --- a/lib/server/WinNamedPipeListener.h +++ b/lib/server/WinNamedPipeListener.h @@ -11,10 +11,10 @@ #ifndef WINNAMEDPIPELISTENER__H #define WINNAMEDPIPELISTENER__H -#include <OverlappedIO.h> -#include <WinNamedPipeStream.h> - -#include "ServerException.h" +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" +#include "OverlappedIO.h" +#include "WinNamedPipeStream.h" #include "MemLeakFindOn.h" @@ -53,8 +53,8 @@ private: socket.c_str(), // pipe name PIPE_ACCESS_DUPLEX | // read/write access FILE_FLAG_OVERLAPPED, // enabled overlapped I/O - PIPE_TYPE_BYTE | // message type pipe - PIPE_READMODE_BYTE | // message-read mode + PIPE_TYPE_BYTE | + PIPE_READMODE_BYTE | PIPE_WAIT, // blocking mode ListenBacklog + 1, // max. instances 4096, // output buffer size @@ -64,9 +64,9 @@ private: if (handle == INVALID_HANDLE_VALUE) { - BOX_LOG_WIN_ERROR("Failed to create named pipe " << - socket); - THROW_EXCEPTION(ServerException, SocketOpenError) + THROW_WIN_FILE_ERRNO("Failed to create named pipe", + socket, GetLastError(), ServerException, + SocketOpenError); } return handle; diff --git a/lib/server/WinNamedPipeStream.cpp b/lib/server/WinNamedPipeStream.cpp index 1179516e..448a3c9d 100644 --- a/lib/server/WinNamedPipeStream.cpp +++ b/lib/server/WinNamedPipeStream.cpp @@ -19,10 +19,12 @@ #include <errno.h> #include <windows.h> -#include "WinNamedPipeStream.h" -#include "ServerException.h" +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" +#include "BoxTime.h" #include "CommonException.h" #include "Socket.h" +#include "WinNamedPipeStream.h" #include "MemLeakFindOn.h" @@ -37,13 +39,14 @@ std::string WinNamedPipeStream::sPipeNamePrefix = "\\\\.\\pipe\\"; // // -------------------------------------------------------------------------- WinNamedPipeStream::WinNamedPipeStream() - : mSocketHandle(INVALID_HANDLE_VALUE), - mReadableEvent(INVALID_HANDLE_VALUE), - mBytesInBuffer(0), - mReadClosed(false), - mWriteClosed(false), - mIsServer(false), - mIsConnected(false) +: mSocketHandle(INVALID_HANDLE_VALUE), + mReadableEvent(INVALID_HANDLE_VALUE), + mBytesInBuffer(0), + mReadClosed(false), + mWriteClosed(false), + mIsServer(false), + mIsConnected(false), + mNeedAnotherRead(false) { } // -------------------------------------------------------------------------- @@ -55,14 +58,21 @@ WinNamedPipeStream::WinNamedPipeStream() // // -------------------------------------------------------------------------- WinNamedPipeStream::WinNamedPipeStream(HANDLE hNamedPipe) - : mSocketHandle(hNamedPipe), - mReadableEvent(INVALID_HANDLE_VALUE), - mBytesInBuffer(0), - mReadClosed(false), - mWriteClosed(false), - mIsServer(true), - mIsConnected(true) +: mSocketHandle(hNamedPipe), + mReadableEvent(INVALID_HANDLE_VALUE), + mBytesInBuffer(0), + mReadClosed(false), + mWriteClosed(false), + mIsServer(true), + mIsConnected(true), + mNeedAnotherRead(false) { + StartFirstRead(); +} + +// Start the first overlapped read +void WinNamedPipeStream::StartFirstRead() +{ // create the Readable event mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL); @@ -74,23 +84,50 @@ WinNamedPipeStream::WinNamedPipeStream(HANDLE hNamedPipe) THROW_EXCEPTION(CommonException, Internal) } - // initialise the OVERLAPPED structure + StartOverlappedRead(); +} + +void WinNamedPipeStream::StartOverlappedRead() +{ + // We should only do this when the buffer is empty. We don't want + // to start an overlapped read anywhere else than the start of the + // buffer, because it could complete at any time and we don't want + // to mess about with interrupting the read already in progress. + ASSERT(mBytesInBuffer == 0); + + // Initialise the OVERLAPPED structure memset(&mReadOverlap, 0, sizeof(mReadOverlap)); mReadOverlap.hEvent = mReadableEvent; - // start the first overlapped read if (!ReadFile(mSocketHandle, mReadBuffer, sizeof(mReadBuffer), NULL, &mReadOverlap)) { DWORD err = GetLastError(); - - if (err != ERROR_IO_PENDING) + if (err == ERROR_IO_PENDING) + { + // Don't reset yet, there might be data + // in the buffer waiting to be read, + // will check below. + // ResetEvent(mReadableEvent); + } + else if (err == ERROR_HANDLE_EOF) + { + BOX_INFO("Control client disconnected"); + mReadClosed = true; + } + else if (err == ERROR_BROKEN_PIPE || + err == ERROR_PIPE_NOT_CONNECTED) + { + BOX_NOTICE("Control client disconnected"); + mReadClosed = true; + mIsConnected = false; + } + else { - BOX_ERROR("Failed to start overlapped read: " << - GetErrorMessage(err)); Close(); - THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError) + THROW_WIN_ERROR_NUMBER("Failed to start overlapped " + "read", err, ConnectionException, + SocketReadError) } } } @@ -105,6 +142,12 @@ WinNamedPipeStream::WinNamedPipeStream(HANDLE hNamedPipe) // -------------------------------------------------------------------------- WinNamedPipeStream::~WinNamedPipeStream() { + for(std::list<WriteInProgress*>::iterator i = mWritesInProgress.begin(); + i != mWritesInProgress.end(); i++) + { + delete *i; + } + if (mSocketHandle != INVALID_HANDLE_VALUE) { try @@ -157,36 +200,7 @@ void WinNamedPipeStream::Accept() mIsServer = true; // must flush and disconnect before closing mIsConnected = true; - // create the Readable event - mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - - if (mReadableEvent == INVALID_HANDLE_VALUE) - { - BOX_ERROR("Failed to create the Readable event: " << - GetErrorMessage(GetLastError())); - Close(); - THROW_EXCEPTION(CommonException, Internal) - } - - // initialise the OVERLAPPED structure - memset(&mReadOverlap, 0, sizeof(mReadOverlap)); - mReadOverlap.hEvent = mReadableEvent; - - // start the first overlapped read - if (!ReadFile(mSocketHandle, mReadBuffer, sizeof(mReadBuffer), - NULL, &mReadOverlap)) - { - DWORD err = GetLastError(); - - if (err != ERROR_IO_PENDING) - { - BOX_ERROR("Failed to start overlapped read: " << - GetErrorMessage(err)); - Close(); - THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError) - } - } + StartFirstRead(); } */ @@ -214,7 +228,7 @@ void WinNamedPipeStream::Connect(const std::string& rName) 0, // no sharing NULL, // default security attributes OPEN_EXISTING, - 0, // default attributes + 0, // FILE_FLAG_OVERLAPPED, // dwFlagsAndAttributes NULL); // no template file if (mSocketHandle == INVALID_HANDLE_VALUE) @@ -237,6 +251,86 @@ void WinNamedPipeStream::Connect(const std::string& rName) mWriteClosed = false; mIsServer = false; // just close the socket mIsConnected = true; + + StartFirstRead(); +} + +// Returns true if the operation is complete (and you will need to start +// another one), or false otherwise (you can wait again). +bool WinNamedPipeStream::WaitForOverlappedOperation(OVERLAPPED& Overlapped, + int Timeout, int64_t* pBytesTransferred) +{ + if (Timeout == IOStream::TimeOutInfinite) + { + Timeout = INFINITE; + } + + // overlapped I/O completed successfully? (wait if needed) + DWORD waitResult = WaitForSingleObject(Overlapped.hEvent, Timeout); + DWORD NumBytesTransferred = -1; + + if (waitResult == WAIT_FAILED) + { + THROW_WIN_ERROR_NUMBER("Failed to wait for overlapped I/O", + GetLastError(), ServerException, Internal); + } + + if (waitResult == WAIT_ABANDONED) + { + THROW_EXCEPTION_MESSAGE(ServerException, Internal, + "Wait for overlapped I/O abandoned by system"); + } + + if (waitResult == WAIT_TIMEOUT) + { + // wait timed out, nothing to read + *pBytesTransferred = 0; + return false; + } + + if (waitResult != WAIT_OBJECT_0) + { + THROW_EXCEPTION_MESSAGE(ServerException, BadSocketHandle, + "Failed to wait for overlapped I/O: unknown " + "result code: " << waitResult); + } + + // Overlapped operation completed successfully. Return the number + // of bytes transferred. + if (GetOverlappedResult(mSocketHandle, &Overlapped, + &NumBytesTransferred, TRUE)) + { + *pBytesTransferred = NumBytesTransferred; + return true; + } + + // We are here because GetOverlappedResult() informed us that the + // overlapped operation encountered an error, so what was it? + DWORD err = GetLastError(); + + if (err == ERROR_HANDLE_EOF) + { + Close(); + *pBytesTransferred = 0; + return true; + } + + // ERROR_NO_DATA is a strange name for + // "The pipe is being closed". No exception wanted. + + if (err == ERROR_NO_DATA || + err == ERROR_PIPE_NOT_CONNECTED || + err == ERROR_BROKEN_PIPE) + { + BOX_INFO(BOX_WIN_ERRNO_MESSAGE(err, + "Named pipe peer disconnected")); + Close(); + *pBytesTransferred = 0; + return true; + } + + THROW_WIN_ERROR_NUMBER("Failed to wait for overlapped I/O " + "to complete", err, ConnectionException, SocketReadError); } // -------------------------------------------------------------------------- @@ -249,192 +343,61 @@ void WinNamedPipeStream::Connect(const std::string& rName) // -------------------------------------------------------------------------- int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) { - // TODO no support for timeouts yet - if (!mIsServer && Timeout != IOStream::TimeOutInfinite) - { - THROW_EXCEPTION(CommonException, AssertFailed) - } - if (mSocketHandle == INVALID_HANDLE_VALUE || !mIsConnected) { - THROW_EXCEPTION(ServerException, BadSocketHandle) + THROW_EXCEPTION_MESSAGE(ServerException, BadSocketHandle, + "Tried to read from closed pipe"); } if (mReadClosed) { - THROW_EXCEPTION(ConnectionException, SocketShutdownError) + THROW_EXCEPTION_MESSAGE(ConnectionException, + SocketShutdownError, "Tried to read from closing pipe"); } // ensure safe to cast NBytes to unsigned if (NBytes < 0) { - THROW_EXCEPTION(CommonException, AssertFailed) + THROW_EXCEPTION(CommonException, AssertFailed); } - DWORD NumBytesRead; + int64_t NumBytesRead; - if (mIsServer) + // Satisfy from buffer if possible, to avoid blocking on read. + if (mBytesInBuffer == 0) { - // satisfy from buffer if possible, to avoid - // blocking on read. - bool needAnotherRead = false; - if (mBytesInBuffer == 0) - { - // overlapped I/O completed successfully? - // (wait if needed) - DWORD waitResult = WaitForSingleObject( - mReadOverlap.hEvent, Timeout); - - if (waitResult == WAIT_ABANDONED) - { - BOX_ERROR("Wait for command socket read " - "abandoned by system"); - THROW_EXCEPTION(ServerException, - BadSocketHandle); - } - else if (waitResult == WAIT_TIMEOUT) - { - // wait timed out, nothing to read - NumBytesRead = 0; - } - else if (waitResult != WAIT_OBJECT_0) - { - BOX_ERROR("Failed to wait for command " - "socket read: unknown result " << - waitResult); - } - // object is ready to read from - else if (GetOverlappedResult(mSocketHandle, - &mReadOverlap, &NumBytesRead, TRUE)) - { - needAnotherRead = true; - } - else - { - DWORD err = GetLastError(); - - if (err == ERROR_HANDLE_EOF) - { - mReadClosed = true; - } - else - { - if (err == ERROR_BROKEN_PIPE) - { - BOX_NOTICE("Control client " - "disconnected"); - } - else - { - BOX_ERROR("Failed to wait for " - "ReadFile to complete: " - << GetErrorMessage(err)); - } - - Close(); - THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError) - } - } - } - else - { - NumBytesRead = 0; - } - - size_t BytesToCopy = NumBytesRead + mBytesInBuffer; - size_t BytesRemaining = 0; - - if (BytesToCopy > (size_t)NBytes) - { - BytesRemaining = BytesToCopy - NBytes; - BytesToCopy = NBytes; - } - - memcpy(pBuffer, mReadBuffer, BytesToCopy); - memmove(mReadBuffer, mReadBuffer + BytesToCopy, BytesRemaining); - - mBytesInBuffer = BytesRemaining; - NumBytesRead = BytesToCopy; - - if (needAnotherRead) + if (mNeedAnotherRead) { - // reinitialise the OVERLAPPED structure - memset(&mReadOverlap, 0, sizeof(mReadOverlap)); - mReadOverlap.hEvent = mReadableEvent; + // Start the next overlapped read + StartOverlappedRead(); } - // start the next overlapped read - if (needAnotherRead && !ReadFile(mSocketHandle, - mReadBuffer + mBytesInBuffer, - sizeof(mReadBuffer) - mBytesInBuffer, - NULL, &mReadOverlap)) - { - DWORD err = GetLastError(); - if (err == ERROR_IO_PENDING) - { - // Don't reset yet, there might be data - // in the buffer waiting to be read, - // will check below. - // ResetEvent(mReadableEvent); - } - else if (err == ERROR_HANDLE_EOF) - { - mReadClosed = true; - } - else if (err == ERROR_BROKEN_PIPE) - { - BOX_ERROR("Control client disconnected"); - mReadClosed = true; - } - else - { - BOX_ERROR("Failed to start overlapped read: " - << GetErrorMessage(err)); - Close(); - THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError) - } - } + mNeedAnotherRead = WaitForOverlappedOperation(mReadOverlap, + Timeout, &NumBytesRead); } else { - if (!ReadFile( - mSocketHandle, // pipe handle - pBuffer, // buffer to receive reply - NBytes, // size of buffer - &NumBytesRead, // number of bytes read - NULL)) // not overlapped - { - DWORD err = GetLastError(); - - Close(); + // Just return the existing data from the buffer + // this time around. The caller should call again, + // and then the buffer will be empty. + NumBytesRead = 0; + } - // ERROR_NO_DATA is a strange name for - // "The pipe is being closed". No exception wanted. - - if (err == ERROR_NO_DATA || - err == ERROR_PIPE_NOT_CONNECTED) - { - NumBytesRead = 0; - } - else - { - BOX_ERROR("Failed to read from control socket: " - << GetErrorMessage(err)); - THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError) - } - } - - // Closed for reading at EOF? - if (NumBytesRead == 0) - { - mReadClosed = true; - } + int BytesToCopy = NumBytesRead + mBytesInBuffer; + + if (NBytes < BytesToCopy) + { + BytesToCopy = NBytes; } - - return NumBytesRead; + + memcpy(pBuffer, mReadBuffer, BytesToCopy); + + size_t BytesRemaining = mBytesInBuffer + NumBytesRead - BytesToCopy; + ASSERT(BytesToCopy + BytesRemaining <= sizeof(mReadBuffer)); + memmove(mReadBuffer, mReadBuffer + BytesToCopy, BytesRemaining); + mBytesInBuffer = BytesRemaining; + + return BytesToCopy; } // -------------------------------------------------------------------------- @@ -445,8 +408,15 @@ int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void WinNamedPipeStream::Write(const void *pBuffer, int NBytes) +void WinNamedPipeStream::Write(const void *pBuffer, int NBytes, int Timeout) { + // Calculate the deadline at the beginning. Not valid if Timeout is + // IOStream::TimeOutInfinite! + ASSERT(Timeout != IOStream::TimeOutInfinite); + + box_time_t deadline = GetCurrentBoxTime() + + MilliSecondsToBoxTime(Timeout); + if (mSocketHandle == INVALID_HANDLE_VALUE || !mIsConnected) { THROW_EXCEPTION(ServerException, BadSocketHandle) @@ -454,41 +424,62 @@ void WinNamedPipeStream::Write(const void *pBuffer, int NBytes) // Buffer in byte sized type. ASSERT(sizeof(char) == 1); - const char *pByteBuffer = (char *)pBuffer; - - int NumBytesWrittenTotal = 0; - - while (NumBytesWrittenTotal < NBytes) + WriteInProgress* new_write = new WriteInProgress( + std::string((char *)pBuffer, NBytes)); + + // Start the WriteFile operation, and add to queue if pending. + BOOL Success = WriteFile( + mSocketHandle, // pipe handle + new_write->mBuffer.c_str(), // message + NBytes, // message length + NULL, // bytes written this time + &(new_write->mOverlap)); + + if (Success == TRUE) { - DWORD NumBytesWrittenThisTime = 0; - - bool Success = WriteFile( - mSocketHandle, // pipe handle - pByteBuffer + NumBytesWrittenTotal, // message - NBytes - NumBytesWrittenTotal, // message length - &NumBytesWrittenThisTime, // bytes written this time - NULL); // not overlapped + // Unfortunately this does happen. We should still call + // GetOverlappedResult() to get the number of bytes written, + // so we can treat it just the same. + // BOX_NOTICE("Write claimed success while overlapped?"); + mWritesInProgress.push_back(new_write); + } + else + { + DWORD err = GetLastError(); - if (!Success) + if (err == ERROR_IO_PENDING) { - // ERROR_NO_DATA is a strange name for - // "The pipe is being closed". - - DWORD err = GetLastError(); - - if (err != ERROR_NO_DATA) - { - BOX_ERROR("Failed to write to control " - "socket: " << GetErrorMessage(err)); - } - + BOX_TRACE("WriteFile is pending, adding to queue"); + mWritesInProgress.push_back(new_write); + } + else + { + // Not in progress any more, pop it Close(); - - THROW_EXCEPTION(ConnectionException, - Conn_SocketWriteError) + THROW_WIN_ERROR_NUMBER("Failed to start overlapped " + "write", err, ConnectionException, + SocketWriteError); } + } + + // Wait for previous WriteFile operations to complete, one at a time, + // until the deadline expires or the pipe becomes disconnected. + for(box_time_t remaining = deadline - GetCurrentBoxTime(); + remaining > 0 && !mWritesInProgress.empty() && mIsConnected; + remaining = deadline - GetCurrentBoxTime()) + { + int new_timeout = BoxTimeToMilliSeconds(remaining); + WriteInProgress* oldest_write = + *(mWritesInProgress.begin()); - NumBytesWrittenTotal += NumBytesWrittenThisTime; + int64_t bytes_written = 0; + if(WaitForOverlappedOperation(oldest_write->mOverlap, + new_timeout, &bytes_written)) + { + // This one is complete, pop it and start a new one + delete oldest_write; + mWritesInProgress.pop_front(); + } } } @@ -513,59 +504,47 @@ void WinNamedPipeStream::Close() THROW_EXCEPTION(ServerException, BadSocketHandle) } - if (mIsServer) + if (!CancelIo(mSocketHandle)) { - if (!CancelIo(mSocketHandle)) - { - BOX_ERROR("Failed to cancel outstanding I/O: " << - GetErrorMessage(GetLastError())); - } + BOX_LOG_WIN_ERROR("Failed to cancel outstanding I/O"); + } - if (mReadableEvent == INVALID_HANDLE_VALUE) - { - BOX_ERROR("Failed to destroy Readable event: " - "invalid handle"); - } - else if (!CloseHandle(mReadableEvent)) - { - BOX_ERROR("Failed to destroy Readable event: " << - GetErrorMessage(GetLastError())); - } + if (mReadableEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to destroy Readable event: " + "invalid handle"); + } + else if (!CloseHandle(mReadableEvent)) + { + BOX_LOG_WIN_ERROR("Failed to destroy Readable event"); + } - mReadableEvent = INVALID_HANDLE_VALUE; + mReadableEvent = INVALID_HANDLE_VALUE; - if (!FlushFileBuffers(mSocketHandle)) - { - BOX_ERROR("Failed to FlushFileBuffers: " << - GetErrorMessage(GetLastError())); - } - - if (!DisconnectNamedPipe(mSocketHandle)) + if (mIsConnected && !FlushFileBuffers(mSocketHandle)) + { + BOX_LOG_WIN_ERROR("Failed to FlushFileBuffers"); + } + + if (mIsServer && mIsConnected && !DisconnectNamedPipe(mSocketHandle)) + { + DWORD err = GetLastError(); + if (err != ERROR_PIPE_NOT_CONNECTED) { - DWORD err = GetLastError(); - if (err != ERROR_PIPE_NOT_CONNECTED) - { - BOX_ERROR("Failed to DisconnectNamedPipe: " << - GetErrorMessage(err)); - } + BOX_LOG_WIN_ERROR("Failed to DisconnectNamedPipe"); } - - mIsServer = false; } - bool result = CloseHandle(mSocketHandle); + if (!CloseHandle(mSocketHandle)) + { + THROW_WIN_ERROR_NUMBER("Failed to CloseHandle", + GetLastError(), ServerException, SocketCloseError); + } mSocketHandle = INVALID_HANDLE_VALUE; mIsConnected = false; mReadClosed = true; mWriteClosed = true; - - if (!result) - { - BOX_ERROR("Failed to CloseHandle: " << - GetErrorMessage(GetLastError())); - THROW_EXCEPTION(ServerException, SocketCloseError) - } } // -------------------------------------------------------------------------- diff --git a/lib/server/WinNamedPipeStream.h b/lib/server/WinNamedPipeStream.h index 386ff7e3..5473c690 100644 --- a/lib/server/WinNamedPipeStream.h +++ b/lib/server/WinNamedPipeStream.h @@ -10,6 +10,8 @@ #if ! defined WINNAMEDPIPESTREAM__H && defined WIN32 #define WINNAMEDPIPESTREAM__H +#include <list> + #include "IOStream.h" // -------------------------------------------------------------------------- @@ -36,15 +38,27 @@ public: // both sides virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual void WriteAllBuffered(); virtual void Close(); virtual bool StreamDataLeft(); virtual bool StreamClosed(); + // Why not inherited from IOStream? Never mind, we want to enforce + // supplying a timeout for network operations anyway. + virtual void Write(const std::string& rBuffer, int Timeout) + { + IOStream::Write(rBuffer, Timeout); + } + protected: void MarkAsReadClosed() {mReadClosed = true;} void MarkAsWriteClosed() {mWriteClosed = true;} + bool WaitForOverlappedOperation(OVERLAPPED& Overlapped, + int Timeout, int64_t* pBytesTransferred); + void StartFirstRead(); + void StartOverlappedRead(); private: WinNamedPipeStream(const WinNamedPipeStream &rToCopy) @@ -59,6 +73,37 @@ private: bool mWriteClosed; bool mIsServer; bool mIsConnected; + bool mNeedAnotherRead; + + class WriteInProgress { + private: + friend class WinNamedPipeStream; + std::string mBuffer; + OVERLAPPED mOverlap; + WriteInProgress(const WriteInProgress& other); // do not call + public: + WriteInProgress(const std::string& dataToWrite) + : mBuffer(dataToWrite) + { + // create the Writable event + HANDLE writable_event = CreateEvent(NULL, TRUE, FALSE, + NULL); + if (writable_event == INVALID_HANDLE_VALUE) + { + BOX_LOG_WIN_ERROR("Failed to create the " + "Writable event"); + THROW_EXCEPTION(CommonException, Internal) + } + + memset(&mOverlap, 0, sizeof(mOverlap)); + mOverlap.hEvent = writable_event; + } + ~WriteInProgress() + { + CloseHandle(mOverlap.hEvent); + } + }; + std::list<WriteInProgress*> mWritesInProgress; public: static std::string sPipeNamePrefix; diff --git a/lib/server/makeprotocol.pl.in b/lib/server/makeprotocol.pl.in index a074b435..d6c0e216 100755 --- a/lib/server/makeprotocol.pl.in +++ b/lib/server/makeprotocol.pl.in @@ -78,7 +78,7 @@ sub add_type my ($protocol_name, $cpp_name, $header_file) = split /\s+/,$_[0]; $translate_type_info{$protocol_name} = [0, $cpp_name]; - push @extra_header_files, $header_file; + push @extra_header_files, $header_file if $header_file; } # check attributes @@ -158,7 +158,10 @@ print CPP <<__E; #include <sstream> #include "$filename_base.h" -#include "IOStream.h" +#include "CollectInBufferStream.h" +#include "MemBlockStream.h" +#include "SelfFlushingStream.h" +#include "SocketStream.h" __E print H <<__E; @@ -174,12 +177,10 @@ print H <<__E; #include <syslog.h> #endif +#include "autogen_ConnectionException.h" #include "Protocol.h" #include "Message.h" -#include "ServerException.h" - -class IOStream; - +#include "SocketStream.h" __E @@ -210,19 +211,26 @@ __E my $request_base_class = "${protocol_name}ProtocolRequest"; my $reply_base_class = "${protocol_name}ProtocolReply"; # the abstract protocol interface -my $protocol_base_class = $protocol_name."ProtocolBase"; +my $custom_protocol_subclass = $protocol_name."Protocol"; +my $client_server_base_class = $protocol_name."ProtocolClientServer"; my $replyable_base_class = $protocol_name."ProtocolReplyable"; +my $callable_base_class = $protocol_name."ProtocolCallable"; +my $send_receive_class = $protocol_name."ProtocolSendReceive"; print H <<__E; -class $protocol_base_class; +class $custom_protocol_subclass; +class $client_server_base_class; +class $callable_base_class; class $replyable_base_class; -class $reply_base_class; class $message_base_class : public Message { public: virtual std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, $context_class &rContext) const; + virtual std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, + $context_class &rContext, IOStream& rDataStream) const; + virtual bool HasStreamWithCommand() const = 0; }; class $reply_base_class @@ -233,17 +241,44 @@ class $request_base_class { }; +class $send_receive_class { +public: + virtual void Send(const $message_base_class &rObject) = 0; + virtual std::auto_ptr<$message_base_class> Receive() = 0; +}; + +class $custom_protocol_subclass : public Protocol +{ +public: + $custom_protocol_subclass(std::auto_ptr<SocketStream> apConn) + : Protocol(apConn) + { } + virtual ~$custom_protocol_subclass() { } + virtual std::auto_ptr<Message> MakeMessage(int ObjType); + virtual const char *GetProtocolIdentString(); + +private: + $custom_protocol_subclass(const $custom_protocol_subclass &rToCopy); +}; + __E print CPP <<__E; std::auto_ptr<$message_base_class> $message_base_class\::DoCommand($replyable_base_class &rProtocol, $context_class &rContext) const { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_TriedToExecuteReplyCommand) + THROW_EXCEPTION(ConnectionException, Protocol_TriedToExecuteReplyCommand) +} + +std::auto_ptr<$message_base_class> $message_base_class\::DoCommand($replyable_base_class &rProtocol, + $context_class &rContext, IOStream& rDataStream) const +{ + THROW_EXCEPTION(ConnectionException, Protocol_TriedToExecuteReplyCommand) } __E -my %cmd_class; +my %cmd_classes; +my $error_message = undef; # output the classes foreach my $cmd (@cmd_list) @@ -262,7 +297,7 @@ foreach my $cmd (@cmd_list) my $cmd_base_class = join(", ", map {"public $_"} @cmd_base_classes); my $cmd_class = $protocol_name."Protocol".$cmd; - $cmd_class{$cmd} = $cmd_class; + $cmd_classes{$cmd} = $cmd_class; print H <<__E; class $cmd_class : $cmd_base_class @@ -294,18 +329,50 @@ __E if(obj_is_type($cmd,'IsError')) { - print H "\tbool IsError(int &rTypeOut, int &rSubTypeOut) const;\n"; - print H "\tstd::string GetMessage() const;\n"; + $error_message = $cmd; + my ($mem_type,$mem_subtype) = split /,/,obj_get_type_params($cmd,'IsError'); + my $error_type = $cmd_constants{"ErrorType"}; + print H <<__E; + $cmd_class(int SubType) : m$mem_type($error_type), m$mem_subtype(SubType) { } + bool IsError(int &rTypeOut, int &rSubTypeOut) const; + std::string GetMessage() const { return GetMessage(m$mem_subtype); }; + static std::string GetMessage(int subtype); +__E } - if(obj_is_type($cmd, 'Command')) + my $has_stream = obj_is_type($cmd, 'StreamWithCommand'); + + if(obj_is_type($cmd, 'Command') && $has_stream) + { + print H <<__E; + std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, + $context_class &rContext, IOStream& rDataStream) const; // IMPLEMENT THIS\n + std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, + $context_class &rContext) const + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "This command requires a stream parameter"); + } +__E + } + elsif(obj_is_type($cmd, 'Command') && !$has_stream) { print H <<__E; std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, $context_class &rContext) const; // IMPLEMENT THIS\n + std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, + $context_class &rContext, IOStream& rDataStream) const + { + THROW_EXCEPTION_MESSAGE(CommonException, NotSupported, + "This command requires no stream parameter"); + } __E } + print H <<__E; + bool HasStreamWithCommand() const { return $has_stream; } +__E + # want to be able to read from streams? print H "\tvoid SetPropertiesFromStreamData(Protocol &rProtocol);\n"; @@ -442,9 +509,9 @@ bool $cmd_class\::IsError(int &rTypeOut, int &rSubTypeOut) const rSubTypeOut = m$mem_subtype; return true; } -std::string $cmd_class\::GetMessage() const +std::string $cmd_class\::GetMessage(int subtype) { - switch(m$mem_subtype) + switch(subtype) { __E foreach my $const (@{$cmd_constants{$cmd}}) @@ -459,7 +526,7 @@ __E print CPP <<__E; default: std::ostringstream out; - out << "Unknown subtype " << m$mem_subtype; + out << "Unknown subtype " << subtype; return out.str(); } } @@ -505,47 +572,44 @@ my $error_class = $protocol_name."ProtocolError"; # the abstract protocol interface print H <<__E; -class $protocol_base_class + +class $client_server_base_class { public: - $protocol_base_class(); - virtual ~$protocol_base_class(); - virtual const char *GetIdentString(); + $client_server_base_class(); + virtual ~$client_server_base_class(); + virtual std::auto_ptr<IOStream> ReceiveStream() = 0; bool GetLastError(int &rTypeOut, int &rSubTypeOut); + int GetLastErrorType() { return mLastErrorSubType; } protected: - void CheckReply(const std::string& requestCommand, - const $message_base_class &rReply, int expectedType); void SetLastError(int Type, int SubType) { mLastErrorType = Type; mLastErrorSubType = SubType; } + std::string mPreviousCommand; + std::string mPreviousReply; private: - $protocol_base_class(const $protocol_base_class &rToCopy); /* do not call */ + $client_server_base_class(const $client_server_base_class &rToCopy); /* do not call */ int mLastErrorType; int mLastErrorSubType; }; -class $replyable_base_class : public virtual $protocol_base_class +class $replyable_base_class : public virtual $client_server_base_class { public: - $replyable_base_class(); + $replyable_base_class() { } virtual ~$replyable_base_class(); - /* - virtual std::auto_ptr<$message_base_class> Receive() = 0; - virtual void Send(const ${message_base_class} &rObject) = 0; - */ - - virtual std::auto_ptr<IOStream> ReceiveStream() = 0; virtual int GetTimeout() = 0; void SendStreamAfterCommand(std::auto_ptr<IOStream> apStream); - + protected: std::list<IOStream*> mStreamsToSend; void DeleteStreamsToSend(); + virtual std::auto_ptr<$message_base_class> HandleException(BoxException& e) const; private: $replyable_base_class(const $replyable_base_class &rToCopy); /* do not call */ @@ -554,24 +618,47 @@ private: __E print CPP <<__E; -$protocol_base_class\::$protocol_base_class() +$client_server_base_class\::$client_server_base_class() : mLastErrorType(Protocol::NoError), mLastErrorSubType(Protocol::NoError) { } -$protocol_base_class\::~$protocol_base_class() +$client_server_base_class\::~$client_server_base_class() { } -const char *$protocol_base_class\::GetIdentString() +const char *$custom_protocol_subclass\::GetProtocolIdentString() { return "$ident_string"; } -$replyable_base_class\::$replyable_base_class() -{ } +std::auto_ptr<Message> $custom_protocol_subclass\::MakeMessage(int ObjType) +{ + switch(ObjType) + { +__E + +# do objects within this +for my $cmd (@cmd_list) +{ + print CPP <<__E; + case $cmd_id{$cmd}: + return std::auto_ptr<Message>(new $cmd_classes{$cmd}()); + break; +__E +} + +print CPP <<__E; + default: + THROW_EXCEPTION(ConnectionException, Protocol_UnknownCommandRecieved) + } +} $replyable_base_class\::~$replyable_base_class() -{ } +{ + // If there were any streams left over, there's no longer any way to + // access them, and we're responsible for them, so we'd better delete them. + DeleteStreamsToSend(); +} void $replyable_base_class\::SendStreamAfterCommand(std::auto_ptr<IOStream> apStream) { @@ -589,12 +676,14 @@ void $replyable_base_class\::DeleteStreamsToSend() mStreamsToSend.clear(); } -void $protocol_base_class\::CheckReply(const std::string& requestCommand, - const $message_base_class &rReply, int expectedType) +void $callable_base_class\::CheckReply(const std::string& requestCommandName, + const $message_base_class &rCommand, const $message_base_class &rReply, + int expectedType) { if(rReply.GetType() == expectedType) { // Correct response, do nothing + SetLastError(Protocol::NoError, Protocol::NoError); } else { @@ -605,8 +694,8 @@ void $protocol_base_class\::CheckReply(const std::string& requestCommand, { SetLastError(type, subType); THROW_EXCEPTION_MESSAGE(ConnectionException, - Conn_Protocol_UnexpectedReply, - requestCommand << " command failed: " + Protocol_UnexpectedReply, + requestCommandName << " command failed: " "received error " << (($error_class&)rReply).GetMessage()); } @@ -614,12 +703,18 @@ void $protocol_base_class\::CheckReply(const std::string& requestCommand, { SetLastError(Protocol::UnknownError, Protocol::UnknownError); THROW_EXCEPTION_MESSAGE(ConnectionException, - Conn_Protocol_UnexpectedReply, - requestCommand << " command failed: " + Protocol_UnexpectedReply, + requestCommandName << " command failed: " "received unexpected response type " << rReply.GetType()); } } + + // As a client, if we get an unexpected reply later, we'll want to know + // the last command that we executed, and the reply, to help debug the + // server. + mPreviousCommand = rCommand.ToString(); + mPreviousReply = rReply.ToString(); } // -------------------------------------------------------------------------- @@ -630,7 +725,7 @@ void $protocol_base_class\::CheckReply(const std::string& requestCommand, // Created: 2003/08/19 // // -------------------------------------------------------------------------- -bool $protocol_base_class\::GetLastError(int &rTypeOut, int &rSubTypeOut) +bool $client_server_base_class\::GetLastError(int &rTypeOut, int &rSubTypeOut) { if(mLastErrorType == Protocol::NoError) { @@ -653,13 +748,19 @@ __E # the callable protocol interface (implemented by Client and Local classes) # with Query methods that don't take a context parameter -my $callable_base_class = $protocol_name."ProtocolCallable"; print H <<__E; -class $callable_base_class : public virtual $protocol_base_class +class $callable_base_class : public virtual $client_server_base_class, + public $send_receive_class { public: - virtual std::auto_ptr<IOStream> ReceiveStream() = 0; virtual int GetTimeout() = 0; + +protected: + void CheckReply(const std::string& requestCommandName, + const $message_base_class &rCommand, + const $message_base_class &rReply, int expectedType); + +public: __E # add plain object taking query functions @@ -671,8 +772,8 @@ for my $cmd (@cmd_list) my $has_stream = obj_is_type($cmd,'StreamWithCommand'); my $argextra = $has_stream?', std::auto_ptr<IOStream> apStream':''; my $queryextra = $has_stream?', apStream':''; - my $request_class = $cmd_class{$cmd}; - my $reply_class = $cmd_class{obj_get_type_params($cmd,'Command')}; + my $request_class = $cmd_classes{$cmd}; + my $reply_class = $cmd_classes{obj_get_type_params($cmd,'Command')}; print H "\tvirtual std::auto_ptr<$reply_class> Query(const $request_class &rQuery$argextra) = 0;\n"; my @a; @@ -720,13 +821,15 @@ foreach my $type ('Client', 'Server', 'Local') { push @base_classes, $replyable_base_class; } + if (not $writing_server) { push @base_classes, $callable_base_class; } + if (not $writing_local) { - push @base_classes, "Protocol"; + push @base_classes, $custom_protocol_subclass; } my $base_classes_str = join(", ", map {"public $_"} @base_classes); @@ -735,6 +838,7 @@ foreach my $type ('Client', 'Server', 'Local') class $server_or_client_class : $base_classes_str { public: + virtual ~$server_or_client_class(); __E if($writing_local) @@ -743,18 +847,12 @@ __E $server_or_client_class($context_class &rContext); __E } - else - { - print H <<__E; - $server_or_client_class(IOStream &rStream); + + print H <<__E; + $server_or_client_class(std::auto_ptr<SocketStream> apConn); std::auto_ptr<$message_base_class> Receive(); void Send(const $message_base_class &rObject); __E - } - - print H <<__E; - virtual ~$server_or_client_class(); -__E if($writing_server) { @@ -775,37 +873,29 @@ __E my $has_stream = obj_is_type($cmd,'StreamWithCommand'); my $argextra = $has_stream?', std::auto_ptr<IOStream> apStream':''; my $queryextra = $has_stream?', apStream':''; - my $request_class = $cmd_class{$cmd}; - my $reply_class = $cmd_class{obj_get_type_params($cmd,'Command')}; + my $request_class = $cmd_classes{$cmd}; + my $reply_class = $cmd_classes{obj_get_type_params($cmd,'Command')}; print H "\tstd::auto_ptr<$reply_class> Query(const $request_class &rQuery$argextra);\n"; } } } - + if($writing_local) { print H <<__E; private: $context_class &mrContext; -__E - } - - print H <<__E; - -protected: - virtual std::auto_ptr<Message> MakeMessage(int ObjType); - -__E - - if($writing_local) - { - print H <<__E; - virtual void InformStreamReceiving(u_int32_t Size) { } - virtual void InformStreamSending(u_int32_t Size) { } - + std::auto_ptr<$message_base_class> mapLastReply; public: virtual std::auto_ptr<IOStream> ReceiveStream() { + if(mStreamsToSend.empty()) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Tried to ReceiveStream when none was sent or " + "made available"); + } + std::auto_ptr<IOStream> apStream(mStreamsToSend.front()); mStreamsToSend.pop_front(); return apStream; @@ -815,29 +905,33 @@ __E else { print H <<__E; - virtual void InformStreamReceiving(u_int32_t Size) - { - this->Protocol::InformStreamReceiving(Size); - } - virtual void InformStreamSending(u_int32_t Size) - { - this->Protocol::InformStreamSending(Size); - } + virtual std::auto_ptr<IOStream> ReceiveStream(); +__E -public: - virtual std::auto_ptr<IOStream> ReceiveStream() + print CPP <<__E; +std::auto_ptr<IOStream> $server_or_client_class\::ReceiveStream() +{ + try { - return this->Protocol::ReceiveStream(); - } -__E + return $custom_protocol_subclass\::ReceiveStream(); } - - print H <<__E; - virtual const char *GetProtocolIdentString() + catch(ConnectionException &e) { - return GetIdentString(); + if(e.GetSubType() == ConnectionException::Protocol_ObjWhenStreamExpected) + { + THROW_EXCEPTION_MESSAGE(ConnectionException, + Protocol_ObjWhenStreamExpected, + "Last exchange was " << mPreviousCommand << + " => " << mPreviousReply); + } + else + { + throw; + } } +} __E + } if($writing_local) { @@ -853,23 +947,13 @@ __E print H <<__E; virtual int GetTimeout() { - return this->Protocol::GetTimeout(); + return $custom_protocol_subclass\::GetTimeout(); } __E } - + print H <<__E; - /* - virtual void Handshake() - { - this->Protocol::Handshake(); - } - virtual bool GetLastError(int &rTypeOut, int &rSubTypeOut) - { - return this->Protocol::GetLastError(rTypeOut, rSubTypeOut); - } - */ - + private: $server_or_client_class(const $server_or_client_class &rToCopy); /* no copies */ }; @@ -890,8 +974,8 @@ __E else { print CPP <<__E; -$server_or_client_class\::$server_or_client_class(IOStream &rStream) -: Protocol(rStream) +$server_or_client_class\::$server_or_client_class(std::auto_ptr<SocketStream> apConn) +: $custom_protocol_subclass(apConn) { } __E } @@ -903,49 +987,58 @@ $server_or_client_class\::~$server_or_client_class() __E # write receive and send functions - print CPP <<__E; -std::auto_ptr<Message> $server_or_client_class\::MakeMessage(int ObjType) -{ - switch(ObjType) - { -__E - - # do objects within this - for my $cmd (@cmd_list) + if($writing_local) { print CPP <<__E; - case $cmd_id{$cmd}: - return std::auto_ptr<Message>(new $cmd_class{$cmd}()); - break; -__E - } - - print CPP <<__E; - default: - THROW_EXCEPTION(ConnectionException, Conn_Protocol_UnknownCommandRecieved) - } +std::auto_ptr<$message_base_class> $server_or_client_class\::Receive() +{ + return mapLastReply; +} +void $server_or_client_class\::Send(const $message_base_class &rObject) +{ + mapLastReply = rObject.DoCommand(*this, mrContext); } __E - - if(not $writing_local) + } + else { print CPP <<__E; std::auto_ptr<$message_base_class> $server_or_client_class\::Receive() { - std::auto_ptr<$message_base_class> preply(($message_base_class *) - Protocol::ReceiveInternal().release()); + std::auto_ptr<$message_base_class> apReply; + + try + { + apReply = std::auto_ptr<$message_base_class>( + static_cast<$message_base_class *> + ($custom_protocol_subclass\::ReceiveInternal().release())); + } + catch(ConnectionException &e) + { + if(e.GetSubType() == ConnectionException::Protocol_StreamWhenObjExpected) + { + THROW_EXCEPTION_MESSAGE(ConnectionException, + Protocol_StreamWhenObjExpected, + "Last exchange was " << mPreviousCommand << + " => " << mPreviousReply); + } + else + { + throw; + } + } if(GetLogToSysLog()) { - preply->LogSysLog("Receive"); + apReply->LogSysLog("Receive"); } if(GetLogToFile() != 0) { - preply->LogFile("Receive", GetLogToFile()); + apReply->LogFile("Receive", GetLogToFile()); } - return preply; + return apReply; } void $server_or_client_class\::Send(const $message_base_class &rObject) @@ -981,10 +1074,40 @@ void $server_or_client_class\::DoServer($context_class &rContext) { // Get an object from the conversation std::auto_ptr<$message_base_class> pobj = Receive(); + std::auto_ptr<$message_base_class> preply; // Run the command - std::auto_ptr<$message_base_class> preply = pobj->DoCommand(*this, rContext); - + try + { + try + { + if(pobj->HasStreamWithCommand()) + { + std::auto_ptr<IOStream> apDataStream = ReceiveStream(); + SelfFlushingStream autoflush(*apDataStream); + preply = pobj->DoCommand(*this, rContext, *apDataStream); + } + else + { + preply = pobj->DoCommand(*this, rContext); + } + } + catch(BoxException &e) + { + // First try a the built-in exception handler + preply = HandleException(e); + } + } + catch (...) + { + // Fallback in case the exception isn't a BoxException + // or the exception handler fails as well. This path + // throws the exception upwards, killing the process + // that handles the current client. + Send($cmd_classes{$error_message}(-1)); + throw; + } + // Send the reply Send(*preply); @@ -995,7 +1118,16 @@ void $server_or_client_class\::DoServer($context_class &rContext) { SendStream(**i); } - + + // As a server, if we get an unexpected message later, we'll + // want to know the last command that we received, and the + // reply, to help debug our response to it. + mPreviousCommand = pobj->ToString(); + std::ostringstream reply; + reply << preply->ToString() << " and " << + mStreamsToSend.size() << " streams"; + mPreviousReply = reply.str(); + // Delete these streams DeleteStreamsToSend(); @@ -1004,7 +1136,7 @@ void $server_or_client_class\::DoServer($context_class &rContext) { inProgress = false; } - } + } } __E @@ -1017,67 +1149,86 @@ __E { if(obj_is_type($cmd,'Command')) { - my $request_class = $cmd_class{$cmd}; + my $request_class = $cmd_classes{$cmd}; my $reply_msg = obj_get_type_params($cmd,'Command'); - my $reply_class = $cmd_class{$reply_msg}; + my $reply_class = $cmd_classes{$reply_msg}; my $reply_id = $cmd_id{$reply_msg}; my $has_stream = obj_is_type($cmd,'StreamWithCommand'); - my $argextra = $has_stream?', std::auto_ptr<IOStream> apStream':''; + my $argextra = $has_stream?', std::auto_ptr<IOStream> apDataStream':''; my $send_stream_extra = ''; - my $send_stream_method = $writing_client ? "SendStream" - : "SendStreamAfterCommand"; - + + print CPP <<__E; +std::auto_ptr<$reply_class> $server_or_client_class\::Query(const $request_class &rQuery$argextra) +{ +__E + if($writing_client) { if($has_stream) { $send_stream_extra = <<__E; // Send stream after the command - SendStream(*apStream); + try + { + SendStream(*apDataStream); + } + catch (BoxException &e) + { + BOX_WARNING("Failed to send stream after command: " << + rQuery.ToString() << ": " << e.what()); + throw; + } __E } print CPP <<__E; -std::auto_ptr<$reply_class> $server_or_client_class\::Query(const $request_class &rQuery$argextra) -{ // Send query Send(rQuery); - $send_stream_extra +$send_stream_extra // Wait for the reply - std::auto_ptr<$message_base_class> preply = Receive(); - - CheckReply("$cmd", *preply, $reply_id); - - // Correct response, if no exception thrown by CheckReply - return std::auto_ptr<$reply_class>(($reply_class *)preply.release()); -} + std::auto_ptr<$message_base_class> apReply = Receive(); __E } elsif($writing_local) { + print CPP <<__E; + std::auto_ptr<$message_base_class> apReply; + try + { +__E if($has_stream) { - $send_stream_extra = <<__E; - // Send stream after the command - SendStreamAfterCommand(apStream); + print CPP <<__E; + apReply = rQuery.DoCommand(*this, mrContext, *apDataStream); +__E + } + else + { + print CPP <<__E; + apReply = rQuery.DoCommand(*this, mrContext); __E } print CPP <<__E; -std::auto_ptr<$reply_class> $server_or_client_class\::Query(const $request_class &rQuery$argextra) -{ - // Send query - $send_stream_extra - std::auto_ptr<$message_base_class> preply = rQuery.DoCommand(*this, mrContext); - - CheckReply("$cmd", *preply, $reply_id); + } + catch(BoxException &e) + { + // First try a the built-in exception handler + apReply = HandleException(e); + } +__E + } + + # Common to both client and local + print CPP <<__E; + CheckReply("$cmd", rQuery, *apReply, $reply_id); // Correct response, if no exception thrown by CheckReply - return std::auto_ptr<$reply_class>(($reply_class *)preply.release()); + return std::auto_ptr<$reply_class>( + static_cast<$reply_class *>(apReply.release())); } __E - } } } } @@ -1110,7 +1261,7 @@ sub obj_get_type_params { return $1 if $_ =~ m/\A$ty\((.+?)\)\Z/; } - die "Can't find attribute $ty\n" + die "Can't find attribute $ty on command $c\n" } # returns (is basic type, typename) diff --git a/lib/win32/box_getopt.h b/lib/win32/box_getopt.h new file mode 100644 index 00000000..f18446d4 --- /dev/null +++ b/lib/win32/box_getopt.h @@ -0,0 +1,14 @@ +#if defined _MSC_VER || defined __MINGW32__ +#define REPLACE_GETOPT 1 /* use this getopt as the system getopt(3) */ +#else +#define REPLACE_GETOPT 0 // force a conflict if included multiple times +#endif + +#if REPLACE_GETOPT +# include "bsd_getopt.h" +# define BOX_BSD_GETOPT +#else +# include <getopt.h> +# undef BOX_BSD_GETOPT +#endif + diff --git a/lib/win32/getopt.h b/lib/win32/bsd_getopt.h index 7c290343..3e2441ca 100755 --- a/lib/win32/getopt.h +++ b/lib/win32/bsd_getopt.h @@ -1,98 +1,105 @@ -/* $OpenBSD: getopt.h,v 1.1 2002/12/03 20:24:29 millert Exp $ */
-/* $NetBSD: getopt.h,v 1.4 2000/07/07 10:43:54 ad Exp $ */
-
-/*-
- * Copyright (c) 2000 The NetBSD Foundation, Inc.
- * All rights reserved.
- *
- * This code is derived from software contributed to The NetBSD Foundation
- * by Dieter Baron and Thomas Klausner.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- * must display the following acknowledgement:
- * This product includes software developed by the NetBSD
- * Foundation, Inc. and its contributors.
- * 4. Neither the name of The NetBSD Foundation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _GETOPT_H_
-#define _GETOPT_H_
-
-// copied from: http://www.la.utexas.edu/lab/software/devtool/gnu/libtool/C_header_files.html
-
-/* __BEGIN_DECLS should be used at the beginning of your declarations,
- so that C++ compilers don't mangle their names. Use __END_DECLS at
- the end of C declarations. */
-#undef __BEGIN_DECLS
-#undef __END_DECLS
-#ifdef __cplusplus
-# define __BEGIN_DECLS extern "C" {
-# define __END_DECLS }
-#else
-# define __BEGIN_DECLS /* empty */
-# define __END_DECLS /* empty */
-#endif
-
-/*
- * GNU-like getopt_long() and 4.4BSD getsubopt()/optreset extensions
- */
-#define no_argument 0
-#define required_argument 1
-#define optional_argument 2
-
-struct option {
- /* name of long option */
- const char *name;
- /*
- * one of no_argument, required_argument, and optional_argument:
- * whether option takes an argument
- */
- int has_arg;
- /* if not NULL, set *flag to val when option found */
- int *flag;
- /* if flag not NULL, value to set *flag to; else return value */
- int val;
-};
-
-__BEGIN_DECLS
-int getopt_long(int, char * const *, const char *,
- const struct option *, int *);
-int getopt_long_only(int, char * const *, const char *,
- const struct option *, int *);
-#ifndef _GETOPT_DEFINED_
-#define _GETOPT_DEFINED_
-int getopt(int, char * const *, const char *);
-int getsubopt(char **, char * const *, char **);
-
-extern char *optarg; /* getopt(3) external variables */
-extern int opterr;
-extern int optind;
-extern int optopt;
-extern int optreset;
-extern char *suboptarg; /* getsubopt(3) external variable */
-#endif
-__END_DECLS
-
-#endif /* !_GETOPT_H_ */
+/* $OpenBSD: getopt.h,v 1.1 2002/12/03 20:24:29 millert Exp $ */ +/* $NetBSD: getopt.h,v 1.4 2000/07/07 10:43:54 ad Exp $ */ + +/*- + * Copyright (c) 2000 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Dieter Baron and Thomas Klausner. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the NetBSD + * Foundation, Inc. and its contributors. + * 4. Neither the name of The NetBSD Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef REPLACE_GETOPT +#error You must include box_getopt.h, not bsd_getopt.h +#endif + +#if REPLACE_GETOPT // defined in box_getopt.h; until end of file + +#ifndef _GETOPT_H_ +#define _GETOPT_H_ + +// copied from: http://www.la.utexas.edu/lab/software/devtool/gnu/libtool/C_header_files.html + +/* __BEGIN_DECLS should be used at the beginning of your declarations, + so that C++ compilers don't mangle their names. Use __END_DECLS at + the end of C declarations. */ +#undef __BEGIN_DECLS +#undef __END_DECLS +#ifdef __cplusplus +# define __BEGIN_DECLS extern "C" { +# define __END_DECLS } +#else +# define __BEGIN_DECLS /* empty */ +# define __END_DECLS /* empty */ +#endif + +/* + * GNU-like getopt_long() and 4.4BSD getsubopt()/optreset extensions + */ +#define no_argument 0 +#define required_argument 1 +#define optional_argument 2 + +struct option { + /* name of long option */ + const char *name; + /* + * one of no_argument, required_argument, and optional_argument: + * whether option takes an argument + */ + int has_arg; + /* if not NULL, set *flag to val when option found */ + int *flag; + /* if flag not NULL, value to set *flag to; else return value */ + int val; +}; + +__BEGIN_DECLS +int getopt_long(int, char * const *, const char *, + const struct option *, int *); +int getopt_long_only(int, char * const *, const char *, + const struct option *, int *); +#ifndef _GETOPT_DEFINED_ +#define _GETOPT_DEFINED_ +int getopt(int, char * const *, const char *); +int getsubopt(char **, char * const *, char **); + +extern char *optarg; /* getopt(3) external variables */ +extern int opterr; +extern int optind; +extern int optopt; +extern int optreset; +extern char *suboptarg; /* getsubopt(3) external variable */ +#endif +__END_DECLS + +#endif /* !_GETOPT_H_ */ +#endif // REPLACE_GETOPT diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp index ef237671..1f6392d5 100644 --- a/lib/win32/emu.cpp +++ b/lib/win32/emu.cpp @@ -32,8 +32,9 @@ bool EnableBackupRights() if(!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &hToken)) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to open process token: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return false; } @@ -45,8 +46,9 @@ bool EnableBackupRights() SE_BACKUP_NAME, //the name of the privilege &( token_priv.Privileges[0].Luid ))) //result { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to lookup backup privilege: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); CloseHandle(hToken); return false; } @@ -68,8 +70,9 @@ bool EnableBackupRights() //this function is a little tricky - if we were adjusting //more than one privilege, it could return success but not //adjust them all - in the general case, you need to trap this + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to enable backup privilege: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); CloseHandle(hToken); return false; @@ -238,9 +241,10 @@ char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) if (len == 0) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to convert wide string to narrow: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EINVAL; return NULL; } @@ -270,9 +274,10 @@ char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) if (len == 0) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to convert wide string to narrow: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EACCES; delete [] buffer; return NULL; @@ -299,9 +304,10 @@ bool ConvertFromWideString(const std::wstring& rInput, if (len == 0) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to convert wide string to narrow: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EINVAL; return false; } @@ -331,9 +337,10 @@ bool ConvertFromWideString(const std::wstring& rInput, if (len == 0) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to convert wide string to narrow: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EACCES; delete [] buffer; return false; @@ -363,10 +370,11 @@ bool ConvertEncoding(const std::string& rSource, int sourceCodePage, true); if (pWide == NULL) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to convert string '%s' from " "current code page %d to wide string: %s", rSource.c_str(), sourceCodePage, - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return false; } @@ -528,7 +536,10 @@ std::string GetErrorMessage(DWORD errorCode) if (chars == 0 || pMsgBuf == NULL) { - return std::string("failed to get error message"); + std::ostringstream oss; + oss << "Failed to get error message for error code " << errorCode << ": error " << + GetLastError(); + return oss.str(); } // remove embedded newline @@ -605,7 +616,7 @@ HANDLE openfile(const char *pFileName, int flags, int mode) createDisposition = CREATE_NEW; } - if (flags & O_LOCK) + if (flags & BOX_OPEN_LOCK) { shareMode = 0; } @@ -641,7 +652,7 @@ HANDLE openfile(const char *pFileName, int flags, int mode) ::syslog(LOG_WARNING, "Failed to open file '%s': " "%s", pFileName, - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return INVALID_HANDLE_VALUE; } @@ -684,16 +695,18 @@ int emu_fstat(HANDLE hdir, struct emu_stat * st) BY_HANDLE_FILE_INFORMATION fi; if (!GetFileInformationByHandle(hdir, &fi)) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to read file information: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EACCES; return -1; } if (INVALID_FILE_ATTRIBUTES == fi.dwFileAttributes) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to get file attributes: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EACCES; return -1; } @@ -826,10 +839,10 @@ HANDLE OpenFileByNameUtf8(const char* pFileName, DWORD flags) if (handle == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); + winerrno = GetLastError(); - if (err == ERROR_FILE_NOT_FOUND || - err == ERROR_PATH_NOT_FOUND) + if (winerrno == ERROR_FILE_NOT_FOUND || + winerrno == ERROR_PATH_NOT_FOUND) { errno = ENOENT; } @@ -837,7 +850,7 @@ HANDLE OpenFileByNameUtf8(const char* pFileName, DWORD flags) { ::syslog(LOG_WARNING, "Failed to open '%s': " "%s", pFileName, - GetErrorMessage(err).c_str()); + GetErrorMessage(winerrno).c_str()); errno = EACCES; } @@ -906,9 +919,10 @@ int statfs(const char * pName, struct statfs * s) BY_HANDLE_FILE_INFORMATION fi; if (!GetFileInformationByHandle(handle, &fi)) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to get file information " "for '%s': %s", pName, - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); CloseHandle(handle); errno = EACCES; return -1; @@ -961,8 +975,9 @@ int emu_utimes(const char * pName, const struct timeval times[]) if (!SetFileTime(handle, &creationTime, NULL, &modificationTime)) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to set times on '%s': %s", pName, - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); CloseHandle(handle); return 1; } @@ -1004,8 +1019,9 @@ int emu_chmod(const char * pName, mode_t mode) DWORD attribs = GetFileAttributesW(pBuffer); if (attribs == INVALID_FILE_ATTRIBUTES) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to get file attributes of '%s': %s", - pName, GetErrorMessage(GetLastError()).c_str()); + pName, GetErrorMessage(winerrno).c_str()); errno = EACCES; free(pBuffer); return -1; @@ -1022,8 +1038,9 @@ int emu_chmod(const char * pName, mode_t mode) if (!SetFileAttributesW(pBuffer, attribs)) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to set file attributes of '%s': %s", - pName, GetErrorMessage(GetLastError()).c_str()); + pName, GetErrorMessage(winerrno).c_str()); errno = EACCES; free(pBuffer); return -1; @@ -1078,7 +1095,6 @@ DIR *opendir(const char *name) } pDir->fd = FindFirstFileW(pDir->name, &pDir->info); - DWORD tmp = GetLastError(); if (pDir->fd == INVALID_HANDLE_VALUE) { @@ -1297,7 +1313,7 @@ int poll (struct pollfd *ufds, unsigned long nfds, int timeout) BOOL AddEventSource ( - LPTSTR pszSrcName, // event source name + const std::string& name, // event source name DWORD dwNum // number of categories ) { @@ -1309,8 +1325,9 @@ BOOL AddEventSource if (len == 0) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to get the program file name: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return FALSE; } @@ -1318,31 +1335,39 @@ BOOL AddEventSource std::string regkey("SYSTEM\\CurrentControlSet\\Services\\EventLog\\" "Application\\"); - regkey += pszSrcName; + regkey += name; HKEY hk; DWORD dwDisp; - if (RegCreateKeyEx(HKEY_LOCAL_MACHINE, regkey.c_str(), - 0, NULL, REG_OPTION_NON_VOLATILE, - KEY_WRITE, NULL, &hk, &dwDisp)) + winerrno = RegCreateKeyEx(HKEY_LOCAL_MACHINE, regkey.c_str(), + 0, NULL, REG_OPTION_NON_VOLATILE, + KEY_WRITE, NULL, &hk, &dwDisp); + if (winerrno == ERROR_ACCESS_DENIED) { - ::syslog(LOG_ERR, "Failed to create the registry key: %s", - GetErrorMessage(GetLastError()).c_str()); + ::syslog(LOG_ERR, "Failed to create the registry key: access denied. You must " + "be an Administrator to register new event sources in %s", regkey.c_str()); + return FALSE; + } + else if (winerrno != ERROR_SUCCESS) + { + ::syslog(LOG_ERR, "Failed to create the registry key: %s: %s", + GetErrorMessage(winerrno).c_str(), regkey.c_str()); return FALSE; } // Set the name of the message file. - if (RegSetValueExW(hk, // subkey handle - L"EventMessageFile", // value name - 0, // must be zero - REG_EXPAND_SZ, // value type - (LPBYTE)cmd, // pointer to value data - len*sizeof(WCHAR))) // data size + winerrno = RegSetValueExW(hk, // subkey handle + L"EventMessageFile", // value name + 0, // must be zero + REG_EXPAND_SZ, // value type + (LPBYTE)cmd, // pointer to value data + len*sizeof(WCHAR)); // data size + if (winerrno != ERROR_SUCCESS) { ::syslog(LOG_ERR, "Failed to set the event message file: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); RegCloseKey(hk); return FALSE; } @@ -1352,43 +1377,46 @@ BOOL AddEventSource DWORD dwData = EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | EVENTLOG_INFORMATION_TYPE; - if (RegSetValueEx(hk, // subkey handle - "TypesSupported", // value name - 0, // must be zero - REG_DWORD, // value type - (LPBYTE) &dwData, // pointer to value data - sizeof(DWORD))) // length of value data + winerrno = RegSetValueEx(hk, // subkey handle + "TypesSupported", // value name + 0, // must be zero + REG_DWORD, // value type + (LPBYTE) &dwData, // pointer to value data + sizeof(DWORD)); // length of value data + if (winerrno != ERROR_SUCCESS) { ::syslog(LOG_ERR, "Failed to set the supported types: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); RegCloseKey(hk); return FALSE; } // Set the category message file and number of categories. - if (RegSetValueExW(hk, // subkey handle - L"CategoryMessageFile", // value name - 0, // must be zero - REG_EXPAND_SZ, // value type - (LPBYTE)cmd, // pointer to value data - len*sizeof(WCHAR))) // data size + winerrno = RegSetValueExW(hk, // subkey handle + L"CategoryMessageFile", // value name + 0, // must be zero + REG_EXPAND_SZ, // value type + (LPBYTE)cmd, // pointer to value data + len*sizeof(WCHAR)); // data size + if (winerrno != ERROR_SUCCESS) { ::syslog(LOG_ERR, "Failed to set the category message file: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); RegCloseKey(hk); return FALSE; } - if (RegSetValueEx(hk, // subkey handle + winerrno = RegSetValueEx(hk, // subkey handle "CategoryCount", // value name 0, // must be zero REG_DWORD, // value type (LPBYTE) &dwNum, // pointer to value data - sizeof(DWORD))) // length of value data + sizeof(DWORD)); // length of value data + if (winerrno != ERROR_SUCCESS) { ::syslog(LOG_ERR, "Failed to set the category count: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); RegCloseKey(hk); return FALSE; } @@ -1397,7 +1425,7 @@ BOOL AddEventSource return TRUE; } -static HANDLE gSyslogH = 0; +static HANDLE gSyslogH = INVALID_HANDLE_VALUE; static bool sHaveWarnedEventLogFull = false; void openlog(const char * daemonName, int, int) @@ -1406,19 +1434,21 @@ void openlog(const char * daemonName, int, int) nameStr += daemonName; nameStr += ")"; - // register a default event source, so that we can - // log errors with the process of adding or registering our own. + // Don't try to open a new handle when one is already open. It will leak handles. + assert(gSyslogH == INVALID_HANDLE_VALUE); + + // Register a default event source, so that we can log errors with the process of + // adding or registering our own, which follows. If this fails, there's not much we + // can do about it, certainly not send anything to the event log! gSyslogH = RegisterEventSource( NULL, // uses local computer nameStr.c_str()); // source name if (gSyslogH == NULL) { + gSyslogH = INVALID_HANDLE_VALUE; } - char* name = strdup(nameStr.c_str()); - BOOL success = AddEventSource(name, 0); - free(name); - + BOOL success = AddEventSource(nameStr, 0); if (!success) { ::syslog(LOG_ERR, "Failed to add our own event source"); @@ -1428,8 +1458,9 @@ void openlog(const char * daemonName, int, int) HANDLE newSyslogH = RegisterEventSource(NULL, nameStr.c_str()); if (newSyslogH == NULL) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to register our own event source: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); return; } @@ -1439,7 +1470,11 @@ void openlog(const char * daemonName, int, int) void closelog(void) { - DeregisterEventSource(gSyslogH); + if(gSyslogH != INVALID_HANDLE_VALUE) + { + DeregisterEventSource(gSyslogH); + gSyslogH = INVALID_HANDLE_VALUE; + } } void syslog(int loglevel, const char *frmt, ...) @@ -1494,7 +1529,7 @@ void syslog(int loglevel, const char *frmt, ...) va_end(args); - if (gSyslogH == 0) + if (gSyslogH == INVALID_HANDLE_VALUE) { printf("%s\r\n", buffer); fflush(stdout); @@ -1541,22 +1576,20 @@ void syslog(int loglevel, const char *frmt, ...) if (result == 0) { - DWORD err = GetLastError(); - if (err == ERROR_LOG_FILE_FULL) + winerrno = GetLastError(); + if (winerrno == ERROR_LOG_FILE_FULL) { if (!sHaveWarnedEventLogFull) { printf("Unable to send message to Event Log " - "(Event Log is full):\r\n"); - fflush(stdout); + "(Event Log is full): %s\r\n", buffer); sHaveWarnedEventLogFull = TRUE; } } else { - printf("Unable to send message to Event Log: %s:\r\n", - GetErrorMessage(err).c_str()); - fflush(stdout); + printf("Unable to send message to Event Log: %s: %s\r\n", + GetErrorMessage(winerrno).c_str(), buffer); } } else @@ -1589,8 +1622,9 @@ int emu_chdir(const char* pDirName) if (result != 0) return 0; errno = EACCES; + winerrno = GetLastError(); fprintf(stderr, "Failed to change directory to '%s': %s\n", - pDirName, GetErrorMessage(GetLastError()).c_str()); + pDirName, GetErrorMessage(winerrno).c_str()); return -1; } @@ -1668,6 +1702,74 @@ int emu_mkdir(const char* pPathName) return 0; } +int emu_link(const char* pOldPath, const char* pNewPath) +{ + std::string AbsOldPathWithUnicode = + ConvertPathToAbsoluteUnicode(pOldPath); + + if (AbsOldPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return -1; + } + + std::string AbsNewPathWithUnicode = + ConvertPathToAbsoluteUnicode(pNewPath); + + if (AbsNewPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return -1; + } + + WCHAR* pOldBuffer = ConvertUtf8ToWideString(AbsOldPathWithUnicode.c_str()); + if (!pOldBuffer) + { + return -1; + } + + WCHAR* pNewBuffer = ConvertUtf8ToWideString(AbsNewPathWithUnicode.c_str()); + if (!pNewBuffer) + { + delete [] pOldBuffer; + return -1; + } + + BOOL result = CreateHardLinkW(pNewBuffer, pOldBuffer, NULL); + winerrno = GetLastError(); + delete [] pOldBuffer; + delete [] pNewBuffer; + + if (!result) + { + if (winerrno == ERROR_FILE_NOT_FOUND || + winerrno == ERROR_PATH_NOT_FOUND) + { + errno = ENOENT; + } + else if (winerrno == ERROR_SHARING_VIOLATION) + { + errno = EBUSY; + } + else if (winerrno == ERROR_ACCESS_DENIED) + { + errno = EACCES; + } + else + { + ::syslog(LOG_WARNING, "Failed to hardlink file " + "'%s' to '%s': %s", pOldPath, pNewPath, + GetErrorMessage(winerrno).c_str()); + errno = ENOSYS; + } + + return -1; + } + + return 0; + +} + int emu_unlink(const char* pFileName) { std::string AbsPathWithUnicode = @@ -1686,20 +1788,21 @@ int emu_unlink(const char* pFileName) } BOOL result = DeleteFileW(pBuffer); - DWORD err = GetLastError(); + winerrno = GetLastError(); delete [] pBuffer; if (!result) { - if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + if (winerrno == ERROR_FILE_NOT_FOUND || + winerrno == ERROR_PATH_NOT_FOUND) { errno = ENOENT; } - else if (err == ERROR_SHARING_VIOLATION) + else if (winerrno == ERROR_SHARING_VIOLATION) { errno = EBUSY; } - else if (err == ERROR_ACCESS_DENIED) + else if (winerrno == ERROR_ACCESS_DENIED) { errno = EACCES; } @@ -1707,9 +1810,10 @@ int emu_unlink(const char* pFileName) { ::syslog(LOG_WARNING, "Failed to delete file " "'%s': %s", pFileName, - GetErrorMessage(err).c_str()); + GetErrorMessage(winerrno).c_str()); errno = ENOSYS; } + return -1; } @@ -1751,21 +1855,22 @@ int emu_rename(const char* pOldFileName, const char* pNewFileName) } BOOL result = MoveFileW(pOldBuffer, pNewBuffer); - DWORD err = GetLastError(); + winerrno = GetLastError(); delete [] pOldBuffer; delete [] pNewBuffer; if (!result) { - if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + if (winerrno == ERROR_FILE_NOT_FOUND || + winerrno == ERROR_PATH_NOT_FOUND) { errno = ENOENT; } - else if (err == ERROR_SHARING_VIOLATION) + else if (winerrno == ERROR_SHARING_VIOLATION) { errno = EBUSY; } - else if (err == ERROR_ACCESS_DENIED) + else if (winerrno == ERROR_ACCESS_DENIED) { errno = EACCES; } @@ -1773,7 +1878,7 @@ int emu_rename(const char* pOldFileName, const char* pNewFileName) { ::syslog(LOG_WARNING, "Failed to rename file " "'%s' to '%s': %s", pOldFileName, pNewFileName, - GetErrorMessage(err).c_str()); + GetErrorMessage(winerrno).c_str()); errno = ENOSYS; } return -1; @@ -1788,8 +1893,9 @@ int console_read(char* pBuffer, size_t BufferSize) if (hConsole == INVALID_HANDLE_VALUE) { + winerrno = GetLastError(); ::fprintf(stderr, "Failed to get a handle on standard input: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); return -1; } @@ -1812,8 +1918,9 @@ int console_read(char* pBuffer, size_t BufferSize) NULL // reserved )) { + winerrno = GetLastError(); ::fprintf(stderr, "Failed to read from console: %s\n", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return -1; } @@ -1864,7 +1971,7 @@ int writev(int filedes, const struct iovec *vector, size_t count) return bytes; } -// need this for conversions +// Need this for conversions. Works in UTC. time_t ConvertFileTimeToTime_t(FILETIME *fileTime) { SYSTEMTIME stUTC; @@ -1883,18 +1990,17 @@ time_t ConvertFileTimeToTime_t(FILETIME *fileTime) // timeinfo.tm_yday = ...; timeinfo.tm_year = stUTC.wYear - 1900; - time_t retVal = mktime(&timeinfo) - _timezone; + time_t retVal = _mkgmtime(&timeinfo); return retVal; } bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo) { - time_t adjusted = from + _timezone; - struct tm *time_breakdown = gmtime(&adjusted); + struct tm *time_breakdown = gmtime(&from); if (time_breakdown == NULL) { ::syslog(LOG_ERR, "Error: failed to convert time format: " - "%d is not a valid time\n", adjusted); + "%d is not a valid time\n", from); return false; } @@ -1911,8 +2017,9 @@ bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo) // Convert the last-write time to local time. if (!SystemTimeToFileTime(&stUTC, pTo)) { + winerrno = GetLastError(); syslog(LOG_ERR, "Failed to convert between time formats: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return false; } diff --git a/lib/win32/emu.h b/lib/win32/emu.h index bf408050..91793004 100644 --- a/lib/win32/emu.h +++ b/lib/win32/emu.h @@ -18,7 +18,14 @@ #define EMU_INCLUDE // Need feature detection macros below -#include "../common/BoxConfig.h" +#if defined BOX_CMAKE +# include "../common/BoxConfig.cmake.h" +#elif defined _MSC_VER +# include "../common/BoxConfig-MSVC.h" +# define NEED_BOX_VERSION_H +#else +# include "../common/BoxConfig.h" +#endif // Shut up stupid new warnings. Thanks MinGW! Ever heard of "compatibility"? #ifdef __MINGW32__ @@ -27,31 +34,21 @@ // basic types, may be required by other headers since we // don't include sys/types.h - -#ifdef __MINGW32__ - #include <stdint.h> -#else // MSVC - typedef unsigned __int64 u_int64_t; - typedef unsigned __int64 uint64_t; - typedef __int64 int64_t; - typedef unsigned __int32 uint32_t; - typedef unsigned __int32 u_int32_t; - typedef __int32 int32_t; - typedef unsigned __int16 uint16_t; - typedef __int16 int16_t; - typedef unsigned __int8 uint8_t; - typedef __int8 int8_t; -#endif +#include <stdint.h> // emulated types, present on MinGW but not MSVC or vice versa -#ifdef __MINGW32__ - typedef uint32_t u_int32_t; -#else +#ifndef __MINGW32__ typedef unsigned int mode_t; typedef unsigned int pid_t; + typedef unsigned int uid_t; + typedef unsigned int gid_t; #endif +// Disable Windows' non-standard implementation of min() and max(): +// http://stackoverflow.com/a/5004874/648162 +#define NOMINMAX + // Windows headers #include <winsock2.h> @@ -76,17 +73,6 @@ #define ITIMER_REAL 0 -#ifdef _MSC_VER -// Microsoft decided to deprecate the standard POSIX functions. Great! -#define open(file,flags,mode) _open(file,flags,mode) -#define close(fd) _close(fd) -#define dup(fd) _dup(fd) -#define read(fd,buf,count) _read(fd,buf,count) -#define write(fd,buf,count) _write(fd,buf,count) -#define lseek(fd,off,whence) _lseek(fd,off,whence) -#define fileno(struct_file) _fileno(struct_file) -#endif - struct passwd { char *pw_name; char *pw_passwd; @@ -110,26 +96,34 @@ inline struct passwd * getpwnam(const char * name) return &gTempPasswd; } -#define S_IRWXG 1 -#define S_IRWXO 2 -#define S_ISUID 4 -#define S_ISGID 8 -#define S_ISVTX 16 - -#ifndef __MINGW32__ +#ifndef S_IRGRP + // these constants are only defined in MinGW64, not the original MinGW headers, + // nor MSVC, so use poor man's feature detection to define them only if needed. //not sure if these are correct //S_IWRITE - writing permitted //_S_IREAD - reading permitted //_S_IREAD | _S_IWRITE - - #define S_IRUSR S_IWRITE - #define S_IWUSR S_IREAD - #define S_IRWXU (S_IREAD|S_IWRITE|S_IEXEC) +# define S_IRUSR S_IWRITE +# define S_IWUSR S_IREAD +# define S_IRGRP S_IWRITE +# define S_IWGRP S_IREAD +# define S_IROTH S_IWRITE | S_IREAD +# define S_IWOTH S_IREAD | S_IREAD +# define S_IRWXU (S_IREAD|S_IWRITE|S_IEXEC) +# define S_IRWXG 1 +# define S_IRWXO 2 +#endif +#define S_ISUID 4 +#define S_ISGID 8 +#define S_ISVTX 16 + +#ifndef __MINGW32__ #define S_ISREG(x) (S_IFREG & x) #define S_ISDIR(x) (S_IFDIR & x) #endif -inline int chown(const char * Filename, u_int32_t uid, u_int32_t gid) +inline int chown(const char * Filename, uint32_t uid, uint32_t gid) { //important - this needs implementing //If a large restore is required then @@ -183,7 +177,7 @@ inline int geteuid(void) // MinGW provides a getopt implementation #ifndef __MINGW32__ -#include "getopt.h" +#include "box_getopt.h" #endif // !__MINGW32__ #define timespec timeval @@ -195,11 +189,6 @@ inline int geteuid(void) typedef int socklen_t; #endif -#define S_IRGRP S_IWRITE -#define S_IWGRP S_IREAD -#define S_IROTH S_IWRITE | S_IREAD -#define S_IWOTH S_IREAD | S_IREAD - //again need to verify these #define S_IFLNK 1 #define S_IFSOCK 0 @@ -209,9 +198,14 @@ inline int geteuid(void) #define vsnprintf _vsnprintf #ifndef __MINGW32__ +#define snprintf _snprintf inline int strcasecmp(const char *s1, const char *s2) { - return _stricmp(s1,s2); + return _stricmp(s1, s2); +} +inline int strncasecmp(const char *s1, const char *s2, size_t count) +{ + return _strnicmp(s1, s2, count); } #endif @@ -238,7 +232,7 @@ struct dirent *readdir(DIR *dp); int closedir(DIR *dp); // local constant to open file exclusively without shared access -#define O_LOCK 0x10000 +#define BOX_OPEN_LOCK 0x10000 extern DWORD winerrno; /* used to report errors from openfile() */ HANDLE openfile(const char *filename, int flags, int mode); @@ -279,7 +273,7 @@ void syslog (int loglevel, const char *fmt, ...); #define strtoll _strtoi64 #endif -inline unsigned int sleep(unsigned int secs) +extern "C" inline unsigned int sleep(unsigned int secs) { Sleep(secs*1000); return(ERROR_SUCCESS); @@ -347,6 +341,7 @@ bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo); int emu_chdir (const char* pDirName); int emu_mkdir (const char* pPathName); +int emu_link (const char* pOldPath, const char* pNewPath); int emu_unlink (const char* pFileName); int emu_fstat (HANDLE file, struct emu_stat* st); int emu_stat (const char* pName, struct emu_stat* st); @@ -357,6 +352,7 @@ int emu_rename (const char* pOldName, const char* pNewName); #define chdir(directory) emu_chdir (directory) #define mkdir(path, mode) emu_mkdir (path) +#define link(oldpath, newpath) emu_link (oldpath, newpath) #define unlink(file) emu_unlink (file) #define utimes(buffer, times) emu_utimes (buffer, times) #define chmod(file, mode) emu_chmod (file, mode) @@ -403,6 +399,7 @@ bool ConvertConsoleToUtf8(const std::string& rSource, std::string& rDest); char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage); bool ConvertFromWideString(const std::wstring& rInput, std::string* pOutput, unsigned int codepage); +WCHAR* ConvertUtf8ToWideString(const char* pString); std::string ConvertPathToAbsoluteUnicode(const char *pFileName); // Utility function which returns a default config file name, diff --git a/lib/win32/getopt_long.cpp b/lib/win32/getopt_long.cpp index 31695aa0..af2833a1 100755 --- a/lib/win32/getopt_long.cpp +++ b/lib/win32/getopt_long.cpp @@ -66,18 +66,15 @@ #include <stdio.h>
#include <string.h>
-#include "getopt.h"
+#include "box_getopt.h"
-#if defined _MSC_VER || defined __MINGW32__
-#define REPLACE_GETOPT /* use this getopt as the system getopt(3) */
+#ifdef REPLACE_GETOPT // until end of file
-#ifdef REPLACE_GETOPT
int opterr = 1; /* if error message should be printed */
int optind = 1; /* index into parent argv vector */
int optopt = '?'; /* character checked for validity */
int optreset; /* reset getopt */
char *optarg; /* argument associated with option */
-#endif
#define PRINT_ERROR ((opterr) && (*options != ':'))
@@ -499,7 +496,6 @@ start: return (optchar);
}
-#ifdef REPLACE_GETOPT
/*
* getopt --
* Parse argc/argv argument vector.
@@ -520,7 +516,6 @@ getopt(int nargc, char * const *nargv, const char *options) */
return (getopt_internal(nargc, nargv, options, NULL, NULL, 0));
}
-#endif /* REPLACE_GETOPT */
/*
* getopt_long --
@@ -548,4 +543,4 @@ getopt_long_only(int nargc, char * const *nargv, const char *options, FLAG_PERMUTE|FLAG_LONGONLY));
}
-#endif // defined _MSC_VER || defined __MINGW32__
+#endif // REPLACE_GETOPT
diff --git a/lib/win32/messages.h b/lib/win32/messages.h index 6959591b..22290226 100755 --- a/lib/win32/messages.h +++ b/lib/win32/messages.h @@ -1,57 +1,57 @@ - // Message source file, to be compiled to a resource file with
- // Microsoft Message Compiler (MC), to an object file with a Resource
- // Compiler, and linked into the application.
-
- // The main reason for this file is to work around Windows' stupid
- // messages in the Event Log, which say:
-
- // The description for Event ID ( 4 ) in Source ( Box Backup (bbackupd) )
- // cannot be found. The local computer may not have the necessary
- // registry information or message DLL files to display messages from a
- // remote computer. The following information is part of the event:
- // Message definitions follow
-//
-// Values are 32 bit values layed out as follows:
-//
-// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
-// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
-// +---+-+-+-----------------------+-------------------------------+
-// |Sev|C|R| Facility | Code |
-// +---+-+-+-----------------------+-------------------------------+
-//
-// where
-//
-// Sev - is the severity code
-//
-// 00 - Success
-// 01 - Informational
-// 10 - Warning
-// 11 - Error
-//
-// C - is the Customer code flag
-//
-// R - is a reserved bit
-//
-// Facility - is the facility code
-//
-// Code - is the facility's status code
-//
-//
-// Define the facility codes
-//
-
-
-//
-// Define the severity codes
-//
-
-
-//
-// MessageId: MSG_ERR
-//
-// MessageText:
-//
-// %1
-//
-#define MSG_ERR ((DWORD)0x40000001L)
-
+ // Message source file, to be compiled to a resource file with + // Microsoft Message Compiler (MC), to an object file with a Resource + // Compiler, and linked into the application. + + // The main reason for this file is to work around Windows' stupid + // messages in the Event Log, which say: + + // The description for Event ID ( 4 ) in Source ( Box Backup (bbackupd) ) + // cannot be found. The local computer may not have the necessary + // registry information or message DLL files to display messages from a + // remote computer. The following information is part of the event: + // Message definitions follow +// +// Values are 32 bit values layed out as follows: +// +// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 +// +---+-+-+-----------------------+-------------------------------+ +// |Sev|C|R| Facility | Code | +// +---+-+-+-----------------------+-------------------------------+ +// +// where +// +// Sev - is the severity code +// +// 00 - Success +// 01 - Informational +// 10 - Warning +// 11 - Error +// +// C - is the Customer code flag +// +// R - is a reserved bit +// +// Facility - is the facility code +// +// Code - is the facility's status code +// +// +// Define the facility codes +// + + +// +// Define the severity codes +// + + +// +// MessageId: MSG_ERR +// +// MessageText: +// +// %1 +// +#define MSG_ERR ((DWORD)0x40000001L) + |