From a84d45498bd861c9225080232948a99c2e317bb8 Mon Sep 17 00:00:00 2001 From: Reinhard Tartler Date: Thu, 2 Apr 2009 13:58:11 +0200 Subject: Import upstream version 0.11~rc3~r2491 --- bin/bbstored/BBStoreDHousekeeping.cpp | 16 +- bin/bbstored/BackupCommands.cpp | 165 +-- bin/bbstored/BackupConstants.h | 2 - bin/bbstored/BackupContext.cpp | 1661 ------------------------------ bin/bbstored/BackupContext.h | 149 --- bin/bbstored/BackupStoreContext.cpp | 1752 ++++++++++++++++++++++++++++++++ bin/bbstored/BackupStoreContext.h | 183 ++++ bin/bbstored/BackupStoreDaemon.cpp | 34 +- bin/bbstored/BackupStoreDaemon.h | 15 +- bin/bbstored/HousekeepStoreAccount.cpp | 274 +++-- bin/bbstored/HousekeepStoreAccount.h | 3 + bin/bbstored/backupprotocol.txt | 8 +- bin/bbstored/bbstored-config.in | 4 +- bin/bbstored/bbstored.cpp | 2 +- 14 files changed, 2267 insertions(+), 2001 deletions(-) delete mode 100644 bin/bbstored/BackupContext.cpp delete mode 100644 bin/bbstored/BackupContext.h create mode 100644 bin/bbstored/BackupStoreContext.cpp create mode 100644 bin/bbstored/BackupStoreContext.h (limited to 'bin/bbstored') diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index 16a1432a..1c1767ca 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -46,6 +46,12 @@ void BackupStoreDaemon::HousekeepingProcess() { RunHousekeepingIfNeeded(); + // Stop early? + if(StopRun()) + { + break; + } + // Calculate how long should wait before doing the next // housekeeping run int64_t timeNow = GetCurrentBoxTime(); @@ -155,7 +161,11 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() void BackupStoreDaemon::OnIdle() { - #ifdef WIN32 + if (!IsSingleProcess()) + { + return; + } + if (!mHousekeepingInited) { HousekeepingInit(); @@ -163,7 +173,6 @@ void BackupStoreDaemon::OnIdle() } RunHousekeepingIfNeeded(); - #endif } // -------------------------------------------------------------------------- @@ -193,7 +202,8 @@ bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitT std::string line; if(mInterProcessComms.GetLine(line, false /* no pre-processing */, MaximumWaitTime)) { - TRACE1("Housekeeping received command '%s' over interprocess comms\n", line.c_str()); + BOX_TRACE("Housekeeping received command '" << line << + "' over interprocess comms"); int account = 0; diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index bca52c04..38cda234 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -13,25 +13,25 @@ #include #include "autogen_BackupProtocolServer.h" +#include "autogen_RaidFileException.h" #include "BackupConstants.h" -#include "BackupContext.h" -#include "CollectInBufferStream.h" +#include "BackupStoreContext.h" +#include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "StreamableMemBlock.h" -#include "BackupStoreConstants.h" -#include "RaidFileController.h" #include "BackupStoreInfo.h" -#include "RaidFileController.h" +#include "BufferedStream.h" +#include "CollectInBufferStream.h" #include "FileStream.h" #include "InvisibleTempFileStream.h" -#include "BufferedStream.h" +#include "RaidFileController.h" +#include "StreamableMemBlock.h" #include "MemLeakFindOn.h" #define CHECK_PHASE(phase) \ - if(rContext.GetPhase() != BackupContext::phase) \ + if(rContext.GetPhase() != BackupStoreContext::phase) \ { \ return std::auto_ptr(new BackupProtocolServerError( \ BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_NotInRightProtocolPhase)); \ @@ -47,12 +47,12 @@ // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerVersion::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerVersion::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Return the current version, or an error if the requested version isn't allowed // Created: 2003/08/20 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerVersion::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerVersion::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Version) @@ -64,7 +64,7 @@ std::auto_ptr BackupProtocolServerVersion::DoCommand(BackupProto } // Mark the next phase - rContext.SetPhase(BackupContext::Phase_Login); + rContext.SetPhase(BackupStoreContext::Phase_Login); // Return our version return std::auto_ptr(new BackupProtocolServerVersion(BACKUP_STORE_SERVER_VERSION)); @@ -73,12 +73,12 @@ std::auto_ptr BackupProtocolServerVersion::DoCommand(BackupProto // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerLogin::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerLogin::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Return the current version, or an error if the requested version isn't allowed // Created: 2003/08/20 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Login) @@ -131,7 +131,7 @@ std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtoco int64_t clientStoreMarker = rContext.GetClientStoreMarker(); // Mark the next phase - rContext.SetPhase(BackupContext::Phase_Commands); + rContext.SetPhase(BackupStoreContext::Phase_Commands); // Log login BOX_NOTICE("Login from Client ID " << @@ -151,12 +151,12 @@ std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtoco // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerFinished::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerFinished::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Marks end of conversation (Protocol framework handles this) // Created: 2003/08/20 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { BOX_NOTICE("Session finished for Client ID " << BOX_FORMAT_ACCOUNT(rContext.GetClientID())); @@ -172,47 +172,72 @@ std::auto_ptr BackupProtocolServerFinished::DoCommand(BackupProt // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerListDirectory::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerListDirectory::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Command to list a directory // Created: 2003/09/02 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerListDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerListDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) - // Ask the context for a directory - const BackupStoreDirectory &rdir(rContext.GetDirectory(mObjectID)); - // Store the listing to a stream std::auto_ptr stream(new CollectInBufferStream); - rdir.WriteToStream(*stream, mFlagsMustBeSet, mFlagsNotToBeSet, mSendAttributes, - false /* never send dependency info to the client */); + + try + { + // Ask the context for a directory + const BackupStoreDirectory &rdir( + rContext.GetDirectory(mObjectID)); + rdir.WriteToStream(*stream, mFlagsMustBeSet, + mFlagsNotToBeSet, mSendAttributes, + false /* never send dependency info to the client */); + } + catch (RaidFileException &e) + { + if (e.GetSubType() == RaidFileException::RaidFileDoesntExist) + { + return std::auto_ptr( + new BackupProtocolServerError( + BackupProtocolServerError::ErrorType, + BackupProtocolServerError::Err_DoesNotExist)); + } + throw; + } + stream->SetForReading(); // Get the protocol to send the stream rProtocol.SendStreamAfterCommand(stream.release()); - return std::auto_ptr(new BackupProtocolServerSuccess(mObjectID)); + return std::auto_ptr( + new BackupProtocolServerSuccess(mObjectID)); } // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerStoreFile::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerStoreFile::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Command to store a file on the server // Created: 2003/09/02 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerStoreFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerStoreFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION + + std::auto_ptr hookResult = + rContext.StartCommandHook(*this); + if(hookResult.get()) + { + return hookResult; + } // Check that the diff from file actually exists, if it's specified if(mDiffFromFileID != 0) { - if(!rContext.ObjectExists(mDiffFromFileID, BackupContext::ObjectExists_File)) + if(!rContext.ObjectExists(mDiffFromFileID, BackupStoreContext::ObjectExists_File)) { return std::auto_ptr(new BackupProtocolServerError( BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DiffFromFileDoesNotExist)); @@ -257,12 +282,12 @@ std::auto_ptr BackupProtocolServerStoreFile::DoCommand(BackupPro // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetObject::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerGetObject::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Command to get an arbitary object from the server // Created: 2003/09/03 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetObject::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetObject::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -285,13 +310,13 @@ std::auto_ptr BackupProtocolServerGetObject::DoCommand(BackupPro // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetFile::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerGetFile::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Command to get an file object from the server -- may have to do a bit of // work to get the object. // Created: 2003/09/03 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -457,12 +482,12 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerCreateDirectory::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerCreateDirectory::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Create directory command // Created: 2003/09/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerCreateDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerCreateDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -500,12 +525,12 @@ std::auto_ptr BackupProtocolServerCreateDirectory::DoCommand(Bac // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerChangeDirAttributes::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerChangeDirAttributes::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Change attributes on directory // Created: 2003/09/06 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerChangeDirAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerChangeDirAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -528,12 +553,12 @@ std::auto_ptr BackupProtocolServerChangeDirAttributes::DoCommand // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerSetReplacementFileAttributes::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerSetReplacementFileAttributes::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Change attributes on directory // Created: 2003/09/06 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerSetReplacementFileAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerSetReplacementFileAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -563,12 +588,12 @@ std::auto_ptr BackupProtocolServerSetReplacementFileAttributes:: // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Delete a file // Created: 2003/10/21 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -585,12 +610,36 @@ std::auto_ptr BackupProtocolServerDeleteFile::DoCommand(BackupPr // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerUndeleteFile::DoCommand( +// BackupProtocolServer &, BackupStoreContext &) +// Purpose: Undelete a file +// Created: 2008-09-12 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupProtocolServerUndeleteFile::DoCommand( + BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Context handles this + bool result = rContext.UndeleteFile(mObjectID, mInDirectory); + + // return the object ID or zero for not found + return std::auto_ptr( + new BackupProtocolServerSuccess(result ? mObjectID : 0)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Delete a directory // Created: 2003/10/21 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -613,12 +662,12 @@ std::auto_ptr BackupProtocolServerDeleteDirectory::DoCommand(Bac // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Undelete a directory // Created: 23/11/03 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -640,12 +689,12 @@ std::auto_ptr BackupProtocolServerUndeleteDirectory::DoCommand(B // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Command to set the client's store marker // Created: 2003/10/29 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -661,12 +710,12 @@ std::auto_ptr BackupProtocolServerSetClientStoreMarker::DoComman // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Command to move an object from one directory to another // Created: 2003/11/12 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -704,12 +753,12 @@ std::auto_ptr BackupProtocolServerMoveObject::DoCommand(BackupPr // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Command to find the name of an object // Created: 12/11/03 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -730,7 +779,7 @@ std::auto_ptr BackupProtocolServerGetObjectName::DoCommand(Backu do { // Check the directory really exists - if(!rContext.ObjectExists(dirID, BackupContext::ObjectExists_Directory)) + if(!rContext.ObjectExists(dirID, BackupStoreContext::ObjectExists_Directory)) { return std::auto_ptr(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0)); } @@ -795,12 +844,12 @@ std::auto_ptr BackupProtocolServerGetObjectName::DoCommand(Backu // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Get the block index from a file, by ID // Created: 19/1/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -821,12 +870,12 @@ std::auto_ptr BackupProtocolServerGetBlockIndexByID::DoCommand(B // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Get the block index from a file, by name within a directory // Created: 19/1/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -873,12 +922,12 @@ std::auto_ptr BackupProtocolServerGetBlockIndexByName::DoCommand // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Return the amount of disc space used // Created: 19/4/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -904,12 +953,12 @@ std::auto_ptr BackupProtocolServerGetAccountUsage::DoCommand(Bac // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Return the amount of disc space used // Created: 19/4/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) diff --git a/bin/bbstored/BackupConstants.h b/bin/bbstored/BackupConstants.h index 515b3bcd..19d06a15 100644 --- a/bin/bbstored/BackupConstants.h +++ b/bin/bbstored/BackupConstants.h @@ -10,8 +10,6 @@ #ifndef BACKUPCONSTANTS__H #define BACKUPCONSTANTS__H -#define BACKUP_STORE_DEFAULT_ACCOUNT_DATABASE_FILE "/etc/box/backupstoreaccounts" - // 15 minutes to timeout (milliseconds) #define BACKUP_STORE_TIMEOUT (15*60*1000) diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp deleted file mode 100644 index 16388099..00000000 --- a/bin/bbstored/BackupContext.cpp +++ /dev/null @@ -1,1661 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupContext.cpp -// Purpose: Context for backup store server -// Created: 2003/08/20 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include - -#include "BackupContext.h" -#include "RaidFileWrite.h" -#include "RaidFileRead.h" -#include "BackupStoreDirectory.h" -#include "BackupStoreException.h" -#include "BackupStoreInfo.h" -#include "BackupConstants.h" -#include "BackupStoreFile.h" -#include "BackupStoreObjectMagic.h" -#include "StoreStructure.h" -#include "BackupStoreDaemon.h" -#include "RaidFileController.h" -#include "FileStream.h" -#include "InvisibleTempFileStream.h" -#include "BufferedStream.h" - -#include "MemLeakFindOn.h" - - -// Maximum number of directories to keep in the cache -// When the cache is bigger than this, everything gets -// deleted. -#ifdef NDEBUG - #define MAX_CACHE_SIZE 32 -#else - #define MAX_CACHE_SIZE 2 -#endif - -// Allow the housekeeping process 4 seconds to release an account -#define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT 4 - -// Maximum amount of store info updates before it's actually saved to disc. -#define STORE_INFO_SAVE_DELAY 96 - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::BackupContext() -// Purpose: Constructor -// Created: 2003/08/20 -// -// -------------------------------------------------------------------------- -BackupContext::BackupContext(int32_t ClientID, BackupStoreDaemon &rDaemon) - : mClientID(ClientID), - mrDaemon(rDaemon), - mProtocolPhase(Phase_START), - mClientHasAccount(false), - mStoreDiscSet(-1), - mReadOnly(true), - mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::~BackupContext() -// Purpose: Destructor -// Created: 2003/08/20 -// -// -------------------------------------------------------------------------- -BackupContext::~BackupContext() -{ - // Delete the objects in the cache - for(std::map::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) - { - delete (i->second); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::CleanUp() -// Purpose: Clean up after a connection -// Created: 16/12/03 -// -// -------------------------------------------------------------------------- -void BackupContext::CleanUp() -{ - // Make sure the store info is saved, if it has been loaded, isn't read only and has been modified - if(mpStoreInfo.get() && !(mpStoreInfo->IsReadOnly()) && mpStoreInfo->IsModified()) - { - mpStoreInfo->Save(); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::ReceivedFinishCommand() -// Purpose: Called when the finish command is received by the protocol -// Created: 16/12/03 -// -// -------------------------------------------------------------------------- -void BackupContext::ReceivedFinishCommand() -{ - if(!mReadOnly && mpStoreInfo.get()) - { - // Save the store info, not delayed - SaveStoreInfo(false); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::AttemptToGetWriteLock() -// Purpose: Attempt to get a write lock for the store, and if so, unset the read only flags -// Created: 2003/09/02 -// -// -------------------------------------------------------------------------- -bool BackupContext::AttemptToGetWriteLock() -{ - // Make the filename of the write lock file - std::string writeLockFile; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); - - // Request the lock - bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); - - if(!gotLock) - { - // The housekeeping process might have the thing open -- ask it to stop - char msg[256]; - int msgLen = sprintf(msg, "r%x\n", mClientID); - // Send message - mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen); - - // Then try again a few times - int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; - do - { - ::sleep(1 /* second */); - --tries; - gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); - - } while(!gotLock && tries > 0); - } - - if(gotLock) - { - // Got the lock, mark as not read only - mReadOnly = false; - } - - return gotLock; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::LoadStoreInfo() -// Purpose: Load the store info from disc -// Created: 2003/09/03 -// -// -------------------------------------------------------------------------- -void BackupContext::LoadStoreInfo() -{ - if(mpStoreInfo.get() != 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded) - } - - // Load it up! - std::auto_ptr i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly)); - - // Check it - if(i->GetAccountID() != mClientID) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount) - } - - // Keep the pointer to it - mpStoreInfo = i; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::SaveStoreInfo(bool) -// Purpose: Potentially delayed saving of the store info -// Created: 16/12/03 -// -// -------------------------------------------------------------------------- -void BackupContext::SaveStoreInfo(bool AllowDelay) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // Can delay saving it a little while? - if(AllowDelay) - { - --mSaveStoreInfoDelay; - if(mSaveStoreInfoDelay > 0) - { - return; - } - } - - // Want to save now - mpStoreInfo->Save(); - - // Set count for next delay - mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::MakeObjectFilename(int64_t, std::string &, bool) -// Purpose: Create the filename of an object in the store, optionally creating the -// containing directory if it doesn't already exist. -// Created: 2003/09/02 -// -// -------------------------------------------------------------------------- -void BackupContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists) -{ - // Delegate to utility function - StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::GetDirectoryInternal(int64_t) -// Purpose: Return a reference to a directory. Valid only until the -// next time a function which affects directories is called. -// Mainly this funciton, and creation of files. -// Private version of this, which returns non-const directories. -// Created: 2003/09/02 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory &BackupContext::GetDirectoryInternal(int64_t ObjectID) -{ - // Get the filename - std::string filename; - MakeObjectFilename(ObjectID, filename); - - // Already in cache? - std::map::iterator item(mDirectoryCache.find(ObjectID)); - if(item != mDirectoryCache.end()) - { - // Check the revision ID of the file -- does it need refreshing? - int64_t revID = 0; - if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID)) - { - THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) - } - - if(revID == item->second->GetRevisionID()) - { - // Looks good... return the cached object - return *(item->second); - } - - // Delete this cached object - delete item->second; - mDirectoryCache.erase(item); - } - - // Need to load it up - - // First check to see if the cache is too big - if(mDirectoryCache.size() > MAX_CACHE_SIZE) - { - // Very simple. Just delete everything! - for(std::map::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) - { - delete (i->second); - } - mDirectoryCache.clear(); - } - - // Get a RaidFileRead to read it - int64_t revID = 0; - std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID)); - ASSERT(revID != 0); - - // New directory object - std::auto_ptr dir(new BackupStoreDirectory); - - // Read it from the stream, then set it's revision ID - BufferedStream buf(*objectFile); - dir->ReadFromStream(buf, IOStream::TimeOutInfinite); - dir->SetRevisionID(revID); - - // Make sure the size of the directory is available for writing the dir back - int64_t dirSize = objectFile->GetDiscUsageInBlocks(); - ASSERT(dirSize > 0); - dir->SetUserInfo1_SizeInBlocks(dirSize); - - // Store in cache - BackupStoreDirectory *pdir = dir.release(); - try - { - mDirectoryCache[ObjectID] = pdir; - } - catch(...) - { - delete pdir; - throw; - } - - // Return it - return *pdir; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::AllocateObjectID() -// Purpose: Allocate a new object ID, tolerant of failures to save store info -// Created: 16/12/03 -// -// -------------------------------------------------------------------------- -int64_t BackupContext::AllocateObjectID() -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - // Given that the store info may not be saved for STORE_INFO_SAVE_DELAY - // times after it has been updated, this is a reasonable number of times - // to try for finding an unused ID. - // (Sizes used in the store info are fixed by the housekeeping process) - int retryLimit = (STORE_INFO_SAVE_DELAY * 2); - - while(retryLimit > 0) - { - // Attempt to allocate an ID from the store - int64_t id = mpStoreInfo->AllocateObjectID(); - - // Generate filename - std::string filename; - MakeObjectFilename(id, filename); - // Check it doesn't exist - if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) - { - // Success! - return id; - } - - // Decrement retry count, and try again - --retryLimit; - - // Mark that the store info should be saved as soon as possible - mSaveStoreInfoDelay = 0; - - TRACE1("When allocating object ID, found that %lld is already in use\n", id); - } - - THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation) -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) -// Purpose: Add a file to the store, from a given stream, into a specified directory. -// Returns object ID of the new file. -// Created: 2003/09/03 -// -// -------------------------------------------------------------------------- -int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, - int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, - bool MarkFileWithSameNameAsOldVersions) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // This is going to be a bit complex to make sure it copes OK - // with things going wrong. - // The only thing which isn't safe is incrementing the object ID - // and keeping the blocks used entirely accurate -- but these - // aren't big problems if they go horribly wrong. The sizes will - // be corrected the next time the account has a housekeeping run, - // and the object ID allocation code is tolerant of missed IDs. - // (the info is written lazily, so these are necessary) - - // Get the directory we want to modify - BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - - // Allocate the next ID - int64_t id = AllocateObjectID(); - - // Stream the file to disc - std::string fn; - MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); - int64_t blocksUsed = 0; - RaidFileWrite *ppreviousVerStoreFile = 0; - bool reversedDiffIsCompletelyDifferent = false; - int64_t oldVersionNewBlocksUsed = 0; - try - { - RaidFileWrite storeFile(mStoreDiscSet, fn); - storeFile.Open(false /* no overwriting */); - int64_t spaceAdjustFromDiff = 0; // size adjustment from use of patch in old file - - // Diff or full file? - if(DiffFromFileID == 0) - { - // A full file, just store to disc - if(!rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT)) - { - THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) - } - } - else - { - // Check that the diffed from ID actually exists in the directory - if(dir.FindEntryByID(DiffFromFileID) == 0) - { - THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory) - } - - // Diff file, needs to be recreated. - // Choose a temporary filename. - std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp", - 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */)); - - try - { - // Open it twice -#ifdef WIN32 - InvisibleTempFileStream diff(tempFn.c_str(), - O_RDWR | O_CREAT | O_BINARY); - InvisibleTempFileStream diff2(tempFn.c_str(), - O_RDWR | O_BINARY); -#else - FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); - FileStream diff2(tempFn.c_str(), O_RDONLY); - - // Unlink it immediately, so it definitely goes away - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } -#endif - - // Stream the incoming diff to this temporary file - if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) - { - THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) - } - - // Verify the diff - diff.Seek(0, IOStream::SeekType_Absolute); - if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) - { - THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) - } - - // Seek to beginning of diff file - diff.Seek(0, IOStream::SeekType_Absolute); - - // Filename of the old version - std::string oldVersionFilename; - MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */); - - // Reassemble that diff -- open previous file, and combine the patch and file - std::auto_ptr from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); - BackupStoreFile::CombineFile(diff, diff2, *from, storeFile); - - // Then... reverse the patch back (open the from file again, and create a write file to overwrite it) - std::auto_ptr from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); - ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename); - ppreviousVerStoreFile->Open(true /* allow overwriting */); - from->Seek(0, IOStream::SeekType_Absolute); - diff.Seek(0, IOStream::SeekType_Absolute); - BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile, - DiffFromFileID, &reversedDiffIsCompletelyDifferent); - - // Store disc space used - oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks(); - - // And make a space adjustment for the size calculation - spaceAdjustFromDiff = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed; - - // Everything cleans up here... - } - catch(...) - { - // Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway - ::unlink(tempFn.c_str()); - throw; - } - } - - // Get the blocks used - blocksUsed = storeFile.GetDiscUsageInBlocks(); - - // Exceeds the hard limit? - if((mpStoreInfo->GetBlocksUsed() + blocksUsed - spaceAdjustFromDiff) > mpStoreInfo->GetBlocksHardLimit()) - { - THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) - // The store file will be deleted automatically by the RaidFile object - } - - // Commit the file - storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - } - catch(...) - { - // Delete any previous version store file - if(ppreviousVerStoreFile != 0) - { - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; - } - - throw; - } - - // Verify the file -- only necessary for non-diffed versions - // NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because - // in the non-diffed code path it's never allocated. - if(DiffFromFileID == 0) - { - std::auto_ptr checkFile(RaidFileRead::Open(mStoreDiscSet, fn)); - if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile)) - { - // Error! Delete the file - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); - - // Exception - THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) - } - } - - // Modify the directory -- first make all files with the same name - // marked as an old version - int64_t blocksInOldFiles = 0; - try - { - if(MarkFileWithSameNameAsOldVersions) - { - BackupStoreDirectory::Iterator i(dir); - - BackupStoreDirectory::Entry *e = 0; - while((e = i.Next()) != 0) - { - // First, check it's not an old version (cheaper comparison) - if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) - { - // Compare name - if(e->GetName() == rFilename) - { - // Check that it's definately not an old version - ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0); - // Set old version flag - e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion); - // Can safely do this, because we know we won't be here if it's already - // an old version - blocksInOldFiles += e->GetSizeInBlocks(); - } - } - } - } - - // Then the new entry - BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename, - ModificationTime, id, blocksUsed, BackupStoreDirectory::Entry::Flags_File, AttributesHash); - - // Adjust for the patch back stuff? - if(DiffFromFileID != 0) - { - // Get old version entry - BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID); - ASSERT(poldEntry != 0); - - // Adjust dependency info of file? - if(!reversedDiffIsCompletelyDifferent) - { - poldEntry->SetDependsNewer(id); - pnewEntry->SetDependsOlder(DiffFromFileID); - } - - // Adjust size of old entry - int64_t oldSize = poldEntry->GetSizeInBlocks(); - poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); - - // And adjust blocks used count, for later adjustment - blocksUsed += (oldVersionNewBlocksUsed - oldSize); - blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize); - } - - // Write the directory back to disc - SaveDirectory(dir, InDirectory); - - // Commit the old version's new patched version, now that the directory safely reflects - // the state of the files on disc. - if(ppreviousVerStoreFile != 0) - { - ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; - } - } - catch(...) - { - // Back out on adding that file - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); - - // Remove this entry from the cache - RemoveDirectoryFromCache(InDirectory); - - // Delete any previous version store file - if(ppreviousVerStoreFile != 0) - { - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; - } - - // Don't worry about the incremented number in the store info - throw; - } - - // Check logic - ASSERT(ppreviousVerStoreFile == 0); - - // Modify the store info - mpStoreInfo->ChangeBlocksUsed(blocksUsed); - mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles); - - // Save the store info -- can cope if this exceptions because infomation - // will be rebuilt by housekeeping, and ID allocation can recover. - SaveStoreInfo(); - - // Return the ID to the caller - return id; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) -// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. -// Created: 2003/10/21 -// -// -------------------------------------------------------------------------- -bool BackupContext::DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut) -{ - // Essential checks! - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // Find the directory the file is in (will exception if it fails) - BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - - // Setup flags - bool fileExisted = false; - bool madeChanges = false; - rObjectIDOut = 0; // not found - - // Count of deleted blocks - int64_t blocksDel = 0; - - try - { - // Iterate through directory, only looking at files which haven't been deleted - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *e = 0; - while((e = i.Next(BackupStoreDirectory::Entry::Flags_File, - BackupStoreDirectory::Entry::Flags_Deleted)) != 0) - { - // Compare name - if(e->GetName() == rFilename) - { - // Check that it's definately not already deleted - ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0); - // Set deleted flag - e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); - // Mark as made a change - madeChanges = true; - // Can safely do this, because we know we won't be here if it's already - // an old version - blocksDel += e->GetSizeInBlocks(); - // Is this the last version? - if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) - { - // Yes. It's been found. - rObjectIDOut = e->GetObjectID(); - fileExisted = true; - } - } - } - - // Save changes? - if(madeChanges) - { - // Save the directory back - SaveDirectory(dir, InDirectory); - - // Modify the store info, and write - mpStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); - - // Maybe postponed save of store info - SaveStoreInfo(); - } - } - catch(...) - { - RemoveDirectoryFromCache(InDirectory); - throw; - } - - - return fileExisted; -} - - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::RemoveDirectoryFromCache(int64_t) -// Purpose: Remove directory from cache -// Created: 2003/09/04 -// -// -------------------------------------------------------------------------- -void BackupContext::RemoveDirectoryFromCache(int64_t ObjectID) -{ - std::map::iterator item(mDirectoryCache.find(ObjectID)); - if(item != mDirectoryCache.end()) - { - // Delete this cached object - delete item->second; - // Erase the entry form the map - mDirectoryCache.erase(item); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::SaveDirectory(BackupStoreDirectory &, int64_t) -// Purpose: Save directory back to disc, update time in cache -// Created: 2003/09/04 -// -// -------------------------------------------------------------------------- -void BackupContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(rDir.GetObjectID() != ObjectID) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - try - { - // Write to disc, adjust size in store info - std::string dirfn; - MakeObjectFilename(ObjectID, dirfn); - { - RaidFileWrite writeDir(mStoreDiscSet, dirfn); - writeDir.Open(true /* allow overwriting */); - rDir.WriteToStream(writeDir); - - // get the disc usage (must do this before commiting it) - int64_t dirSize = writeDir.GetDiscUsageInBlocks(); - - // Commit directory - writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - - // Make sure the size of the directory is available for writing the dir back - ASSERT(dirSize > 0); - int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks(); - mpStoreInfo->ChangeBlocksUsed(sizeAdjustment); - mpStoreInfo->ChangeBlocksInDirectories(sizeAdjustment); - // Update size stored in directory - rDir.SetUserInfo1_SizeInBlocks(dirSize); - } - // Refresh revision ID in cache - { - int64_t revid = 0; - if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid)) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - rDir.SetRevisionID(revid); - } - } - catch(...) - { - // Remove it from the cache if anything went wrong - RemoveDirectoryFromCache(ObjectID); - throw; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &) -// Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely. -// Created: 2003/09/04 -// -// -------------------------------------------------------------------------- -int64_t BackupContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // Flags as not already existing - rAlreadyExists = false; - - // Get the directory we want to modify - BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - - // Scan the directory for the name (only looking for directories which already exist) - { - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, - BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) // Ignore deleted and old directories - { - if(en->GetName() == rFilename) - { - // Already exists - rAlreadyExists = true; - return en->GetObjectID(); - } - } - } - - // Allocate the next ID - int64_t id = AllocateObjectID(); - - // Create a blank directory with the given attributes on disc - std::string fn; - MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); - { - BackupStoreDirectory emptyDir(id, InDirectory); - // add the atttribues - emptyDir.SetAttributes(Attributes, AttributesModTime); - - // Write... - RaidFileWrite dirFile(mStoreDiscSet, fn); - dirFile.Open(false /* no overwriting */); - emptyDir.WriteToStream(dirFile); - // Get disc usage, before it's commited - int64_t dirSize = dirFile.GetDiscUsageInBlocks(); - // Commit the file - dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - - // Make sure the size of the directory is added to the usage counts in the info - ASSERT(dirSize > 0); - mpStoreInfo->ChangeBlocksUsed(dirSize); - mpStoreInfo->ChangeBlocksInDirectories(dirSize); - // Not added to cache, so don't set the size in the directory - } - - // Then add it into the directory - try - { - dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */); - SaveDirectory(dir, InDirectory); - } - catch(...) - { - // Back out on adding that directory - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); - - // Remove this entry from the cache - RemoveDirectoryFromCache(InDirectory); - - // Don't worry about the incremented number in the store info - throw; - } - - // Save the store info (may be postponed) - SaveStoreInfo(); - - // tell caller what the ID was - return id; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &, bool) -// Purpose: Recusively deletes a directory (or undeletes if Undelete = true) -// Created: 2003/10/21 -// -// -------------------------------------------------------------------------- -void BackupContext::DeleteDirectory(int64_t ObjectID, bool Undelete) -{ - // Essential checks! - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // Containing directory - int64_t InDirectory = 0; - - // Count of blocks deleted - int64_t blocksDeleted = 0; - - try - { - // Get the directory that's to be deleted - { - // In block, because dir may not be valid after the delete directory call - BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - - // Store the directory it's in for later - InDirectory = dir.GetContainerID(); - - // Depth first delete of contents - DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete); - } - - // Remove the entry from the directory it's in - ASSERT(InDirectory != 0); - BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory)); - - BackupStoreDirectory::Iterator i(parentDir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), - Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) - { - if(en->GetObjectID() == ObjectID) - { - // This is the one to delete - if(Undelete) - { - en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); - } - else - { - en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); - } - - // Save it - SaveDirectory(parentDir, InDirectory); - - // Done - break; - } - } - - // Update blocks deleted count - mpStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted)); - - // Save store info, may be postponed - SaveStoreInfo(); - } - catch(...) - { - RemoveDirectoryFromCache(InDirectory); - throw; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::DeleteDirectoryRecurse(BackupStoreDirectory &, int64_t) -// Purpose: Private. Deletes a directory depth-first recusively. -// Created: 2003/10/21 -// -// -------------------------------------------------------------------------- -void BackupContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete) -{ - try - { - // Does things carefully to avoid using a directory in the cache after recursive call - // because it may have been deleted. - - // Do sub directories - { - // Get the directory... - BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - - // Then scan it for directories - std::vector subDirs; - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - if(Undelete) - { - while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted, // deleted dirs - BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING)) != 0) - { - // Store the directory ID. - subDirs.push_back(en->GetObjectID()); - } - } - else - { - while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir, // dirs only - BackupStoreDirectory::Entry::Flags_Deleted)) != 0) // but not deleted ones - { - // Store the directory ID. - subDirs.push_back(en->GetObjectID()); - } - } - - // Done with the directory for now. Recurse to sub directories - for(std::vector::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i) - { - DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete); - } - } - - // Then, delete the files. Will need to load the directory again because it might have - // been removed from the cache. - { - // Get the directory... - BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - - // Changes made? - bool changesMade = false; - - // Run through files - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - - while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), - Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) - { - // Add/remove the deleted flags - if(Undelete) - { - en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); - } - else - { - en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); - } - - // Keep count of the deleted blocks - if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) - { - rBlocksDeletedOut += en->GetSizeInBlocks(); - } - - // Did something - changesMade = true; - } - - // Save the directory - if(changesMade) - { - SaveDirectory(dir, ObjectID); - } - } - } - catch(...) - { - RemoveDirectoryFromCache(ObjectID); - throw; - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::ChangeDirAttributes(int64_t, const StreamableMemBlock &, int64_t) -// Purpose: Change the attributes of a directory -// Created: 2003/09/06 -// -// -------------------------------------------------------------------------- -void BackupContext::ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - try - { - // Get the directory we want to modify - BackupStoreDirectory &dir(GetDirectoryInternal(Directory)); - - // Set attributes - dir.SetAttributes(Attributes, AttributesModTime); - - // Save back - SaveDirectory(dir, Directory); - } - catch(...) - { - RemoveDirectoryFromCache(Directory); - throw; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::ChangeFileAttributes(int64_t, int64_t, const StreamableMemBlock &, int64_t) -// Purpose: Sets the attributes on a directory entry. Returns true if the object existed, false if it didn't. -// Created: 2003/09/06 -// -// -------------------------------------------------------------------------- -bool BackupContext::ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - try - { - // Get the directory we want to modify - BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - - // Find the file entry - BackupStoreDirectory::Entry *en = 0; - // Iterate through current versions of files, only - BackupStoreDirectory::Iterator i(dir); - while((en = i.Next( - BackupStoreDirectory::Entry::Flags_File, - BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion) - ) != 0) - { - if(en->GetName() == rFilename) - { - // Set attributes - en->SetAttributes(Attributes, AttributesHash); - - // Tell caller the object ID - rObjectIDOut = en->GetObjectID(); - - // Done - break; - } - } - if(en == 0) - { - // Didn't find it - return false; - } - - // Save back - SaveDirectory(dir, InDirectory); - } - catch(...) - { - RemoveDirectoryFromCache(InDirectory); - throw; - } - - // Changed, everything OK - return true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::ObjectExists(int64_t) -// Purpose: Test to see if an object of this ID exists in the store -// Created: 2003/09/03 -// -// -------------------------------------------------------------------------- -bool BackupContext::ObjectExists(int64_t ObjectID, int MustBe) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - // Note that we need to allow object IDs a little bit greater than the last one in the store info, - // because the store info may not have got saved in an error condition. Max greater ID is - // STORE_INFO_SAVE_DELAY in this case, *2 to be safe. - if(ObjectID <= 0 || ObjectID > (mpStoreInfo->GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2))) - { - // Obviously bad object ID - return false; - } - - // Test to see if it exists on the disc - std::string filename; - MakeObjectFilename(ObjectID, filename); - if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) - { - // RaidFile reports no file there - return false; - } - - // Do we need to be more specific? - if(MustBe != ObjectExists_Anything) - { - // Open the file - std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, filename)); - - // Read the first integer - u_int32_t magic; - if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */)) - { - // Failed to get any bytes, must have failed - return false; - } - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - if(MustBe == ObjectExists_File && ntohl(magic) == OBJECTMAGIC_FILE_MAGIC_VALUE_V0) - { - // Old version detected - return true; - } -#endif - - // Right one? - u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; - - // Check - if(ntohl(magic) != requiredMagic) - { - return false; - } - - // File is implicitly closed - } - - return true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::OpenObject(int64_t) -// Purpose: Opens an object -// Created: 2003/09/03 -// -// -------------------------------------------------------------------------- -std::auto_ptr BackupContext::OpenObject(int64_t ObjectID) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - // Attempt to open the file - std::string fn; - MakeObjectFilename(ObjectID, fn); - return std::auto_ptr(RaidFileRead::Open(mStoreDiscSet, fn).release()); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::GetClientStoreMarker() -// Purpose: Retrieve the client store marker -// Created: 2003/10/29 -// -// -------------------------------------------------------------------------- -int64_t BackupContext::GetClientStoreMarker() -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - return mpStoreInfo->GetClientStoreMarker(); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::GetStoreDiscUsageInfo(int64_t &, int64_t &, int64_t &) -// Purpose: Get disc usage info from store info -// Created: 1/1/04 -// -// -------------------------------------------------------------------------- -void BackupContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - rBlocksUsed = mpStoreInfo->GetBlocksUsed(); - rBlocksSoftLimit = mpStoreInfo->GetBlocksSoftLimit(); - rBlocksHardLimit = mpStoreInfo->GetBlocksHardLimit(); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::HardLimitExceeded() -// Purpose: Returns true if the hard limit has been exceeded -// Created: 1/1/04 -// -// -------------------------------------------------------------------------- -bool BackupContext::HardLimitExceeded() -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - return mpStoreInfo->GetBlocksUsed() > mpStoreInfo->GetBlocksHardLimit(); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::SetClientStoreMarker(int64_t) -// Purpose: Sets the client store marker, and commits it to disc -// Created: 2003/10/29 -// -// -------------------------------------------------------------------------- -void BackupContext::SetClientStoreMarker(int64_t ClientStoreMarker) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - mpStoreInfo->SetClientStoreMarker(ClientStoreMarker); - SaveStoreInfo(false /* don't delay saving this */); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::MoveObject(int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) -// Purpose: Move an object (and all objects with the same name) from one directory to another -// Created: 12/11/03 -// -// -------------------------------------------------------------------------- -void BackupContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) -{ - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // Should deleted files be excluded when checking for the existance of objects with the target name? - int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject) - ?(BackupStoreDirectory::Entry::Flags_Deleted) - :(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); - - // Special case if the directories are the same... - if(MoveFromDirectory == MoveToDirectory) - { - try - { - // Get the first directory - BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory)); - - // Find the file entry - BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); - - // Error if not found - if(en == 0) - { - THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) - } - - // Check the new name doens't already exist (optionally ignoring deleted files) - { - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *c = 0; - while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0) - { - if(c->GetName() == rNewFilename) - { - THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory) - } - } - } - - // Need to get all the entries with the same name? - if(MoveAllWithSameName) - { - // Iterate through the directory, copying all with matching names - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *c = 0; - while((c = i.Next()) != 0) - { - if(c->GetName() == en->GetName()) - { - // Rename this one - c->SetName(rNewFilename); - } - } - } - else - { - // Just copy this one - en->SetName(rNewFilename); - } - - // Save the directory back - SaveDirectory(dir, MoveFromDirectory); - } - catch(...) - { - RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same - throw; - } - - return; - } - - // Got to be careful how this is written, as we can't guarentte that if we have two - // directories open, the first won't be deleted as the second is opened. (cache) - - // List of entries to move - std::vector moving; - - // list of directory IDs which need to have containing dir id changed - std::vector dirsToChangeContainingID; - - try - { - // First of all, get copies of the entries to move to the to directory. - - { - // Get the first directory - BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); - - // Find the file entry - BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID); - - // Error if not found - if(en == 0) - { - THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) - } - - // Need to get all the entries with the same name? - if(MoveAllWithSameName) - { - // Iterate through the directory, copying all with matching names - BackupStoreDirectory::Iterator i(from); - BackupStoreDirectory::Entry *c = 0; - while((c = i.Next()) != 0) - { - if(c->GetName() == en->GetName()) - { - // Copy - moving.push_back(new BackupStoreDirectory::Entry(*c)); - - // Check for containing directory correction - if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID()); - } - } - ASSERT(!moving.empty()); - } - else - { - // Just copy this one - moving.push_back(new BackupStoreDirectory::Entry(*en)); - - // Check for containing directory correction - if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID()); - } - } - - // Secondly, insert them into the to directory, and save it - - { - // To directory - BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); - - // Check the new name doens't already exist - { - BackupStoreDirectory::Iterator i(to); - BackupStoreDirectory::Entry *c = 0; - while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0) - { - if(c->GetName() == rNewFilename) - { - THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory) - } - } - } - - // Copy the entries into it, changing the name as we go - for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) - { - BackupStoreDirectory::Entry *en = (*i); - en->SetName(rNewFilename); - to.AddEntry(*en); // adds copy - } - - // Save back - SaveDirectory(to, MoveToDirectory); - } - - // Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory - try - { - // Get directory - BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); - - // Delete each one - for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) - { - from.DeleteEntry((*i)->GetObjectID()); - } - - // Save back - SaveDirectory(from, MoveFromDirectory); - } - catch(...) - { - // UNDO modification to To directory - - // Get directory - BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); - - // Delete each one - for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) - { - to.DeleteEntry((*i)->GetObjectID()); - } - - // Save back - SaveDirectory(to, MoveToDirectory); - - // Throw the error - throw; - } - - // Finally... for all the directories we moved, modify their containing directory ID - for(std::vector::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) - { - // Load the directory - BackupStoreDirectory &change(GetDirectoryInternal(*i)); - - // Modify containing dir ID - change.SetContainerID(MoveToDirectory); - - // Save it back - SaveDirectory(change, *i); - } - } - catch(...) - { - // Make sure directories aren't in the cache, as they may have been modified - RemoveDirectoryFromCache(MoveToDirectory); - RemoveDirectoryFromCache(MoveFromDirectory); - for(std::vector::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) - { - RemoveDirectoryFromCache(*i); - } - - while(!moving.empty()) - { - delete moving.back(); - moving.pop_back(); - } - throw; - } - - // Clean up - while(!moving.empty()) - { - delete moving.back(); - moving.pop_back(); - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::GetBackupStoreInfo() -// Purpose: Return the backup store info object, exception if it isn't loaded -// Created: 19/4/04 -// -// -------------------------------------------------------------------------- -const BackupStoreInfo &BackupContext::GetBackupStoreInfo() const -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - return *(mpStoreInfo.get()); -} - - diff --git a/bin/bbstored/BackupContext.h b/bin/bbstored/BackupContext.h deleted file mode 100644 index 18f2f25c..00000000 --- a/bin/bbstored/BackupContext.h +++ /dev/null @@ -1,149 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupContext.h -// Purpose: Context for backup store server -// Created: 2003/08/20 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCONTEXT__H -#define BACKUPCONTEXT__H - -#include -#include -#include - -#include "NamedLock.h" -#include "Utils.h" - -class BackupStoreDirectory; -class BackupStoreFilename; -class BackupStoreDaemon; -class BackupStoreInfo; -class IOStream; -class StreamableMemBlock; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupContext -// Purpose: Context for backup store server -// Created: 2003/08/20 -// -// -------------------------------------------------------------------------- -class BackupContext -{ -public: - BackupContext(int32_t ClientID, BackupStoreDaemon &rDaemon); - ~BackupContext(); -private: - BackupContext(const BackupContext &rToCopy); -public: - - void ReceivedFinishCommand(); - void CleanUp(); - - int32_t GetClientID() {return mClientID;} - - enum - { - Phase_START = 0, - Phase_Version = 0, - Phase_Login = 1, - Phase_Commands = 2 - }; - - int GetPhase() const {return mProtocolPhase;} - void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;} - - // Read only locking - bool SessionIsReadOnly() {return mReadOnly;} - bool AttemptToGetWriteLock(); - - void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;} - bool GetClientHasAccount() const {return mClientHasAccount;} - const std::string &GetStoreRoot() const {return mStoreRoot;} - int GetStoreDiscSet() const {return mStoreDiscSet;} - - // Store info - void LoadStoreInfo(); - void SaveStoreInfo(bool AllowDelay = true); - const BackupStoreInfo &GetBackupStoreInfo() const; - - // Client marker - int64_t GetClientStoreMarker(); - void SetClientStoreMarker(int64_t ClientStoreMarker); - - // Usage information - void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit); - bool HardLimitExceeded(); - - // Reading directories - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupContext::GetDirectory(int64_t) - // Purpose: Return a reference to a directory. Valid only until the - // next time a function which affects directories is called. - // Mainly this funciton, and creation of files. - // Created: 2003/09/02 - // - // -------------------------------------------------------------------------- - const BackupStoreDirectory &GetDirectory(int64_t ObjectID) - { - // External callers aren't allowed to change it -- this function - // merely turns the the returned directory const. - return GetDirectoryInternal(ObjectID); - } - - // Manipulating files/directories - int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions); - int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists); - void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime); - bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut); - bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut); - void DeleteDirectory(int64_t ObjectID, bool Undelete = false); - void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject); - - // Manipulating objects - enum - { - ObjectExists_Anything = 0, - ObjectExists_File = 1, - ObjectExists_Directory = 2 - }; - bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything); - std::auto_ptr OpenObject(int64_t ObjectID); - - // Info - int32_t GetClientID() const {return mClientID;} - -private: - void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false); - BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID); - void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID); - void RemoveDirectoryFromCache(int64_t ObjectID); - void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete); - int64_t AllocateObjectID(); - -private: - int32_t mClientID; - BackupStoreDaemon &mrDaemon; - int mProtocolPhase; - bool mClientHasAccount; - std::string mStoreRoot; // has final directory separator - int mStoreDiscSet; - bool mReadOnly; - NamedLock mWriteLock; - int mSaveStoreInfoDelay; // how many times to delay saving the store info - - // Store info - std::auto_ptr mpStoreInfo; - - // Directory cache - std::map mDirectoryCache; -}; - -#endif // BACKUPCONTEXT__H - diff --git a/bin/bbstored/BackupStoreContext.cpp b/bin/bbstored/BackupStoreContext.cpp new file mode 100644 index 00000000..990be05d --- /dev/null +++ b/bin/bbstored/BackupStoreContext.cpp @@ -0,0 +1,1752 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreContext.cpp +// Purpose: Context for backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#include "BackupStoreContext.h" +#include "RaidFileWrite.h" +#include "RaidFileRead.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreInfo.h" +#include "BackupConstants.h" +#include "BackupStoreFile.h" +#include "BackupStoreObjectMagic.h" +#include "StoreStructure.h" +#include "BackupStoreDaemon.h" +#include "RaidFileController.h" +#include "FileStream.h" +#include "InvisibleTempFileStream.h" +#include "BufferedStream.h" + +#include "MemLeakFindOn.h" + + +// Maximum number of directories to keep in the cache +// When the cache is bigger than this, everything gets +// deleted. +#ifdef BOX_RELEASE_BUILD + #define MAX_CACHE_SIZE 32 +#else + #define MAX_CACHE_SIZE 2 +#endif + +// Allow the housekeeping process 4 seconds to release an account +#define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT 4 + +// Maximum amount of store info updates before it's actually saved to disc. +#define STORE_INFO_SAVE_DELAY 96 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::BackupStoreContext() +// Purpose: Constructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreContext::BackupStoreContext(int32_t ClientID, + HousekeepingInterface &rDaemon) + : mClientID(ClientID), + mrDaemon(rDaemon), + mProtocolPhase(Phase_START), + mClientHasAccount(false), + mStoreDiscSet(-1), + mReadOnly(true), + mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), + mpTestHook(NULL) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::~BackupStoreContext() +// Purpose: Destructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreContext::~BackupStoreContext() +{ + // Delete the objects in the cache + for(std::map::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + { + delete (i->second); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::CleanUp() +// Purpose: Clean up after a connection +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::CleanUp() +{ + // Make sure the store info is saved, if it has been loaded, isn't read only and has been modified + if(mpStoreInfo.get() && !(mpStoreInfo->IsReadOnly()) && mpStoreInfo->IsModified()) + { + mpStoreInfo->Save(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ReceivedFinishCommand() +// Purpose: Called when the finish command is received by the protocol +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::ReceivedFinishCommand() +{ + if(!mReadOnly && mpStoreInfo.get()) + { + // Save the store info, not delayed + SaveStoreInfo(false); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AttemptToGetWriteLock() +// Purpose: Attempt to get a write lock for the store, and if so, unset the read only flags +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::AttemptToGetWriteLock() +{ + // Make the filename of the write lock file + std::string writeLockFile; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); + + // Request the lock + bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); + + if(!gotLock) + { + // The housekeeping process might have the thing open -- ask it to stop + char msg[256]; + int msgLen = sprintf(msg, "r%x\n", mClientID); + // Send message + mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen); + + // Then try again a few times + int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; + do + { + ::sleep(1 /* second */); + --tries; + gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); + + } while(!gotLock && tries > 0); + } + + if(gotLock) + { + // Got the lock, mark as not read only + mReadOnly = false; + } + + return gotLock; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::LoadStoreInfo() +// Purpose: Load the store info from disc +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::LoadStoreInfo() +{ + if(mpStoreInfo.get() != 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded) + } + + // Load it up! + std::auto_ptr i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly)); + + // Check it + if(i->GetAccountID() != mClientID) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount) + } + + // Keep the pointer to it + mpStoreInfo = i; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::SaveStoreInfo(bool) +// Purpose: Potentially delayed saving of the store info +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::SaveStoreInfo(bool AllowDelay) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Can delay saving it a little while? + if(AllowDelay) + { + --mSaveStoreInfoDelay; + if(mSaveStoreInfoDelay > 0) + { + return; + } + } + + // Want to save now + mpStoreInfo->Save(); + + // Set count for next delay + mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::MakeObjectFilename(int64_t, std::string &, bool) +// Purpose: Create the filename of an object in the store, optionally creating the +// containing directory if it doesn't already exist. +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists) +{ + // Delegate to utility function + StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetDirectoryInternal(int64_t) +// Purpose: Return a reference to a directory. Valid only until the +// next time a function which affects directories is called. +// Mainly this funciton, and creation of files. +// Private version of this, which returns non-const directories. +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) +{ + // Get the filename + std::string filename; + MakeObjectFilename(ObjectID, filename); + + // Already in cache? + std::map::iterator item(mDirectoryCache.find(ObjectID)); + if(item != mDirectoryCache.end()) + { + // Check the revision ID of the file -- does it need refreshing? + int64_t revID = 0; + if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID)) + { + THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) + } + + if(revID == item->second->GetRevisionID()) + { + // Looks good... return the cached object + BOX_TRACE("Returning object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " from cache, modtime = " << revID); + return *(item->second); + } + + BOX_TRACE("Refreshing object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " in cache, modtime changed from " << + item->second->GetRevisionID() << " to " << revID); + + // Delete this cached object + delete item->second; + mDirectoryCache.erase(item); + } + + // Need to load it up + + // First check to see if the cache is too big + if(mDirectoryCache.size() > MAX_CACHE_SIZE) + { + // Very simple. Just delete everything! + for(std::map::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + { + delete (i->second); + } + mDirectoryCache.clear(); + } + + // Get a RaidFileRead to read it + int64_t revID = 0; + std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID)); + ASSERT(revID != 0); + + // New directory object + std::auto_ptr dir(new BackupStoreDirectory); + + // Read it from the stream, then set it's revision ID + BufferedStream buf(*objectFile); + dir->ReadFromStream(buf, IOStream::TimeOutInfinite); + dir->SetRevisionID(revID); + + // Make sure the size of the directory is available for writing the dir back + int64_t dirSize = objectFile->GetDiscUsageInBlocks(); + ASSERT(dirSize > 0); + dir->SetUserInfo1_SizeInBlocks(dirSize); + + // Store in cache + BackupStoreDirectory *pdir = dir.release(); + try + { + mDirectoryCache[ObjectID] = pdir; + } + catch(...) + { + delete pdir; + throw; + } + + // Return it + return *pdir; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AllocateObjectID() +// Purpose: Allocate a new object ID, tolerant of failures to save store info +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::AllocateObjectID() +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + // Given that the store info may not be saved for STORE_INFO_SAVE_DELAY + // times after it has been updated, this is a reasonable number of times + // to try for finding an unused ID. + // (Sizes used in the store info are fixed by the housekeeping process) + int retryLimit = (STORE_INFO_SAVE_DELAY * 2); + + while(retryLimit > 0) + { + // Attempt to allocate an ID from the store + int64_t id = mpStoreInfo->AllocateObjectID(); + + // Generate filename + std::string filename; + MakeObjectFilename(id, filename); + // Check it doesn't exist + if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) + { + // Success! + return id; + } + + // Decrement retry count, and try again + --retryLimit; + + // Mark that the store info should be saved as soon as possible + mSaveStoreInfoDelay = 0; + + BOX_WARNING("When allocating object ID, found that " << + BOX_FORMAT_OBJECTID(id) << " is already in use"); + } + + THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) +// Purpose: Add a file to the store, from a given stream, into a specified directory. +// Returns object ID of the new file. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, + int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, + bool MarkFileWithSameNameAsOldVersions) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // This is going to be a bit complex to make sure it copes OK + // with things going wrong. + // The only thing which isn't safe is incrementing the object ID + // and keeping the blocks used entirely accurate -- but these + // aren't big problems if they go horribly wrong. The sizes will + // be corrected the next time the account has a housekeeping run, + // and the object ID allocation code is tolerant of missed IDs. + // (the info is written lazily, so these are necessary) + + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Allocate the next ID + int64_t id = AllocateObjectID(); + + // Stream the file to disc + std::string fn; + MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); + int64_t blocksUsed = 0; + RaidFileWrite *ppreviousVerStoreFile = 0; + bool reversedDiffIsCompletelyDifferent = false; + int64_t oldVersionNewBlocksUsed = 0; + try + { + RaidFileWrite storeFile(mStoreDiscSet, fn); + storeFile.Open(false /* no overwriting */); + int64_t spaceAdjustFromDiff = 0; // size adjustment from use of patch in old file + + // Diff or full file? + if(DiffFromFileID == 0) + { + // A full file, just store to disc + if(!rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT)) + { + THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) + } + } + else + { + // Check that the diffed from ID actually exists in the directory + if(dir.FindEntryByID(DiffFromFileID) == 0) + { + THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory) + } + + // Diff file, needs to be recreated. + // Choose a temporary filename. + std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp", + 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */)); + + try + { + // Open it twice +#ifdef WIN32 + InvisibleTempFileStream diff(tempFn.c_str(), + O_RDWR | O_CREAT | O_BINARY); + InvisibleTempFileStream diff2(tempFn.c_str(), + O_RDWR | O_BINARY); +#else + FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); + FileStream diff2(tempFn.c_str(), O_RDONLY); + + // Unlink it immediately, so it definitely goes away + if(::unlink(tempFn.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } +#endif + + // Stream the incoming diff to this temporary file + if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) + { + THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) + } + + // Verify the diff + diff.Seek(0, IOStream::SeekType_Absolute); + if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) + { + THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) + } + + // Seek to beginning of diff file + diff.Seek(0, IOStream::SeekType_Absolute); + + // Filename of the old version + std::string oldVersionFilename; + MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */); + + // Reassemble that diff -- open previous file, and combine the patch and file + std::auto_ptr from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); + BackupStoreFile::CombineFile(diff, diff2, *from, storeFile); + + // Then... reverse the patch back (open the from file again, and create a write file to overwrite it) + std::auto_ptr from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); + ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename); + ppreviousVerStoreFile->Open(true /* allow overwriting */); + from->Seek(0, IOStream::SeekType_Absolute); + diff.Seek(0, IOStream::SeekType_Absolute); + BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile, + DiffFromFileID, &reversedDiffIsCompletelyDifferent); + + // Store disc space used + oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks(); + + // And make a space adjustment for the size calculation + spaceAdjustFromDiff = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed; + + // Everything cleans up here... + } + catch(...) + { + // Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway + ::unlink(tempFn.c_str()); + throw; + } + } + + // Get the blocks used + blocksUsed = storeFile.GetDiscUsageInBlocks(); + + // Exceeds the hard limit? + if((mpStoreInfo->GetBlocksUsed() + blocksUsed - spaceAdjustFromDiff) > mpStoreInfo->GetBlocksHardLimit()) + { + THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) + // The store file will be deleted automatically by the RaidFile object + } + + // Commit the file + storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + } + catch(...) + { + // Delete any previous version store file + if(ppreviousVerStoreFile != 0) + { + delete ppreviousVerStoreFile; + ppreviousVerStoreFile = 0; + } + + throw; + } + + // Verify the file -- only necessary for non-diffed versions + // NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because + // in the non-diffed code path it's never allocated. + if(DiffFromFileID == 0) + { + std::auto_ptr checkFile(RaidFileRead::Open(mStoreDiscSet, fn)); + if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile)) + { + // Error! Delete the file + RaidFileWrite del(mStoreDiscSet, fn); + del.Delete(); + + // Exception + THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) + } + } + + // Modify the directory -- first make all files with the same name + // marked as an old version + int64_t blocksInOldFiles = 0; + try + { + if(MarkFileWithSameNameAsOldVersions) + { + BackupStoreDirectory::Iterator i(dir); + + BackupStoreDirectory::Entry *e = 0; + while((e = i.Next()) != 0) + { + // First, check it's not an old version (cheaper comparison) + if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) + { + // Compare name + if(e->GetName() == rFilename) + { + // Check that it's definately not an old version + ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0); + // Set old version flag + e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion); + // Can safely do this, because we know we won't be here if it's already + // an old version + blocksInOldFiles += e->GetSizeInBlocks(); + } + } + } + } + + // Then the new entry + BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename, + ModificationTime, id, blocksUsed, BackupStoreDirectory::Entry::Flags_File, AttributesHash); + + // Adjust for the patch back stuff? + if(DiffFromFileID != 0) + { + // Get old version entry + BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID); + ASSERT(poldEntry != 0); + + // Adjust dependency info of file? + if(!reversedDiffIsCompletelyDifferent) + { + poldEntry->SetDependsNewer(id); + pnewEntry->SetDependsOlder(DiffFromFileID); + } + + // Adjust size of old entry + int64_t oldSize = poldEntry->GetSizeInBlocks(); + poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); + + // And adjust blocks used count, for later adjustment + blocksUsed += (oldVersionNewBlocksUsed - oldSize); + blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize); + } + + // Write the directory back to disc + SaveDirectory(dir, InDirectory); + + // Commit the old version's new patched version, now that the directory safely reflects + // the state of the files on disc. + if(ppreviousVerStoreFile != 0) + { + ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + delete ppreviousVerStoreFile; + ppreviousVerStoreFile = 0; + } + } + catch(...) + { + // Back out on adding that file + RaidFileWrite del(mStoreDiscSet, fn); + del.Delete(); + + // Remove this entry from the cache + RemoveDirectoryFromCache(InDirectory); + + // Delete any previous version store file + if(ppreviousVerStoreFile != 0) + { + delete ppreviousVerStoreFile; + ppreviousVerStoreFile = 0; + } + + // Don't worry about the incremented number in the store info + throw; + } + + // Check logic + ASSERT(ppreviousVerStoreFile == 0); + + // Modify the store info + mpStoreInfo->ChangeBlocksUsed(blocksUsed); + mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles); + + // Save the store info -- can cope if this exceptions because infomation + // will be rebuilt by housekeeping, and ID allocation can recover. + SaveStoreInfo(); + + // Return the ID to the caller + return id; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) +// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut) +{ + // Essential checks! + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Find the directory the file is in (will exception if it fails) + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Setup flags + bool fileExisted = false; + bool madeChanges = false; + rObjectIDOut = 0; // not found + + // Count of deleted blocks + int64_t blocksDel = 0; + + try + { + // Iterate through directory, only looking at files which haven't been deleted + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *e = 0; + while((e = i.Next(BackupStoreDirectory::Entry::Flags_File, + BackupStoreDirectory::Entry::Flags_Deleted)) != 0) + { + // Compare name + if(e->GetName() == rFilename) + { + // Check that it's definately not already deleted + ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0); + // Set deleted flag + e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); + // Mark as made a change + madeChanges = true; + // Can safely do this, because we know we won't be here if it's already + // an old version + blocksDel += e->GetSizeInBlocks(); + // Is this the last version? + if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) + { + // Yes. It's been found. + rObjectIDOut = e->GetObjectID(); + fileExisted = true; + } + } + } + + // Save changes? + if(madeChanges) + { + // Save the directory back + SaveDirectory(dir, InDirectory); + + // Modify the store info, and write + mpStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); + + // Maybe postponed save of store info + SaveStoreInfo(); + } + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } + + return fileExisted; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) +// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory) +{ + // Essential checks! + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Find the directory the file is in (will exception if it fails) + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Setup flags + bool fileExisted = false; + bool madeChanges = false; + + // Count of deleted blocks + int64_t blocksDel = 0; + + try + { + // Iterate through directory, only looking at files which have been deleted + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *e = 0; + while((e = i.Next(BackupStoreDirectory::Entry::Flags_File | + BackupStoreDirectory::Entry::Flags_Deleted, 0)) != 0) + { + // Compare name + if(e->GetObjectID() == ObjectID) + { + // Check that it's definitely already deleted + ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0); + // Clear deleted flag + e->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); + // Mark as made a change + madeChanges = true; + blocksDel -= e->GetSizeInBlocks(); + + // Is this the last version? + if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) + { + // Yes. It's been found. + fileExisted = true; + } + } + } + + // Save changes? + if(madeChanges) + { + // Save the directory back + SaveDirectory(dir, InDirectory); + + // Modify the store info, and write + mpStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); + + // Maybe postponed save of store info + SaveStoreInfo(); + } + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } + + return fileExisted; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::RemoveDirectoryFromCache(int64_t) +// Purpose: Remove directory from cache +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID) +{ + std::map::iterator item(mDirectoryCache.find(ObjectID)); + if(item != mDirectoryCache.end()) + { + // Delete this cached object + delete item->second; + // Erase the entry form the map + mDirectoryCache.erase(item); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &, int64_t) +// Purpose: Save directory back to disc, update time in cache +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(rDir.GetObjectID() != ObjectID) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + try + { + // Write to disc, adjust size in store info + std::string dirfn; + MakeObjectFilename(ObjectID, dirfn); + { + RaidFileWrite writeDir(mStoreDiscSet, dirfn); + writeDir.Open(true /* allow overwriting */); + rDir.WriteToStream(writeDir); + + // get the disc usage (must do this before commiting it) + int64_t dirSize = writeDir.GetDiscUsageInBlocks(); + + // Commit directory + writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + + // Make sure the size of the directory is available for writing the dir back + ASSERT(dirSize > 0); + int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks(); + mpStoreInfo->ChangeBlocksUsed(sizeAdjustment); + mpStoreInfo->ChangeBlocksInDirectories(sizeAdjustment); + // Update size stored in directory + rDir.SetUserInfo1_SizeInBlocks(dirSize); + } + // Refresh revision ID in cache + { + int64_t revid = 0; + if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + rDir.SetRevisionID(revid); + } + } + catch(...) + { + // Remove it from the cache if anything went wrong + RemoveDirectoryFromCache(ObjectID); + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &) +// Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely. +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Flags as not already existing + rAlreadyExists = false; + + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Scan the directory for the name (only looking for directories which already exist) + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, + BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) // Ignore deleted and old directories + { + if(en->GetName() == rFilename) + { + // Already exists + rAlreadyExists = true; + return en->GetObjectID(); + } + } + } + + // Allocate the next ID + int64_t id = AllocateObjectID(); + + // Create a blank directory with the given attributes on disc + std::string fn; + MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); + { + BackupStoreDirectory emptyDir(id, InDirectory); + // add the atttribues + emptyDir.SetAttributes(Attributes, AttributesModTime); + + // Write... + RaidFileWrite dirFile(mStoreDiscSet, fn); + dirFile.Open(false /* no overwriting */); + emptyDir.WriteToStream(dirFile); + // Get disc usage, before it's commited + int64_t dirSize = dirFile.GetDiscUsageInBlocks(); + // Commit the file + dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + + // Make sure the size of the directory is added to the usage counts in the info + ASSERT(dirSize > 0); + mpStoreInfo->ChangeBlocksUsed(dirSize); + mpStoreInfo->ChangeBlocksInDirectories(dirSize); + // Not added to cache, so don't set the size in the directory + } + + // Then add it into the directory + try + { + dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */); + SaveDirectory(dir, InDirectory); + } + catch(...) + { + // Back out on adding that directory + RaidFileWrite del(mStoreDiscSet, fn); + del.Delete(); + + // Remove this entry from the cache + RemoveDirectoryFromCache(InDirectory); + + // Don't worry about the incremented number in the store info + throw; + } + + // Save the store info (may be postponed) + SaveStoreInfo(); + + // tell caller what the ID was + return id; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &, bool) +// Purpose: Recusively deletes a directory (or undeletes if Undelete = true) +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) +{ + // Essential checks! + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Containing directory + int64_t InDirectory = 0; + + // Count of blocks deleted + int64_t blocksDeleted = 0; + + try + { + // Get the directory that's to be deleted + { + // In block, because dir may not be valid after the delete directory call + BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); + + // Store the directory it's in for later + InDirectory = dir.GetContainerID(); + + // Depth first delete of contents + DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete); + } + + // Remove the entry from the directory it's in + ASSERT(InDirectory != 0); + BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory)); + + BackupStoreDirectory::Iterator i(parentDir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), + Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) + { + if(en->GetObjectID() == ObjectID) + { + // This is the one to delete + if(Undelete) + { + en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + else + { + en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + + // Save it + SaveDirectory(parentDir, InDirectory); + + // Done + break; + } + } + + // Update blocks deleted count + mpStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted)); + + // Save store info, may be postponed + SaveStoreInfo(); + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteDirectoryRecurse(BackupStoreDirectory &, int64_t) +// Purpose: Private. Deletes a directory depth-first recusively. +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete) +{ + try + { + // Does things carefully to avoid using a directory in the cache after recursive call + // because it may have been deleted. + + // Do sub directories + { + // Get the directory... + BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); + + // Then scan it for directories + std::vector subDirs; + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + if(Undelete) + { + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted, // deleted dirs + BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING)) != 0) + { + // Store the directory ID. + subDirs.push_back(en->GetObjectID()); + } + } + else + { + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir, // dirs only + BackupStoreDirectory::Entry::Flags_Deleted)) != 0) // but not deleted ones + { + // Store the directory ID. + subDirs.push_back(en->GetObjectID()); + } + } + + // Done with the directory for now. Recurse to sub directories + for(std::vector::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i) + { + DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete); + } + } + + // Then, delete the files. Will need to load the directory again because it might have + // been removed from the cache. + { + // Get the directory... + BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); + + // Changes made? + bool changesMade = false; + + // Run through files + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + + while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), + Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) + { + // Add/remove the deleted flags + if(Undelete) + { + en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + else + { + en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + + // Keep count of the deleted blocks + if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) + { + rBlocksDeletedOut += en->GetSizeInBlocks(); + } + + // Did something + changesMade = true; + } + + // Save the directory + if(changesMade) + { + SaveDirectory(dir, ObjectID); + } + } + } + catch(...) + { + RemoveDirectoryFromCache(ObjectID); + throw; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ChangeDirAttributes(int64_t, const StreamableMemBlock &, int64_t) +// Purpose: Change the attributes of a directory +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + try + { + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(Directory)); + + // Set attributes + dir.SetAttributes(Attributes, AttributesModTime); + + // Save back + SaveDirectory(dir, Directory); + } + catch(...) + { + RemoveDirectoryFromCache(Directory); + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ChangeFileAttributes(int64_t, int64_t, const StreamableMemBlock &, int64_t) +// Purpose: Sets the attributes on a directory entry. Returns true if the object existed, false if it didn't. +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + try + { + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Find the file entry + BackupStoreDirectory::Entry *en = 0; + // Iterate through current versions of files, only + BackupStoreDirectory::Iterator i(dir); + while((en = i.Next( + BackupStoreDirectory::Entry::Flags_File, + BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion) + ) != 0) + { + if(en->GetName() == rFilename) + { + // Set attributes + en->SetAttributes(Attributes, AttributesHash); + + // Tell caller the object ID + rObjectIDOut = en->GetObjectID(); + + // Done + break; + } + } + if(en == 0) + { + // Didn't find it + return false; + } + + // Save back + SaveDirectory(dir, InDirectory); + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } + + // Changed, everything OK + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ObjectExists(int64_t) +// Purpose: Test to see if an object of this ID exists in the store +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + // Note that we need to allow object IDs a little bit greater than the last one in the store info, + // because the store info may not have got saved in an error condition. Max greater ID is + // STORE_INFO_SAVE_DELAY in this case, *2 to be safe. + if(ObjectID <= 0 || ObjectID > (mpStoreInfo->GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2))) + { + // Obviously bad object ID + return false; + } + + // Test to see if it exists on the disc + std::string filename; + MakeObjectFilename(ObjectID, filename); + if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) + { + // RaidFile reports no file there + return false; + } + + // Do we need to be more specific? + if(MustBe != ObjectExists_Anything) + { + // Open the file + std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, filename)); + + // Read the first integer + u_int32_t magic; + if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */)) + { + // Failed to get any bytes, must have failed + return false; + } + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(MustBe == ObjectExists_File && ntohl(magic) == OBJECTMAGIC_FILE_MAGIC_VALUE_V0) + { + // Old version detected + return true; + } +#endif + + // Right one? + u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; + + // Check + if(ntohl(magic) != requiredMagic) + { + return false; + } + + // File is implicitly closed + } + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::OpenObject(int64_t) +// Purpose: Opens an object +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupStoreContext::OpenObject(int64_t ObjectID) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + // Attempt to open the file + std::string fn; + MakeObjectFilename(ObjectID, fn); + return std::auto_ptr(RaidFileRead::Open(mStoreDiscSet, fn).release()); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetClientStoreMarker() +// Purpose: Retrieve the client store marker +// Created: 2003/10/29 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::GetClientStoreMarker() +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + return mpStoreInfo->GetClientStoreMarker(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetStoreDiscUsageInfo(int64_t &, int64_t &, int64_t &) +// Purpose: Get disc usage info from store info +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + rBlocksUsed = mpStoreInfo->GetBlocksUsed(); + rBlocksSoftLimit = mpStoreInfo->GetBlocksSoftLimit(); + rBlocksHardLimit = mpStoreInfo->GetBlocksHardLimit(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::HardLimitExceeded() +// Purpose: Returns true if the hard limit has been exceeded +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::HardLimitExceeded() +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + return mpStoreInfo->GetBlocksUsed() > mpStoreInfo->GetBlocksHardLimit(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::SetClientStoreMarker(int64_t) +// Purpose: Sets the client store marker, and commits it to disc +// Created: 2003/10/29 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + mpStoreInfo->SetClientStoreMarker(ClientStoreMarker); + SaveStoreInfo(false /* don't delay saving this */); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::MoveObject(int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) +// Purpose: Move an object (and all objects with the same name) from one directory to another +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Should deleted files be excluded when checking for the existance of objects with the target name? + int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject) + ?(BackupStoreDirectory::Entry::Flags_Deleted) + :(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); + + // Special case if the directories are the same... + if(MoveFromDirectory == MoveToDirectory) + { + try + { + // Get the first directory + BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory)); + + // Find the file entry + BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); + + // Error if not found + if(en == 0) + { + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) + } + + // Check the new name doens't already exist (optionally ignoring deleted files) + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0) + { + if(c->GetName() == rNewFilename) + { + THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory) + } + } + } + + // Need to get all the entries with the same name? + if(MoveAllWithSameName) + { + // Iterate through the directory, copying all with matching names + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next()) != 0) + { + if(c->GetName() == en->GetName()) + { + // Rename this one + c->SetName(rNewFilename); + } + } + } + else + { + // Just copy this one + en->SetName(rNewFilename); + } + + // Save the directory back + SaveDirectory(dir, MoveFromDirectory); + } + catch(...) + { + RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same + throw; + } + + return; + } + + // Got to be careful how this is written, as we can't guarentte that if we have two + // directories open, the first won't be deleted as the second is opened. (cache) + + // List of entries to move + std::vector moving; + + // list of directory IDs which need to have containing dir id changed + std::vector dirsToChangeContainingID; + + try + { + // First of all, get copies of the entries to move to the to directory. + + { + // Get the first directory + BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); + + // Find the file entry + BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID); + + // Error if not found + if(en == 0) + { + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) + } + + // Need to get all the entries with the same name? + if(MoveAllWithSameName) + { + // Iterate through the directory, copying all with matching names + BackupStoreDirectory::Iterator i(from); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next()) != 0) + { + if(c->GetName() == en->GetName()) + { + // Copy + moving.push_back(new BackupStoreDirectory::Entry(*c)); + + // Check for containing directory correction + if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID()); + } + } + ASSERT(!moving.empty()); + } + else + { + // Just copy this one + moving.push_back(new BackupStoreDirectory::Entry(*en)); + + // Check for containing directory correction + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID()); + } + } + + // Secondly, insert them into the to directory, and save it + + { + // To directory + BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); + + // Check the new name doens't already exist + { + BackupStoreDirectory::Iterator i(to); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0) + { + if(c->GetName() == rNewFilename) + { + THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory) + } + } + } + + // Copy the entries into it, changing the name as we go + for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) + { + BackupStoreDirectory::Entry *en = (*i); + en->SetName(rNewFilename); + to.AddEntry(*en); // adds copy + } + + // Save back + SaveDirectory(to, MoveToDirectory); + } + + // Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory + try + { + // Get directory + BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); + + // Delete each one + for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) + { + from.DeleteEntry((*i)->GetObjectID()); + } + + // Save back + SaveDirectory(from, MoveFromDirectory); + } + catch(...) + { + // UNDO modification to To directory + + // Get directory + BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); + + // Delete each one + for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) + { + to.DeleteEntry((*i)->GetObjectID()); + } + + // Save back + SaveDirectory(to, MoveToDirectory); + + // Throw the error + throw; + } + + // Finally... for all the directories we moved, modify their containing directory ID + for(std::vector::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) + { + // Load the directory + BackupStoreDirectory &change(GetDirectoryInternal(*i)); + + // Modify containing dir ID + change.SetContainerID(MoveToDirectory); + + // Save it back + SaveDirectory(change, *i); + } + } + catch(...) + { + // Make sure directories aren't in the cache, as they may have been modified + RemoveDirectoryFromCache(MoveToDirectory); + RemoveDirectoryFromCache(MoveFromDirectory); + for(std::vector::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) + { + RemoveDirectoryFromCache(*i); + } + + while(!moving.empty()) + { + delete moving.back(); + moving.pop_back(); + } + throw; + } + + // Clean up + while(!moving.empty()) + { + delete moving.back(); + moving.pop_back(); + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetBackupStoreInfo() +// Purpose: Return the backup store info object, exception if it isn't loaded +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +const BackupStoreInfo &BackupStoreContext::GetBackupStoreInfo() const +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + return *(mpStoreInfo.get()); +} + + diff --git a/bin/bbstored/BackupStoreContext.h b/bin/bbstored/BackupStoreContext.h new file mode 100644 index 00000000..4cfdb601 --- /dev/null +++ b/bin/bbstored/BackupStoreContext.h @@ -0,0 +1,183 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreContext.h +// Purpose: Context for backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCONTEXT__H +#define BACKUPCONTEXT__H + +#include +#include +#include + +#include "NamedLock.h" +#include "ProtocolObject.h" +#include "Utils.h" + +class BackupStoreDirectory; +class BackupStoreFilename; +class BackupStoreDaemon; +class BackupStoreInfo; +class IOStream; +class BackupProtocolObject; +class StreamableMemBlock; + +class HousekeepingInterface +{ + public: + virtual ~HousekeepingInterface() { } + virtual void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreContext +// Purpose: Context for backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +class BackupStoreContext +{ +public: + BackupStoreContext(int32_t ClientID, HousekeepingInterface &rDaemon); + ~BackupStoreContext(); +private: + BackupStoreContext(const BackupStoreContext &rToCopy); +public: + + void ReceivedFinishCommand(); + void CleanUp(); + + int32_t GetClientID() {return mClientID;} + + enum + { + Phase_START = 0, + Phase_Version = 0, + Phase_Login = 1, + Phase_Commands = 2 + }; + + int GetPhase() const {return mProtocolPhase;} + void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;} + + // Read only locking + bool SessionIsReadOnly() {return mReadOnly;} + bool AttemptToGetWriteLock(); + + void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;} + bool GetClientHasAccount() const {return mClientHasAccount;} + const std::string &GetStoreRoot() const {return mStoreRoot;} + int GetStoreDiscSet() const {return mStoreDiscSet;} + + // Store info + void LoadStoreInfo(); + void SaveStoreInfo(bool AllowDelay = true); + const BackupStoreInfo &GetBackupStoreInfo() const; + + // Client marker + int64_t GetClientStoreMarker(); + void SetClientStoreMarker(int64_t ClientStoreMarker); + + // Usage information + void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit); + bool HardLimitExceeded(); + + // Reading directories + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupStoreContext::GetDirectory(int64_t) + // Purpose: Return a reference to a directory. Valid only until the + // next time a function which affects directories is called. + // Mainly this funciton, and creation of files. + // Created: 2003/09/02 + // + // -------------------------------------------------------------------------- + const BackupStoreDirectory &GetDirectory(int64_t ObjectID) + { + // External callers aren't allowed to change it -- this function + // merely turns the the returned directory const. + return GetDirectoryInternal(ObjectID); + } + + // Manipulating files/directories + int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions); + int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists); + void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime); + bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut); + bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut); + bool UndeleteFile(int64_t ObjectID, int64_t InDirectory); + void DeleteDirectory(int64_t ObjectID, bool Undelete = false); + void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject); + + // Manipulating objects + enum + { + ObjectExists_Anything = 0, + ObjectExists_File = 1, + ObjectExists_Directory = 2 + }; + bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything); + std::auto_ptr OpenObject(int64_t ObjectID); + + // Info + int32_t GetClientID() const {return mClientID;} + +private: + void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false); + BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID); + void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID); + void RemoveDirectoryFromCache(int64_t ObjectID); + void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete); + int64_t AllocateObjectID(); + +private: + int32_t mClientID; + HousekeepingInterface &mrDaemon; + int mProtocolPhase; + bool mClientHasAccount; + std::string mStoreRoot; // has final directory separator + int mStoreDiscSet; + bool mReadOnly; + NamedLock mWriteLock; + int mSaveStoreInfoDelay; // how many times to delay saving the store info + + // Store info + std::auto_ptr mpStoreInfo; + + // Directory cache + std::map mDirectoryCache; + +public: + class TestHook + { + public: + virtual std::auto_ptr StartCommand(BackupProtocolObject& + rCommand) = 0; + virtual ~TestHook() { } + }; + void SetTestHook(TestHook& rTestHook) + { + mpTestHook = &rTestHook; + } + std::auto_ptr StartCommandHook(BackupProtocolObject& rCommand) + { + if(mpTestHook) + { + return mpTestHook->StartCommand(rCommand); + } + return std::auto_ptr(); + } + +private: + TestHook* mpTestHook; +}; + +#endif // BACKUPCONTEXT__H + diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 28e28176..4de0a078 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -17,7 +17,7 @@ #include #endif -#include "BackupContext.h" +#include "BackupStoreContext.h" #include "BackupStoreDaemon.h" #include "BackupStoreConfigVerify.h" #include "autogen_BackupProtocolServer.h" @@ -43,7 +43,8 @@ BackupStoreDaemon::BackupStoreDaemon() mHaveForkedHousekeeping(false), mIsHousekeepingProcess(false), mHousekeepingInited(false), - mInterProcessComms(mInterProcessCommsSocket) + mInterProcessComms(mInterProcessCommsSocket), + mpTestHook(NULL) { } @@ -171,11 +172,12 @@ void BackupStoreDaemon::Run() const Configuration &config(GetConfiguration()); mExtendedLogging = config.GetKeyValueBool("ExtendedLogging"); -#ifdef WIN32 - // Housekeeping runs synchronously on Win32 -#else - // Fork off housekeeping daemon -- must only do this the first time Run() is called - if(!mHaveForkedHousekeeping) + // Fork off housekeeping daemon -- must only do this the first + // time Run() is called. Housekeeping runs synchronously on Win32 + // because IsSingleProcess() is always true + +#ifndef WIN32 + if(!IsSingleProcess() && !mHaveForkedHousekeeping) { // Open a socket pair for communication int sv[2] = {-1,-1}; @@ -205,7 +207,9 @@ void BackupStoreDaemon::Run() // Log that housekeeping started BOX_INFO("Housekeeping process started"); // Ignore term and hup - // Parent will handle these and alert the child via the socket, don't want to randomly die + // Parent will handle these and alert the + // child via the socket, don't want to + // randomly die! ::signal(SIGHUP, SIG_IGN); ::signal(SIGTERM, SIG_IGN); } @@ -317,10 +321,18 @@ void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) } // Make ps listings clearer - SetProcessTitle("client %08x", id); + std::ostringstream tag; + tag << "client=" << BOX_FORMAT_ACCOUNT(id); + SetProcessTitle(tag.str().c_str()); + Logging::Tagger tagWithClientID(tag.str()); // Create a context, using this ID - BackupContext context(id, *this); + BackupStoreContext context(id, *this); + + if (mpTestHook) + { + context.SetTestHook(*mpTestHook); + } // See if the client has an account? if(mpAccounts && mpAccounts->AccountExists(id)) @@ -352,7 +364,7 @@ void BackupStoreDaemon::LogConnectionStats(const char *commonName, const SocketStreamTLS &s) { // Log the amount of data transferred - BOX_INFO("Connection statistics for " << commonName << ":" + BOX_NOTICE("Connection statistics for " << commonName << ":" " IN=" << s.GetBytesRead() << " OUT=" << s.GetBytesWritten() << " TOTAL=" << (s.GetBytesRead() + s.GetBytesWritten())); diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 387a1d5b..a5d216f4 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -13,6 +13,7 @@ #include "ServerTLS.h" #include "BoxPortsAndFiles.h" #include "BackupConstants.h" +#include "BackupStoreContext.h" #include "IOStreamGetLine.h" class BackupStoreAccounts; @@ -27,7 +28,8 @@ class HousekeepStoreAccount; // Created: 2003/08/20 // // -------------------------------------------------------------------------- -class BackupStoreDaemon : public ServerTLS +class BackupStoreDaemon : public ServerTLS, + HousekeepingInterface { friend class HousekeepStoreAccount; @@ -38,7 +40,7 @@ private: BackupStoreDaemon(const BackupStoreDaemon &rToCopy); public: - // For BackupContext to communicate with housekeeping process + // For BackupStoreContext to communicate with housekeeping process void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) { #ifndef WIN32 @@ -81,6 +83,15 @@ private: void HousekeepingInit(); void RunHousekeepingIfNeeded(); int64_t mLastHousekeepingRun; + +public: + void SetTestHook(BackupStoreContext::TestHook& rTestHook) + { + mpTestHook = &rTestHook; + } + +private: + BackupStoreContext::TestHook* mpTestHook; }; diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp index 9f4239e7..dbb9b544 100644 --- a/bin/bbstored/HousekeepStoreAccount.cpp +++ b/bin/bbstored/HousekeepStoreAccount.cpp @@ -85,16 +85,19 @@ void HousekeepStoreAccount::DoHousekeeping() { // Attempt to lock the account std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFilename); + StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, + writeLockFilename); NamedLock writeLock; - if(!writeLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */)) + if(!writeLock.TryAndGetLock(writeLockFilename.c_str(), + 0600 /* restrictive file permissions */)) { // Couldn't lock the account -- just stop now return; } // Load the store info to find necessary info for the housekeeping - std::auto_ptr info(BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet, false /* Read/Write */)); + std::auto_ptr info(BackupStoreInfo::Load(mAccountID, + mStoreRoot, mStoreDiscSet, false /* Read/Write */)); // Calculate how much should be deleted mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit(); @@ -104,14 +107,18 @@ void HousekeepStoreAccount::DoHousekeeping() } // Scan the directory for potential things to delete - // This will also remove elegiable items marked with RemoveASAP + // This will also remove eligible items marked with RemoveASAP bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID); - // If scan directory stopped for some reason, probably parent instructed to teminate, stop now. + // If scan directory stopped for some reason, probably parent + // instructed to terminate, stop now. if(!continueHousekeeping) { - // If any files were marked "delete now", then update the size of the store. - if(mBlocksUsedDelta != 0 || mBlocksInOldFilesDelta != 0 || mBlocksInDeletedFilesDelta != 0) + // If any files were marked "delete now", then update + // the size of the store. + if(mBlocksUsedDelta != 0 || + mBlocksInOldFilesDelta != 0 || + mBlocksInDeletedFilesDelta != 0) { info->ChangeBlocksUsed(mBlocksUsedDelta); info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); @@ -124,8 +131,8 @@ void HousekeepStoreAccount::DoHousekeeping() return; } - // Log any difference in opinion between the values recorded in the store info, and - // the values just calculated for space usage. + // Log any difference in opinion between the values recorded in + // the store info, and the values just calculated for space usage. // BLOCK { int64_t used = info->GetBlocksUsed(); @@ -133,9 +140,12 @@ void HousekeepStoreAccount::DoHousekeeping() int64_t usedDeleted = info->GetBlocksInDeletedFiles(); int64_t usedDirectories = info->GetBlocksInDirectories(); - // If the counts were wrong, taking into account RemoveASAP items deleted, log a message - if((used + mBlocksUsedDelta) != mBlocksUsed || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles - || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles || usedDirectories != mBlocksInDirectories) + // If the counts were wrong, taking into account RemoveASAP + // items deleted, log a message + if((used + mBlocksUsedDelta) != mBlocksUsed + || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles + || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles + || usedDirectories != mBlocksInDirectories) { // Log this BOX_ERROR("Housekeeping on account " << @@ -153,18 +163,25 @@ void HousekeepStoreAccount::DoHousekeeping() } // If the current values don't match, store them - if(used != mBlocksUsed || usedOld != mBlocksInOldFiles - || usedDeleted != mBlocksInDeletedFiles || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta)) + if(used != mBlocksUsed + || usedOld != mBlocksInOldFiles + || usedDeleted != mBlocksInDeletedFiles + || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta)) { // Set corrected values in store info - info->CorrectAllUsedValues(mBlocksUsed, mBlocksInOldFiles, mBlocksInDeletedFiles, mBlocksInDirectories + mBlocksInDirectoriesDelta); + info->CorrectAllUsedValues(mBlocksUsed, + mBlocksInOldFiles, mBlocksInDeletedFiles, + mBlocksInDirectories + mBlocksInDirectoriesDelta); info->Save(); } } - // Reset the delta counts for files, as they will include RemoveASAP flagged files deleted - // during the initial scan. - int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta; // keep for reporting + // Reset the delta counts for files, as they will include + // RemoveASAP flagged files deleted during the initial scan. + + // keep for reporting + int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta; + mBlocksUsedDelta = 0; mBlocksInOldFilesDelta = 0; mBlocksInDeletedFilesDelta = 0; @@ -172,7 +189,8 @@ void HousekeepStoreAccount::DoHousekeeping() // Go and delete items from the accounts bool deleteInterrupted = DeleteFiles(); - // If that wasn't interrupted, remove any empty directories which are also marked as deleted in their containing directory + // If that wasn't interrupted, remove any empty directories which + // are also marked as deleted in their containing directory if(!deleteInterrupted) { deleteInterrupted = DeleteEmptyDirectories(); @@ -190,8 +208,9 @@ void HousekeepStoreAccount::DoHousekeeping() (deleteInterrupted?" and was interrupted":"")); } - // Make sure the delta's won't cause problems if the counts are really wrong, and - // it wasn't fixed because the store was updated during the scan. + // Make sure the delta's won't cause problems if the counts are + // really wrong, and it wasn't fixed because the store was + // updated during the scan. if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) { mBlocksUsedDelta = (0 - info->GetBlocksUsed()); @@ -218,7 +237,8 @@ void HousekeepStoreAccount::DoHousekeeping() // Save the store info back info->Save(); - // Explicity release the lock (would happen automatically on going out of scope, included for code clarity) + // Explicity release the lock (would happen automatically on + // going out of scope, included for code clarity) writeLock.ReleaseLock(); } @@ -243,8 +263,9 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF // // Function // Name: HousekeepStoreAccount::ScanDirectory(int64_t) -// Purpose: Private. Scan a directory for potenitally deleteable items, and -// add them to the list. Returns true if the scan should continue. +// Purpose: Private. Scan a directory for potentially deleteable +// items, and add them to the list. Returns true if the +// scan should continue. // Created: 11/12/03 // // -------------------------------------------------------------------------- @@ -253,9 +274,12 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) #ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { - mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; + mCountUntilNextInterprocessMsgCheck = + POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; + // Check for having to stop - if(mrDaemon.CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is locked + // Include account ID here as the specified account is locked + if(mrDaemon.CheckForInterProcessMsg(mAccountID)) { // Need to abort now return false; @@ -268,7 +292,8 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) MakeObjectFilename(ObjectID, objectFilename); // Open it. - std::auto_ptr dirStream(RaidFileRead::Open(mStoreDiscSet, objectFilename)); + std::auto_ptr dirStream(RaidFileRead::Open(mStoreDiscSet, + objectFilename)); // Add the size of the directory on disc to the size being calculated int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); @@ -290,8 +315,8 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // BLOCK { - // Remove any files which are marked for removal as soon as they become old - // or deleted. + // Remove any files which are marked for removal as soon + // as they become old or deleted. bool deletedSomething = false; do { @@ -324,7 +349,8 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Add files to the list of potential deletions // map to count the distance from the mark - std::map, int32_t> markVersionAges; + typedef std::pair version_t; + std::map markVersionAges; // map of pair (filename, mark number) -> version age // NOTE: use a reverse iterator to allow the distance from mark stuff to work @@ -342,7 +368,10 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Work out ages of this version from the last mark int32_t enVersionAge = 0; - std::map, int32_t>::iterator enVersionAgeI(markVersionAges.find(std::pair(en->GetName(), en->GetMarkNumber()))); + std::map::iterator enVersionAgeI( + markVersionAges.find( + version_t(en->GetName().GetEncodedFilename(), + en->GetMarkNumber()))); if(enVersionAgeI != markVersionAges.end()) { enVersionAge = enVersionAgeI->second + 1; @@ -350,7 +379,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } else { - markVersionAges[std::pair(en->GetName(), en->GetMarkNumber())] = enVersionAge; + markVersionAges[version_t(en->GetName().GetEncodedFilename(), en->GetMarkNumber())] = enVersionAge; } // enVersionAge is now the age of this version. @@ -364,6 +393,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) d.mSizeInBlocks = en->GetSizeInBlocks(); d.mMarkNumber = en->GetMarkNumber(); d.mVersionAgeWithinMark = enVersionAge; + d.mIsFlagDeleted = (enFlags & + BackupStoreDirectory::Entry::Flags_Deleted) + ? true : false; // Add it to the list mPotentialDeletions.insert(d); @@ -541,6 +573,10 @@ bool HousekeepStoreAccount::DeleteFiles() // Delete the file DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig); + BOX_INFO("Housekeeping removed " << + (i->mIsFlagDeleted ? "deleted" : "old") << + " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << + " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); // Stop if the deletion target has been matched or exceeded // (checking here rather than at the beginning will tend to reduce the @@ -786,99 +822,115 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() continue; } - // Load up the directory to potentially delete - std::string dirFilename; - BackupStoreDirectory dir; - int64_t dirSizeInBlocks = 0; - { - MakeObjectFilename(*i, dirFilename); - // Check it actually exists (just in case it gets added twice to the list) - if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename)) - { - // doesn't exist, next! - continue; - } - // load - std::auto_ptr dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename)); - dirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); - dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); - } + DeleteEmptyDirectory(*i, toExamine); + } - // Make sure this directory is actually empty - if(dir.GetNumberOfEntries() != 0) - { - // Not actually empty, try next one - continue; - } + // Remove contents of empty directories + mEmptyDirectories.clear(); + // Swap in new, so it's examined next time round + mEmptyDirectories.swap(toExamine); + } + + // Not interrupted + return false; +} - // Candiate for deletion... open containing directory - std::string containingDirFilename; - BackupStoreDirectory containingDir; - int64_t containingDirSizeInBlocksOrig = 0; - { - MakeObjectFilename(dir.GetContainerID(), containingDirFilename); - std::auto_ptr containingDirStream(RaidFileRead::Open(mStoreDiscSet, containingDirFilename)); - containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks(); - containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite); - } +void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, + std::vector& rToExamine) +{ + // Load up the directory to potentially delete + std::string dirFilename; + BackupStoreDirectory dir; + int64_t dirSizeInBlocks = 0; - // Find the entry - BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID()); - if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)) - { - // Should be deleted - containingDir.DeleteEntry(dir.GetObjectID()); + // BLOCK + { + MakeObjectFilename(dirId, dirFilename); + // Check it actually exists (just in case it gets + // added twice to the list) + if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename)) + { + // doesn't exist, next! + return; + } + // load + std::auto_ptr dirStream( + RaidFileRead::Open(mStoreDiscSet, dirFilename)); + dirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); + dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + } - // Is the containing dir now a candidate for deletion? - if(containingDir.GetNumberOfEntries() == 0) - { - toExamine.push_back(containingDir.GetObjectID()); - } + // Make sure this directory is actually empty + if(dir.GetNumberOfEntries() != 0) + { + // Not actually empty, try next one + return; + } - // Write revised parent directory - RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename); - writeDir.Open(true /* allow overwriting */); - containingDir.WriteToStream(writeDir); + // Candidate for deletion... open containing directory + std::string containingDirFilename; + BackupStoreDirectory containingDir; + int64_t containingDirSizeInBlocksOrig = 0; + { + MakeObjectFilename(dir.GetContainerID(), containingDirFilename); + std::auto_ptr containingDirStream( + RaidFileRead::Open(mStoreDiscSet, + containingDirFilename)); + containingDirSizeInBlocksOrig = + containingDirStream->GetDiscUsageInBlocks(); + containingDir.ReadFromStream(*containingDirStream, + IOStream::TimeOutInfinite); + } - // get the disc usage (must do this before commiting it) - int64_t dirSize = writeDir.GetDiscUsageInBlocks(); + // Find the entry + BackupStoreDirectory::Entry *pdirentry = + containingDir.FindEntryByID(dir.GetObjectID()); + if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)) + { + // Should be deleted + containingDir.DeleteEntry(dir.GetObjectID()); - // Commit directory - writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + // Is the containing dir now a candidate for deletion? + if(containingDir.GetNumberOfEntries() == 0) + { + rToExamine.push_back(containingDir.GetObjectID()); + } - // adjust usage counts for this directory - if(dirSize > 0) - { - int64_t adjust = dirSize - containingDirSizeInBlocksOrig; - mBlocksUsedDelta += adjust; - mBlocksInDirectoriesDelta += adjust; - } + // Write revised parent directory + RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename); + writeDir.Open(true /* allow overwriting */); + containingDir.WriteToStream(writeDir); - // Delete the directory itself - { - RaidFileWrite del(mStoreDiscSet, dirFilename); - del.Delete(); - } + // get the disc usage (must do this before commiting it) + int64_t dirSize = writeDir.GetDiscUsageInBlocks(); - // And adjust usage counts for the directory that's just been deleted - mBlocksUsedDelta -= dirSizeInBlocks; - mBlocksInDirectoriesDelta -= dirSizeInBlocks; + // Commit directory + writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - // Update count - ++mEmptyDirectoriesDeleted; - } + // adjust usage counts for this directory + if(dirSize > 0) + { + int64_t adjust = dirSize - containingDirSizeInBlocksOrig; + mBlocksUsedDelta += adjust; + mBlocksInDirectoriesDelta += adjust; } - // Remove contents of empty directories - mEmptyDirectories.clear(); - // Swap in new, so it's examined next time round - mEmptyDirectories.swap(toExamine); - } - - // Not interrupted - return false; -} + // Delete the directory itself + { + RaidFileWrite del(mStoreDiscSet, dirFilename); + del.Delete(); + } + BOX_INFO("Housekeeping removed empty deleted dir " << + BOX_FORMAT_OBJECTID(dirId)); + // And adjust usage counts for the directory that's + // just been deleted + mBlocksUsedDelta -= dirSizeInBlocks; + mBlocksInDirectoriesDelta -= dirSizeInBlocks; + // Update count + ++mEmptyDirectoriesDeleted; + } +} diff --git a/bin/bbstored/HousekeepStoreAccount.h b/bin/bbstored/HousekeepStoreAccount.h index 6c8f251d..5c2a9885 100644 --- a/bin/bbstored/HousekeepStoreAccount.h +++ b/bin/bbstored/HousekeepStoreAccount.h @@ -42,6 +42,8 @@ private: bool ScanDirectory(int64_t ObjectID); bool DeleteFiles(); bool DeleteEmptyDirectories(); + void DeleteEmptyDirectory(int64_t dirId, + std::vector& rToExamine); void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks); private: @@ -52,6 +54,7 @@ private: int64_t mSizeInBlocks; int32_t mMarkNumber; int32_t mVersionAgeWithinMark; // 0 == current, 1 latest old version, etc + bool mIsFlagDeleted; // false for files flagged "Old" } DelEn; struct DelEnCompare diff --git a/bin/bbstored/backupprotocol.txt b/bin/bbstored/backupprotocol.txt index 62d837ff..3eca514a 100644 --- a/bin/bbstored/backupprotocol.txt +++ b/bin/bbstored/backupprotocol.txt @@ -4,7 +4,7 @@ Name Backup IdentString Box-Backup:v=C -ServerContextClass BackupContext BackupContext.h +ServerContextClass BackupStoreContext BackupStoreContext.h ClientType Filename BackupStoreFilenameClear BackupStoreFilenameClear.h ServerType Filename BackupStoreFilename BackupStoreFilename.h @@ -204,6 +204,12 @@ GetBlockIndexByName 35 Command(Success) # stream of the block index follows the reply if found ID != 0 +UndeleteFile 36 Command(Success) + int64 InDirectory + int64 ObjectID + # will return 0 if the object couldn't be found in the specified directory + + # ------------------------------------------------------------------------------------- # Information commands # ------------------------------------------------------------------------------------- diff --git a/bin/bbstored/bbstored-config.in b/bin/bbstored/bbstored-config.in index 33cfb39a..5ad96d50 100755 --- a/bin/bbstored/bbstored-config.in +++ b/bin/bbstored/bbstored-config.in @@ -196,7 +196,7 @@ TimeBetweenHousekeeping = 900 Server { - PidFile = @localstatedir_expanded@/bbstored.pid + PidFile = @localstatedir_expanded@/run/bbstored.pid User = $username ListenAddresses = inet:$server CertificateFile = $certificate @@ -234,7 +234,7 @@ What you need to do now... 4) Create accounts with bbstoreaccounts 5) Start the backup store daemon with the command - @bindir_expanded@/bbstored$daemon_args + @sbindir_expanded@/bbstored$daemon_args in /etc/rc.local, or your local equivalent. =================================================================== diff --git a/bin/bbstored/bbstored.cpp b/bin/bbstored/bbstored.cpp index 54858dd4..21a9e5f1 100644 --- a/bin/bbstored/bbstored.cpp +++ b/bin/bbstored/bbstored.cpp @@ -18,7 +18,7 @@ int main(int argc, const char *argv[]) { MAINHELPER_START - Logging::SetProgramName("Box Backup (bbstored)"); + Logging::SetProgramName("bbstored"); Logging::ToConsole(true); Logging::ToSyslog (true); -- cgit v1.2.3