diff options
Diffstat (limited to 'bin/bbackupd')
-rw-r--r-- | bin/bbackupd/BackupClientContext.cpp | 578 | ||||
-rw-r--r-- | bin/bbackupd/BackupClientContext.h | 252 | ||||
-rw-r--r-- | bin/bbackupd/BackupClientDeleteList.cpp | 229 | ||||
-rw-r--r-- | bin/bbackupd/BackupClientDeleteList.h | 75 | ||||
-rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.cpp | 2302 | ||||
-rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.h | 244 | ||||
-rw-r--r-- | bin/bbackupd/BackupClientInodeToIDMap.cpp | 320 | ||||
-rw-r--r-- | bin/bbackupd/BackupClientInodeToIDMap.h | 59 | ||||
-rw-r--r-- | bin/bbackupd/BackupDaemon.cpp | 3654 | ||||
-rw-r--r-- | bin/bbackupd/BackupDaemon.h | 539 | ||||
-rw-r--r-- | bin/bbackupd/BackupDaemonInterface.h | 166 | ||||
-rw-r--r-- | bin/bbackupd/Win32BackupService.cpp | 48 | ||||
-rw-r--r-- | bin/bbackupd/Win32BackupService.h | 21 | ||||
-rw-r--r-- | bin/bbackupd/Win32ServiceFunctions.cpp | 384 | ||||
-rw-r--r-- | bin/bbackupd/Win32ServiceFunctions.h | 19 |
15 files changed, 0 insertions, 8890 deletions
diff --git a/bin/bbackupd/BackupClientContext.cpp b/bin/bbackupd/BackupClientContext.cpp deleted file mode 100644 index 4c0b01ce..00000000 --- a/bin/bbackupd/BackupClientContext.cpp +++ /dev/null @@ -1,578 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientContext.cpp -// Purpose: Keep track of context -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#ifdef HAVE_SIGNAL_H - #include <signal.h> -#endif - -#ifdef HAVE_SYS_TIME_H - #include <sys/time.h> -#endif - -#include "BoxPortsAndFiles.h" -#include "BoxTime.h" -#include "BackupClientContext.h" -#include "SocketStreamTLS.h" -#include "Socket.h" -#include "BackupStoreConstants.h" -#include "BackupStoreException.h" -#include "BackupDaemon.h" -#include "autogen_BackupProtocol.h" -#include "BackupStoreFile.h" -#include "Logging.h" -#include "TcpNice.h" - -#include "MemLeakFindOn.h" - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::BackupClientContext(BackupDaemon &, TLSContext &, const std::string &, int32_t, bool, bool, std::string) -// Purpose: Constructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupClientContext::BackupClientContext -( - LocationResolver &rResolver, - TLSContext &rTLSContext, - const std::string &rHostname, - int Port, - uint32_t AccountNumber, - bool ExtendedLogging, - bool ExtendedLogToFile, - std::string ExtendedLogFile, - ProgressNotifier& rProgressNotifier, - bool TcpNiceMode -) -: mExperimentalSnapshotMode(false), - mrResolver(rResolver), - mrTLSContext(rTLSContext), - mHostname(rHostname), - mPort(Port), - mAccountNumber(AccountNumber), - mExtendedLogging(ExtendedLogging), - mExtendedLogToFile(ExtendedLogToFile), - mExtendedLogFile(ExtendedLogFile), - mpExtendedLogFileHandle(NULL), - mClientStoreMarker(ClientStoreMarker_NotKnown), - mpDeleteList(0), - mpCurrentIDMap(0), - mpNewIDMap(0), - mStorageLimitExceeded(false), - mpExcludeFiles(0), - mpExcludeDirs(0), - mKeepAliveTimer(0, "KeepAliveTime"), - mbIsManaged(false), - mrProgressNotifier(rProgressNotifier), - mTcpNiceMode(TcpNiceMode), - mpNice(NULL) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::~BackupClientContext() -// Purpose: Destructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupClientContext::~BackupClientContext() -{ - CloseAnyOpenConnection(); - - // Delete delete list - if(mpDeleteList != 0) - { - delete mpDeleteList; - mpDeleteList = 0; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetConnection() -// Purpose: Returns the connection, making the connection and logging into -// the backup store if necessary. -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupProtocolCallable &BackupClientContext::GetConnection() -{ - // Already got it? Just return it. - if(mapConnection.get()) - { - return *mapConnection; - } - - // Defensive. Must close connection before releasing any old socket. - mapConnection.reset(); - - std::auto_ptr<SocketStream> apSocket(new SocketStreamTLS); - - try - { - // Defensive. - mapConnection.reset(); - - // Log intention - BOX_INFO("Opening connection to server '" << mHostname << - "'..."); - - // Connect! - ((SocketStreamTLS *)(apSocket.get()))->Open(mrTLSContext, - Socket::TypeINET, mHostname, mPort); - - if(mTcpNiceMode) - { - // Pass control of apSocket to NiceSocketStream, - // which will take care of destroying it for us. - // But we need to hang onto a pointer to the nice - // socket, so we can enable and disable nice mode. - // This is scary, it could be deallocated under us. - mpNice = new NiceSocketStream(apSocket); - apSocket.reset(mpNice); - } - - // We need to call some methods that aren't defined in - // BackupProtocolCallable, so we need to hang onto a more - // strongly typed pointer (to avoid far too many casts). - BackupProtocolClient *pClient = new BackupProtocolClient(apSocket); - mapConnection.reset(pClient); - - // Set logging option - pClient->SetLogToSysLog(mExtendedLogging); - - if (mExtendedLogToFile) - { - ASSERT(mpExtendedLogFileHandle == NULL); - - mpExtendedLogFileHandle = fopen( - mExtendedLogFile.c_str(), "a+"); - - if (!mpExtendedLogFileHandle) - { - BOX_LOG_SYS_ERROR("Failed to open extended " - "log file: " << mExtendedLogFile); - } - else - { - pClient->SetLogToFile(mpExtendedLogFileHandle); - } - } - - // Handshake - pClient->Handshake(); - - // Check the version of the server - { - std::auto_ptr<BackupProtocolVersion> serverVersion( - mapConnection->QueryVersion(BACKUP_STORE_SERVER_VERSION)); - if(serverVersion->GetVersion() != BACKUP_STORE_SERVER_VERSION) - { - THROW_EXCEPTION(BackupStoreException, WrongServerVersion) - } - } - - // Login -- if this fails, the Protocol will exception - std::auto_ptr<BackupProtocolLoginConfirmed> loginConf( - mapConnection->QueryLogin(mAccountNumber, 0 /* read/write */)); - - // Check that the client store marker is the one we expect - if(mClientStoreMarker != ClientStoreMarker_NotKnown) - { - if(loginConf->GetClientStoreMarker() != mClientStoreMarker) - { - // Not good... finish the connection, abort, etc, ignoring errors - try - { - mapConnection->QueryFinished(); - } - catch(...) - { - // IGNORE - } - - // Then throw an exception about this - THROW_EXCEPTION_MESSAGE(BackupStoreException, - ClientMarkerNotAsExpected, - "Expected " << mClientStoreMarker << - " but found " << loginConf->GetClientStoreMarker() << - ": is someone else writing to the " - "same account?"); - } - } - else // mClientStoreMarker == ClientStoreMarker_NotKnown - { - // Yes, choose one, the current time will do - box_time_t marker = GetCurrentBoxTime(); - - // Set it on the store - mapConnection->QuerySetClientStoreMarker(marker); - - // Record it so that it can be picked up later. - mClientStoreMarker = marker; - } - - // Log success - BOX_INFO("Connection made, login successful"); - - // Check to see if there is any space available on the server - if(loginConf->GetBlocksUsed() >= loginConf->GetBlocksHardLimit()) - { - // no -- flag so only things like deletions happen - mStorageLimitExceeded = true; - // Log - BOX_WARNING("Exceeded storage hard-limit on server, " - "not uploading changes to files"); - } - } - catch(...) - { - // Clean up. - mapConnection.reset(); - throw; - } - - return *mapConnection; -} - -BackupProtocolCallable* BackupClientContext::GetOpenConnection() const -{ - return mapConnection.get(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::CloseAnyOpenConnection() -// Purpose: Closes a connection, if it's open -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupClientContext::CloseAnyOpenConnection() -{ - BackupProtocolCallable* pConnection(GetOpenConnection()); - if(pConnection) - { - try - { - // Quit nicely - pConnection->QueryFinished(); - } - catch(...) - { - // Ignore errors here - } - - // Delete it anyway. - mapConnection.reset(); - } - - // Delete any pending list - if(mpDeleteList != 0) - { - delete mpDeleteList; - mpDeleteList = 0; - } - - if (mpExtendedLogFileHandle != NULL) - { - fclose(mpExtendedLogFileHandle); - mpExtendedLogFileHandle = NULL; - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetTimeout() -// Purpose: Gets the current timeout time. -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -int BackupClientContext::GetTimeout() const -{ - BackupProtocolCallable* pConnection(GetOpenConnection()); - if(pConnection) - { - return pConnection->GetTimeout(); - } - - return (15*60*1000); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetDeleteList() -// Purpose: Returns the delete list, creating one if necessary -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -BackupClientDeleteList &BackupClientContext::GetDeleteList() -{ - // Already created? - if(mpDeleteList == 0) - { - mpDeleteList = new BackupClientDeleteList; - } - - // Return reference to object - return *mpDeleteList; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::PerformDeletions() -// Purpose: Perform any pending file deletions. -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientContext::PerformDeletions() -{ - // Got a list? - if(mpDeleteList == 0) - { - // Nothing to do - return; - } - - // Delegate to the delete list object - mpDeleteList->PerformDeletions(*this); - - // Delete the object - delete mpDeleteList; - mpDeleteList = 0; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetCurrentIDMap() const -// Purpose: Return a (const) reference to the current ID map -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -const BackupClientInodeToIDMap &BackupClientContext::GetCurrentIDMap() const -{ - ASSERT(mpCurrentIDMap != 0); - if(mpCurrentIDMap == 0) - { - THROW_EXCEPTION(CommonException, Internal) - } - return *mpCurrentIDMap; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetNewIDMap() const -// Purpose: Return a reference to the new ID map -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -BackupClientInodeToIDMap &BackupClientContext::GetNewIDMap() const -{ - ASSERT(mpNewIDMap != 0); - if(mpNewIDMap == 0) - { - THROW_EXCEPTION(CommonException, Internal) - } - return *mpNewIDMap; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::FindFilename(int64_t, int64_t, std::string &, bool &) const -// Purpose: Attempts to find the pathname of an object with a given ID on the server. -// Returns true if it can be found, in which case rPathOut is the local filename, -// and rIsDirectoryOut == true if the local object is a directory. -// Created: 12/11/03 -// -// -------------------------------------------------------------------------- -bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut, - bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer, box_time_t *pAttributesHashOnServer, BackupStoreFilenameClear *pLeafname) -{ - // Make a connection to the server - BackupProtocolCallable &connection(GetConnection()); - - // Request filenames from the server, in a "safe" manner to ignore errors properly - { - BackupProtocolGetObjectName send(ObjectID, ContainingDirectory); - connection.Send(send); - } - std::auto_ptr<BackupProtocolMessage> preply(connection.Receive()); - - // Is it of the right type? - if(preply->GetType() != BackupProtocolObjectName::TypeID) - { - // Was an error or something - return false; - } - - // Cast to expected type. - BackupProtocolObjectName *names = (BackupProtocolObjectName *)(preply.get()); - - // Anything found? - int32_t numElements = names->GetNumNameElements(); - if(numElements <= 0) - { - // No. - return false; - } - - // Get the stream containing all the names - std::auto_ptr<IOStream> nameStream(connection.ReceiveStream()); - - // Path - std::string path; - - // Remember this is in reverse order! - for(int l = 0; l < numElements; ++l) - { - BackupStoreFilenameClear elementName; - elementName.ReadFromStream(*nameStream, GetTimeout()); - - // Store leafname for caller? - if(l == 0 && pLeafname) - { - *pLeafname = elementName; - } - - // Is it part of the filename in the location? - if(l < (numElements - 1)) - { - // Part of filename within - path = (path.empty())?(elementName.GetClearFilename()):(elementName.GetClearFilename() + DIRECTORY_SEPARATOR_ASCHAR + path); - } - else - { - // Location name -- look up in daemon's records - std::string locPath; - if(!mrResolver.FindLocationPathName(elementName.GetClearFilename(), locPath)) - { - // Didn't find the location... so can't give the local filename - return false; - } - - // Add in location path - path = (path.empty())?(locPath):(locPath + DIRECTORY_SEPARATOR_ASCHAR + path); - } - } - - // Is it a directory? - rIsDirectoryOut = ((names->GetFlags() & BackupProtocolListDirectory::Flags_Dir) == BackupProtocolListDirectory::Flags_Dir); - - // Is it the current version? - rIsCurrentVersionOut = ((names->GetFlags() & (BackupProtocolListDirectory::Flags_OldVersion | BackupProtocolListDirectory::Flags_Deleted)) == 0); - - // And other information which may be required - if(pModTimeOnServer) *pModTimeOnServer = names->GetModificationTime(); - if(pAttributesHashOnServer) *pAttributesHashOnServer = names->GetAttributesHash(); - - // Tell caller about the pathname - rPathOut = path; - - // Found - return true; -} - -void BackupClientContext::SetMaximumDiffingTime(int iSeconds) -{ - mMaximumDiffingTime = iSeconds < 0 ? 0 : iSeconds; - BOX_TRACE("Set maximum diffing time to " << mMaximumDiffingTime << - " seconds"); -} - -void BackupClientContext::SetKeepAliveTime(int iSeconds) -{ - mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; - BOX_TRACE("Set keep-alive time to " << mKeepAliveTime << " seconds"); - mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::ManageDiffProcess() -// Purpose: Initiates a file diff control timer -// Created: 04/19/2005 -// -// -------------------------------------------------------------------------- -void BackupClientContext::ManageDiffProcess() -{ - ASSERT(!mbIsManaged); - mbIsManaged = true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::UnManageDiffProcess() -// Purpose: suspends file diff control timer -// Created: 04/19/2005 -// -// -------------------------------------------------------------------------- -void BackupClientContext::UnManageDiffProcess() -{ - // ASSERT(mbIsManaged); - mbIsManaged = false; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::DoKeepAlive() -// Purpose: Check whether it's time to send a KeepAlive -// message over the SSL link, and if so, send it. -// Created: 04/19/2005 -// -// -------------------------------------------------------------------------- -void BackupClientContext::DoKeepAlive() -{ - BackupProtocolCallable* pConnection(GetOpenConnection()); - if (!pConnection) - { - return; - } - - if (mKeepAliveTime == 0) - { - return; - } - - if (!mKeepAliveTimer.HasExpired()) - { - return; - } - - BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); - pConnection->QueryGetIsAlive(); - - mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); -} - -int BackupClientContext::GetMaximumDiffingTime() -{ - return mMaximumDiffingTime; -} diff --git a/bin/bbackupd/BackupClientContext.h b/bin/bbackupd/BackupClientContext.h deleted file mode 100644 index df43a232..00000000 --- a/bin/bbackupd/BackupClientContext.h +++ /dev/null @@ -1,252 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientContext.h -// Purpose: Keep track of context -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCLIENTCONTEXT__H -#define BACKUPCLIENTCONTEXT__H - -#include "BoxTime.h" -#include "BackupClientDeleteList.h" -#include "BackupClientDirectoryRecord.h" -#include "BackupDaemonInterface.h" -#include "BackupStoreFile.h" -#include "ExcludeList.h" -#include "TcpNice.h" -#include "Timer.h" - -class TLSContext; -class BackupProtocolClient; -class SocketStreamTLS; -class BackupClientInodeToIDMap; -class BackupDaemon; -class BackupStoreFilenameClear; - -#include <string> - - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupClientContext -// Purpose: -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -class BackupClientContext : public DiffTimer -{ -public: - BackupClientContext - ( - LocationResolver &rResolver, - TLSContext &rTLSContext, - const std::string &rHostname, - int32_t Port, - uint32_t AccountNumber, - bool ExtendedLogging, - bool ExtendedLogToFile, - std::string ExtendedLogFile, - ProgressNotifier &rProgressNotifier, - bool TcpNiceMode - ); - virtual ~BackupClientContext(); - -private: - BackupClientContext(const BackupClientContext &); - -public: - // GetConnection() will open a connection if none is currently open. - virtual BackupProtocolCallable& GetConnection(); - // GetOpenConnection() will not open a connection, just return NULL if there is - // no connection already open. - virtual BackupProtocolCallable* GetOpenConnection() const; - void CloseAnyOpenConnection(); - int GetTimeout() const; - BackupClientDeleteList &GetDeleteList(); - void PerformDeletions(); - - enum - { - ClientStoreMarker_NotKnown = 0 - }; - - void SetClientStoreMarker(int64_t ClientStoreMarker) {mClientStoreMarker = ClientStoreMarker;} - int64_t GetClientStoreMarker() const {return mClientStoreMarker;} - - bool StorageLimitExceeded() {return mStorageLimitExceeded;} - void SetStorageLimitExceeded() {mStorageLimitExceeded = true;} - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::SetIDMaps(const BackupClientInodeToIDMap *, BackupClientInodeToIDMap *) - // Purpose: Store pointers to the Current and New ID maps - // Created: 11/11/03 - // - // -------------------------------------------------------------------------- - void SetIDMaps(const BackupClientInodeToIDMap *pCurrent, BackupClientInodeToIDMap *pNew) - { - ASSERT(pCurrent != 0); - ASSERT(pNew != 0); - mpCurrentIDMap = pCurrent; - mpNewIDMap = pNew; - } - const BackupClientInodeToIDMap &GetCurrentIDMap() const; - BackupClientInodeToIDMap &GetNewIDMap() const; - - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::SetExcludeLists(ExcludeList *, ExcludeList *) - // Purpose: Sets the exclude lists for the operation. Can be 0. - // Created: 28/1/04 - // - // -------------------------------------------------------------------------- - void SetExcludeLists(ExcludeList *pExcludeFiles, ExcludeList *pExcludeDirs) - { - mpExcludeFiles = pExcludeFiles; - mpExcludeDirs = pExcludeDirs; - } - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::ExcludeFile(const std::string &) - // Purpose: Returns true is this file should be excluded from the backup - // Created: 28/1/04 - // - // -------------------------------------------------------------------------- - inline bool ExcludeFile(const std::string &rFullFilename) - { - if(mpExcludeFiles != 0) - { - return mpExcludeFiles->IsExcluded(rFullFilename); - } - // If no list, don't exclude anything - return false; - } - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::ExcludeDir(const std::string &) - // Purpose: Returns true is this directory should be excluded from the backup - // Created: 28/1/04 - // - // -------------------------------------------------------------------------- - inline bool ExcludeDir(const std::string &rFullDirName) - { - if(mpExcludeDirs != 0) - { - return mpExcludeDirs->IsExcluded(rFullDirName); - } - // If no list, don't exclude anything - return false; - } - - // Utility functions -- may do a lot of work - bool FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut, - bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer = 0, box_time_t *pAttributesHashOnServer = 0, - BackupStoreFilenameClear *pLeafname = 0); // not const as may connect to server - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::SetMaximumDiffingTime() - // Purpose: Sets the maximum time that will be spent diffing a file - // Created: 04/19/2005 - // - // -------------------------------------------------------------------------- - void SetMaximumDiffingTime(int iSeconds); - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::SetKeepAliveTime() - // Purpose: Sets the time interval for repetitive keep-alive operation - // Created: 04/19/2005 - // - // -------------------------------------------------------------------------- - virtual void SetKeepAliveTime(int iSeconds); - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::ManageDiffProcess() - // Purpose: Initiates an SSL connection/session keep-alive process - // Created: 04/19/2005 - // - // -------------------------------------------------------------------------- - void ManageDiffProcess(); - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::UnManageDiffProcess() - // Purpose: Suspends an SSL connection/session keep-alive process - // Created: 04/19/2005 - // - // -------------------------------------------------------------------------- - void UnManageDiffProcess(); - - // ------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::DoKeepAlive() - // Purpose: Check whether it's time to send a KeepAlive - // message over the SSL link, and if so, send it. - // Created: 04/19/2005 - // - // ------------------------------------------------------------------- - virtual void DoKeepAlive(); - virtual int GetMaximumDiffingTime(); - virtual bool IsManaged() { return mbIsManaged; } - - ProgressNotifier& GetProgressNotifier() const - { - return mrProgressNotifier; - } - - void SetNiceMode(bool enabled) - { - if(mTcpNiceMode) - { - mpNice->SetEnabled(enabled); - } - } - - bool mExperimentalSnapshotMode; - -private: - LocationResolver &mrResolver; - TLSContext &mrTLSContext; - std::string mHostname; - int mPort; - uint32_t mAccountNumber; - std::auto_ptr<BackupProtocolCallable> mapConnection; - bool mExtendedLogging; - bool mExtendedLogToFile; - std::string mExtendedLogFile; - FILE* mpExtendedLogFileHandle; - int64_t mClientStoreMarker; - BackupClientDeleteList *mpDeleteList; - const BackupClientInodeToIDMap *mpCurrentIDMap; - BackupClientInodeToIDMap *mpNewIDMap; - bool mStorageLimitExceeded; - ExcludeList *mpExcludeFiles; - ExcludeList *mpExcludeDirs; - Timer mKeepAliveTimer; - bool mbIsManaged; - int mKeepAliveTime; - int mMaximumDiffingTime; - ProgressNotifier &mrProgressNotifier; - bool mTcpNiceMode; - NiceSocketStream *mpNice; -}; - -#endif // BACKUPCLIENTCONTEXT__H diff --git a/bin/bbackupd/BackupClientDeleteList.cpp b/bin/bbackupd/BackupClientDeleteList.cpp deleted file mode 100644 index ce5e6264..00000000 --- a/bin/bbackupd/BackupClientDeleteList.cpp +++ /dev/null @@ -1,229 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientDeleteList.cpp -// Purpose: List of pending deletes for backup -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <algorithm> - -#include "BackupClientDeleteList.h" -#include "BackupClientContext.h" -#include "autogen_BackupProtocol.h" - -#include "MemLeakFindOn.h" - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::BackupClientDeleteList() -// Purpose: Constructor -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -BackupClientDeleteList::BackupClientDeleteList() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::~BackupClientDeleteList() -// Purpose: Destructor -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -BackupClientDeleteList::~BackupClientDeleteList() -{ -} - -BackupClientDeleteList::FileToDelete::FileToDelete(int64_t DirectoryID, - const BackupStoreFilename& rFilename, - const std::string& rLocalPath) -: mDirectoryID(DirectoryID), - mFilename(rFilename), - mLocalPath(rLocalPath) -{ } - -BackupClientDeleteList::DirToDelete::DirToDelete(int64_t ObjectID, - const std::string& rLocalPath) -: mObjectID(ObjectID), - mLocalPath(rLocalPath) -{ } - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::AddDirectoryDelete(int64_t, -// const BackupStoreFilename&) -// Purpose: Add a directory to the list of directories to be deleted. -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID, - const std::string& rLocalPath) -{ - // Only add the delete to the list if it's not in the "no delete" set - if(mDirectoryNoDeleteList.find(ObjectID) == - mDirectoryNoDeleteList.end()) - { - // Not in the list, so should delete it - mDirectoryList.push_back(DirToDelete(ObjectID, rLocalPath)); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::AddFileDelete(int64_t, -// const BackupStoreFilename &) -// Purpose: -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, - const BackupStoreFilename &rFilename, const std::string& rLocalPath) -{ - // Try to find it in the no delete list - std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator - delEntry(mFileNoDeleteList.begin()); - while(delEntry != mFileNoDeleteList.end()) - { - if((delEntry)->first == DirectoryID - && (delEntry)->second == rFilename) - { - // Found! - break; - } - ++delEntry; - } - - // Only add it to the delete list if it wasn't in the no delete list - if(delEntry == mFileNoDeleteList.end()) - { - mFileList.push_back(FileToDelete(DirectoryID, rFilename, - rLocalPath)); - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) -// Purpose: Perform all the pending deletes -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) -{ - // Anything to do? - if(mDirectoryList.empty() && mFileList.empty()) - { - // Nothing! - return; - } - - // Get a connection - BackupProtocolCallable &connection(rContext.GetConnection()); - - // Do the deletes - for(std::vector<DirToDelete>::iterator i(mDirectoryList.begin()); - i != mDirectoryList.end(); ++i) - { - connection.QueryDeleteDirectory(i->mObjectID); - rContext.GetProgressNotifier().NotifyDirectoryDeleted( - i->mObjectID, i->mLocalPath); - } - - // Clear the directory list - mDirectoryList.clear(); - - // Delete the files - for(std::vector<FileToDelete>::iterator i(mFileList.begin()); - i != mFileList.end(); ++i) - { - connection.QueryDeleteFile(i->mDirectoryID, i->mFilename); - rContext.GetProgressNotifier().NotifyFileDeleted( - i->mDirectoryID, i->mLocalPath); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::StopDirectoryDeletion(int64_t) -// Purpose: Stop a directory being deleted -// Created: 19/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientDeleteList::StopDirectoryDeletion(int64_t ObjectID) -{ - // First of all, is it in the delete vector? - std::vector<DirToDelete>::iterator delEntry(mDirectoryList.begin()); - for(; delEntry != mDirectoryList.end(); delEntry++) - { - if(delEntry->mObjectID == ObjectID) - { - // Found! - break; - } - } - if(delEntry != mDirectoryList.end()) - { - // erase this entry - mDirectoryList.erase(delEntry); - } - else - { - // Haven't been asked to delete it yet, put it in the - // no delete list - mDirectoryNoDeleteList.insert(ObjectID); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::StopFileDeletion(int64_t, const BackupStoreFilename &) -// Purpose: Stop a file from being deleted -// Created: 19/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, - const BackupStoreFilename &rFilename) -{ - // Find this in the delete list - std::vector<FileToDelete>::iterator delEntry(mFileList.begin()); - while(delEntry != mFileList.end()) - { - if(delEntry->mDirectoryID == DirectoryID - && delEntry->mFilename == rFilename) - { - // Found! - break; - } - ++delEntry; - } - - if(delEntry != mFileList.end()) - { - // erase this entry - mFileList.erase(delEntry); - } - else - { - // Haven't been asked to delete it yet, put it in the no delete list - mFileNoDeleteList.push_back(std::pair<int64_t, BackupStoreFilename>(DirectoryID, rFilename)); - } -} - diff --git a/bin/bbackupd/BackupClientDeleteList.h b/bin/bbackupd/BackupClientDeleteList.h deleted file mode 100644 index b0fbf51a..00000000 --- a/bin/bbackupd/BackupClientDeleteList.h +++ /dev/null @@ -1,75 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientDeleteList.h -// Purpose: List of pending deletes for backup -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCLIENTDELETELIST__H -#define BACKUPCLIENTDELETELIST__H - -#include "BackupStoreFilename.h" - -class BackupClientContext; - -#include <vector> -#include <utility> -#include <set> - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupClientDeleteList -// Purpose: List of pending deletes for backup -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -class BackupClientDeleteList -{ -private: - class FileToDelete - { - public: - int64_t mDirectoryID; - BackupStoreFilename mFilename; - std::string mLocalPath; - FileToDelete(int64_t DirectoryID, - const BackupStoreFilename& rFilename, - const std::string& rLocalPath); - }; - - class DirToDelete - { - public: - int64_t mObjectID; - std::string mLocalPath; - DirToDelete(int64_t ObjectID, const std::string& rLocalPath); - }; - -public: - BackupClientDeleteList(); - ~BackupClientDeleteList(); - - void AddDirectoryDelete(int64_t ObjectID, - const std::string& rLocalPath); - void AddFileDelete(int64_t DirectoryID, - const BackupStoreFilename &rFilename, - const std::string& rLocalPath); - - void StopDirectoryDeletion(int64_t ObjectID); - void StopFileDeletion(int64_t DirectoryID, - const BackupStoreFilename &rFilename); - - void PerformDeletions(BackupClientContext &rContext); - -private: - std::vector<DirToDelete> mDirectoryList; - std::set<int64_t> mDirectoryNoDeleteList; // note: things only get in this list if they're not present in mDirectoryList when they are 'added' - std::vector<FileToDelete> mFileList; - std::vector<std::pair<int64_t, BackupStoreFilename> > mFileNoDeleteList; -}; - -#endif // BACKUPCLIENTDELETELIST__H - diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp deleted file mode 100644 index 94cb7965..00000000 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ /dev/null @@ -1,2302 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientDirectoryRecord.cpp -// Purpose: Implementation of record about directory for -// backup client -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#ifdef HAVE_DIRENT_H - #include <dirent.h> -#endif - -#include <errno.h> -#include <string.h> - -#include "autogen_BackupProtocol.h" -#include "autogen_CipherException.h" -#include "autogen_ClientException.h" -#include "Archive.h" -#include "BackupClientContext.h" -#include "BackupClientDirectoryRecord.h" -#include "BackupClientInodeToIDMap.h" -#include "BackupDaemon.h" -#include "BackupStoreException.h" -#include "BackupStoreFile.h" -#include "BackupStoreFileEncodeStream.h" -#include "BufferedStream.h" -#include "CommonException.h" -#include "CollectInBufferStream.h" -#include "FileModificationTime.h" -#include "IOStream.h" -#include "Logging.h" -#include "MemBlockStream.h" -#include "PathUtils.h" -#include "RateLimitingStream.h" -#include "ReadLoggingStream.h" - -#include "MemLeakFindOn.h" - -typedef std::map<std::string, BackupStoreDirectory::Entry *> DecryptedEntriesMap_t; - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::BackupClientDirectoryRecord() -// Purpose: Constructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupClientDirectoryRecord::BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName) - : mObjectID(ObjectID), - mSubDirName(rSubDirName), - mInitialSyncDone(false), - mSyncDone(false), - mpPendingEntries(0) -{ - ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::~BackupClientDirectoryRecord() -// Purpose: Destructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupClientDirectoryRecord::~BackupClientDirectoryRecord() -{ - // Make deletion recursive - DeleteSubDirectories(); - - // Delete maps - if(mpPendingEntries != 0) - { - delete mpPendingEntries; - mpPendingEntries = 0; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::DeleteSubDirectories(); -// Purpose: Delete all sub directory entries -// Created: 2003/10/09 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::DeleteSubDirectories() -{ - // Delete all pointers - for(std::map<std::string, BackupClientDirectoryRecord *>::iterator i = mSubDirectories.begin(); - i != mSubDirectories.end(); ++i) - { - delete i->second; - } - - // Empty list - mSubDirectories.clear(); -} - -std::string BackupClientDirectoryRecord::ConvertVssPathToRealPath( - const std::string &rVssPath, - const Location& rBackupLocation) -{ -#ifdef ENABLE_VSS - BOX_TRACE("VSS: ConvertVssPathToRealPath: mIsSnapshotCreated = " << - rBackupLocation.mIsSnapshotCreated); - BOX_TRACE("VSS: ConvertVssPathToRealPath: File/Directory Path = " << - rVssPath.substr(0, rBackupLocation.mSnapshotPath.length())); - BOX_TRACE("VSS: ConvertVssPathToRealPath: Snapshot Path = " << - rBackupLocation.mSnapshotPath); - if (rBackupLocation.mIsSnapshotCreated && - rVssPath.substr(0, rBackupLocation.mSnapshotPath.length()) == - rBackupLocation.mSnapshotPath) - { - std::string convertedPath = rBackupLocation.mPath + - rVssPath.substr(rBackupLocation.mSnapshotPath.length()); - BOX_TRACE("VSS: ConvertVssPathToRealPath: Converted Path = " << - convertedPath); - return convertedPath; - } -#endif - - return rVssPath; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::SyncDirectory(i -// BackupClientDirectoryRecord::SyncParams &, -// int64_t, const std::string &, -// const std::string &, bool) -// Purpose: Recursively synchronise a local directory -// with the server. -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::SyncDirectory( - BackupClientDirectoryRecord::SyncParams &rParams, - int64_t ContainingDirectoryID, - const std::string &rLocalPath, - const std::string &rRemotePath, - const Location& rBackupLocation, - bool ThisDirHasJustBeenCreated) -{ - BackupClientContext& rContext(rParams.mrContext); - ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); - - // Signal received by daemon? - if(rParams.mrRunStatusProvider.StopRun()) - { - // Yes. Stop now. - THROW_EXCEPTION(BackupStoreException, SignalReceived) - } - - // Start by making some flag changes, marking this sync as not done, - // and on the immediate sub directories. - mSyncDone = false; - for(std::map<std::string, BackupClientDirectoryRecord *>::iterator - i = mSubDirectories.begin(); - i != mSubDirectories.end(); ++i) - { - i->second->mSyncDone = false; - } - - // Work out the time in the future after which the file should - // be uploaded regardless. This is a simple way to avoid having - // too many problems with file servers when they have clients - // with badly out of sync clocks. - rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + - rParams.mMaxFileTimeInFuture; - - // Build the current state checksum to compare against while - // getting info from dirs. Note checksum is used locally only, - // so byte order isn't considered. - MD5Digest currentStateChecksum; - - EMU_STRUCT_STAT dest_st; - // Stat the directory, to get attribute info - // If it's a symbolic link, we want the link target here - // (as we're about to back up the contents of the directory) - { - if(EMU_STAT(rLocalPath.c_str(), &dest_st) != 0) - { - // The directory has probably been deleted, so - // just ignore this error. In a future scan, this - // deletion will be noticed, deleted from server, - // and this object deleted. - rNotifier.NotifyDirStatFailed(this, - ConvertVssPathToRealPath(rLocalPath, rBackupLocation), - strerror(errno)); - return; - } - - BOX_TRACE("Stat dir '" << rLocalPath << "' " - "found device/inode " << - dest_st.st_dev << "/" << dest_st.st_ino); - - // Store inode number in map so directories are tracked - // in case they're renamed - { - BackupClientInodeToIDMap &idMap( - rParams.mrContext.GetNewIDMap()); - idMap.AddToMap(dest_st.st_ino, mObjectID, ContainingDirectoryID, - ConvertVssPathToRealPath(rLocalPath, rBackupLocation)); - } - // Add attributes to checksum - currentStateChecksum.Add(&dest_st.st_mode, - sizeof(dest_st.st_mode)); - currentStateChecksum.Add(&dest_st.st_uid, - sizeof(dest_st.st_uid)); - currentStateChecksum.Add(&dest_st.st_gid, - sizeof(dest_st.st_gid)); - // Inode to be paranoid about things moving around - currentStateChecksum.Add(&dest_st.st_ino, - sizeof(dest_st.st_ino)); -#ifdef HAVE_STRUCT_STAT_ST_FLAGS - currentStateChecksum.Add(&dest_st.st_flags, - sizeof(dest_st.st_flags)); -#endif - - StreamableMemBlock xattr; - BackupClientFileAttributes::FillExtendedAttr(xattr, - rLocalPath.c_str()); - currentStateChecksum.Add(xattr.GetBuffer(), xattr.GetSize()); - } - - // Read directory entries, building arrays of names - // First, need to read the contents of the directory. - std::vector<std::string> dirs; - std::vector<std::string> files; - bool downloadDirectoryRecordBecauseOfFutureFiles = false; - - // BLOCK - { - // read the contents... - DIR *dirHandle = 0; - try - { - std::string nonVssDirPath = ConvertVssPathToRealPath(rLocalPath, - rBackupLocation); - rNotifier.NotifyScanDirectory(this, nonVssDirPath); - - dirHandle = ::opendir(rLocalPath.c_str()); - if(dirHandle == 0) - { - // Report the error (logs and eventual email to administrator) - if (errno == EACCES) - { - rNotifier.NotifyDirListFailed(this, - nonVssDirPath, - "Access denied"); - } - else - { - rNotifier.NotifyDirListFailed(this, - nonVssDirPath, - strerror(errno)); - } - - // Report the error (logs and eventual email - // to administrator) - SetErrorWhenReadingFilesystemObject(rParams, - nonVssDirPath); - // Ignore this directory for now. - return; - } - - struct dirent *en = 0; - int num_entries_found = 0; - - while((en = ::readdir(dirHandle)) != 0) - { - num_entries_found++; - rParams.mrContext.DoKeepAlive(); - if(rParams.mpBackgroundTask) - { - rParams.mpBackgroundTask->RunBackgroundTask( - BackgroundTask::Scanning_Dirs, - num_entries_found, 0); - } - - if (!SyncDirectoryEntry(rParams, rNotifier, - rBackupLocation, rLocalPath, - currentStateChecksum, en, dest_st, dirs, - files, downloadDirectoryRecordBecauseOfFutureFiles)) - { - // This entry is not to be backed up. - continue; - } - } - - if(::closedir(dirHandle) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError) - } - dirHandle = 0; - } - catch(...) - { - if(dirHandle != 0) - { - ::closedir(dirHandle); - } - throw; - } - } - - // Finish off the checksum, and compare with the one currently stored - bool checksumDifferent = true; - currentStateChecksum.Finish(); - if(mInitialSyncDone && currentStateChecksum.DigestMatches(mStateChecksum)) - { - // The checksum is the same, and there was one to compare with - checksumDifferent = false; - } - - // Pointer to potentially downloaded store directory info - std::auto_ptr<BackupStoreDirectory> apDirOnStore; - - try - { - // Want to get the directory listing? - if(ThisDirHasJustBeenCreated) - { - // Avoid sending another command to the server when we know it's empty - apDirOnStore.reset(new BackupStoreDirectory(mObjectID, - ContainingDirectoryID)); - } - // Consider asking the store for it - else if(!mInitialSyncDone || checksumDifferent || - downloadDirectoryRecordBecauseOfFutureFiles) - { - apDirOnStore = FetchDirectoryListing(rParams); - } - - // Make sure the attributes are up to date -- if there's space - // on the server and this directory has not just been created - // (because it's attributes will be correct in this case) and - // the checksum is different, implying they *MIGHT* be - // different. - if((!ThisDirHasJustBeenCreated) && checksumDifferent && - !rParams.mrContext.StorageLimitExceeded()) - { - UpdateAttributes(rParams, apDirOnStore.get(), rLocalPath); - } - - // Create the list of pointers to directory entries - std::vector<BackupStoreDirectory::Entry *> entriesLeftOver; - if(apDirOnStore.get()) - { - entriesLeftOver.resize(apDirOnStore->GetNumberOfEntries(), 0); - BackupStoreDirectory::Iterator i(*apDirOnStore); - // Copy in pointers to all the entries - for(unsigned int l = 0; l < apDirOnStore->GetNumberOfEntries(); ++l) - { - entriesLeftOver[l] = i.Next(); - } - } - - // Do the directory reading - bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, - rRemotePath, rBackupLocation, apDirOnStore.get(), - entriesLeftOver, files, dirs); - - // LAST THING! (think exception safety) - // Store the new checksum -- don't fetch things unnecessarily - // in the future But... only if 1) the storage limit isn't - // exceeded -- make sure things are done again if the directory - // is modified later and 2) All the objects within the - // directory were stored successfully. - if(!rParams.mrContext.StorageLimitExceeded() && - updateCompleteSuccess) - { - currentStateChecksum.CopyDigestTo(mStateChecksum); - } - } - catch(...) - { - // Bad things have happened -- clean up - // Set things so that we get a full go at stuff later - ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); - - throw; - } - - // Flag things as having happened. - mInitialSyncDone = true; - mSyncDone = true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::SyncDirectorEntry( -// BackupClientDirectoryRecord::SyncParams &, -// int64_t, const std::string &, -// const std::string &, bool) -// Purpose: Recursively synchronise a local directory -// with the server. -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -bool BackupClientDirectoryRecord::SyncDirectoryEntry( - BackupClientDirectoryRecord::SyncParams &rParams, - ProgressNotifier& rNotifier, - const Location& rBackupLocation, - const std::string &rDirLocalPath, - MD5Digest& currentStateChecksum, - struct dirent *en, - EMU_STRUCT_STAT dir_st, - std::vector<std::string>& rDirs, - std::vector<std::string>& rFiles, - bool& rDownloadDirectoryRecordBecauseOfFutureFiles) -{ - std::string entry_name = en->d_name; - if(entry_name == "." || entry_name == "..") - { - // ignore parent directory entries - return false; - } - - // Stat file to get info - std::string filename = MakeFullPath(rDirLocalPath, entry_name); - std::string realFileName = ConvertVssPathToRealPath(filename, - rBackupLocation); - EMU_STRUCT_STAT file_st; - -#ifdef WIN32 - // Don't stat the file just yet, to ensure that users can exclude - // unreadable files to suppress warnings that they are not accessible. - // - // Our emulated readdir() abuses en->d_type, which would normally - // contain DT_REG, DT_DIR, etc, but we only use it here and prefer to - // have the full file attributes. - - int type; - if (en->d_type & FILE_ATTRIBUTE_DIRECTORY) - { - type = S_IFDIR; - } - else - { - type = S_IFREG; - } -#else // !WIN32 - if(EMU_LSTAT(filename.c_str(), &file_st) != 0) - { - // We don't know whether it's a file or a directory, so check - // both. This only affects whether a warning message is - // displayed; the file is not backed up in either case. - if(!(rParams.mrContext.ExcludeFile(filename)) && - !(rParams.mrContext.ExcludeDir(filename))) - { - // Report the error (logs and eventual email to - // administrator) - rNotifier.NotifyFileStatFailed(this, filename, - strerror(errno)); - - // FIXME move to NotifyFileStatFailed() - SetErrorWhenReadingFilesystemObject(rParams, filename); - } - - // Ignore this entry for now. - return false; - } - - BOX_TRACE("Stat entry '" << filename << "' found device/inode " << - file_st.st_dev << "/" << file_st.st_ino); - - // Workaround for apparent btrfs bug, where symlinks appear to be on - // a different filesystem than their containing directory, thanks to - // Toke Hoiland-Jorgensen. - - int type = file_st.st_mode & S_IFMT; - if(type == S_IFDIR && file_st.st_dev != dir_st.st_dev) - { - if(!(rParams.mrContext.ExcludeDir(filename))) - { - rNotifier.NotifyMountPointSkipped(this, filename); - } - return false; - } -#endif - - if(type == S_IFREG || type == S_IFLNK) - { - // File or symbolic link - - // Exclude it? - if(rParams.mrContext.ExcludeFile(realFileName)) - { - rNotifier.NotifyFileExcluded(this, realFileName); - // Next item! - return false; - } - } - else if(type == S_IFDIR) - { - // Directory - - // Exclude it? - if(rParams.mrContext.ExcludeDir(realFileName)) - { - rNotifier.NotifyDirExcluded(this, realFileName); - - // Next item! - return false; - } - - #ifdef WIN32 - // exclude reparse points, as Application Data points to the - // parent directory under Vista and later, and causes an - // infinite loop: - // http://social.msdn.microsoft.com/forums/en-US/windowscompatibility/thread/05d14368-25dd-41c8-bdba-5590bf762a68/ - if (en->d_type & FILE_ATTRIBUTE_REPARSE_POINT) - { - rNotifier.NotifyMountPointSkipped(this, realFileName); - return false; - } - #endif - } - else // not a file or directory, what is it? - { - if (type == S_IFSOCK -#ifndef WIN32 - || type == S_IFIFO -#endif - ) - { - // removed notification for these types - // see Debian bug 479145, no objections - } - else if(rParams.mrContext.ExcludeFile(realFileName)) - { - rNotifier.NotifyFileExcluded(this, realFileName); - } - else - { - rNotifier.NotifyUnsupportedFileType(this, realFileName); - SetErrorWhenReadingFilesystemObject(rParams, - realFileName); - } - - return false; - } - - // The object should be backed up (file, symlink or dir, not excluded). - // So make the information for adding to the checksum. - - #ifdef WIN32 - // We didn't stat the file before, but now we need the information. - if(emu_stat(filename.c_str(), &file_st) != 0) - { - rNotifier.NotifyFileStatFailed(this, - ConvertVssPathToRealPath(filename, rBackupLocation), - strerror(errno)); - - // Report the error (logs and eventual email to administrator) - SetErrorWhenReadingFilesystemObject(rParams, filename); - - // Ignore this entry for now. - return false; - } - - if(file_st.st_dev != dir_st.st_dev) - { - rNotifier.NotifyMountPointSkipped(this, - ConvertVssPathToRealPath(filename, rBackupLocation)); - return false; - } - #endif - - // Basic structure for checksum info - struct { - box_time_t mModificationTime; - box_time_t mAttributeModificationTime; - int64_t mSize; - // And then the name follows - } checksum_info; - - // Be paranoid about structure packing - ::memset(&checksum_info, 0, sizeof(checksum_info)); - - checksum_info.mModificationTime = FileModificationTime(file_st); - checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st); - checksum_info.mSize = file_st.st_size; - currentStateChecksum.Add(&checksum_info, sizeof(checksum_info)); - currentStateChecksum.Add(en->d_name, strlen(en->d_name)); - - // If the file has been modified madly into the future, download the - // directory record anyway to ensure that it doesn't get uploaded - // every single time the disc is scanned. - if(checksum_info.mModificationTime > rParams.mUploadAfterThisTimeInTheFuture) - { - rDownloadDirectoryRecordBecauseOfFutureFiles = true; - // Log that this has happened - if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) - { - rNotifier.NotifyFileModifiedInFuture(this, - ConvertVssPathToRealPath(filename, rBackupLocation)); - rParams.mHaveLoggedWarningAboutFutureFileTimes = true; - } - } - - // We've decided to back it up, so add to file or directory list. - if(type == S_IFREG || type == S_IFLNK) - { - rFiles.push_back(entry_name); - } - else if(type == S_IFDIR) - { - rDirs.push_back(entry_name); - } - - return true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::FetchDirectoryListing( -// BackupClientDirectoryRecord::SyncParams &) -// Purpose: Fetch the directory listing of this directory from -// the store. -// Created: 2003/10/09 -// -// -------------------------------------------------------------------------- -std::auto_ptr<BackupStoreDirectory> -BackupClientDirectoryRecord::FetchDirectoryListing( - BackupClientDirectoryRecord::SyncParams &rParams) -{ - std::auto_ptr<BackupStoreDirectory> apDir; - - // Get connection to store - BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); - - // Query the directory - std::auto_ptr<BackupProtocolSuccess> dirreply(connection.QueryListDirectory( - mObjectID, - // both files and directories - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - // exclude old/deleted stuff - BackupProtocolListDirectory::Flags_Deleted | - BackupProtocolListDirectory::Flags_OldVersion, - true /* want attributes */)); - - // Retrieve the directory from the stream following - apDir.reset(new BackupStoreDirectory(connection.ReceiveStream(), - connection.GetTimeout())); - return apDir; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::UpdateAttributes( -// BackupClientDirectoryRecord::SyncParams &, -// const std::string &) -// Purpose: Sets the attributes of the directory on the store, -// if necessary. -// Created: 2003/10/09 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::UpdateAttributes( - BackupClientDirectoryRecord::SyncParams &rParams, - BackupStoreDirectory *pDirOnStore, - const std::string &rLocalPath) -{ - // Get attributes for the directory - BackupClientFileAttributes attr; - box_time_t attrModTime = 0; - attr.ReadAttributes(rLocalPath.c_str(), true /* directories have zero mod times */, - 0 /* no modification time */, &attrModTime); - - // Assume attributes need updating, unless proved otherwise - bool updateAttr = true; - - // Got a listing to compare with? - ASSERT(pDirOnStore == 0 || (pDirOnStore != 0 && pDirOnStore->HasAttributes())); - if(pDirOnStore != 0 && pDirOnStore->HasAttributes()) - { - const StreamableMemBlock &storeAttrEnc(pDirOnStore->GetAttributes()); - // Explict decryption - BackupClientFileAttributes storeAttr(storeAttrEnc); - - // Compare the attributes - if(attr.Compare(storeAttr, true, - true /* ignore both modification times */)) - { - // No update necessary - updateAttr = false; - } - } - - // Update them? - if(updateAttr) - { - // Get connection to store - BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); - - // Exception thrown if this doesn't work - std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); - connection.QueryChangeDirAttributes(mObjectID, attrModTime, attrStream); - } -} - -std::string BackupClientDirectoryRecord::DecryptFilename( - BackupStoreDirectory::Entry *en, - const std::string& rRemoteDirectoryPath) -{ - BackupStoreFilenameClear fn(en->GetName()); - return DecryptFilename(fn, en->GetObjectID(), rRemoteDirectoryPath); -} - -std::string BackupClientDirectoryRecord::DecryptFilename( - BackupStoreFilenameClear fn, int64_t filenameObjectID, - const std::string& rRemoteDirectoryPath) -{ - std::string filenameClear; - try - { - filenameClear = fn.GetClearFilename(); - } - catch(BoxException &e) - { - BOX_ERROR("Failed to decrypt filename for object " << - BOX_FORMAT_OBJECTID(filenameObjectID) << " in " - "directory " << BOX_FORMAT_OBJECTID(mObjectID) << - " (" << rRemoteDirectoryPath << ")"); - throw; - } - return filenameClear; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &, const std::string &, BackupStoreDirectory *, std::vector<BackupStoreDirectory::Entry *> &) -// Purpose: Update the items stored on the server. The rFiles vector will be erased after it's used to save space. -// Returns true if all items were updated successfully. (If not, the failures will have been logged). -// Created: 2003/10/09 -// -// -------------------------------------------------------------------------- -bool BackupClientDirectoryRecord::UpdateItems( - BackupClientDirectoryRecord::SyncParams &rParams, - const std::string &rLocalPath, - const std::string &rRemotePath, - const Location& rBackupLocation, - BackupStoreDirectory *pDirOnStore, - std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, - std::vector<std::string> &rFiles, - const std::vector<std::string> &rDirs) -{ - BackupClientContext& rContext(rParams.mrContext); - ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); - - bool allUpdatedSuccessfully = true; - - // Decrypt all the directory entries. - // It would be nice to be able to just compare the encrypted versions, however this doesn't work - // in practise because there can be multiple encodings of the same filename using different - // methods (although each method will result in the same string for the same filename.) This - // happens when the server fixes a broken store, and gives plain text generated filenames. - // So if we didn't do things like this, then you wouldn't be able to recover from bad things - // happening with the server. - DecryptedEntriesMap_t decryptedEntries; - if(pDirOnStore != 0) - { - BackupStoreDirectory::Iterator i(*pDirOnStore); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next()) != 0) - { - std::string filenameClear; - try - { - filenameClear = DecryptFilename(en, - rRemotePath); - decryptedEntries[filenameClear] = en; - } - catch (CipherException &e) - { - BOX_ERROR("Failed to decrypt a filename, " - "pretending that the file doesn't " - "exist"); - } - } - } - - // Do files - for(std::vector<std::string>::const_iterator f = rFiles.begin(); - f != rFiles.end(); ++f) - { - // Send keep-alive message if needed - rContext.DoKeepAlive(); - - // Filename of this file - std::string filename(MakeFullPath(rLocalPath, *f)); - std::string nonVssFilePath = ConvertVssPathToRealPath(filename, - rBackupLocation); - - // Get relevant info about file - box_time_t modTime = 0; - uint64_t attributesHash = 0; - int64_t fileSize = 0; - InodeRefType inodeNum = 0; - // BLOCK - { - // Stat the file - EMU_STRUCT_STAT st; - if(EMU_LSTAT(filename.c_str(), &st) != 0) - { - rNotifier.NotifyFileStatFailed(this, nonVssFilePath, - strerror(errno)); - - // Report the error (logs and - // eventual email to administrator) - SetErrorWhenReadingFilesystemObject(rParams, nonVssFilePath); - - // Ignore this entry for now. - continue; - } - - // Extract required data - modTime = FileModificationTime(st); - fileSize = st.st_size; - inodeNum = st.st_ino; - attributesHash = BackupClientFileAttributes::GenerateAttributeHash(st, filename, *f); - } - - // See if it's in the listing (if we have one) - BackupStoreFilenameClear storeFilename(*f); - BackupStoreDirectory::Entry *en = 0; - int64_t latestObjectID = 0; - if(pDirOnStore != 0) - { - DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*f)); - if(i != decryptedEntries.end()) - { - en = i->second; - latestObjectID = en->GetObjectID(); - } - } - - // Check that the entry which might have been found is in fact a file - if((en != 0) && !(en->IsFile())) - { - // Directory exists in the place of this file -- sort it out - RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, - en, *f); - en = 0; - } - - // Check for renaming? - if(pDirOnStore != 0 && en == 0) - { - // We now know... - // 1) File has just been added - // 2) It's not in the store - - // Do we know about the inode number? - const BackupClientInodeToIDMap &idMap(rContext.GetCurrentIDMap()); - int64_t renameObjectID = 0, renameInDirectory = 0; - if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) - { - // Look up on the server to get the name, to build the local filename - std::string localPotentialOldName; - bool isDir = false; - bool isCurrentVersion = false; - box_time_t srvModTime = 0, srvAttributesHash = 0; - BackupStoreFilenameClear oldLeafname; - if(rContext.FindFilename(renameObjectID, renameInDirectory, - localPotentialOldName, isDir, isCurrentVersion, - &srvModTime, &srvAttributesHash, &oldLeafname)) - { - // Only interested if it's a file and the latest version - if(!isDir && isCurrentVersion) - { - // Check that the object we found in the ID map doesn't exist on disc - EMU_STRUCT_STAT st; - if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) - { - // Doesn't exist locally, but does exist on the server. - // Therefore we can safely rename it to this new file. - - // Get the connection to the server - BackupProtocolCallable &connection(rContext.GetConnection()); - - // Only do this step if there is room on the server. - // This step will be repeated later when there is space available - if(!rContext.StorageLimitExceeded()) - { - // Rename the existing files (ie include old versions) on the server - connection.QueryMoveObject(renameObjectID, - renameInDirectory, - mObjectID /* move to this directory */, - BackupProtocolMoveObject::Flags_MoveAllWithSameName | - BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, - storeFilename); - - // Stop the attempt to delete the file in the original location - BackupClientDeleteList &rdelList(rContext.GetDeleteList()); - rdelList.StopFileDeletion(renameInDirectory, oldLeafname); - - // Create new entry in the directory for it - // -- will be near enough what's actually on the server for the rest to work. - en = pDirOnStore->AddEntry(storeFilename, - srvModTime, renameObjectID, - 0 /* size in blocks unknown, but not needed */, - BackupStoreDirectory::Entry::Flags_File, - srvAttributesHash); - - // Store the object ID for the inode lookup map later - latestObjectID = renameObjectID; - } - } - } - } - } - } - - // Is it in the mPendingEntries list? - box_time_t pendingFirstSeenTime = 0; // ie not seen - if(mpPendingEntries != 0) - { - std::map<std::string, box_time_t>::const_iterator i(mpPendingEntries->find(*f)); - if(i != mpPendingEntries->end()) - { - // found it -- set flag - pendingFirstSeenTime = i->second; - } - } - - // If pDirOnStore == 0, then this must have been after an initial sync: - ASSERT(pDirOnStore != 0 || mInitialSyncDone); - // So, if pDirOnStore == 0, then we know that everything before syncPeriodStart - // is either on the server, or in the toupload list. If the directory had changed, - // we'd have got a directory listing. - // - // At this point, if (pDirOnStore == 0 && en == 0), we can assume it's on the server with a - // mod time < syncPeriodStart, or didn't exist before that time. - // - // But if en != 0, then we need to compare modification times to avoid uploading it again. - - // Need to update? - // - // Condition for upload: - // modification time within sync period - // if it's been seen before but not uploaded, is the time from this first sight longer than the MaxUploadWait - // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store - - bool doUpload = false; - std::string decisionReason = "unknown"; - - // Only upload a file if the mod time locally is - // different to that on the server. - - if (en == 0 || en->GetModificationTime() != modTime) - { - // Check the file modified within the acceptable time period we're checking - // If the file isn't on the server, the acceptable time starts at zero. - // Check pDirOnStore and en, because if we didn't download a directory listing, - // pDirOnStore will be zero, but we know it's on the server. - if (modTime < rParams.mSyncPeriodEnd) - { - if (pDirOnStore != 0 && en == 0) - { - doUpload = true; - decisionReason = "not on server"; - } - else if (modTime >= rParams.mSyncPeriodStart) - { - doUpload = true; - decisionReason = "modified since last sync"; - } - } - - // However, just in case things are continually - // modified, we check the first seen time. - // The two compares of syncPeriodEnd and - // pendingFirstSeenTime are because the values - // are unsigned. - - if (!doUpload && - pendingFirstSeenTime != 0 && - rParams.mSyncPeriodEnd > pendingFirstSeenTime && - (rParams.mSyncPeriodEnd - pendingFirstSeenTime) - > rParams.mMaxUploadWait) - { - doUpload = true; - decisionReason = "continually modified"; - } - - // Then make sure that if files are added with a - // time less than the sync period start - // (which can easily happen on file server), it - // gets uploaded. The directory contents checksum - // will pick up the fact it has been added, so the - // store listing will be available when this happens. - - if (!doUpload && - modTime <= rParams.mSyncPeriodStart && - en != 0 && - en->GetModificationTime() != modTime) - { - doUpload = true; - decisionReason = "mod time changed"; - } - - // And just to catch really badly off clocks in - // the future for file server clients, - // just upload the file if it's madly in the future. - - if (!doUpload && modTime > - rParams.mUploadAfterThisTimeInTheFuture) - { - doUpload = true; - decisionReason = "mod time in the future"; - } - } - - if (en != 0 && en->GetModificationTime() == modTime) - { - doUpload = false; - decisionReason = "not modified since last upload"; - } - else if (!doUpload) - { - if (modTime > rParams.mSyncPeriodEnd) - { - box_time_t now = GetCurrentBoxTime(); - int age = BoxTimeToSeconds(now - - modTime); - std::ostringstream s; - s << "modified too recently: only " << - age << " seconds ago"; - decisionReason = s.str(); - } - else - { - std::ostringstream s; - s << "mod time is " << modTime << - " which is outside sync window, " - << rParams.mSyncPeriodStart << " to " - << rParams.mSyncPeriodEnd; - decisionReason = s.str(); - } - } - - BOX_TRACE("Upload decision: " << nonVssFilePath << ": " << - (doUpload ? "will upload" : "will not upload") << - " (" << decisionReason << ")"); - - bool fileSynced = true; - - if (doUpload) - { - // Upload needed, don't mark sync success until - // we've actually done it - fileSynced = false; - - // Make sure we're connected -- must connect here so we know whether - // the storage limit has been exceeded, and hence whether or not - // to actually upload the file. - rContext.GetConnection(); - - // Only do this step if there is room on the server. - // This step will be repeated later when there is space available - if(!rContext.StorageLimitExceeded()) - { - // Upload the file to the server, recording the - // object ID it returns - bool noPreviousVersionOnServer = - ((pDirOnStore != 0) && (en == 0)); - - // Surround this in a try/catch block, to - // catch errors, but still continue - bool uploadSuccess = false; - try - { - latestObjectID = UploadFile(rParams, - filename, - nonVssFilePath, - rRemotePath + "/" + *f, - storeFilename, - fileSize, modTime, - attributesHash, - noPreviousVersionOnServer); - - if (latestObjectID == 0) - { - // storage limit exceeded - rParams.mrContext.SetStorageLimitExceeded(); - uploadSuccess = false; - allUpdatedSuccessfully = false; - } - else - { - uploadSuccess = true; - } - } - catch(ConnectionException &e) - { - // Connection errors should just be - // passed on to the main handler, - // retries would probably just cause - // more problems. - rNotifier.NotifyFileUploadException( - this, nonVssFilePath, e); - throw; - } - catch(BoxException &e) - { - if (e.GetType() == BackupStoreException::ExceptionType && - e.GetSubType() == BackupStoreException::SignalReceived) - { - // abort requested, pass the - // exception on up. - throw; - } - - // an error occured -- make return - // code false, to show error in directory - allUpdatedSuccessfully = false; - // Log it. - SetErrorWhenReadingFilesystemObject(rParams, - nonVssFilePath); - rNotifier.NotifyFileUploadException(this, - nonVssFilePath, e); - } - - // Update structures if the file was uploaded - // successfully. - if(uploadSuccess) - { - fileSynced = true; - - // delete from pending entries - if(pendingFirstSeenTime != 0 && mpPendingEntries != 0) - { - mpPendingEntries->erase(*f); - } - } - } - else - { - rNotifier.NotifyFileSkippedServerFull(this, nonVssFilePath); - } - } - else if(en != 0 && en->GetAttributesHash() != attributesHash) - { - // Attributes have probably changed, upload them again. - // If the attributes have changed enough, the directory - // hash will have changed too, and so the dir will have - // been downloaded, and the entry will be available. - - // Get connection - BackupProtocolCallable &connection(rContext.GetConnection()); - - // Only do this step if there is room on the server. - // This step will be repeated later when there is - // space available - if(!rContext.StorageLimitExceeded()) - { - try - { - rNotifier.NotifyFileUploadingAttributes(this, - nonVssFilePath); - - // Update store - BackupClientFileAttributes attr; - attr.ReadAttributes(filename, - false /* put mod times in the attributes, please */); - std::auto_ptr<IOStream> attrStream( - new MemBlockStream(attr)); - connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream); - fileSynced = true; - } - catch (BoxException &e) - { - BOX_ERROR("Failed to read or store file attributes " - "for '" << nonVssFilePath << "', will try again " - "later"); - } - } - } - - if(modTime >= rParams.mSyncPeriodEnd) - { - // Allocate? - if(mpPendingEntries == 0) - { - mpPendingEntries = new std::map<std::string, box_time_t>; - } - // Adding to mPendingEntries list - if(pendingFirstSeenTime == 0) - { - // Haven't seen this before -- add to list! - (*mpPendingEntries)[*f] = modTime; - } - } - - // Zero pointer in rEntriesLeftOver, if we have a pointer to zero - if(en != 0) - { - for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) - { - if(rEntriesLeftOver[l] == en) - { - rEntriesLeftOver[l] = 0; - break; - } - } - } - - // Does this file need an entry in the ID map? - if(fileSize >= rParams.mFileTrackingSizeThreshold) - { - // Get the map - BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap()); - - // Need to get an ID from somewhere... - if(latestObjectID == 0) - { - // Don't know it -- haven't sent anything to the store, and didn't get a listing. - // Look it up in the current map, and if it's there, use that. - const BackupClientInodeToIDMap ¤tIDMap(rContext.GetCurrentIDMap()); - int64_t objid = 0, dirid = 0; - if(currentIDMap.Lookup(inodeNum, objid, dirid)) - { - // Found - if (dirid != mObjectID) - { - BOX_WARNING("Found conflicting parent ID for " - "file ID " << inodeNum << " (" << - nonVssFilePath << "): expected " << - mObjectID << " but found " << dirid << - " (same directory used in two different " - "locations?)"); - } - - ASSERT(dirid == mObjectID); - - // NOTE: If the above assert fails, an inode number has been reused by the OS, - // or there is a problem somewhere. If this happened on a short test run, look - // into it. However, in a long running process this may happen occasionally and - // not indicate anything wrong. - // Run the release version for real life use, where this check is not made. - - latestObjectID = objid; - } - } - - if(latestObjectID != 0) - { - BOX_TRACE("Storing uploaded file ID " << - inodeNum << " (" << nonVssFilePath << ") " - "in ID map as object " << - latestObjectID << " with parent " << - mObjectID); - idMap.AddToMap(inodeNum, latestObjectID, - mObjectID /* containing directory */, - nonVssFilePath); - } - - } - - if (fileSynced) - { - rNotifier.NotifyFileSynchronised(this, nonVssFilePath, - fileSize); - } - } - - // Erase contents of files to save space when recursing - rFiles.clear(); - - // Delete the pending entries, if the map is empty - if(mpPendingEntries != 0 && mpPendingEntries->size() == 0) - { - BOX_TRACE("Deleting mpPendingEntries from dir ID " << - BOX_FORMAT_OBJECTID(mObjectID)); - delete mpPendingEntries; - mpPendingEntries = 0; - } - - // Do directories - for(std::vector<std::string>::const_iterator d = rDirs.begin(); - d != rDirs.end(); ++d) - { - // Send keep-alive message if needed - rContext.DoKeepAlive(); - - // Get the local filename - std::string dirname(MakeFullPath(rLocalPath, *d)); - std::string nonVssDirPath = ConvertVssPathToRealPath(dirname, - rBackupLocation); - - // See if it's in the listing (if we have one) - BackupStoreFilenameClear storeFilename(*d); - BackupStoreDirectory::Entry *en = 0; - if(pDirOnStore != 0) - { - DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*d)); - if(i != decryptedEntries.end()) - { - en = i->second; - } - } - - // Check that the entry which might have been found is in fact a directory - if((en != 0) && !(en->IsDir())) - { - // Entry exists, but is not a directory. Bad. - // Get rid of it. - BackupProtocolCallable &connection(rContext.GetConnection()); - connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); - - std::string filenameClear = DecryptFilename(en, - rRemotePath); - rNotifier.NotifyFileDeleted(en->GetObjectID(), - filenameClear); - - // Nothing found - en = 0; - } - - // Zero pointer in rEntriesLeftOver, if we have a pointer to zero - if(en != 0) - { - for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) - { - if(rEntriesLeftOver[l] == en) - { - rEntriesLeftOver[l] = 0; - break; - } - } - } - - // Flag for having created directory, so can optimise the - // recursive call not to read it again, because we know - // it's empty. - bool haveJustCreatedDirOnServer = false; - - // Next, see if it's in the list of sub directories - BackupClientDirectoryRecord *psubDirRecord = 0; - std::map<std::string, BackupClientDirectoryRecord *>::iterator - e(mSubDirectories.find(*d)); - - if(e != mSubDirectories.end()) - { - // In the list, just use this pointer - psubDirRecord = e->second; - } - else - { - // Note: if we have exceeded our storage limit, then - // we should not upload any more data, nor create any - // DirectoryRecord representing data that would have - // been uploaded. This step will be repeated when - // there is some space available. - bool doCreateDirectoryRecord = true; - - // Need to create the record. But do we need to create the directory on the server? - int64_t subDirObjectID = 0; - if(en != 0) - { - // No. Exists on the server, and we know about it from the listing. - subDirObjectID = en->GetObjectID(); - } - else if(rContext.StorageLimitExceeded()) - // know we've got a connection if we get this far, - // as dir will have been modified. - { - doCreateDirectoryRecord = false; - } - else - { - // Yes, creation required! - // It is known that it doesn't exist: - // - // if en == 0 and pDirOnStore == 0, then the - // directory has had an initial sync, and - // hasn't been modified (Really? then why - // are we here? TODO FIXME) - // so it has definitely been created already - // (so why create it again?) - // - // if en == 0 but pDirOnStore != 0, well... obviously it doesn't exist. - // - subDirObjectID = CreateRemoteDir(dirname, - nonVssDirPath, rRemotePath + "/" + *d, - storeFilename, &haveJustCreatedDirOnServer, - rParams); - doCreateDirectoryRecord = (subDirObjectID != 0); - } - - if (doCreateDirectoryRecord) - { - // New an object for this - psubDirRecord = new BackupClientDirectoryRecord(subDirObjectID, *d); - - // Store in list - try - { - mSubDirectories[*d] = psubDirRecord; - } - catch(...) - { - delete psubDirRecord; - psubDirRecord = 0; - throw; - } - } - } - - // ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded()); - // There's another possible reason now: the directory no longer - // existed when we finally got around to checking its - // attributes. See for example Brendon Baumgartner's reported - // error with Wordpress cache directories. - - if(psubDirRecord) - { - // Sync this sub directory too - psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, - rRemotePath + "/" + *d, rBackupLocation, - haveJustCreatedDirOnServer); - } - } - - // Delete everything which is on the store, but not on disc - for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) - { - if(rEntriesLeftOver[l] != 0) - { - BackupStoreDirectory::Entry *en = rEntriesLeftOver[l]; - - // These entries can't be deleted immediately, as it would prevent - // renaming and moving of objects working properly. So we add them - // to a list, which is actually deleted at the very end of the session. - // If there's an error during the process, it doesn't matter if things - // aren't actually deleted, as the whole state will be reset anyway. - BackupClientDeleteList &rdel(rContext.GetDeleteList()); - std::string filenameClear; - bool isCorruptFilename = false; - - try - { - filenameClear = DecryptFilename(en, - rRemotePath); - } - catch (CipherException &e) - { - BOX_ERROR("Failed to decrypt a filename, " - "scheduling that file for deletion"); - filenameClear = "<corrupt filename>"; - isCorruptFilename = true; - } - - std::string localName = MakeFullPath(rLocalPath, - filenameClear); - std::string nonVssLocalName = ConvertVssPathToRealPath(localName, - rBackupLocation); - - // Delete this entry -- file or directory? - if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) - { - // Set a pending deletion for the file - rdel.AddFileDelete(mObjectID, en->GetName(), - localName); - } - else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) - { - // Set as a pending deletion for the directory - rdel.AddDirectoryDelete(en->GetObjectID(), - localName); - - // If there's a directory record for it in - // the sub directory map, delete it now - BackupStoreFilenameClear dirname(en->GetName()); - std::map<std::string, BackupClientDirectoryRecord *>::iterator - e(mSubDirectories.find(filenameClear)); - if(e != mSubDirectories.end() && !isCorruptFilename) - { - // Carefully delete the entry from the map - BackupClientDirectoryRecord *rec = e->second; - mSubDirectories.erase(e); - delete rec; - - BOX_TRACE("Deleted directory record for " << - nonVssLocalName); - } - } - } - } - - // Return success flag (will be false if some files failed) - return allUpdatedSuccessfully; -} - -int64_t BackupClientDirectoryRecord::CreateRemoteDir(const std::string& localDirPath, - const std::string& nonVssDirPath, const std::string& remoteDirPath, - BackupStoreFilenameClear& storeFilename, bool* pHaveJustCreatedDirOnServer, - BackupClientDirectoryRecord::SyncParams &rParams) -{ - // Get attributes - box_time_t attrModTime = 0; - InodeRefType inodeNum = 0; - BackupClientFileAttributes attr; - *pHaveJustCreatedDirOnServer = false; - ProgressNotifier& rNotifier(rParams.mrContext.GetProgressNotifier()); - - try - { - attr.ReadAttributes(localDirPath, - true /* directories have zero mod times */, - 0 /* not interested in mod time */, - &attrModTime, 0 /* not file size */, - &inodeNum); - } - catch (BoxException &e) - { - // We used to try to recover from this, but we'd need an - // attributes block to upload to the server, so we have to - // skip creating the directory instead. - BOX_WARNING("Failed to read attributes of directory, " - "ignoring it for now: " << nonVssDirPath); - return 0; // no object ID - } - - // Check to see if the directory been renamed - // First, do we have a record in the ID map? - int64_t renameObjectID = 0, renameInDirectory = 0; - bool renameDir = false; - const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap()); - - if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) - { - // Look up on the server to get the name, to build the local filename - std::string localPotentialOldName; - bool isDir = false; - bool isCurrentVersion = false; - if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, - localPotentialOldName, isDir, isCurrentVersion)) - { - // Only interested if it's a directory - if(isDir && isCurrentVersion) - { - // Check that the object doesn't exist already - EMU_STRUCT_STAT st; - if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && - errno == ENOENT) - { - // Doesn't exist locally, but does exist - // on the server. Therefore we can - // safely rename it. - renameDir = true; - } - } - } - } - - // Get connection - BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); - - // Don't do a check for storage limit exceeded here, because if we get to this - // stage, a connection will have been opened, and the status known, so the check - // in the else if(...) above will be correct. - - // Build attribute stream for sending - std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); - - if(renameDir) - { - // Rename the existing directory on the server - connection.QueryMoveObject(renameObjectID, - renameInDirectory, - mObjectID /* move to this directory */, - BackupProtocolMoveObject::Flags_MoveAllWithSameName | - BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, - storeFilename); - - // Put the latest attributes on it - connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream); - - // Stop it being deleted later - BackupClientDeleteList &rdelList( - rParams.mrContext.GetDeleteList()); - rdelList.StopDirectoryDeletion(renameObjectID); - - // This is the ID for the renamed directory - return renameObjectID; - } - else - { - int64_t subDirObjectID = 0; // no object ID - - // Create a new directory - try - { - subDirObjectID = connection.QueryCreateDirectory( - mObjectID, attrModTime, storeFilename, - attrStream)->GetObjectID(); - // Flag as having done this for optimisation later - *pHaveJustCreatedDirOnServer = true; - } - catch(BoxException &e) - { - int type, subtype; - connection.GetLastError(type, subtype); - rNotifier.NotifyFileUploadServerError(this, nonVssDirPath, - type, subtype); - if(e.GetType() == ConnectionException::ExceptionType && - e.GetSubType() == ConnectionException::Protocol_UnexpectedReply && - type == BackupProtocolError::ErrorType && - subtype == BackupProtocolError::Err_StorageLimitExceeded) - { - // The hard limit was exceeded on the server, notify! - rParams.mrContext.SetStorageLimitExceeded(); - rParams.mrSysadminNotifier.NotifySysadmin( - SysadminNotifier::StoreFull); - } - else - { - throw; - } - } - - if(*pHaveJustCreatedDirOnServer) - { - rNotifier.NotifyDirectoryCreated(subDirObjectID, - nonVssDirPath, remoteDirPath); - } - - return subDirObjectID; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &, BackupStoreDirectory *, int64_t, const std::string &) -// Purpose: Called to resolve difficulties when a directory is found on the -// store where a file is to be uploaded. -// Created: 9/7/04 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile( - SyncParams &rParams, - BackupStoreDirectory* pDirOnStore, - BackupStoreDirectory::Entry* pEntry, - const std::string &rFilename) -{ - // First, delete the directory - BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); - connection.QueryDeleteDirectory(pEntry->GetObjectID()); - - BackupStoreFilenameClear clear(pEntry->GetName()); - rParams.mrContext.GetProgressNotifier().NotifyDirectoryDeleted( - pEntry->GetObjectID(), clear.GetClearFilename()); - - // Then, delete any directory record - std::map<std::string, BackupClientDirectoryRecord *>::iterator - e(mSubDirectories.find(rFilename)); - - if(e != mSubDirectories.end()) - { - // A record exists for this, remove it - BackupClientDirectoryRecord *psubDirRecord = e->second; - mSubDirectories.erase(e); - - // And delete the object - delete psubDirRecord; - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::UploadFile( -// BackupClientDirectoryRecord::SyncParams &, -// const std::string &, -// const BackupStoreFilename &, -// int64_t, box_time_t, box_time_t, bool) -// Purpose: Private. Upload a file to the server. May send -// a patch instead of the whole thing -// Created: 20/1/04 -// -// -------------------------------------------------------------------------- -int64_t BackupClientDirectoryRecord::UploadFile( - BackupClientDirectoryRecord::SyncParams &rParams, - const std::string &rLocalPath, - const std::string &rNonVssFilePath, - const std::string &rRemotePath, - const BackupStoreFilenameClear &rStoreFilename, - int64_t FileSize, - box_time_t ModificationTime, - box_time_t AttributesHash, - bool NoPreviousVersionOnServer) -{ - BackupClientContext& rContext(rParams.mrContext); - ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); - - // Get the connection - BackupProtocolCallable &connection(rContext.GetConnection()); - - // Info - int64_t objID = 0; - int64_t uploadedSize = -1; - - // Use a try block to catch store full errors - try - { - std::auto_ptr<BackupStoreFileEncodeStream> apStreamToUpload; - int64_t diffFromID = 0; - - // Might an old version be on the server, and is the file - // size over the diffing threshold? - if(!NoPreviousVersionOnServer && - FileSize >= rParams.mDiffingUploadSizeThreshold) - { - // YES -- try to do diff, if possible - // First, query the server to see if there's an old version available - std::auto_ptr<BackupProtocolSuccess> getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename)); - diffFromID = getBlockIndex->GetObjectID(); - - if(diffFromID != 0) - { - // Found an old version - - // Get the index - std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream()); - - // - // Diff the file - // - - rContext.ManageDiffProcess(); - - bool isCompletelyDifferent = false; - - apStreamToUpload = BackupStoreFile::EncodeFileDiff( - rLocalPath, - mObjectID, /* containing directory */ - rStoreFilename, diffFromID, *blockIndexStream, - connection.GetTimeout(), - &rContext, // DiffTimer implementation - 0 /* not interested in the modification time */, - &isCompletelyDifferent, - rParams.mpBackgroundTask); - - if(isCompletelyDifferent) - { - diffFromID = 0; - } - - rContext.UnManageDiffProcess(); - } - } - - if(apStreamToUpload.get()) - { - rNotifier.NotifyFileUploadingPatch(this, rNonVssFilePath, - apStreamToUpload->GetBytesToUpload()); - } - else // No patch upload, so do a normal upload - { - // below threshold or nothing to diff from, so upload whole - rNotifier.NotifyFileUploading(this, rNonVssFilePath); - - // Prepare to upload, getting a stream which will encode the file as we go along - apStreamToUpload = BackupStoreFile::EncodeFile( - rLocalPath, mObjectID, /* containing directory */ - rStoreFilename, NULL, &rParams, - &(rParams.mrRunStatusProvider), - rParams.mpBackgroundTask); - } - - rContext.SetNiceMode(true); - std::auto_ptr<IOStream> apWrappedStream; - - if(rParams.mMaxUploadRate > 0) - { - apWrappedStream.reset(new RateLimitingStream( - *apStreamToUpload, rParams.mMaxUploadRate)); - } - else - { - // Wrap the stream in *something*, so that - // QueryStoreFile() doesn't delete the original - // stream (upload object) and we can retrieve - // the byte counter. - apWrappedStream.reset(new BufferedStream( - *apStreamToUpload)); - } - - // Send to store - std::auto_ptr<BackupProtocolSuccess> stored( - connection.QueryStoreFile(mObjectID, ModificationTime, - AttributesHash, diffFromID, rStoreFilename, - apWrappedStream)); - - rContext.SetNiceMode(false); - - // Get object ID from the result - objID = stored->GetObjectID(); - uploadedSize = apStreamToUpload->GetTotalBytesSent(); - } - catch(BoxException &e) - { - rContext.UnManageDiffProcess(); - - if(e.GetType() == ConnectionException::ExceptionType && - e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) - { - // Check and see what error the protocol has, - // this is more useful to users than the exception. - int type, subtype; - if(connection.GetLastError(type, subtype)) - { - if(type == BackupProtocolError::ErrorType - && subtype == BackupProtocolError::Err_StorageLimitExceeded) - { - // The hard limit was exceeded on the server, notify! - rParams.mrSysadminNotifier.NotifySysadmin( - SysadminNotifier::StoreFull); - // return an error code instead of - // throwing an exception that we - // can't debug. - return 0; - } - rNotifier.NotifyFileUploadServerError(this, - rNonVssFilePath, type, subtype); - } - } - - // Send the error on it's way - throw; - } - - rNotifier.NotifyFileUploaded(this, rNonVssFilePath, FileSize, - uploadedSize, objID); - - // Return the new object ID of this file - return objID; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( -// SyncParams &, const char *) -// Purpose: Sets the error state when there were problems -// reading an object from the filesystem. -// Created: 29/3/04 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( - BackupClientDirectoryRecord::SyncParams &rParams, - const std::string& rFilename) -{ - // Zero hash, so it gets synced properly next time round. - ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); - - // More detailed logging was already done by the caller, but if we - // have a read error reported, we need to be able to search the logs - // to find out which file it was, so we need to log a consistent and - // clear error message. - BOX_WARNING("Failed to backup file, see above for details: " << - rFilename); - - // Mark that an error occured in the parameters object - rParams.mReadErrorsOnFilesystemObjects = true; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::SyncParams::SyncParams(BackupClientContext &) -// Purpose: Constructor -// Created: 8/3/04 -// -// -------------------------------------------------------------------------- -BackupClientDirectoryRecord::SyncParams::SyncParams( - RunStatusProvider &rRunStatusProvider, - SysadminNotifier &rSysadminNotifier, - ProgressNotifier &rProgressNotifier, - BackupClientContext &rContext, - BackgroundTask *pBackgroundTask) -: mSyncPeriodStart(0), - mSyncPeriodEnd(0), - mMaxUploadWait(0), - mMaxFileTimeInFuture(99999999999999999LL), - mFileTrackingSizeThreshold(16*1024), - mDiffingUploadSizeThreshold(16*1024), - mpBackgroundTask(pBackgroundTask), - mrRunStatusProvider(rRunStatusProvider), - mrSysadminNotifier(rSysadminNotifier), - mrProgressNotifier(rProgressNotifier), - mrContext(rContext), - mReadErrorsOnFilesystemObjects(false), - mMaxUploadRate(0), - mUploadAfterThisTimeInTheFuture(99999999999999999LL), - mHaveLoggedWarningAboutFutureFileTimes(false) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::SyncParams::~SyncParams() -// Purpose: Destructor -// Created: 8/3/04 -// -// -------------------------------------------------------------------------- -BackupClientDirectoryRecord::SyncParams::~SyncParams() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::Deserialize(Archive & rArchive) -// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. -// -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::Deserialize(Archive & rArchive) -{ - // Make deletion recursive - DeleteSubDirectories(); - - // Delete maps - if(mpPendingEntries != 0) - { - delete mpPendingEntries; - mpPendingEntries = 0; - } - - // - // - // - rArchive.Read(mObjectID); - rArchive.Read(mSubDirName); - rArchive.Read(mInitialSyncDone); - rArchive.Read(mSyncDone); - - // - // - // - int64_t iCount = 0; - rArchive.Read(iCount); - - if (iCount != sizeof(mStateChecksum)/sizeof(mStateChecksum[0])) - { - // we have some kind of internal system representation change: throw for now - THROW_EXCEPTION(CommonException, Internal) - } - - for (int v = 0; v < iCount; v++) - { - // Load each checksum entry - rArchive.Read(mStateChecksum[v]); - } - - // - // - // - iCount = 0; - rArchive.Read(iCount); - - if (iCount > 0) - { - // load each pending entry - mpPendingEntries = new std::map<std::string, box_time_t>; - if (!mpPendingEntries) - { - throw std::bad_alloc(); - } - - for (int v = 0; v < iCount; v++) - { - std::string strItem; - box_time_t btItem; - - rArchive.Read(strItem); - rArchive.Read(btItem); - (*mpPendingEntries)[strItem] = btItem; - } - } - - // - // - // - iCount = 0; - rArchive.Read(iCount); - - if (iCount > 0) - { - for (int v = 0; v < iCount; v++) - { - std::string strItem; - rArchive.Read(strItem); - - BackupClientDirectoryRecord* pSubDirRecord = - new BackupClientDirectoryRecord(0, ""); - // will be deserialized anyway, give it id 0 for now - - if (!pSubDirRecord) - { - throw std::bad_alloc(); - } - - /***** RECURSE *****/ - pSubDirRecord->Deserialize(rArchive); - mSubDirectories[strItem] = pSubDirRecord; - } - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::Serialize(Archive & rArchive) -// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. -// -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const -{ - // - // - // - rArchive.Write(mObjectID); - rArchive.Write(mSubDirName); - rArchive.Write(mInitialSyncDone); - rArchive.Write(mSyncDone); - - // - // - // - int64_t iCount = 0; - - // when reading back the archive, we will - // need to know how many items there are. - iCount = sizeof(mStateChecksum) / sizeof(mStateChecksum[0]); - rArchive.Write(iCount); - - for (int v = 0; v < iCount; v++) - { - rArchive.Write(mStateChecksum[v]); - } - - // - // - // - if (!mpPendingEntries) - { - iCount = 0; - rArchive.Write(iCount); - } - else - { - iCount = mpPendingEntries->size(); - rArchive.Write(iCount); - - for (std::map<std::string, box_time_t>::const_iterator - i = mpPendingEntries->begin(); - i != mpPendingEntries->end(); i++) - { - rArchive.Write(i->first); - rArchive.Write(i->second); - } - } - // - // - // - iCount = mSubDirectories.size(); - rArchive.Write(iCount); - - for (std::map<std::string, BackupClientDirectoryRecord*>::const_iterator - i = mSubDirectories.begin(); - i != mSubDirectories.end(); i++) - { - const BackupClientDirectoryRecord* pSubItem = i->second; - ASSERT(pSubItem); - - rArchive.Write(i->first); - pSubItem->Serialize(rArchive); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: Location::Location() -// Purpose: Constructor -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -Location::Location() -: mIDMapIndex(0) -{ } - -// -------------------------------------------------------------------------- -// -// Function -// Name: Location::~Location() -// Purpose: Destructor -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -Location::~Location() -{ } - -// -------------------------------------------------------------------------- -// -// Function -// Name: Location::Serialize(Archive & rArchive) -// Purpose: Serializes this object instance into a stream of bytes, -// using an Archive abstraction. -// -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -void Location::Serialize(Archive & rArchive) const -{ - // - // - // - rArchive.Write(mName); - rArchive.Write(mPath); - rArchive.Write(mIDMapIndex); - - // - // - // - if(!mapDirectoryRecord.get()) - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; - rArchive.Write(aMagicMarker); - } - else - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows - rArchive.Write(aMagicMarker); - - mapDirectoryRecord->Serialize(rArchive); - } - - // - // - // - if(!mapExcludeFiles.get()) - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; - rArchive.Write(aMagicMarker); - } - else - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows - rArchive.Write(aMagicMarker); - - mapExcludeFiles->Serialize(rArchive); - } - - // - // - // - if(!mapExcludeDirs.get()) - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; - rArchive.Write(aMagicMarker); - } - else - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows - rArchive.Write(aMagicMarker); - - mapExcludeDirs->Serialize(rArchive); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: Location::Deserialize(Archive & rArchive) -// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. -// -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -void Location::Deserialize(Archive &rArchive) -{ - // - // - // - mapDirectoryRecord.reset(); - mapExcludeFiles.reset(); - mapExcludeDirs.reset(); - - // - // - // - rArchive.Read(mName); - rArchive.Read(mPath); - rArchive.Read(mIDMapIndex); - - // - // - // - int64_t aMagicMarker = 0; - rArchive.Read(aMagicMarker); - - if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) - { - // NOOP - } - else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) - { - BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, ""); - if(!pSubRecord) - { - throw std::bad_alloc(); - } - - mapDirectoryRecord.reset(pSubRecord); - mapDirectoryRecord->Deserialize(rArchive); - } - else - { - // there is something going on here - THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); - } - - // - // - // - rArchive.Read(aMagicMarker); - - if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) - { - // NOOP - } - else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) - { - mapExcludeFiles.reset(new ExcludeList); - if(!mapExcludeFiles.get()) - { - throw std::bad_alloc(); - } - - mapExcludeFiles->Deserialize(rArchive); - } - else - { - // there is something going on here - THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); - } - - // - // - // - rArchive.Read(aMagicMarker); - - if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) - { - // NOOP - } - else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) - { - mapExcludeDirs.reset(new ExcludeList); - if(!mapExcludeDirs.get()) - { - throw std::bad_alloc(); - } - - mapExcludeDirs->Deserialize(rArchive); - } - else - { - // there is something going on here - THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); - } -} diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/bin/bbackupd/BackupClientDirectoryRecord.h deleted file mode 100644 index 865fc747..00000000 --- a/bin/bbackupd/BackupClientDirectoryRecord.h +++ /dev/null @@ -1,244 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientDirectoryRecord.h -// Purpose: Implementation of record about directory for backup client -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCLIENTDIRECTORYRECORD__H -#define BACKUPCLIENTDIRECTORYRECORD__H - -#include <string> -#include <map> -#include <memory> - -#include "BackgroundTask.h" -#include "BackupClientFileAttributes.h" -#include "BackupDaemonInterface.h" -#include "BackupStoreDirectory.h" -#include "BoxTime.h" -#include "MD5Digest.h" -#include "ReadLoggingStream.h" -#include "RunStatusProvider.h" - -#ifdef ENABLE_VSS -# include <comdef.h> -# include <Vss.h> -# include <VsWriter.h> -# include <VsBackup.h> -#endif - -class Archive; -class BackupClientContext; -class BackupDaemon; -class ExcludeList; -class Location; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupClientDirectoryRecord -// Purpose: Implementation of record about directory for backup client -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -class BackupClientDirectoryRecord -{ -public: - BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName); - virtual ~BackupClientDirectoryRecord(); - - void Deserialize(Archive & rArchive); - void Serialize(Archive & rArchive) const; -private: - BackupClientDirectoryRecord(const BackupClientDirectoryRecord &); -public: - - enum - { - UnknownDirectoryID = 0 - }; - - // -------------------------------------------------------------------------- - // - // Class - // Name: BackupClientDirectoryRecord::SyncParams - // Purpose: Holds parameters etc for directory syncing. Not passed as - // const, some parameters may be modified during sync. - // Created: 8/3/04 - // - // -------------------------------------------------------------------------- - class SyncParams : public ReadLoggingStream::Logger - { - public: - SyncParams( - RunStatusProvider &rRunStatusProvider, - SysadminNotifier &rSysadminNotifier, - ProgressNotifier &rProgressNotifier, - BackupClientContext &rContext, - BackgroundTask *pBackgroundTask); - ~SyncParams(); - private: - // No copying - SyncParams(const SyncParams&); - SyncParams &operator=(const SyncParams&); - - public: - // Data members are public, as accessors are not justified here - box_time_t mSyncPeriodStart; - box_time_t mSyncPeriodEnd; - box_time_t mMaxUploadWait; - box_time_t mMaxFileTimeInFuture; - int32_t mFileTrackingSizeThreshold; - int32_t mDiffingUploadSizeThreshold; - BackgroundTask *mpBackgroundTask; - RunStatusProvider &mrRunStatusProvider; - SysadminNotifier &mrSysadminNotifier; - ProgressNotifier &mrProgressNotifier; - BackupClientContext &mrContext; - bool mReadErrorsOnFilesystemObjects; - int64_t mMaxUploadRate; - - // Member variables modified by syncing process - box_time_t mUploadAfterThisTimeInTheFuture; - bool mHaveLoggedWarningAboutFutureFileTimes; - - bool StopRun() { return mrRunStatusProvider.StopRun(); } - void NotifySysadmin(SysadminNotifier::EventCode Event) - { - mrSysadminNotifier.NotifySysadmin(Event); - } - ProgressNotifier& GetProgressNotifier() const - { - return mrProgressNotifier; - } - - /* ReadLoggingStream::Logger implementation */ - virtual void Log(int64_t readSize, int64_t offset, - int64_t length, box_time_t elapsed, box_time_t finish) - { - mrProgressNotifier.NotifyReadProgress(readSize, offset, - length, elapsed, finish); - } - virtual void Log(int64_t readSize, int64_t offset, - int64_t length) - { - mrProgressNotifier.NotifyReadProgress(readSize, offset, - length); - } - virtual void Log(int64_t readSize, int64_t offset) - { - mrProgressNotifier.NotifyReadProgress(readSize, offset); - } - }; - - void SyncDirectory(SyncParams &rParams, - int64_t ContainingDirectoryID, - const std::string &rLocalPath, - const std::string &rRemotePath, - const Location& rBackupLocation, - bool ThisDirHasJustBeenCreated = false); - - bool SyncDirectoryEntry(SyncParams &rParams, - ProgressNotifier& rNotifier, - const Location& rBackupLocation, - const std::string &rDirLocalPath, - MD5Digest& currentStateChecksum, - struct dirent *en, - EMU_STRUCT_STAT dir_st, - std::vector<std::string>& rDirs, - std::vector<std::string>& rFiles, - bool& rDownloadDirectoryRecordBecauseOfFutureFiles); - - std::string ConvertVssPathToRealPath(const std::string &rVssPath, - const Location& rBackupLocation); - - int64_t GetObjectID() const { return mObjectID; } - -private: - void DeleteSubDirectories(); - std::auto_ptr<BackupStoreDirectory> FetchDirectoryListing(SyncParams &rParams); - void UpdateAttributes(SyncParams &rParams, - BackupStoreDirectory *pDirOnStore, - const std::string &rLocalPath); -protected: // to allow tests to hook in before UpdateItems() runs - virtual bool UpdateItems(SyncParams &rParams, - const std::string &rLocalPath, - const std::string &rRemotePath, - const Location& rBackupLocation, - BackupStoreDirectory *pDirOnStore, - std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, - std::vector<std::string> &rFiles, - const std::vector<std::string> &rDirs); -private: - int64_t CreateRemoteDir(const std::string& localDirPath, - const std::string& nonVssDirPath, - const std::string& remoteDirPath, - BackupStoreFilenameClear& storeFilename, - bool* pHaveJustCreatedDirOnServer, - BackupClientDirectoryRecord::SyncParams &rParams); - int64_t UploadFile(SyncParams &rParams, - const std::string &rFilename, - const std::string &rNonVssFilePath, - const std::string &rRemotePath, - const BackupStoreFilenameClear &rStoreFilename, - int64_t FileSize, box_time_t ModificationTime, - box_time_t AttributesHash, bool NoPreviousVersionOnServer); - void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, - const std::string& rFilename); - void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, - BackupStoreDirectory* pDirOnStore, - BackupStoreDirectory::Entry* pEntry, - const std::string &rFilename); - std::string DecryptFilename(BackupStoreDirectory::Entry *en, - const std::string& rRemoteDirectoryPath); - std::string DecryptFilename(BackupStoreFilenameClear fn, - int64_t filenameObjectID, - const std::string& rRemoteDirectoryPath); - - int64_t mObjectID; - std::string mSubDirName; - bool mInitialSyncDone; - bool mSyncDone; - - // Checksum of directory contents and attributes, used to detect changes - uint8_t mStateChecksum[MD5Digest::DigestLength]; - - std::map<std::string, box_time_t> *mpPendingEntries; - std::map<std::string, BackupClientDirectoryRecord *> mSubDirectories; - // mpPendingEntries is a pointer rather than simple a member - // variable, because most of the time it'll be empty. This would - // waste a lot of memory because of STL allocation policies. -}; - -class Location -{ -public: - Location(); - ~Location(); - - void Deserialize(Archive & rArchive); - void Serialize(Archive & rArchive) const; -private: - Location(const Location &); // copy not allowed - Location &operator=(const Location &); -public: - std::string mName; - std::string mPath; - std::auto_ptr<BackupClientDirectoryRecord> mapDirectoryRecord; - std::auto_ptr<ExcludeList> mapExcludeFiles; - std::auto_ptr<ExcludeList> mapExcludeDirs; - int mIDMapIndex; - -#ifdef ENABLE_VSS - bool mIsSnapshotCreated; - VSS_ID mSnapshotVolumeId; - std::string mSnapshotPath; -#endif -}; - -#endif // BACKUPCLIENTDIRECTORYRECORD__H - - diff --git a/bin/bbackupd/BackupClientInodeToIDMap.cpp b/bin/bbackupd/BackupClientInodeToIDMap.cpp deleted file mode 100644 index 6eaf7394..00000000 --- a/bin/bbackupd/BackupClientInodeToIDMap.cpp +++ /dev/null @@ -1,320 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientInodeToIDMap.cpp -// Purpose: Map of inode numbers to file IDs on the store -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <stdlib.h> -#include <depot.h> - -#define BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION -#include "BackupClientInodeToIDMap.h" -#undef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION - -#include "Archive.h" -#include "BackupStoreException.h" -#include "CollectInBufferStream.h" -#include "MemBlockStream.h" -#include "autogen_CommonException.h" - -#include "MemLeakFindOn.h" - -#define BOX_DBM_INODE_DB_VERSION_KEY "BackupClientInodeToIDMap.Version" -#define BOX_DBM_INODE_DB_VERSION_CURRENT 2 - -#define BOX_DBM_MESSAGE(stuff) stuff << " (qdbm): " << dperrmsg(dpecode) - -#define BOX_LOG_DBM_ERROR(stuff) \ - BOX_ERROR(BOX_DBM_MESSAGE(stuff)) - -#define THROW_DBM_ERROR(message, filename, exception, subtype) \ - BOX_LOG_DBM_ERROR(message << ": " << filename); \ - THROW_EXCEPTION_MESSAGE(exception, subtype, \ - BOX_DBM_MESSAGE(message << ": " << filename)); - -#define ASSERT_DBM_OK(operation, message, filename, exception, subtype) \ - if(!(operation)) \ - { \ - THROW_DBM_ERROR(message, filename, exception, subtype); \ - } - -#define ASSERT_DBM_OPEN() \ - if(mpDepot == 0) \ - { \ - THROW_EXCEPTION_MESSAGE(BackupStoreException, InodeMapNotOpen, \ - "Inode database not open"); \ - } - -#define ASSERT_DBM_CLOSED() \ - if(mpDepot != 0) \ - { \ - THROW_EXCEPTION_MESSAGE(CommonException, Internal, \ - "Inode database already open: " << mFilename); \ - } - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::BackupClientInodeToIDMap() -// Purpose: Constructor -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -BackupClientInodeToIDMap::BackupClientInodeToIDMap() - : mReadOnly(true), - mEmpty(false), - mpDepot(0) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::~BackupClientInodeToIDMap() -// Purpose: Destructor -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -BackupClientInodeToIDMap::~BackupClientInodeToIDMap() -{ - if(mpDepot != 0) - { - Close(); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::Open(const char *, bool, bool) -// Purpose: Open the database map, creating a file on disc to store everything -// Created: 20/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientInodeToIDMap::Open(const char *Filename, bool ReadOnly, - bool CreateNew) -{ - mFilename = Filename; - - // Correct arguments? - ASSERT(!(CreateNew && ReadOnly)); - - // Correct usage? - ASSERT_DBM_CLOSED(); - ASSERT(!mEmpty); - - // Open the database file - int mode = ReadOnly ? DP_OREADER : DP_OWRITER; - if(CreateNew) - { - mode |= DP_OCREAT; - } - - mpDepot = dpopen(Filename, mode, 0); - - if(!mpDepot) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, - BOX_DBM_MESSAGE("Failed to open inode database: " << - mFilename)); - } - - const char* version_key = BOX_DBM_INODE_DB_VERSION_KEY; - int32_t version = 0; - - if(CreateNew) - { - version = BOX_DBM_INODE_DB_VERSION_CURRENT; - - int ret = dpput(mpDepot, version_key, strlen(version_key), - (char *)(&version), sizeof(version), DP_DKEEP); - - if(!ret) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, - BOX_DBM_MESSAGE("Failed to write version number to inode " - "database: " << mFilename)); - } - } - else - { - int ret = dpgetwb(mpDepot, version_key, strlen(version_key), 0, - sizeof(version), (char *)(&version)); - - if(ret == -1) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, - "Missing version number in inode database. Perhaps it " - "needs to be recreated: " << mFilename); - } - - if(ret != sizeof(version)) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, - "Wrong size version number in inode database: expected " - << sizeof(version) << " bytes but found " << ret); - } - - if(version != BOX_DBM_INODE_DB_VERSION_CURRENT) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, - "Wrong version number in inode database: expected " << - BOX_DBM_INODE_DB_VERSION_CURRENT << " but found " << - version << ". Perhaps it needs to be recreated: " << - mFilename); - } - - // By this point the version number has been checked and is OK. - } - - // Read only flag - mReadOnly = ReadOnly; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::OpenEmpty() -// Purpose: 'Open' this map. Not associated with a disc file. -// Useful for when a map is required, but is against -// an empty file on disc which shouldn't be created. -// Implies read only. -// Created: 20/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientInodeToIDMap::OpenEmpty() -{ - ASSERT_DBM_CLOSED(); - ASSERT(mpDepot == 0); - mEmpty = true; - mReadOnly = true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::Close() -// Purpose: Close the database file -// Created: 20/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientInodeToIDMap::Close() -{ - ASSERT_DBM_OPEN(); - ASSERT_DBM_OK(dpclose(mpDepot), "Failed to close inode database", - mFilename, BackupStoreException, BerkelyDBFailure); - mpDepot = 0; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::AddToMap(InodeRefType, -// int64_t, int64_t) -// Purpose: Adds an entry to the map. Overwrites any existing -// entry. -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, - int64_t InDirectory, const std::string& LocalPath) -{ - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, InodeMapIsReadOnly); - } - - if(mpDepot == 0) - { - THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); - } - - ASSERT_DBM_OPEN(); - - // Setup structures - CollectInBufferStream buf; - Archive arc(buf, IOStream::TimeOutInfinite); - arc.WriteExact((uint64_t)ObjectID); - arc.WriteExact((uint64_t)InDirectory); - arc.Write(LocalPath); - buf.SetForReading(); - - ASSERT_DBM_OK(dpput(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), - (const char *)buf.GetBuffer(), buf.GetSize(), DP_DOVER), - "Failed to add record to inode database", mFilename, - BackupStoreException, BerkelyDBFailure); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::Lookup(InodeRefType, -// int64_t &, int64_t &) const -// Purpose: Looks up an inode in the map, returning true if it -// exists, and the object ids of it and the directory -// it's in the reference arguments. -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, - int64_t &rInDirectoryOut, std::string* pLocalPathOut) const -{ - if(mEmpty) - { - // Map is empty - return false; - } - - if(mpDepot == 0) - { - THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); - } - - ASSERT_DBM_OPEN(); - int size; - char* data = dpget(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), - 0, -1, &size); - if(data == NULL) - { - // key not in file - return false; - } - - // Free data automatically when the guard goes out of scope. - MemoryBlockGuard<char *> guard(data); - MemBlockStream stream(data, size); - Archive arc(stream, IOStream::TimeOutInfinite); - - // Return data - try - { - arc.Read(rObjectIDOut); - arc.Read(rInDirectoryOut); - if(pLocalPathOut) - { - arc.Read(*pLocalPathOut); - } - } - catch(CommonException &e) - { - if(e.GetSubType() == CommonException::ArchiveBlockIncompleteRead) - { - THROW_FILE_ERROR("Failed to lookup record in inode database: " - << InodeRef << ": not enough data in record", mFilename, - BackupStoreException, BerkelyDBFailure); - // Need to throw precisely that exception to ensure that the - // invalid database is deleted, so that we don't hit the same - // error next time. - } - - throw; - } - - // Found - return true; -} diff --git a/bin/bbackupd/BackupClientInodeToIDMap.h b/bin/bbackupd/BackupClientInodeToIDMap.h deleted file mode 100644 index 4bb1e085..00000000 --- a/bin/bbackupd/BackupClientInodeToIDMap.h +++ /dev/null @@ -1,59 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientInodeToIDMap.h -// Purpose: Map of inode numbers to file IDs on the store -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCLIENTINODETOIDMAP_H -#define BACKUPCLIENTINODETOIDMAP_H - -#include <sys/types.h> - -#include <map> -#include <utility> - -// avoid having to include the DB files when not necessary -#ifndef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION - class DEPOT; -#endif - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupClientInodeToIDMap -// Purpose: Map of inode numbers to file IDs on the store -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -class BackupClientInodeToIDMap -{ -public: - BackupClientInodeToIDMap(); - ~BackupClientInodeToIDMap(); -private: - BackupClientInodeToIDMap(const BackupClientInodeToIDMap &rToCopy); // not allowed -public: - - void Open(const char *Filename, bool ReadOnly, bool CreateNew); - void OpenEmpty(); - - void AddToMap(InodeRefType InodeRef, int64_t ObjectID, - int64_t InDirectory, const std::string& LocalPath); - bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, - int64_t &rInDirectoryOut, std::string* pLocalPathOut = NULL) const; - - void Close(); - -private: - bool mReadOnly; - bool mEmpty; - std::string mFilename; - DEPOT *mpDepot; -}; - -#endif // BACKUPCLIENTINODETOIDMAP_H - - diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp deleted file mode 100644 index 03ce7454..00000000 --- a/bin/bbackupd/BackupDaemon.cpp +++ /dev/null @@ -1,3654 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupDaemon.cpp -// Purpose: Backup daemon -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif -#ifdef HAVE_SIGNAL_H - #include <signal.h> -#endif -#ifdef HAVE_SYS_PARAM_H - #include <sys/param.h> -#endif -#ifdef HAVE_SYS_WAIT_H - #include <sys/wait.h> -#endif -#ifdef HAVE_SYS_MOUNT_H - #include <sys/mount.h> -#endif -#ifdef HAVE_MNTENT_H - #include <mntent.h> -#endif -#ifdef HAVE_SYS_MNTTAB_H - #include <cstdio> - #include <sys/mnttab.h> -#endif -#ifdef HAVE_PROCESS_H - #include <process.h> -#endif - -#include <iostream> -#include <set> -#include <sstream> - -#include "Configuration.h" -#include "IOStream.h" -#include "MemBlockStream.h" -#include "CommonException.h" -#include "BoxPortsAndFiles.h" - -#include "SSLLib.h" - -#include "autogen_BackupProtocol.h" -#include "autogen_ClientException.h" -#include "autogen_CommonException.h" -#include "autogen_ConversionException.h" -#include "Archive.h" -#include "BackupClientContext.h" -#include "BackupClientCryptoKeys.h" -#include "BackupClientDirectoryRecord.h" -#include "BackupClientFileAttributes.h" -#include "BackupClientInodeToIDMap.h" -#include "BackupClientMakeExcludeList.h" -#include "BackupConstants.h" -#include "BackupDaemon.h" -#include "BackupDaemonConfigVerify.h" -#include "BackupStoreConstants.h" -#include "BackupStoreDirectory.h" -#include "BackupStoreException.h" -#include "BackupStoreFile.h" -#include "BackupStoreFilenameClear.h" -#include "BannerText.h" -#include "Conversion.h" -#include "ExcludeList.h" -#include "FileStream.h" -#include "IOStreamGetLine.h" -#include "LocalProcessStream.h" -#include "Logging.h" -#include "Random.h" -#include "Timer.h" -#include "Utils.h" - -#ifdef WIN32 - #include "Win32ServiceFunctions.h" - #include "Win32BackupService.h" - - extern Win32BackupService* gpDaemonService; - -# ifdef ENABLE_VSS -# include <comdef.h> -# include <Vss.h> -# include <VsWriter.h> -# include <VsBackup.h> - - // http://www.flounder.com/cstring.htm - std::string GetMsgForHresult(HRESULT hr) - { - std::ostringstream buf; - - if(hr == VSS_S_ASYNC_CANCELLED) - { - buf << "VSS async operation cancelled"; - } - else if(hr == VSS_S_ASYNC_FINISHED) - { - buf << "VSS async operation finished"; - } - else if(hr == VSS_S_ASYNC_PENDING) - { - buf << "VSS async operation pending"; - } - else - { - buf << _com_error(hr).ErrorMessage(); - } - - buf << " (" << BOX_FORMAT_HEX32(hr) << ")"; - return buf.str(); - } - - std::string WideStringToString(WCHAR *buf) - { - if (buf == NULL) - { - return "(null)"; - } - - char* pStr = ConvertFromWideString(buf, CP_UTF8); - - if(pStr == NULL) - { - return "(conversion failed)"; - } - - std::string result(pStr); - free(pStr); - return result; - } - - std::string GuidToString(GUID guid) - { - wchar_t buf[64]; - StringFromGUID2(guid, buf, sizeof(buf)); - return WideStringToString(buf); - } - - std::string BstrToString(const BSTR arg) - { - if(arg == NULL) - { - return std::string("(null)"); - } - else - { - // Extract the *long* before where the arg points to - long len = ((long *)arg)[-1] / 2; - std::wstring wstr((WCHAR *)arg, len); - std::string str; - if(!ConvertFromWideString(wstr, &str, CP_UTF8)) - { - throw std::exception("string conversion failed"); - } - return str; - } - } -# endif - - // Mutex support by Achim: see https://www.boxbackup.org/ticket/67 - - // Creates the two mutexes checked for by the installer/uninstaller to - // see if the program is still running. One of the mutexes is created - // in the global name space (which makes it possible to access the - // mutex across user sessions in Windows XP); the other is created in - // the session name space (because versions of Windows NT prior to - // 4.0 TSE don't have a global name space and don't support the - // 'Global\' prefix). - - void CreateMutexes(const std::string& rName) - { - SECURITY_DESCRIPTOR SecurityDesc; - SECURITY_ATTRIBUTES SecurityAttr; - - /* By default on Windows NT, created mutexes are accessible only by the user - running the process. We need our mutexes to be accessible to all users, so - that the mutex detection can work across user sessions in Windows XP. To - do this we use a security descriptor with a null DACL. - */ - - InitializeSecurityDescriptor(&SecurityDesc, SECURITY_DESCRIPTOR_REVISION); - SetSecurityDescriptorDacl(&SecurityDesc, TRUE, NULL, FALSE); - SecurityAttr.nLength = sizeof(SecurityAttr); - SecurityAttr.lpSecurityDescriptor = &SecurityDesc; - SecurityAttr.bInheritHandle = FALSE; - // We don't care if this succeeds or fails. It's only used to - // ensure that an installer can detect if Box Backup is running. - CreateMutexA(&SecurityAttr, FALSE, rName.c_str()); - std::string global_name = "Global\\" + rName; - CreateMutexA(&SecurityAttr, FALSE, global_name.c_str()); - } -#endif - -#include "MemLeakFindOn.h" - -static const time_t MAX_SLEEP_TIME = 1024; - -// Make the actual sync period have a little bit of extra time, up to a 64th of the main sync period. -// This prevents repetative cycles of load on the server -#define SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY 6 - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::BackupDaemon() -// Purpose: constructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupDaemon::BackupDaemon() - : mState(BackupDaemon::State_Initialising), - mDeleteRedundantLocationsAfter(0), - mLastNotifiedEvent(SysadminNotifier::MAX), - mDeleteUnusedRootDirEntriesAfter(0), - mClientStoreMarker(BackupClientContext::ClientStoreMarker_NotKnown), - mStorageLimitExceeded(false), - mReadErrorsOnFilesystemObjects(false), - mLastSyncTime(0), - mNextSyncTime(0), - mCurrentSyncStartTime(0), - mUpdateStoreInterval(0), - mDeleteStoreObjectInfoFile(false), - mDoSyncForcedByPreviousSyncError(false), - mNumFilesUploaded(-1), - mNumDirsCreated(-1), - mMaxBandwidthFromSyncAllowScript(0), - mLogAllFileAccess(false), - mpProgressNotifier(this), - mpLocationResolver(this), - mpRunStatusProvider(this), - mpSysadminNotifier(this), - mapCommandSocketPollTimer(NULL) - #ifdef WIN32 - , mInstallService(false), - mRemoveService(false), - mRunAsService(false), - mServiceName("bbackupd") - #endif -#ifdef ENABLE_VSS - , mpVssBackupComponents(NULL) -#endif -{ - // Only ever one instance of a daemon - SSLLib::Initialise(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::~BackupDaemon() -// Purpose: Destructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupDaemon::~BackupDaemon() -{ - DeleteAllLocations(); - DeleteAllIDMaps(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DaemonName() -// Purpose: Get name of daemon -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -const char *BackupDaemon::DaemonName() const -{ - return "bbackupd"; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DaemonBanner() -// Purpose: Daemon banner -// Created: 1/1/04 -// -// -------------------------------------------------------------------------- -std::string BackupDaemon::DaemonBanner() const -{ - return BANNER_TEXT("Backup Client"); -} - -void BackupDaemon::Usage() -{ - this->Daemon::Usage(); - -#ifdef WIN32 - std::cout << - " -s Run as a Windows Service, for internal use only\n" - " -i Install Windows Service (you may want to specify a config file)\n" - " -r Remove Windows Service\n" - " -S <name> Service name for -i and -r options\n"; -#endif -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::GetConfigVerify() -// Purpose: Get configuration specification -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -const ConfigurationVerify *BackupDaemon::GetConfigVerify() const -{ - // Defined elsewhere - return &BackupDaemonConfigVerify; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::SetupInInitialProcess() -// Purpose: Platforms with non-checkable credentials on -// local sockets only. -// Prints a warning if the command socket is used. -// Created: 25/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::SetupInInitialProcess() -{ - const Configuration& config(GetConfiguration()); - - // These keys may or may not be required, depending on the configured - // store type (e.g. not when using Amazon S3 stores), so they can't be - // verified by BackupDaemonConfigVerify. - std::vector<std::string> requiredKeys; - requiredKeys.push_back("StoreHostname"); - requiredKeys.push_back("AccountNumber"); - requiredKeys.push_back("CertificateFile"); - requiredKeys.push_back("PrivateKeyFile"); - requiredKeys.push_back("TrustedCAsFile"); - bool missingRequiredKeys = false; - - for(std::vector<std::string>::const_iterator i = requiredKeys.begin(); - i != requiredKeys.end(); i++) - { - if(!config.KeyExists(*i)) - { - BOX_ERROR("Missing required configuration key: " << *i); - missingRequiredKeys = true; - } - } - - if(missingRequiredKeys) - { - THROW_EXCEPTION_MESSAGE(CommonException, InvalidConfiguration, - "Some required configuration keys are missing in " << - GetConfigFileName()); - } - -#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET - // Print a warning on this platform if the CommandSocket is used. - if(GetConfiguration().KeyExists("CommandSocket")) - { - BOX_WARNING( - "==============================================================================\n" - "SECURITY WARNING: This platform cannot check the credentials of connections to\n" - "the command socket. This is a potential DoS security problem.\n" - "Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl\n" - "is not used.\n" - "==============================================================================\n" - ); - } -#endif -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeleteAllLocations() -// Purpose: Deletes all records stored -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupDaemon::DeleteAllLocations() -{ - // Run through, and delete everything - for(Locations::iterator i = mLocations.begin(); - i != mLocations.end(); ++i) - { - delete *i; - } - - // Clear the contents of the map, so it is empty - mLocations.clear(); - - // And delete everything from the associated mount vector - mIDMapMounts.clear(); -} - -#ifdef WIN32 -std::string BackupDaemon::GetOptionString() -{ - std::string oldOpts = this->Daemon::GetOptionString(); - ASSERT(oldOpts.find("s") == std::string::npos); - ASSERT(oldOpts.find("S") == std::string::npos); - ASSERT(oldOpts.find("i") == std::string::npos); - ASSERT(oldOpts.find("r") == std::string::npos); - return oldOpts + "sS:ir"; -} - -int BackupDaemon::ProcessOption(signed int option) -{ - switch(option) - { - case 's': - { - mRunAsService = true; - return 0; - } - - case 'S': - { - mServiceName = optarg; - Logging::SetProgramName(mServiceName); - return 0; - } - - case 'i': - { - mInstallService = true; - return 0; - } - - case 'r': - { - mRemoveService = true; - return 0; - } - - default: - { - return this->Daemon::ProcessOption(option); - } - } -} - -int BackupDaemon::Main(const std::string &rConfigFileName) -{ - if (mInstallService) - { - return InstallService(rConfigFileName.c_str(), mServiceName); - } - - if (mRemoveService) - { - return RemoveService(mServiceName); - } - -#ifdef ENABLE_VSS - HRESULT result = CoInitialize(NULL); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to initialize COM: " << - GetMsgForHresult(result)); - return 1; - } -#endif - - CreateMutexes("__boxbackup_mutex__"); - - int returnCode; - - if (mRunAsService) - { - // We will be called reentrantly by the Service Control - // Manager, and we had better not call OurService again! - mRunAsService = false; - - BOX_INFO("Box Backup service starting"); - returnCode = OurService(rConfigFileName.c_str()); - BOX_INFO("Box Backup service shut down"); - } - else - { - returnCode = this->Daemon::Main(rConfigFileName); - } - - return returnCode; -} -#endif // WIN32 - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::Run() -// Purpose: Run function for daemon -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::Run() -{ - // initialise global timer mechanism - Timers::Init(); - - mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, - "CommandSocketPollTimer")); - - #ifndef WIN32 - // Ignore SIGPIPE so that if a command connection is broken, - // the daemon doesn't terminate. - ::signal(SIGPIPE, SIG_IGN); - #endif - - // Create a command socket? - const Configuration &conf(GetConfiguration()); - if(conf.KeyExists("CommandSocket")) - { - // Yes, create a local UNIX socket - mapCommandSocketInfo.reset(new CommandSocketInfo); - const char *socketName = - conf.GetKeyValue("CommandSocket").c_str(); - #ifdef WIN32 - mapCommandSocketInfo->mListeningSocket.Listen( - socketName); - #else - ::unlink(socketName); - mapCommandSocketInfo->mListeningSocket.Listen( - Socket::TypeUNIX, socketName); - #endif - } - - // Handle things nicely on exceptions - try - { - Run2(); - } - catch(...) - { - try - { - mapCommandSocketInfo.reset(); - } - catch(std::exception &e) - { - BOX_WARNING("Internal error while closing command " - "socket after another exception, ignored: " << - e.what()); - } - catch(...) - { - BOX_WARNING("Error closing command socket after " - "exception, ignored."); - } - - mapCommandSocketPollTimer.reset(); - Timers::Cleanup(); - - throw; - } - - // Clean up - mapCommandSocketInfo.reset(); - mapCommandSocketPollTimer.reset(); - Timers::Cleanup(); -} - -void BackupDaemon::InitCrypto() -{ - // Read in the certificates creating a TLS context - const Configuration &conf(GetConfiguration()); - std::string certFile(conf.GetKeyValue("CertificateFile")); - std::string keyFile(conf.GetKeyValue("PrivateKeyFile")); - std::string caFile(conf.GetKeyValue("TrustedCAsFile")); - mTlsContext.Initialise(false /* as client */, certFile.c_str(), - keyFile.c_str(), caFile.c_str()); - - // Set up the keys for various things - BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile")); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::Run2() -// Purpose: Run function for daemon (second stage) -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupDaemon::Run2() -{ - InitCrypto(); - - const Configuration &conf(GetConfiguration()); - - // How often to connect to the store (approximate) - mUpdateStoreInterval = SecondsToBoxTime( - conf.GetKeyValueInt("UpdateStoreInterval")); - mBackupErrorDelay = conf.GetKeyValueInt("BackupErrorDelay"); - - // But are we connecting automatically? - bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup"); - - // When the next sync should take place -- which is ASAP - mNextSyncTime = 0; - - // When the last sync started (only updated if the store was not full when the sync ended) - mLastSyncTime = 0; - - // -------------------------------------------------------------------------------------------- - - mDeleteStoreObjectInfoFile = DeserializeStoreObjectInfo(mLastSyncTime, - mNextSyncTime); - - // -------------------------------------------------------------------------------------------- - - - // Set state - SetState(State_Idle); - - mDoSyncForcedByPreviousSyncError = false; - - // Loop around doing backups - do - { - // Flags used below - bool storageLimitExceeded = false; - bool doSync = false; - bool mDoSyncForcedByCommand = false; - - // Check whether we should be stopping, and if so, - // don't hang around waiting on the command socket. - if(StopRun()) - { - BOX_INFO("Skipping command socket polling " - "due to shutdown request"); - break; - } - - // Is a delay necessary? - box_time_t currentTime = GetCurrentBoxTime(); - box_time_t requiredDelay = (mNextSyncTime < currentTime) - ? (0) : (mNextSyncTime - currentTime); - mNextSyncTime = currentTime + requiredDelay; - - if (mDoSyncForcedByPreviousSyncError) - { - BOX_INFO("Last backup was not successful, " - "next one starting at " << - FormatTime(mNextSyncTime, false, true)); - } - else if (automaticBackup) - { - BOX_INFO("Automatic backups are enabled, " - "next one starting at " << - FormatTime(mNextSyncTime, false, true)); - } - else - { - BOX_INFO("No automatic backups, waiting for " - "bbackupctl snapshot command"); - requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); - } - - if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME)) - { - requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); - } - - // Only delay if necessary - if(requiredDelay == 0) - { - // No sleep necessary, so don't listen on the command - // socket at all right now. - } - else if(mapCommandSocketInfo.get() != 0) - { - // A command socket exists, so sleep by waiting for a - // connection or command on it. - WaitOnCommandSocket(requiredDelay, doSync, - mDoSyncForcedByCommand); - } - else - { - // No command socket or connection, just do a normal - // sleep. - time_t sleepSeconds = - BoxTimeToSeconds(requiredDelay); - ::sleep((sleepSeconds <= 0) - ? 1 : sleepSeconds); - } - - // We have now slept, so if automaticBackup is enabled then - // it's time for a backup now. - - if(StopRun()) - { - BOX_INFO("Stopping idle loop due to shutdown request"); - break; - } - else if(doSync) - { - BOX_INFO("Starting a backup immediately due to " - "bbackupctl sync command"); - } - else if(GetCurrentBoxTime() < mNextSyncTime) - { - BOX_TRACE("Deadline not reached, sleeping again"); - continue; - } - else if(mDoSyncForcedByPreviousSyncError) - { - BOX_INFO("Last backup was not successful, next one " - "starting now"); - } - else if(!automaticBackup) - { - BOX_TRACE("Sleeping again because automatic backups " - "are not enabled"); - continue; - } - else - { - BOX_INFO("Automatic backups are enabled, next one " - "starting now"); - } - - // If we pass this point, or exit the loop, we should have - // logged something at INFO level or higher to explain why. - - // Use a script to see if sync is allowed now? - if(mDoSyncForcedByCommand) - { - BOX_INFO("Skipping SyncAllowScript due to bbackupctl " - "force-sync command"); - } - else - { - int d = UseScriptToSeeIfSyncAllowed(); - if(d > 0) - { - // Script has asked for a delay - mNextSyncTime = GetCurrentBoxTime() + - SecondsToBoxTime(d); - BOX_INFO("Impending backup stopped by " - "SyncAllowScript, next attempt " - "scheduled for " << - FormatTime(mNextSyncTime, false)); - continue; - } - } - - mCurrentSyncStartTime = GetCurrentBoxTime(); - RunSyncNowWithExceptionHandling(); - - // Set state - SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle); - } - while(!StopRun()); - - // Make sure we have a clean start next time round (if restart) - DeleteAllLocations(); - DeleteAllIDMaps(); -} - -std::auto_ptr<BackupClientContext> BackupDaemon::RunSyncNowWithExceptionHandling() -{ - bool errorOccurred = false; - int errorCode = 0, errorSubCode = 0; - std::string errorString = "unknown"; - - try - { - OnBackupStart(); - // Do sync - RunSyncNow(); - } - catch(BoxException &e) - { - errorOccurred = true; - errorString = e.what(); - errorCode = e.GetType(); - errorSubCode = e.GetSubType(); - } - catch(std::exception &e) - { - BOX_ERROR("Internal error during backup run: " << e.what()); - errorOccurred = true; - errorString = e.what(); - } - catch(...) - { - // TODO: better handling of exceptions here... - // need to be very careful - errorOccurred = true; - } - - // do not retry immediately without a good reason - mDoSyncForcedByPreviousSyncError = false; - - // Is it a berkely db failure? - bool isBerkelyDbFailure = false; - - // Notify system administrator about the final state of the backup - if(errorOccurred) - { - if (errorCode == BackupStoreException::ExceptionType - && errorSubCode == BackupStoreException::BerkelyDBFailure) - { - isBerkelyDbFailure = true; - } - - if(isBerkelyDbFailure) - { - // Delete corrupt files - DeleteCorruptBerkelyDbFiles(); - } - - ResetCachedState(); - - // Handle restart? - if(StopRun()) - { - BOX_NOTICE("Exception (" << errorCode << "/" << - errorSubCode << ") due to signal"); - OnBackupFinish(); - return mapClientContext; // releases mapClientContext - } - - NotifySysadmin(SysadminNotifier::BackupError); - - // If the Berkely db files get corrupted, - // delete them and try again immediately. - if(isBerkelyDbFailure) - { - BOX_ERROR("Berkely db inode map files corrupted, " - "deleting and restarting scan. Renamed files " - "and directories will not be tracked until " - "after this scan."); - ::sleep(1); - } - else - { - // Not restart/terminate, pause and retry - // Notify administrator - SetState(State_Error); - BOX_ERROR("Exception caught (" << errorString << - " " << errorCode << "/" << errorSubCode << - "), reset state and waiting to retry..."); - ::sleep(10); - mNextSyncTime = GetCurrentBoxTime() + - SecondsToBoxTime(mBackupErrorDelay) + - Random::RandomInt(mUpdateStoreInterval >> - SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); - } - } - - if(mReadErrorsOnFilesystemObjects) - { - NotifySysadmin(SysadminNotifier::ReadError); - } - - if(mStorageLimitExceeded) - { - NotifySysadmin(SysadminNotifier::StoreFull); - } - - if (!errorOccurred && !mReadErrorsOnFilesystemObjects && - !mStorageLimitExceeded) - { - NotifySysadmin(SysadminNotifier::BackupOK); - } - - // If we were retrying after an error, and this backup succeeded, - // then now would be a good time to stop :-) - mDoSyncForcedByPreviousSyncError = errorOccurred && !isBerkelyDbFailure; - - OnBackupFinish(); - return mapClientContext; // releases mapClientContext -} - -void BackupDaemon::ResetCachedState() -{ - // Clear state data - // Go back to beginning of time - mLastSyncTime = 0; - mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything - DeleteAllLocations(); - DeleteAllIDMaps(); -} - -std::auto_ptr<BackupClientContext> BackupDaemon::GetNewContext -( - LocationResolver &rResolver, - TLSContext &rTLSContext, - const std::string &rHostname, - int32_t Port, - uint32_t AccountNumber, - bool ExtendedLogging, - bool ExtendedLogToFile, - std::string ExtendedLogFile, - ProgressNotifier &rProgressNotifier, - bool TcpNiceMode -) -{ - std::auto_ptr<BackupClientContext> context(new BackupClientContext( - rResolver, rTLSContext, rHostname, Port, AccountNumber, - ExtendedLogging, ExtendedLogToFile, ExtendedLogFile, - rProgressNotifier, TcpNiceMode)); - return context; -} - -// Returns the BackupClientContext so that tests can use it to hold the -// connection open and prevent housekeeping from running. Otherwise don't use -// it, let it be destroyed and close the connection. -std::auto_ptr<BackupClientContext> BackupDaemon::RunSyncNow() -{ - Timers::AssertInitialised(); - - // Delete the serialised store object file, - // so that we don't try to reload it after a - // partially completed backup - if(mDeleteStoreObjectInfoFile && !DeleteStoreObjectInfo()) - { - BOX_ERROR("Failed to delete the StoreObjectInfoFile, " - "backup cannot continue safely."); - THROW_EXCEPTION(ClientException, - FailedToDeleteStoreObjectInfoFile); - } - - // In case the backup throws an exception, - // we should not try to delete the store info - // object file again. - mDeleteStoreObjectInfoFile = false; - - const Configuration &conf(GetConfiguration()); - - std::auto_ptr<FileLogger> fileLogger; - - if (conf.KeyExists("LogFile")) - { - bool overwrite = false; - if (conf.KeyExists("LogFileOverwrite")) - { - overwrite = conf.GetKeyValueBool("LogFileOverwrite"); - } - - Log::Level level = Log::INFO; - if (conf.KeyExists("LogFileLevel")) - { - level = Logging::GetNamedLevel( - conf.GetKeyValue("LogFileLevel")); - } - - fileLogger.reset(new FileLogger(conf.GetKeyValue("LogFile"), - level, !overwrite)); - } - - std::string extendedLogFile; - if (conf.KeyExists("ExtendedLogFile")) - { - extendedLogFile = conf.GetKeyValue("ExtendedLogFile"); - } - - if (conf.KeyExists("LogAllFileAccess")) - { - mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess"); - } - - // Then create a client context object (don't - // just connect, as this may be unnecessary) - mapClientContext = GetNewContext( - *mpLocationResolver, - mTlsContext, - conf.GetKeyValue("StoreHostname"), - conf.GetKeyValueInt("StorePort"), - conf.GetKeyValueUint32("AccountNumber"), - conf.GetKeyValueBool("ExtendedLogging"), - conf.KeyExists("ExtendedLogFile"), - extendedLogFile, - *mpProgressNotifier, - conf.GetKeyValueBool("TcpNice") - ); - - // The minimum age a file needs to be before it will be - // considered for uploading - box_time_t minimumFileAge = SecondsToBoxTime( - conf.GetKeyValueInt("MinimumFileAge")); - - // The maximum time we'll wait to upload a file, regardless - // of how often it's modified - box_time_t maxUploadWait = SecondsToBoxTime( - conf.GetKeyValueInt("MaxUploadWait")); - // Adjust by subtracting the minimum file age, so is relative - // to sync period end in comparisons - if (maxUploadWait > minimumFileAge) - { - maxUploadWait -= minimumFileAge; - } - else - { - maxUploadWait = 0; - } - - // Calculate the sync period of files to examine - box_time_t syncPeriodStart = mLastSyncTime; - box_time_t syncPeriodEnd = GetCurrentBoxTime() - minimumFileAge; - - if(syncPeriodStart >= syncPeriodEnd && - syncPeriodStart - syncPeriodEnd < minimumFileAge) - { - // This can happen if we receive a force-sync command less - // than minimumFileAge after the last sync. Deal with it by - // moving back syncPeriodStart, which should not do any - // damage. - syncPeriodStart = syncPeriodEnd - SecondsToBoxTime(1); - } - - if(syncPeriodStart >= syncPeriodEnd) - { - BOX_ERROR("Invalid (negative) sync period: perhaps your clock " - "is going backwards? (" << syncPeriodStart << " to " << - syncPeriodEnd << ")"); - THROW_EXCEPTION(ClientException, ClockWentBackwards); - } - - // Check logic - ASSERT(syncPeriodEnd > syncPeriodStart); - // Paranoid check on sync times - if(syncPeriodStart >= syncPeriodEnd) - { - return mapClientContext; // releases mapClientContext - } - - // Adjust syncPeriodEnd to emulate snapshot behaviour properly - box_time_t syncPeriodEndExtended = syncPeriodEnd; - - // Using zero min file age? - if(minimumFileAge == 0) - { - // Add a year on to the end of the end time, - // to make sure we sync files which are - // modified after the scan run started. - // Of course, they may be eligible to be - // synced again the next time round, - // but this should be OK, because the changes - // only upload should upload no data. - syncPeriodEndExtended += SecondsToBoxTime( - (time_t)(356*24*3600)); - } - - // Set up the sync parameters - BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider, - *mpSysadminNotifier, *mpProgressNotifier, *mapClientContext, this); - params.mSyncPeriodStart = syncPeriodStart; - params.mSyncPeriodEnd = syncPeriodEndExtended; - // use potentially extended end time - params.mMaxUploadWait = maxUploadWait; - params.mFileTrackingSizeThreshold = - conf.GetKeyValueInt("FileTrackingSizeThreshold"); - params.mDiffingUploadSizeThreshold = - conf.GetKeyValueInt("DiffingUploadSizeThreshold"); - params.mMaxFileTimeInFuture = - SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture")); - mNumFilesUploaded = 0; - mNumDirsCreated = 0; - - if(conf.KeyExists("MaxUploadRate")) - { - params.mMaxUploadRate = conf.GetKeyValueInt("MaxUploadRate"); - } - - if(mMaxBandwidthFromSyncAllowScript != 0) - { - params.mMaxUploadRate = mMaxBandwidthFromSyncAllowScript; - } - - mDeleteRedundantLocationsAfter = - conf.GetKeyValueInt("DeleteRedundantLocationsAfter"); - mStorageLimitExceeded = false; - mReadErrorsOnFilesystemObjects = false; - - // Setup various timings - int maximumDiffingTime = 600; - int keepAliveTime = 60; - - // max diffing time, keep-alive time - if(conf.KeyExists("MaximumDiffingTime")) - { - maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime"); - } - if(conf.KeyExists("KeepAliveTime")) - { - keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); - } - - mapClientContext->SetMaximumDiffingTime(maximumDiffingTime); - mapClientContext->SetKeepAliveTime(keepAliveTime); - - // Set store marker - mapClientContext->SetClientStoreMarker(mClientStoreMarker); - - // Set up the locations, if necessary -- need to do it here so we have - // a (potential) connection to use. - { - const Configuration &locations( - conf.GetSubConfiguration( - "BackupLocations")); - - // Make sure all the directory records - // are set up - SetupLocations(*mapClientContext, locations); - } - - mpProgressNotifier->NotifyIDMapsSetup(*mapClientContext); - - // Get some ID maps going - SetupIDMapsForSync(); - - // Delete any unused directories? - DeleteUnusedRootDirEntries(*mapClientContext); - -#ifdef ENABLE_VSS - CreateVssBackupComponents(); -#endif - - // Go through the records, syncing them - for(Locations::const_iterator - i(mLocations.begin()); - i != mLocations.end(); ++i) - { - // Set current and new ID map pointers - // in the context - mapClientContext->SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], - mNewIDMaps[(*i)->mIDMapIndex]); - - // Set exclude lists (context doesn't - // take ownership) - mapClientContext->SetExcludeLists( - (*i)->mapExcludeFiles.get(), - (*i)->mapExcludeDirs.get()); - - // Sync the directory - std::string locationPath = (*i)->mPath; -#ifdef ENABLE_VSS - if((*i)->mIsSnapshotCreated) - { - locationPath = (*i)->mSnapshotPath; - } -#endif - - (*i)->mapDirectoryRecord->SyncDirectory(params, - BackupProtocolListDirectory::RootDirectory, - locationPath, std::string("/") + (*i)->mName, **i); - - // Unset exclude lists (just in case) - mapClientContext->SetExcludeLists(0, 0); - } - - // Perform any deletions required -- these are - // delayed until the end to allow renaming to - // happen neatly. - mapClientContext->PerformDeletions(); - -#ifdef ENABLE_VSS - CleanupVssBackupComponents(); -#endif - - // Get the new store marker - mClientStoreMarker = mapClientContext->GetClientStoreMarker(); - mStorageLimitExceeded = mapClientContext->StorageLimitExceeded(); - mReadErrorsOnFilesystemObjects |= - params.mReadErrorsOnFilesystemObjects; - - if(!mStorageLimitExceeded) - { - // The start time of the next run is the end time of this - // run. This is only done if the storage limit wasn't - // exceeded (as things won't have been done properly if - // it was) - mLastSyncTime = syncPeriodEnd; - } - - // Commit the ID Maps - CommitIDMapsAfterSync(); - - // Calculate when the next sync run should be - mNextSyncTime = mCurrentSyncStartTime + - mUpdateStoreInterval + - Random::RandomInt(mUpdateStoreInterval >> - SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); - - // -------------------------------------------------------------------------------------------- - - // We had a successful backup, save the store - // info. If we save successfully, we must - // delete the file next time we start a backup - - mDeleteStoreObjectInfoFile = - SerializeStoreObjectInfo(mLastSyncTime, mNextSyncTime); - - // -------------------------------------------------------------------------------------------- - - return mapClientContext; // releases mapClientContext -} - -#ifdef ENABLE_VSS -bool BackupDaemon::WaitForAsync(IVssAsync *pAsync, - const std::string& description) -{ - BOX_INFO("VSS: waiting for " << description << " to complete"); - HRESULT result; - - do - { - result = pAsync->Wait(1000); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to wait for " << description << - " to complete: " << GetMsgForHresult(result)); - break; - } - - HRESULT result2; - result = pAsync->QueryStatus(&result2, NULL); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to query " << description << - " status: " << GetMsgForHresult(result)); - break; - } - - result = result2; - BOX_INFO("VSS: " << description << " status: " << - GetMsgForHresult(result)); - } - while(result == VSS_S_ASYNC_PENDING); - - pAsync->Release(); - - return (result == VSS_S_ASYNC_FINISHED); -} - -#define CALL_MEMBER_FN(object, method) ((object).*(method)) - -bool BackupDaemon::CallAndWaitForAsync(AsyncMethod method, - const std::string& description) -{ - IVssAsync *pAsync; - HRESULT result = CALL_MEMBER_FN(*mpVssBackupComponents, method)(&pAsync); - if(result != S_OK) - { - BOX_ERROR("VSS: " << description << " failed: " << - GetMsgForHresult(result)); - return false; - } - - return WaitForAsync(pAsync, description); -} - -void BackupDaemon::CreateVssBackupComponents() -{ - std::map<char, VSS_ID> volumesIncluded; - - HRESULT result = ::CreateVssBackupComponents(&mpVssBackupComponents); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to create backup components: " << - GetMsgForHresult(result)); - return; - } - - result = mpVssBackupComponents->InitializeForBackup(NULL); - if(result != S_OK) - { - std::string message = GetMsgForHresult(result); - - if (result == VSS_E_UNEXPECTED) - { - message = "Check the Application Log for details, and ensure " - "that the Volume Shadow Copy, COM+ System Application, " - "and Distributed Transaction Coordinator services " - "are running"; - } - - BOX_ERROR("VSS: Failed to initialize for backup: " << message); - return; - } - - result = mpVssBackupComponents->SetContext(VSS_CTX_BACKUP); - if(result == E_NOTIMPL) - { - BOX_INFO("VSS: Failed to set context to VSS_CTX_BACKUP: " - "not implemented, probably Windows XP, ignored."); - } - else if(result != S_OK) - { - BOX_ERROR("VSS: Failed to set context to VSS_CTX_BACKUP: " << - GetMsgForHresult(result)); - return; - } - - result = mpVssBackupComponents->SetBackupState( - false, /* no components for now */ - true, /* might as well ask for a bootable backup */ - VSS_BT_FULL, - false /* what is Partial File Support? */); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to set backup state: " << - GetMsgForHresult(result)); - return; - } - - if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterMetadata, - "GatherWriterMetadata()")) - { - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - - UINT writerCount; - result = mpVssBackupComponents->GetWriterMetadataCount(&writerCount); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get writer count: " << - GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - - for(UINT iWriter = 0; iWriter < writerCount; iWriter++) - { - BOX_INFO("VSS: Getting metadata from writer " << iWriter); - VSS_ID writerInstance; - IVssExamineWriterMetadata* pMetadata; - result = mpVssBackupComponents->GetWriterMetadata(iWriter, - &writerInstance, &pMetadata); - if(result != S_OK) - { - BOX_ERROR("Failed to get VSS metadata from writer " << iWriter << - ": " << GetMsgForHresult(result)); - continue; - } - - UINT includeFiles, excludeFiles, numComponents; - result = pMetadata->GetFileCounts(&includeFiles, &excludeFiles, - &numComponents); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get metadata file counts from " - "writer " << iWriter << ": " << - GetMsgForHresult(result)); - pMetadata->Release(); - continue; - } - - for(UINT iComponent = 0; iComponent < numComponents; iComponent++) - { - IVssWMComponent* pComponent; - result = pMetadata->GetComponent(iComponent, &pComponent); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get metadata component " << - iComponent << " from writer " << iWriter << ": " << - GetMsgForHresult(result)); - continue; - } - - PVSSCOMPONENTINFO pComponentInfo; - result = pComponent->GetComponentInfo(&pComponentInfo); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get metadata component " << - iComponent << " info from writer " << iWriter << ": " << - GetMsgForHresult(result)); - pComponent->Release(); - continue; - } - - BOX_TRACE("VSS: writer " << iWriter << " component " << - iComponent << " info:"); - switch(pComponentInfo->type) - { - case VSS_CT_UNDEFINED: BOX_TRACE("VSS: type: undefined"); break; - case VSS_CT_DATABASE: BOX_TRACE("VSS: type: database"); break; - case VSS_CT_FILEGROUP: BOX_TRACE("VSS: type: filegroup"); break; - default: - BOX_WARNING("VSS: type: unknown (" << pComponentInfo->type << ")"); - } - - BOX_TRACE("VSS: logical path: " << - BstrToString(pComponentInfo->bstrLogicalPath)); - BOX_TRACE("VSS: component name: " << - BstrToString(pComponentInfo->bstrComponentName)); - BOX_TRACE("VSS: caption: " << - BstrToString(pComponentInfo->bstrCaption)); - BOX_TRACE("VSS: restore metadata: " << - pComponentInfo->bRestoreMetadata); - BOX_TRACE("VSS: notify on complete: " << - pComponentInfo->bRestoreMetadata); - BOX_TRACE("VSS: selectable: " << - pComponentInfo->bSelectable); - BOX_TRACE("VSS: selectable for restore: " << - pComponentInfo->bSelectableForRestore); - BOX_TRACE("VSS: component flags: " << - BOX_FORMAT_HEX32(pComponentInfo->dwComponentFlags)); - BOX_TRACE("VSS: file count: " << - pComponentInfo->cFileCount); - BOX_TRACE("VSS: databases: " << - pComponentInfo->cDatabases); - BOX_TRACE("VSS: log files: " << - pComponentInfo->cLogFiles); - BOX_TRACE("VSS: dependencies: " << - pComponentInfo->cDependencies); - - pComponent->FreeComponentInfo(pComponentInfo); - pComponent->Release(); - } - - pMetadata->Release(); - } - - VSS_ID snapshotSetId; - result = mpVssBackupComponents->StartSnapshotSet(&snapshotSetId); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to start snapshot set: " << - GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - - // Add all volumes included as backup locations to the snapshot set - for(Locations::iterator - iLocation = mLocations.begin(); - iLocation != mLocations.end(); - iLocation++) - { - Location& rLocation(**iLocation); - std::string path = rLocation.mPath; - // convert to absolute and remove Unicode prefix - path = ConvertPathToAbsoluteUnicode(path.c_str()).substr(4); - - if(path.length() >= 3 && path[1] == ':' && path[2] == '\\') - { - std::string volumeRoot = path.substr(0, 3); - - std::map<char, VSS_ID>::iterator i = - volumesIncluded.find(path[0]); - - if(i == volumesIncluded.end()) - { - std::wstring volumeRootWide; - volumeRootWide.push_back((WCHAR) path[0]); - volumeRootWide.push_back((WCHAR) ':'); - volumeRootWide.push_back((WCHAR) '\\'); - VSS_ID newVolumeId; - result = mpVssBackupComponents->AddToSnapshotSet( - (VSS_PWSZ)(volumeRootWide.c_str()), GUID_NULL, - &newVolumeId); - if(result == S_OK) - { - BOX_TRACE("VSS: Added volume " << volumeRoot << - " for backup location " << path << - " to snapshot set"); - volumesIncluded[path[0]] = newVolumeId; - rLocation.mSnapshotVolumeId = newVolumeId; - } - else - { - BOX_ERROR("VSS: Failed to add volume " << - volumeRoot << " to snapshot set: " << - GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - } - else - { - BOX_TRACE("VSS: Skipping already included volume " << - volumeRoot << " for backup location " << path); - rLocation.mSnapshotVolumeId = i->second; - } - - rLocation.mIsSnapshotCreated = true; - - // If the snapshot path starts with the volume root - // (drive letter), because the path is absolute (as it - // should be), then remove it so that the resulting - // snapshot path can be appended to the snapshot device - // object to make a real path, without a spurious drive - // letter in it. - - if (path.substr(0, volumeRoot.length()) == volumeRoot) - { - path = path.substr(volumeRoot.length()); - } - - rLocation.mSnapshotPath = path; - } - else - { - BOX_WARNING("VSS: Skipping backup location " << path << - " which does not start with a volume specification"); - } - } - - if(!CallAndWaitForAsync(&IVssBackupComponents::PrepareForBackup, - "PrepareForBackup()")) - { - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - - if(!CallAndWaitForAsync(&IVssBackupComponents::DoSnapshotSet, - "DoSnapshotSet()")) - { - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - - if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterStatus, - "GatherWriterStatus()")) - { - goto CreateVssBackupComponents_cleanup_WriterStatus; - } - - result = mpVssBackupComponents->GetWriterStatusCount(&writerCount); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get writer status count: " << - GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterStatus; - } - - for(UINT iWriter = 0; iWriter < writerCount; iWriter++) - { - VSS_ID instance, writer; - BSTR writerNameBstr; - VSS_WRITER_STATE writerState; - HRESULT writerResult; - - result = mpVssBackupComponents->GetWriterStatus(iWriter, - &instance, &writer, &writerNameBstr, &writerState, - &writerResult); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to query writer " << iWriter << - " status: " << GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterStatus; - } - - std::string writerName = BstrToString(writerNameBstr); - ::SysFreeString(writerNameBstr); - - if(writerResult != S_OK) - { - BOX_ERROR("VSS: Writer " << iWriter << " (" << - writerName << ") failed: " << - GetMsgForHresult(writerResult)); - continue; - } - - std::string stateName; - - switch(writerState) - { -#define WRITER_STATE(code) \ - case code: stateName = #code; break; - WRITER_STATE(VSS_WS_UNKNOWN); - WRITER_STATE(VSS_WS_STABLE); - WRITER_STATE(VSS_WS_WAITING_FOR_FREEZE); - WRITER_STATE(VSS_WS_WAITING_FOR_THAW); - WRITER_STATE(VSS_WS_WAITING_FOR_POST_SNAPSHOT); - WRITER_STATE(VSS_WS_WAITING_FOR_BACKUP_COMPLETE); - WRITER_STATE(VSS_WS_FAILED_AT_IDENTIFY); - WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_BACKUP); - WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_SNAPSHOT); - WRITER_STATE(VSS_WS_FAILED_AT_FREEZE); - WRITER_STATE(VSS_WS_FAILED_AT_THAW); - WRITER_STATE(VSS_WS_FAILED_AT_POST_SNAPSHOT); - WRITER_STATE(VSS_WS_FAILED_AT_BACKUP_COMPLETE); - WRITER_STATE(VSS_WS_FAILED_AT_PRE_RESTORE); - WRITER_STATE(VSS_WS_FAILED_AT_POST_RESTORE); - WRITER_STATE(VSS_WS_FAILED_AT_BACKUPSHUTDOWN); -#undef WRITER_STATE - default: - std::ostringstream o; - o << "unknown (" << writerState << ")"; - stateName = o.str(); - } - - BOX_TRACE("VSS: Writer " << iWriter << " (" << - writerName << ") is in state " << stateName); - } - - // lookup new snapshot volume for each location that has a snapshot - for(Locations::iterator - iLocation = mLocations.begin(); - iLocation != mLocations.end(); - iLocation++) - { - Location& rLocation(**iLocation); - if(rLocation.mIsSnapshotCreated) - { - VSS_SNAPSHOT_PROP prop; - result = mpVssBackupComponents->GetSnapshotProperties( - rLocation.mSnapshotVolumeId, &prop); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get snapshot properties " - "for volume " << GuidToString(rLocation.mSnapshotVolumeId) << - " for location " << rLocation.mPath << ": " << - GetMsgForHresult(result)); - rLocation.mIsSnapshotCreated = false; - continue; - } - - rLocation.mSnapshotPath = - WideStringToString(prop.m_pwszSnapshotDeviceObject) + - DIRECTORY_SEPARATOR + rLocation.mSnapshotPath; - VssFreeSnapshotProperties(&prop); - - BOX_INFO("VSS: Location " << rLocation.mPath << " using " - "snapshot path " << rLocation.mSnapshotPath); - } - } - - IVssEnumObject *pEnum; - result = mpVssBackupComponents->Query(GUID_NULL, VSS_OBJECT_NONE, - VSS_OBJECT_SNAPSHOT, &pEnum); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to query snapshot list: " << - GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterStatus; - } - - while(result == S_OK) - { - VSS_OBJECT_PROP rgelt; - ULONG count; - result = pEnum->Next(1, &rgelt, &count); - - if(result == S_FALSE) - { - // end of list, break out of the loop - break; - } - else if(result != S_OK) - { - BOX_ERROR("VSS: Failed to enumerate snapshot: " << - GetMsgForHresult(result)); - } - else if(count != 1) - { - BOX_ERROR("VSS: Failed to enumerate snapshot: " << - "Next() returned " << count << " objects instead of 1"); - } - else if(rgelt.Type != VSS_OBJECT_SNAPSHOT) - { - BOX_ERROR("VSS: Failed to enumerate snapshot: " << - "Next() returned a type " << rgelt.Type << " object " - "instead of VSS_OBJECT_SNAPSHOT"); - } - else - { - VSS_SNAPSHOT_PROP *pSnap = &rgelt.Obj.Snap; - BOX_TRACE("VSS: Snapshot ID: " << - GuidToString(pSnap->m_SnapshotId)); - BOX_TRACE("VSS: Snapshot set ID: " << - GuidToString(pSnap->m_SnapshotSetId)); - BOX_TRACE("VSS: Number of volumes: " << - pSnap->m_lSnapshotsCount); - BOX_TRACE("VSS: Snapshot device object: " << - WideStringToString(pSnap->m_pwszSnapshotDeviceObject)); - BOX_TRACE("VSS: Original volume name: " << - WideStringToString(pSnap->m_pwszOriginalVolumeName)); - BOX_TRACE("VSS: Originating machine: " << - WideStringToString(pSnap->m_pwszOriginatingMachine)); - BOX_TRACE("VSS: Service machine: " << - WideStringToString(pSnap->m_pwszServiceMachine)); - BOX_TRACE("VSS: Exposed name: " << - WideStringToString(pSnap->m_pwszExposedName)); - BOX_TRACE("VSS: Exposed path: " << - WideStringToString(pSnap->m_pwszExposedPath)); - BOX_TRACE("VSS: Provider ID: " << - GuidToString(pSnap->m_ProviderId)); - BOX_TRACE("VSS: Snapshot attributes: " << - BOX_FORMAT_HEX32(pSnap->m_lSnapshotAttributes)); - BOX_TRACE("VSS: Snapshot creation time: " << - BOX_FORMAT_HEX32(pSnap->m_tsCreationTimestamp)); - - std::string status; - switch(pSnap->m_eStatus) - { - case VSS_SS_UNKNOWN: status = "Unknown (error)"; break; - case VSS_SS_PREPARING: status = "Preparing"; break; - case VSS_SS_PROCESSING_PREPARE: status = "Preparing (processing)"; break; - case VSS_SS_PREPARED: status = "Prepared"; break; - case VSS_SS_PROCESSING_PRECOMMIT: status = "Precommitting"; break; - case VSS_SS_PRECOMMITTED: status = "Precommitted"; break; - case VSS_SS_PROCESSING_COMMIT: status = "Commiting"; break; - case VSS_SS_COMMITTED: status = "Committed"; break; - case VSS_SS_PROCESSING_POSTCOMMIT: status = "Postcommitting"; break; - case VSS_SS_PROCESSING_PREFINALCOMMIT: status = "Pre final committing"; break; - case VSS_SS_PREFINALCOMMITTED: status = "Pre final committed"; break; - case VSS_SS_PROCESSING_POSTFINALCOMMIT: status = "Post final committing"; break; - case VSS_SS_CREATED: status = "Created"; break; - case VSS_SS_ABORTED: status = "Aborted"; break; - case VSS_SS_DELETED: status = "Deleted"; break; - case VSS_SS_POSTCOMMITTED: status = "Postcommitted"; break; - default: - std::ostringstream buf; - buf << "Unknown code: " << pSnap->m_eStatus; - status = buf.str(); - } - - BOX_TRACE("VSS: Snapshot status: " << status); - VssFreeSnapshotProperties(pSnap); - } - } - - pEnum->Release(); - -CreateVssBackupComponents_cleanup_WriterStatus: - result = mpVssBackupComponents->FreeWriterStatus(); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to free writer status: " << - GetMsgForHresult(result)); - } - -CreateVssBackupComponents_cleanup_WriterMetadata: - result = mpVssBackupComponents->FreeWriterMetadata(); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to free writer metadata: " << - GetMsgForHresult(result)); - } -} - -void BackupDaemon::CleanupVssBackupComponents() -{ - if(mpVssBackupComponents == NULL) - { - return; - } - - CallAndWaitForAsync(&IVssBackupComponents::BackupComplete, - "BackupComplete()"); - - mpVssBackupComponents->Release(); - mpVssBackupComponents = NULL; -} -#endif - -void BackupDaemon::OnBackupStart() -{ - ResetLogFile(); - - // Touch a file to record times in filesystem - TouchFileInWorkingDir("last_sync_start"); - - // Reset statistics on uploads - BackupStoreFile::ResetStats(); - - // Tell anything connected to the command socket - SendSyncStartOrFinish(true /* start */); - - // Notify administrator - NotifySysadmin(SysadminNotifier::BackupStart); - - // Setup timer for polling the command socket - mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, - "CommandSocketPollTimer")); - - // Set state and log start - SetState(State_Connected); - BOX_NOTICE("Beginning scan of local files"); -} - -void BackupDaemon::OnBackupFinish() -{ - try - { - // Log - BOX_NOTICE("Finished scan of local files"); - - // Log the stats - BOX_NOTICE("File statistics: total file size uploaded " - << BackupStoreFile::msStats.mBytesInEncodedFiles - << ", bytes already on server " - << BackupStoreFile::msStats.mBytesAlreadyOnServer - << ", encoded size " - << BackupStoreFile::msStats.mTotalFileStreamSize - << ", " << mNumFilesUploaded << " files uploaded, " - << mNumDirsCreated << " dirs created"); - - // Reset statistics again - BackupStoreFile::ResetStats(); - - // Notify administrator - NotifySysadmin(SysadminNotifier::BackupFinish); - - // Stop the timer for polling the command socket, - // to prevent needless alarms while sleeping. - mapCommandSocketPollTimer.reset(); - - // Tell anything connected to the command socket - SendSyncStartOrFinish(false /* finish */); - - // Touch a file to record times in filesystem - TouchFileInWorkingDir("last_sync_finish"); - } - catch (std::exception &e) - { - BOX_ERROR("Failed to perform backup finish actions: " << e.what()); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::UseScriptToSeeIfSyncAllowed() -// Purpose: Private. Use a script to see if the sync should be -// allowed now (if configured). Returns -1 if it's -// allowed, time in seconds to wait otherwise. -// Created: 21/6/04 -// -// -------------------------------------------------------------------------- -int BackupDaemon::UseScriptToSeeIfSyncAllowed() -{ - const Configuration &conf(GetConfiguration()); - - // Got a script to run? - if(!conf.KeyExists("SyncAllowScript")) - { - // No. Do sync. - return -1; - } - - // If there's no result, try again in five minutes - int waitInSeconds = (60*5); - - std::string script(conf.GetKeyValue("SyncAllowScript") + - " \"" + GetConfigFileName() + "\""); - - // Run it? - pid_t pid = 0; - try - { - std::auto_ptr<IOStream> pscript(LocalProcessStream(script, - pid)); - - // Read in the result - IOStreamGetLine getLine(*pscript); - std::string line; - if(getLine.GetLine(line, true, 30000)) // 30 seconds should be enough - { - waitInSeconds = BackupDaemon::ParseSyncAllowScriptOutput(script, line); - } - else - { - BOX_ERROR("SyncAllowScript output nothing within " - "30 seconds, waiting 5 minutes to try again" - " (" << script << ")"); - } - } - catch(std::exception &e) - { - BOX_ERROR("Internal error running SyncAllowScript: " - << e.what() << " (" << script << ")"); - } - catch(...) - { - // Ignore any exceptions - // Log that something bad happened - BOX_ERROR("Unknown error running SyncAllowScript (" << - script << ")"); - } - - // Wait and then cleanup child process, if any - if(pid != 0) - { - int status = 0; - ::waitpid(pid, &status, 0); - } - - return waitInSeconds; -} - -int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, - const std::string& output) -{ - int waitInSeconds = (60*5); - std::istringstream iss(output); - - std::string delay; - iss >> delay; - - if(delay == "") - { - BOX_ERROR("SyncAllowScript output an empty line, sleeping for " - << waitInSeconds << " seconds (" << script << ")"); - return waitInSeconds; - } - - // Got a string, interpret - if(delay == "now") - { - // Script says do it now. Obey. - waitInSeconds = -1; - - BOX_NOTICE("SyncAllowScript requested a backup now " - "(" << script << ")"); - } - else - { - try - { - // How many seconds to wait? - waitInSeconds = BoxConvert::Convert<int32_t, const std::string&>(delay); - } - catch(ConversionException &e) - { - BOX_ERROR("SyncAllowScript output an invalid " - "number: '" << output << "' (" << - script << ")"); - throw; - } - - BOX_NOTICE("SyncAllowScript requested a delay of " << - waitInSeconds << " seconds (" << script << ")"); - } - - if(iss.eof()) - { - // No bandwidth limit requested - mMaxBandwidthFromSyncAllowScript = 0; - BOX_NOTICE("SyncAllowScript did not set a maximum bandwidth " - "(" << script << ")"); - } - else - { - std::string maxBandwidth; - iss >> maxBandwidth; - - try - { - // How many seconds to wait? - mMaxBandwidthFromSyncAllowScript = - BoxConvert::Convert<int32_t, const std::string&>(maxBandwidth); - } - catch(ConversionException &e) - { - BOX_ERROR("Invalid maximum bandwidth from " - "SyncAllowScript: '" << - output << "' (" << script << ")"); - throw; - } - - BOX_NOTICE("SyncAllowScript set maximum bandwidth to " << - mMaxBandwidthFromSyncAllowScript << " kB/s (" << - script << ")"); - } - - return waitInSeconds; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::RunBackgroundTask() -// Purpose: Checks for connections or commands on the command -// socket and handles them with minimal delay. Polled -// during lengthy operations such as file uploads. -// Created: 07/04/14 -// -// -------------------------------------------------------------------------- -bool BackupDaemon::RunBackgroundTask(State state, uint64_t progress, - uint64_t maximum) -{ - BOX_TRACE("BackupDaemon::RunBackgroundTask: state = " << state << - ", progress = " << progress << "/" << maximum); - - if(!mapCommandSocketPollTimer.get()) - { - return true; // no background task - } - - if(mapCommandSocketPollTimer->HasExpired()) - { - mapCommandSocketPollTimer->Reset(COMMAND_SOCKET_POLL_INTERVAL); - } - else - { - // Do no more work right now - return true; - } - - if(mapCommandSocketInfo.get()) - { - BOX_TRACE("BackupDaemon::RunBackgroundTask: polling command socket"); - - bool sync_flag_out, sync_is_forced_out; - - WaitOnCommandSocket(0, // RequiredDelay - sync_flag_out, sync_is_forced_out); - - if(sync_flag_out) - { - BOX_WARNING("Ignoring request to sync while " - "already syncing."); - } - } - - return true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::WaitOnCommandSocket(box_time_t, bool &, bool &) -// Purpose: Waits on a the command socket for a time of UP TO -// the required time but may be much less, and handles -// a command if necessary. -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut) -{ - DoSyncFlagOut = false; - SyncIsForcedOut = false; - - ASSERT(mapCommandSocketInfo.get()); - if(!mapCommandSocketInfo.get()) - { - // failure case isn't too bad - ::sleep(1); - return; - } - - BOX_TRACE("Wait on command socket, delay = " << - BOX_FORMAT_MICROSECONDS(RequiredDelay)); - - try - { - // Timeout value for connections and things - int timeout = ((int)BoxTimeToMilliSeconds(RequiredDelay)) + 1; - // Handle bad boundary cases - if(timeout <= 0) timeout = 1; - if(timeout == INFTIM) timeout = 100000; - - // Wait for socket connection, or handle a command? - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - // No connection, listen for a new one - mapCommandSocketInfo->mpConnectedSocket.reset(mapCommandSocketInfo->mListeningSocket.Accept(timeout).release()); - - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - // If a connection didn't arrive, there was a timeout, which means we've - // waited long enough and it's time to go. - return; - } - else - { -#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET - bool uidOK = true; - BOX_WARNING("On this platform, no security check can be made on the credentials of peers connecting to the command socket. (bbackupctl)"); -#else - // Security check -- does the process connecting to this socket have - // the same UID as this process? - bool uidOK = false; - // BLOCK - { - uid_t remoteEUID = 0xffff; - gid_t remoteEGID = 0xffff; - if(mapCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID)) - { - // Credentials are available -- check UID - if(remoteEUID == ::getuid()) - { - // Acceptable - uidOK = true; - } - } - } -#endif // PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET - - // Is this an acceptable connection? - if(!uidOK) - { - // Dump the connection - BOX_ERROR("Incoming command connection from peer had different user ID than this process, or security check could not be completed."); - mapCommandSocketInfo->mpConnectedSocket.reset(); - return; - } - else - { - // Log - BOX_INFO("Connection from command socket"); - - // Send a header line summarising the configuration and current state - const Configuration &conf(GetConfiguration()); - std::ostringstream hello; - hello << "bbackupd: " << - (conf.GetKeyValueBool("AutomaticBackup") ? 1 : 0) - << " " << - conf.GetKeyValueInt("UpdateStoreInterval") - << " " << - conf.GetKeyValueInt("MinimumFileAge") - << " " << - conf.GetKeyValueInt("MaxUploadWait") - << "\nstate " << mState << "\n"; - mapCommandSocketInfo->mpConnectedSocket->Write( - hello.str(), timeout); - - // Set the timeout to something very small, so we don't wait too long on waiting - // for any incoming data - timeout = 10; // milliseconds - } - } - } - - // So there must be a connection now. - ASSERT(mapCommandSocketInfo->mpConnectedSocket.get() != 0); - - // Is there a getline object ready? - if(mapCommandSocketInfo->mpGetLine == 0) - { - // Create a new one - mapCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mapCommandSocketInfo->mpConnectedSocket.get())); - } - - // Ping the remote side, to provide errors which will mean the socket gets closed - mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5, - timeout); - - // Wait for a command or something on the socket - std::string command; - while(mapCommandSocketInfo->mpGetLine != 0 && !mapCommandSocketInfo->mpGetLine->IsEOF() - && mapCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) - { - BOX_TRACE("Receiving command '" << command - << "' over command socket"); - - bool sendOK = false; - bool sendResponse = true; - - // Command to process! - if(command == "quit" || command == "") - { - // Close the socket. - CloseCommandConnection(); - sendResponse = false; - } - else if(command == "sync") - { - // Sync now! - DoSyncFlagOut = true; - SyncIsForcedOut = false; - sendOK = true; - } - else if(command == "force-sync") - { - // Sync now (forced -- overrides any SyncAllowScript) - DoSyncFlagOut = true; - SyncIsForcedOut = true; - sendOK = true; - } - else if(command == "reload") - { - // Reload the configuration - SetReloadConfigWanted(); - sendOK = true; - } - else if(command == "terminate") - { - // Terminate the daemon cleanly - SetTerminateWanted(); - sendOK = true; - } - - // Send a response back? - if(sendResponse) - { - std::string response = sendOK ? "ok\n" : "error\n"; - mapCommandSocketInfo->mpConnectedSocket->Write( - response, timeout); - } - - // Set timeout to something very small, so this just checks for data which is waiting - timeout = 1; - } - - // Close on EOF? - if(mapCommandSocketInfo->mpGetLine != 0 && mapCommandSocketInfo->mpGetLine->IsEOF()) - { - CloseCommandConnection(); - } - } - catch(ConnectionException &ce) - { - BOX_NOTICE("Failed to write to command socket: " << ce.what()); - - // If an error occurs, and there is a connection active, - // just close that connection and continue. Otherwise, - // let the error propagate. - - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - throw; // thread will die - } - else - { - // Close socket and ignore error - CloseCommandConnection(); - } - } - catch(std::exception &e) - { - BOX_ERROR("Failed to write to command socket: " << - e.what()); - - // If an error occurs, and there is a connection active, - // just close that connection and continue. Otherwise, - // let the error propagate. - - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - throw; // thread will die - } - else - { - // Close socket and ignore error - CloseCommandConnection(); - } - } - catch(...) - { - BOX_ERROR("Failed to write to command socket: unknown error"); - - // If an error occurs, and there is a connection active, - // just close that connection and continue. Otherwise, - // let the error propagate. - - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - throw; // thread will die - } - else - { - // Close socket and ignore error - CloseCommandConnection(); - } - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::CloseCommandConnection() -// Purpose: Close the command connection, ignoring any errors -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::CloseCommandConnection() -{ - try - { - BOX_TRACE("Closing command connection"); - - if(mapCommandSocketInfo->mpGetLine) - { - delete mapCommandSocketInfo->mpGetLine; - mapCommandSocketInfo->mpGetLine = 0; - } - mapCommandSocketInfo->mpConnectedSocket.reset(); - } - catch(std::exception &e) - { - BOX_ERROR("Internal error while closing command " - "socket: " << e.what()); - } - catch(...) - { - // Ignore any errors - } -} - - -// -------------------------------------------------------------------------- -// -// File -// Name: BackupDaemon.cpp -// Purpose: Send a start or finish sync message to the command socket, if it's connected. -// -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::SendSyncStartOrFinish(bool SendStart) -{ - // The bbackupctl program can't rely on a state change, because it - // may never change if the server doesn't need to be contacted. - - if(mapCommandSocketInfo.get() && - mapCommandSocketInfo->mpConnectedSocket.get() != 0) - { - std::string message = SendStart ? "start-sync" : "finish-sync"; - try - { - message += "\n"; - mapCommandSocketInfo->mpConnectedSocket->Write(message, - 1); // short timeout, it's overlapped - } - catch(std::exception &e) - { - BOX_ERROR("Internal error while sending to " - "command socket client: " << e.what()); - CloseCommandConnection(); - } - catch(...) - { - CloseCommandConnection(); - } - } -} - - - - -#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_NMTONNAME) - // string comparison ordering for when mount points are handled - // by code, rather than the OS. - typedef struct - { - bool operator()(const std::string &s1, const std::string &s2) - { - if(s1.size() == s2.size()) - { - // Equal size, sort according to natural sort order - return s1 < s2; - } - else - { - // Make sure longer strings go first - return s1.size() > s2.size(); - } - } - } mntLenCompare; -#endif - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::SetupLocations(BackupClientContext &, const Configuration &) -// Purpose: Makes sure that the list of directories records is correctly set up -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf) -{ - // Going to need a copy of the root directory. Get a connection, - // and fetch it. - BackupProtocolCallable& connection(rClientContext.GetConnection()); - - // Ask server for a list of everything in the root directory, - // which is a directory itself - std::auto_ptr<BackupProtocolSuccess> dirreply( - connection.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - // only directories - BackupProtocolListDirectory::Flags_Dir, - // exclude old/deleted stuff - BackupProtocolListDirectory::Flags_Deleted | - BackupProtocolListDirectory::Flags_OldVersion, - false /* no attributes */)); - - // Retrieve the directory from the stream following - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(connection.ReceiveStream()); - dir.ReadFromStream(*dirstream, connection.GetTimeout()); - - // Map of mount names to ID map index - std::map<std::string, int> mounts; - int numIDMaps = 0; - -#ifdef HAVE_MOUNTS -#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_MNTONNAME) - // Linux and others can't tell you where a directory is mounted. So we - // have to read the mount entries from /etc/mtab! Bizarre that the OS - // itself can't tell you, but there you go. - std::set<std::string, mntLenCompare> mountPoints; - // BLOCK - FILE *mountPointsFile = 0; - -#ifdef HAVE_STRUCT_MNTENT_MNT_DIR - // Open mounts file - mountPointsFile = ::setmntent("/proc/mounts", "r"); - if(mountPointsFile == 0) - { - mountPointsFile = ::setmntent("/etc/mtab", "r"); - } - if(mountPointsFile == 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } - - try - { - // Read all the entries, and put them in the set - struct mntent *entry = 0; - while((entry = ::getmntent(mountPointsFile)) != 0) - { - BOX_TRACE("Found mount point at " << entry->mnt_dir); - mountPoints.insert(std::string(entry->mnt_dir)); - } - - // Close mounts file - ::endmntent(mountPointsFile); - } - catch(...) - { - ::endmntent(mountPointsFile); - throw; - } -#else // ! HAVE_STRUCT_MNTENT_MNT_DIR - // Open mounts file - mountPointsFile = ::fopen("/etc/mnttab", "r"); - if(mountPointsFile == 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } - - try - { - // Read all the entries, and put them in the set - struct mnttab entry; - while(getmntent(mountPointsFile, &entry) == 0) - { - BOX_TRACE("Found mount point at " << entry.mnt_mountp); - mountPoints.insert(std::string(entry.mnt_mountp)); - } - - // Close mounts file - ::fclose(mountPointsFile); - } - catch(...) - { - ::fclose(mountPointsFile); - throw; - } -#endif // HAVE_STRUCT_MNTENT_MNT_DIR - // Check sorting and that things are as we expect - ASSERT(mountPoints.size() > 0); -#ifndef BOX_RELEASE_BUILD - { - std::set<std::string, mntLenCompare>::reverse_iterator i(mountPoints.rbegin()); - ASSERT(*i == "/"); - } -#endif // n BOX_RELEASE_BUILD -#endif // n HAVE_STRUCT_STATFS_F_MNTONNAME || n HAVE_STRUCT_STATVFS_F_MNTONNAME -#endif // HAVE_MOUNTS - - // Then... go through each of the entries in the configuration, - // making sure there's a directory created for it. - std::vector<std::string> locNames = - rLocationsConf.GetSubConfigurationNames(); - - // We only want completely configured locations to be in the list - // when this function exits, so move them all to a temporary list. - // Entries matching a properly configured location will be moved - // back to mLocations. Anything left in this list after the loop - // finishes will be deleted. - Locations tmpLocations = mLocations; - mLocations.clear(); - - // The ID map list will be repopulated automatically by this loop - mIDMapMounts.clear(); - - for(std::vector<std::string>::iterator - pLocName = locNames.begin(); - pLocName != locNames.end(); - pLocName++) - { - Location* pLoc = NULL; - - // Try to find and reuse an existing Location object - for(Locations::const_iterator - i = tmpLocations.begin(); - i != tmpLocations.end(); i++) - { - if ((*i)->mName == *pLocName) - { - BOX_TRACE("Location already configured: " << *pLocName); - pLoc = *i; - break; - } - } - - const Configuration& rConfig( - rLocationsConf.GetSubConfiguration(*pLocName)); - std::auto_ptr<Location> apLoc; - - try - { - if(pLoc == NULL) - { - // Create a record for it - BOX_TRACE("New location: " << *pLocName); - pLoc = new Location; - - // ensure deletion if setup fails - apLoc.reset(pLoc); - - // Setup names in the location record - pLoc->mName = *pLocName; - pLoc->mPath = rConfig.GetKeyValue("Path"); - } - - // Read the exclude lists from the Configuration - pLoc->mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rConfig)); - pLoc->mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rConfig)); - - // Does this exist on the server? - // Remove from dir object early, so that if we fail - // to stat the local directory, we still don't - // consider to remote one for deletion. - BackupStoreDirectory::Iterator iter(dir); - BackupStoreFilenameClear dirname(pLoc->mName); // generate the filename - BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname); - int64_t oid = 0; - if(en != 0) - { - oid = en->GetObjectID(); - - // Delete the entry from the directory, so we get a list of - // unused root directories at the end of this. - dir.DeleteEntry(oid); - } - - // Do a fsstat on the pathname to find out which mount it's on - { - -#if defined HAVE_STRUCT_STATFS_F_MNTONNAME || defined HAVE_STRUCT_STATVFS_F_MNTONNAME || defined WIN32 - - // BSD style statfs -- includes mount point, which is nice. -#ifdef HAVE_STRUCT_STATVFS_F_MNTONNAME - struct statvfs s; - if(::statvfs(pLoc->mPath.c_str(), &s) != 0) -#else // HAVE_STRUCT_STATVFS_F_MNTONNAME - struct statfs s; - if(::statfs(pLoc->mPath.c_str(), &s) != 0) -#endif // HAVE_STRUCT_STATVFS_F_MNTONNAME - { - THROW_SYS_ERROR("Failed to stat path " - "'" << pLoc->mPath << "' " - "for location " - "'" << pLoc->mName << "'", - CommonException, OSFileError); - } - - // Where the filesystem is mounted - std::string mountName(s.f_mntonname); - -#else // !HAVE_STRUCT_STATFS_F_MNTONNAME && !WIN32 - - // Warn in logs if the directory isn't absolute - if(pLoc->mPath[0] != '/') - { - BOX_WARNING("Location path '" - << pLoc->mPath - << "' is not absolute"); - } - // Go through the mount points found, and find a suitable one - std::string mountName("/"); - { - std::set<std::string, mntLenCompare>::const_iterator i(mountPoints.begin()); - BOX_TRACE(mountPoints.size() - << " potential mount points"); - for(; i != mountPoints.end(); ++i) - { - // Compare first n characters with the filename - // If it matches, the file belongs in that mount point - // (sorting order ensures this) - BOX_TRACE("checking against mount point " << *i); - if(::strncmp(i->c_str(), pLoc->mPath.c_str(), i->size()) == 0) - { - // Match - mountName = *i; - break; - } - } - BOX_TRACE("mount point chosen for " - << pLoc->mPath << " is " - << mountName); - } - -#endif - - // Got it? - std::map<std::string, int>::iterator f(mounts.find(mountName)); - if(f != mounts.end()) - { - // Yes -- store the index - pLoc->mIDMapIndex = f->second; - } - else - { - // No -- new index - pLoc->mIDMapIndex = numIDMaps; - mounts[mountName] = numIDMaps; - - // Store the mount name - mIDMapMounts.push_back(mountName); - - // Increment number of maps - ++numIDMaps; - } - } - - // Does this exist on the server? - if(en == 0) - { - // Doesn't exist, so it has to be created on the server. Let's go! - // First, get the directory's attributes and modification time - box_time_t attrModTime = 0; - BackupClientFileAttributes attr; - try - { - attr.ReadAttributes(pLoc->mPath.c_str(), - true /* directories have zero mod times */, - 0 /* not interested in mod time */, - &attrModTime /* get the attribute modification time */); - } - catch (BoxException &e) - { - BOX_ERROR("Failed to get attributes " - "for path '" << pLoc->mPath - << "', skipping location '" << - pLoc->mName << "'"); - throw; - } - - // Execute create directory command - try - { - std::auto_ptr<IOStream> attrStream( - new MemBlockStream(attr)); - std::auto_ptr<BackupProtocolSuccess> - dirCreate(connection.QueryCreateDirectory( - BACKUPSTORE_ROOT_DIRECTORY_ID, // containing directory - attrModTime, dirname, attrStream)); - - // Object ID for later creation - oid = dirCreate->GetObjectID(); - } - catch (BoxException &e) - { - BOX_ERROR("Failed to create remote " - "directory '/" << pLoc->mName << - "', skipping location '" << - pLoc->mName << "'"); - throw; - } - - } - - // Create and store the directory object for the root of this location - ASSERT(oid != 0); - if(pLoc->mapDirectoryRecord.get() == NULL) - { - pLoc->mapDirectoryRecord.reset( - new BackupClientDirectoryRecord(oid, *pLocName)); - } - - // Remove it from the temporary list to avoid deletion - tmpLocations.remove(pLoc); - - // Push it back on the vector of locations - mLocations.push_back(pLoc); - - if(apLoc.get() != NULL) - { - // Don't delete it now! - apLoc.release(); - } - } - catch (std::exception &e) - { - BOX_ERROR("Failed to configure location '" - << pLoc->mName << "' path '" - << pLoc->mPath << "': " << e.what() << - ": please check for previous errors"); - mReadErrorsOnFilesystemObjects = true; - } - catch(...) - { - BOX_ERROR("Failed to configure location '" - << pLoc->mName << "' path '" - << pLoc->mPath << "': please check for " - "previous errors"); - mReadErrorsOnFilesystemObjects = true; - } - } - - // Now remove any leftovers - for(BackupDaemon::Locations::iterator - i = tmpLocations.begin(); - i != tmpLocations.end(); i++) - { - BOX_INFO("Removing obsolete location from memory: " << - (*i)->mName); - delete *i; - } - - tmpLocations.clear(); - - // Any entries in the root directory which need deleting? - if(dir.GetNumberOfEntries() > 0 && - mDeleteRedundantLocationsAfter == 0) - { - BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " - "in root directory found, but will not delete because " - "DeleteRedundantLocationsAfter = 0"); - } - else if(dir.GetNumberOfEntries() > 0) - { - box_time_t now = GetCurrentBoxTime(); - - // This should reset the timer if the list of unused - // locations changes, but it will not if the number of - // unused locations does not change, but the locations - // do change, e.g. one mysteriously appears and another - // mysteriously appears. (FIXME) - if (dir.GetNumberOfEntries() != mUnusedRootDirEntries.size() || - mDeleteUnusedRootDirEntriesAfter == 0) - { - mDeleteUnusedRootDirEntriesAfter = now + - SecondsToBoxTime(mDeleteRedundantLocationsAfter); - } - - int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter - - now); - - BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " - "in root directory found, will delete from store " - "after " << secs << " seconds."); - - // Store directories in list of things to delete - mUnusedRootDirEntries.clear(); - BackupStoreDirectory::Iterator iter(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = iter.Next()) != 0) - { - // Add name to list - BackupStoreFilenameClear clear(en->GetName()); - const std::string &name(clear.GetClearFilename()); - mUnusedRootDirEntries.push_back( - std::pair<int64_t,std::string> - (en->GetObjectID(), name)); - // Log this - BOX_INFO("Unused location in root: " << name); - } - ASSERT(mUnusedRootDirEntries.size() > 0); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::SetupIDMapsForSync() -// Purpose: Sets up ID maps for the sync process -- make sure they're all there -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::SetupIDMapsForSync() -{ - // Make sure we have some blank, empty ID maps - DeleteIDMapVector(mNewIDMaps); - FillIDMapVector(mNewIDMaps, true /* new maps */); - DeleteIDMapVector(mCurrentIDMaps); - FillIDMapVector(mCurrentIDMaps, false /* new maps */); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &) -// Purpose: Fills the vector with the right number of empty ID maps -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps) -{ - ASSERT(rVector.size() == 0); - rVector.reserve(mIDMapMounts.size()); - - for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) - { - // Create the object - BackupClientInodeToIDMap *pmap = new BackupClientInodeToIDMap(); - try - { - // Get the base filename of this map - std::string filename; - MakeMapBaseName(l, filename); - - // If it's a new one, add a suffix - if(NewMaps) - { - filename += ".n"; - } - - // The new map file should not exist yet. If there's - // one left over from a previous failed run, it's not - // useful to us because we never read from it and will - // overwrite the entries of all files that still - // exist, so we should just delete it and start afresh. - if(NewMaps && FileExists(filename.c_str())) - { - BOX_NOTICE("Found an incomplete ID map " - "database, deleting it to start " - "afresh: " << filename); - if(unlink(filename.c_str()) != 0) - { - BOX_LOG_NATIVE_ERROR(BOX_FILE_MESSAGE( - filename, "Failed to delete " - "incomplete ID map database")); - } - } - - // If it's not a new map, it may not exist in which case an empty map should be created - if(!NewMaps && !FileExists(filename.c_str())) - { - pmap->OpenEmpty(); - } - else - { - // Open the map - pmap->Open(filename.c_str(), !NewMaps /* read only */, NewMaps /* create new */); - } - - // Store on vector - rVector.push_back(pmap); - } - catch(...) - { - delete pmap; - throw; - } - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeleteCorruptBerkelyDbFiles() -// Purpose: Delete the Berkely db files from disc after they have been corrupted. -// Created: 14/9/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::DeleteCorruptBerkelyDbFiles() -{ - for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) - { - // Get the base filename of this map - std::string filename; - MakeMapBaseName(l, filename); - - // Delete the file - BOX_TRACE("Deleting " << filename); - ::unlink(filename.c_str()); - - // Add a suffix for the new map - filename += ".n"; - - // Delete that too - BOX_TRACE("Deleting " << filename); - ::unlink(filename.c_str()); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: MakeMapBaseName(unsigned int, std::string &) -// Purpose: Makes the base name for a inode map -// Created: 20/11/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const -{ - // Get the directory for the maps - const Configuration &config(GetConfiguration()); - std::string dir(config.GetKeyValue("DataDirectory")); - - // Make a leafname - std::string leaf(mIDMapMounts[MountNumber]); - for(unsigned int z = 0; z < leaf.size(); ++z) - { - if(leaf[z] == DIRECTORY_SEPARATOR_ASCHAR) - { - leaf[z] = '_'; - } - } - - // Build the final filename - rNameOut = dir + DIRECTORY_SEPARATOR "mnt" + leaf; -} - - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::CommitIDMapsAfterSync() -// Purpose: Commits the new ID maps, so the 'new' maps are now the 'current' maps. -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::CommitIDMapsAfterSync() -{ - // Get rid of the maps in memory (leaving them on disc of course) - DeleteIDMapVector(mCurrentIDMaps); - DeleteIDMapVector(mNewIDMaps); - - // Then move the old maps into the new places - for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) - { - std::string target; - MakeMapBaseName(l, target); - std::string newmap(target + ".n"); - - // Try to rename -#ifdef WIN32 - // win32 rename doesn't overwrite existing files - ::remove(target.c_str()); -#endif - if(::rename(newmap.c_str(), target.c_str()) != 0) - { - BOX_LOG_SYS_ERROR("Failed to rename ID map: " << - newmap << " to " << target); - THROW_EXCEPTION(CommonException, OSFileError) - } - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &) -// Purpose: Deletes the contents of a vector of ID maps -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector) -{ - while(!rVector.empty()) - { - // Pop off list - BackupClientInodeToIDMap *toDel = rVector.back(); - rVector.pop_back(); - - // Close and delete - delete toDel; - } - ASSERT(rVector.size() == 0); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::FindLocationPathName(const std::string &, std::string &) const -// Purpose: Tries to find the path of the root of a backup location. Returns true (and path in rPathOut) -// if it can be found, false otherwise. -// Created: 12/11/03 -// -// -------------------------------------------------------------------------- -bool BackupDaemon::FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const -{ - // Search for the location - for(Locations::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i) - { - if((*i)->mName == rLocationName) - { - rPathOut = (*i)->mPath; - return true; - } - } - - // Didn't find it - return false; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::SetState(int) -// Purpose: Record current action of daemon, and update process title to reflect this -// Created: 11/12/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::SetState(int State) -{ - // Two little checks - if(State == mState) return; - if(State < 0) return; - - // Update - mState = State; - - // Set process title - const static char *stateText[] = {"idle", "connected", "error -- waiting for retry", "over limit on server -- not backing up"}; - SetProcessTitle(stateText[State]); - - // If there's a command socket connected, then inform it -- disconnecting from the - // command socket if there's an error - - std::ostringstream msg; - msg << "state " << State << "\n"; - - if(!mapCommandSocketInfo.get()) - { - return; - } - - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - return; - } - - // Something connected to the command socket, tell it about the new state - try - { - mapCommandSocketInfo->mpConnectedSocket->Write(msg.str(), - 1); // very short timeout, it's overlapped anyway - } - catch(ConnectionException &ce) - { - BOX_NOTICE("Failed to write state to command socket: " << - ce.what()); - CloseCommandConnection(); - } - catch(std::exception &e) - { - BOX_ERROR("Failed to write state to command socket: " << - e.what()); - CloseCommandConnection(); - } - catch(...) - { - BOX_ERROR("Failed to write state to command socket: " - "unknown error"); - CloseCommandConnection(); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::TouchFileInWorkingDir(const char *) -// Purpose: Make sure a zero length file of the name exists in the working directory. -// Use for marking times of events in the filesystem. -// Created: 21/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::TouchFileInWorkingDir(const char *Filename) -{ - // Filename - const Configuration &config(GetConfiguration()); - std::string fn(config.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); - fn += Filename; - - // Open and close it to update the timestamp - try - { - FileStream touch(fn, O_WRONLY | O_CREAT | O_TRUNC, - S_IRUSR | S_IWUSR); - } - catch (std::exception &e) - { - BOX_ERROR("Failed to write to timestamp file: " << fn << ": " << - e.what()); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::NotifySysadmin(int) -// Purpose: Run the script to tell the sysadmin about events -// which need attention. -// Created: 25/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::NotifySysadmin(SysadminNotifier::EventCode Event) -{ - static const char *sEventNames[] = - { - "store-full", - "read-error", - "backup-error", - "backup-start", - "backup-finish", - "backup-ok", - 0 - }; - - // BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames)); - // BOX_TRACE("sizeof(*sEventNames) == " << sizeof(*sEventNames)); - // BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX); - ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == SysadminNotifier::MAX + 1); - - if(Event < 0 || Event >= SysadminNotifier::MAX) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, - BadNotifySysadminEventCode, "NotifySysadmin() called " - "for unknown event code " << Event); - } - - BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << - sEventNames[Event]); - - if(!GetConfiguration().KeyExists("NotifyAlways") || - !GetConfiguration().GetKeyValueBool("NotifyAlways")) - { - // Don't send lots of repeated messages - // Note: backup-start and backup-finish will always be - // logged, because mLastNotifiedEvent is never set to - // these values and therefore they are never "duplicates". - if(mLastNotifiedEvent == Event) - { - if(Event == SysadminNotifier::BackupOK) - { - BOX_INFO("Suppressing duplicate notification " - "about " << sEventNames[Event]); - } - else - { - BOX_WARNING("Suppressing duplicate notification " - "about " << sEventNames[Event]); - } - return; - } - } - - // Is there a notification script? - const Configuration &conf(GetConfiguration()); - if(!conf.KeyExists("NotifyScript")) - { - // Log, and then return - if(Event != SysadminNotifier::BackupStart && - Event != SysadminNotifier::BackupFinish) - { - BOX_INFO("Not notifying administrator about event " - << sEventNames[Event] << ", set NotifyScript " - "to do this in future"); - } - return; - } - - // Script to run - std::string script(conf.GetKeyValue("NotifyScript") + " " + - sEventNames[Event] + " \"" + GetConfigFileName() + "\""); - - // Log what we're about to do - BOX_INFO("About to notify administrator about event " - << sEventNames[Event] << ", running script '" << script << "'"); - - // Then do it - int returnCode = ::system(script.c_str()); - if(returnCode != 0) - { - BOX_WARNING("Notify script returned error code: " << - returnCode << " (" << script << ")"); - } - else if(Event != SysadminNotifier::BackupStart && - Event != SysadminNotifier::BackupFinish) - { - mLastNotifiedEvent = Event; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &) -// Purpose: Deletes any unused entries in the root directory, if they're scheduled to be deleted. -// Created: 13/5/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) -{ - if(mUnusedRootDirEntries.empty()) - { - BOX_INFO("Not deleting unused entries - none in list"); - return; - } - - if(mDeleteUnusedRootDirEntriesAfter == 0) - { - BOX_INFO("Not deleting unused entries - " - "zero delete time (bad)"); - return; - } - - // Check time - box_time_t now = GetCurrentBoxTime(); - if(now < mDeleteUnusedRootDirEntriesAfter) - { - int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter - - now); - BOX_INFO("Not deleting unused entries - too early (" - << secs << " seconds remaining)"); - return; - } - - // Entries to delete, and it's the right time to do so... - BOX_NOTICE("Deleting unused locations from store root..."); - BackupProtocolCallable &connection(rContext.GetConnection()); - for(std::vector<std::pair<int64_t,std::string> >::iterator - i(mUnusedRootDirEntries.begin()); - i != mUnusedRootDirEntries.end(); ++i) - { - connection.QueryDeleteDirectory(i->first); - rContext.GetProgressNotifier().NotifyFileDeleted( - i->first, i->second); - } - - // Reset state - mDeleteUnusedRootDirEntriesAfter = 0; - mUnusedRootDirEntries.clear(); -} - -// -------------------------------------------------------------------------- - -typedef struct -{ - int32_t mMagicValue; // also the version number - int32_t mNumEntries; - int64_t mObjectID; // this object ID - int64_t mContainerID; // ID of container - uint64_t mAttributesModTime; - int32_t mOptionsPresent; // bit mask of optional sections / features present - -} loc_StreamFormat; - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::CommandSocketInfo::CommandSocketInfo() -// Purpose: Constructor -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -BackupDaemon::CommandSocketInfo::CommandSocketInfo() - : mpGetLine(0) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::CommandSocketInfo::~CommandSocketInfo() -// Purpose: Destructor -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -BackupDaemon::CommandSocketInfo::~CommandSocketInfo() -{ - if(mpGetLine) - { - delete mpGetLine; - mpGetLine = 0; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::SerializeStoreObjectInfo( -// box_time_t theLastSyncTime, -// box_time_t theNextSyncTime) -// Purpose: Serializes remote directory and file information -// into a stream of bytes, using an Archive -// abstraction. -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- - -static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; -static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; -static const int STOREOBJECTINFO_VERSION = 2; - -bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime, - box_time_t theNextSyncTime) const -{ - if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) - { - return false; - } - - std::string StoreObjectInfoFile = - GetConfiguration().GetKeyValue("StoreObjectInfoFile"); - - if(StoreObjectInfoFile.size() <= 0) - { - return false; - } - - bool created = false; - - try - { - FileStream aFile(StoreObjectInfoFile.c_str(), - O_WRONLY | O_CREAT | O_TRUNC); - created = true; - - Archive anArchive(aFile, 0); - - anArchive.Write(STOREOBJECTINFO_MAGIC_ID_VALUE); - anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); - anArchive.Write(STOREOBJECTINFO_VERSION); - anArchive.Write(GetLoadedConfigModifiedTime()); - anArchive.Write(mClientStoreMarker); - anArchive.Write(theLastSyncTime); - anArchive.Write(theNextSyncTime); - - // - // - // - int64_t iCount = mLocations.size(); - anArchive.Write(iCount); - - for(Locations::const_iterator i = mLocations.begin(); - i != mLocations.end(); i++) - { - ASSERT(*i); - (*i)->Serialize(anArchive); - } - - // - // - // - iCount = mIDMapMounts.size(); - anArchive.Write(iCount); - - for(int v = 0; v < iCount; v++) - anArchive.Write(mIDMapMounts[v]); - - // - // - // - iCount = mUnusedRootDirEntries.size(); - anArchive.Write(iCount); - - for(int v = 0; v < iCount; v++) - { - anArchive.Write(mUnusedRootDirEntries[v].first); - anArchive.Write(mUnusedRootDirEntries[v].second); - } - - if (iCount > 0) - { - anArchive.Write(mDeleteUnusedRootDirEntriesAfter); - } - - // - // - // - aFile.Close(); - BOX_INFO("Saved store object info file version " << - STOREOBJECTINFO_VERSION << " (" << - StoreObjectInfoFile << ")"); - } - catch(std::exception &e) - { - BOX_ERROR("Failed to write StoreObjectInfoFile: " << - StoreObjectInfoFile << ": " << e.what()); - } - catch(...) - { - BOX_ERROR("Failed to write StoreObjectInfoFile: " << - StoreObjectInfoFile << ": unknown error"); - } - - return created; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeserializeStoreObjectInfo( -// box_time_t & theLastSyncTime, -// box_time_t & theNextSyncTime) -// Purpose: Deserializes remote directory and file information -// from a stream of bytes, using an Archive -// abstraction. -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, - box_time_t & theNextSyncTime) -{ - // - // - // - DeleteAllLocations(); - - // - // - // - if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) - { - BOX_NOTICE("Store object info file is not enabled. Will " - "download directory listings from store."); - return false; - } - - std::string StoreObjectInfoFile = - GetConfiguration().GetKeyValue("StoreObjectInfoFile"); - - if(StoreObjectInfoFile.size() <= 0) - { - return false; - } - - int64_t fileSize; - if (!FileExists(StoreObjectInfoFile, &fileSize) || fileSize == 0) - { - BOX_NOTICE(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Store object info file does not exist or is empty")); - } - else - { - try - { - FileStream aFile(StoreObjectInfoFile, O_RDONLY); - Archive anArchive(aFile, 0); - - // - // see if the content looks like a valid serialised archive - // - int iMagicValue = 0; - anArchive.Read(iMagicValue); - - if(iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE) - { - BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Store object info file is not a valid " - "or compatible serialised archive")); - return false; - } - - // - // get a bit optimistic and read in a string identifier - // - std::string strMagicValue; - anArchive.Read(strMagicValue); - - if(strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING) - { - BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Store object info file is not a valid " - "or compatible serialised archive")); - return false; - } - - // - // check if we are loading some future format - // version by mistake - // - int iVersion = 0; - anArchive.Read(iVersion); - - if(iVersion != STOREOBJECTINFO_VERSION) - { - BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Store object info file version " << - iVersion << " is not supported")); - return false; - } - - // - // check if this state file is even valid - // for the loaded bbackupd.conf file - // - box_time_t lastKnownConfigModTime; - anArchive.Read(lastKnownConfigModTime); - - if(lastKnownConfigModTime != GetLoadedConfigModifiedTime()) - { - BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Store object info file is older than " - "configuration file")); - return false; - } - - // - // this is it, go at it - // - anArchive.Read(mClientStoreMarker); - anArchive.Read(theLastSyncTime); - anArchive.Read(theNextSyncTime); - - // - // - // - int64_t iCount = 0; - anArchive.Read(iCount); - - for(int v = 0; v < iCount; v++) - { - Location* pLocation = new Location; - if(!pLocation) - { - throw std::bad_alloc(); - } - - pLocation->Deserialize(anArchive); - mLocations.push_back(pLocation); - } - - // - // - // - iCount = 0; - anArchive.Read(iCount); - - for(int v = 0; v < iCount; v++) - { - std::string strItem; - anArchive.Read(strItem); - - mIDMapMounts.push_back(strItem); - } - - // - // - // - iCount = 0; - anArchive.Read(iCount); - - for(int v = 0; v < iCount; v++) - { - int64_t anId; - anArchive.Read(anId); - - std::string aName; - anArchive.Read(aName); - - mUnusedRootDirEntries.push_back(std::pair<int64_t, std::string>(anId, aName)); - } - - if (iCount > 0) - anArchive.Read(mDeleteUnusedRootDirEntriesAfter); - - // - // - // - aFile.Close(); - - BOX_INFO(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Loaded store object info file version " << iVersion)); - return true; - } - catch(std::exception &e) - { - BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Internal error reading store object info " - "file: " << e.what())); - } - catch(...) - { - BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Internal error reading store object info " - "file: unknown error")); - } - } - - BOX_NOTICE("No usable cache, will download directory listings from " - "server."); - - DeleteAllLocations(); - - mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; - theLastSyncTime = 0; - theNextSyncTime = 0; - - return false; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeleteStoreObjectInfo() -// Purpose: Deletes the serialised state file, to prevent us -// from using it again if a backup is interrupted. -// -// Created: 2006/02/12 -// -// -------------------------------------------------------------------------- - -bool BackupDaemon::DeleteStoreObjectInfo() const -{ - if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) - { - return false; - } - - std::string storeObjectInfoFile(GetConfiguration().GetKeyValue("StoreObjectInfoFile")); - - // Check to see if the file exists - if(!FileExists(storeObjectInfoFile.c_str())) - { - // File doesn't exist -- so can't be deleted. But something - // isn't quite right, so log a message - BOX_WARNING("StoreObjectInfoFile did not exist when it " - "was supposed to: " << storeObjectInfoFile); - - // Return true to stop things going around in a loop - return true; - } - - // Actually delete it - if(::unlink(storeObjectInfoFile.c_str()) != 0) - { - BOX_LOG_SYS_ERROR("Failed to delete the old " - "StoreObjectInfoFile: " << storeObjectInfoFile); - return false; - } - - return true; -} diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h deleted file mode 100644 index ffe31247..00000000 --- a/bin/bbackupd/BackupDaemon.h +++ /dev/null @@ -1,539 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupDaemon.h -// Purpose: Backup daemon -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPDAEMON__H -#define BACKUPDAEMON__H - -#include <vector> -#include <string> -#include <memory> - -#include "BackupClientContext.h" -#include "BackupClientDirectoryRecord.h" -#include "BoxTime.h" -#include "Daemon.h" -#include "Logging.h" -#include "Socket.h" -#include "SocketListen.h" -#include "SocketStream.h" -#include "TLSContext.h" - -#include "autogen_BackupProtocol.h" -#include "autogen_BackupStoreException.h" - -#ifdef WIN32 - #include "WinNamedPipeListener.h" - #include "WinNamedPipeStream.h" -#endif - -#ifdef ENABLE_VSS -# include <comdef.h> -# include <Vss.h> -# include <VsWriter.h> -# include <VsBackup.h> -#endif - -#define COMMAND_SOCKET_POLL_INTERVAL 1000 - -class BackupClientDirectoryRecord; -class BackupClientContext; -class Configuration; -class BackupClientInodeToIDMap; -class ExcludeList; -class IOStreamGetLine; -class Archive; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupDaemon -// Purpose: Backup daemon -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -class BackupDaemon : public Daemon, public ProgressNotifier, public LocationResolver, -public RunStatusProvider, public SysadminNotifier, public BackgroundTask -{ -public: - BackupDaemon(); - ~BackupDaemon(); - -private: - // methods below do partial (specialized) serialization of - // client state only - bool SerializeStoreObjectInfo(box_time_t theLastSyncTime, - box_time_t theNextSyncTime) const; - bool DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, - box_time_t & theNextSyncTime); - bool DeleteStoreObjectInfo() const; - BackupDaemon(const BackupDaemon &); - -public: - #ifdef WIN32 - // add command-line options to handle Windows services - std::string GetOptionString(); - int ProcessOption(signed int option); - int Main(const std::string &rConfigFileName); - - // This shouldn't be here, but apparently gcc on - // Windows has no idea about inherited methods... - virtual int Main(const char *DefaultConfigFile, int argc, - const char *argv[]) - { - return Daemon::Main(DefaultConfigFile, argc, argv); - } - #endif - - void Run(); - virtual const char *DaemonName() const; - virtual std::string DaemonBanner() const; - virtual void Usage(); - const ConfigurationVerify *GetConfigVerify() const; - - bool FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const; - - enum - { - // Add stuff to this, make sure the textual equivalents in SetState() are changed too. - State_Initialising = -1, - State_Idle = 0, - State_Connected = 1, - State_Error = 2, - State_StorageLimitExceeded = 3 - }; - - int GetState() {return mState;} - static std::string GetStateName(int state) - { - std::string stateName; - - #define STATE(x) case BackupDaemon::State_ ## x: stateName = #x; break; - switch (state) - { - STATE(Initialising); - STATE(Idle); - STATE(Connected); - STATE(Error); - STATE(StorageLimitExceeded); - default: - stateName = "unknown"; - } - #undef STATE - - return stateName; - } - - // Allow other classes to call this too - void NotifySysadmin(SysadminNotifier::EventCode Event); - -private: - void Run2(); - -public: - void InitCrypto(); - std::auto_ptr<BackupClientContext> RunSyncNowWithExceptionHandling(); - std::auto_ptr<BackupClientContext> RunSyncNow(); - void ResetCachedState(); - void OnBackupStart(); - void OnBackupFinish(); - // TouchFileInWorkingDir is only here for use by Boxi. - // This does NOT constitute an API! - void TouchFileInWorkingDir(const char *Filename); - -protected: - virtual std::auto_ptr<BackupClientContext> GetNewContext - ( - LocationResolver &rResolver, - TLSContext &rTLSContext, - const std::string &rHostname, - int32_t Port, - uint32_t AccountNumber, - bool ExtendedLogging, - bool ExtendedLogToFile, - std::string ExtendedLogFile, - ProgressNotifier &rProgressNotifier, - bool TcpNiceMode - ); - -private: - void DeleteAllLocations(); - void SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf); - - void DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector); - void DeleteAllIDMaps() - { - DeleteIDMapVector(mCurrentIDMaps); - DeleteIDMapVector(mNewIDMaps); - } - void FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps); - - void SetupIDMapsForSync(); - void CommitIDMapsAfterSync(); - void DeleteCorruptBerkelyDbFiles(); - - void MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const; - - void SetState(int State); - - void WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut); - void CloseCommandConnection(); - void SendSyncStartOrFinish(bool SendStart); - - void DeleteUnusedRootDirEntries(BackupClientContext &rContext); - - // For warning user about potential security hole - virtual void SetupInInitialProcess(); - - int UseScriptToSeeIfSyncAllowed(); - -public: - int ParseSyncAllowScriptOutput(const std::string& script, - const std::string& output); - typedef std::list<Location *> Locations; - Locations GetLocations() { return mLocations; } - -private: - int mState; // what the daemon is currently doing - - Locations mLocations; - - std::vector<std::string> mIDMapMounts; - std::vector<BackupClientInodeToIDMap *> mCurrentIDMaps; - std::vector<BackupClientInodeToIDMap *> mNewIDMaps; - - int mDeleteRedundantLocationsAfter; - - // For the command socket - class CommandSocketInfo - { - public: - CommandSocketInfo(); - ~CommandSocketInfo(); - private: - CommandSocketInfo(const CommandSocketInfo &); // no copying - CommandSocketInfo &operator=(const CommandSocketInfo &); - public: -#ifdef WIN32 - WinNamedPipeListener<1 /* listen backlog */> mListeningSocket; - std::auto_ptr<WinNamedPipeStream> mpConnectedSocket; -#else - SocketListen<SocketStream, 1 /* listen backlog */> mListeningSocket; - std::auto_ptr<SocketStream> mpConnectedSocket; -#endif - IOStreamGetLine *mpGetLine; - }; - - // Using a socket? - std::auto_ptr<CommandSocketInfo> mapCommandSocketInfo; - - // Stop notifications being repeated. - SysadminNotifier::EventCode mLastNotifiedEvent; - - // Unused entries in the root directory wait a while before being deleted - box_time_t mDeleteUnusedRootDirEntriesAfter; // time to delete them - std::vector<std::pair<int64_t,std::string> > mUnusedRootDirEntries; - - int64_t mClientStoreMarker; - bool mStorageLimitExceeded; - bool mReadErrorsOnFilesystemObjects; - box_time_t mLastSyncTime, mNextSyncTime; - box_time_t mCurrentSyncStartTime, mUpdateStoreInterval, - mBackupErrorDelay; - TLSContext mTlsContext; - bool mDeleteStoreObjectInfoFile; - bool mDoSyncForcedByPreviousSyncError; - int64_t mNumFilesUploaded, mNumDirsCreated; - int mMaxBandwidthFromSyncAllowScript; - -public: - int GetMaxBandwidthFromSyncAllowScript() { return mMaxBandwidthFromSyncAllowScript; } - bool StopRun() { return this->Daemon::StopRun(); } - bool StorageLimitExceeded() { return mStorageLimitExceeded; } - -private: - bool mLogAllFileAccess; - -public: - ProgressNotifier* GetProgressNotifier() { return mpProgressNotifier; } - LocationResolver* GetLocationResolver() { return mpLocationResolver; } - RunStatusProvider* GetRunStatusProvider() { return mpRunStatusProvider; } - SysadminNotifier* GetSysadminNotifier() { return mpSysadminNotifier; } - void SetProgressNotifier (ProgressNotifier* p) { mpProgressNotifier = p; } - void SetLocationResolver (LocationResolver* p) { mpLocationResolver = p; } - void SetRunStatusProvider(RunStatusProvider* p) { mpRunStatusProvider = p; } - void SetSysadminNotifier (SysadminNotifier* p) { mpSysadminNotifier = p; } - virtual bool RunBackgroundTask(State state, uint64_t progress, - uint64_t maximum); - -private: - ProgressNotifier* mpProgressNotifier; - LocationResolver* mpLocationResolver; - RunStatusProvider* mpRunStatusProvider; - SysadminNotifier* mpSysadminNotifier; - std::auto_ptr<Timer> mapCommandSocketPollTimer; - std::auto_ptr<BackupClientContext> mapClientContext; - - /* ProgressNotifier implementation */ -public: - virtual void NotifyIDMapsSetup(BackupClientContext& rContext) { } - - virtual void NotifyScanDirectory( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - if (mLogAllFileAccess) - { - BOX_INFO("Scanning directory: " << rLocalPath); - } - - if (!RunBackgroundTask(BackgroundTask::Scanning_Dirs, 0, 0)) - { - THROW_EXCEPTION(BackupStoreException, - CancelledByBackgroundTask); - } - } - virtual void NotifyDirStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { - BOX_WARNING("Failed to access directory: " << rLocalPath - << ": " << rErrorMsg); - } - virtual void NotifyFileStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { - BOX_WARNING("Failed to access file: " << rLocalPath - << ": " << rErrorMsg); - } - virtual void NotifyDirListFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { - BOX_WARNING("Failed to list directory: " << rLocalPath - << ": " << rErrorMsg); - } - virtual void NotifyMountPointSkipped( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - #ifdef WIN32 - BOX_WARNING("Ignored directory: " << rLocalPath << - ": is an NTFS junction/reparse point; create " - "a new location if you want to back it up"); - #else - BOX_WARNING("Ignored directory: " << rLocalPath << - ": is a mount point; create a new location " - "if you want to back it up"); - #endif - } - virtual void NotifyFileExcluded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - if (mLogAllFileAccess) - { - BOX_INFO("Skipping excluded file: " << rLocalPath); - } - } - virtual void NotifyDirExcluded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - if (mLogAllFileAccess) - { - BOX_INFO("Skipping excluded directory: " << rLocalPath); - } - } - virtual void NotifyUnsupportedFileType( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - BOX_WARNING("Ignoring file of unknown type: " << rLocalPath); - } - virtual void NotifyFileReadFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { - BOX_WARNING("Error reading file: " << rLocalPath - << ": " << rErrorMsg); - } - virtual void NotifyFileModifiedInFuture( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - BOX_WARNING("Some files have modification times excessively " - "in the future. Check clock synchronisation. " - "Example file (only one shown): " << rLocalPath); - } - virtual void NotifyFileSkippedServerFull( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - BOX_WARNING("Skipped file: server is full: " << rLocalPath); - } - virtual void NotifyFileUploadException( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const BoxException& rException) - { - if (rException.GetType() == CommonException::ExceptionType && - rException.GetSubType() == CommonException::AccessDenied) - { - BOX_ERROR("Failed to upload file: " << rLocalPath - << ": Access denied"); - } - else - { - BOX_ERROR("Failed to upload file: " << rLocalPath - << ": caught exception: " << rException.what() - << " (" << rException.GetType() - << "/" << rException.GetSubType() << ")"); - } - } - virtual void NotifyFileUploadServerError( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int type, int subtype) - { - BOX_ERROR("Failed to upload file: " << rLocalPath << - ": server error: " << - BackupProtocolError::GetMessage(subtype)); - } - virtual void NotifyFileUploading( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Uploading complete file: " << rLocalPath); - } - } - virtual void NotifyFileUploadingPatch( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t EstimatedBytesToUpload) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Uploading patch to file: " << rLocalPath << - ", estimated upload size = " << - EstimatedBytesToUpload); - } - } - virtual void NotifyFileUploadingAttributes( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Uploading new file attributes: " << - rLocalPath); - } - } - virtual void NotifyFileUploaded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Uploaded file: " << rLocalPath << - " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << - "): total size = " << FileSize << ", " - "uploaded size = " << UploadedSize); - } - mNumFilesUploaded++; - } - virtual void NotifyFileSynchronised( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize) - { - if (mLogAllFileAccess) - { - BOX_INFO("Synchronised file: " << rLocalPath); - } - } - virtual void NotifyDirectoryCreated( - int64_t ObjectID, - const std::string& rLocalPath, - const std::string& rRemotePath) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Created directory: " << rRemotePath << - " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << - ")"); - } - mNumDirsCreated++; - } - virtual void NotifyDirectoryDeleted( - int64_t ObjectID, - const std::string& rRemotePath) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Deleted directory: " << rRemotePath << - " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << - ")"); - } - } - virtual void NotifyFileDeleted( - int64_t ObjectID, - const std::string& rRemotePath) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Deleted file: " << rRemotePath << - " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << - ")"); - } - } - virtual void NotifyReadProgress(int64_t readSize, int64_t offset, - int64_t length, box_time_t elapsed, box_time_t finish) - { - BOX_TRACE("Read " << readSize << " bytes at " << offset << - ", " << (length - offset) << " remain, eta " << - BoxTimeToSeconds(finish - elapsed) << "s"); - } - virtual void NotifyReadProgress(int64_t readSize, int64_t offset, - int64_t length) - { - BOX_TRACE("Read " << readSize << " bytes at " << offset << - ", " << (length - offset) << " remain"); - } - virtual void NotifyReadProgress(int64_t readSize, int64_t offset) - { - BOX_TRACE("Read " << readSize << " bytes at " << offset << - ", unknown bytes remaining"); - } - -#ifdef WIN32 - private: - bool mInstallService, mRemoveService, mRunAsService; - std::string mServiceName; -#endif - -#ifdef ENABLE_VSS - IVssBackupComponents* mpVssBackupComponents; - void CreateVssBackupComponents(); - bool WaitForAsync(IVssAsync *pAsync, const std::string& description); - typedef HRESULT (__stdcall IVssBackupComponents::*AsyncMethod)(IVssAsync**); - bool CallAndWaitForAsync(AsyncMethod method, - const std::string& description); - void CleanupVssBackupComponents(); -#endif -}; - -#endif // BACKUPDAEMON__H diff --git a/bin/bbackupd/BackupDaemonInterface.h b/bin/bbackupd/BackupDaemonInterface.h deleted file mode 100644 index d5c47c85..00000000 --- a/bin/bbackupd/BackupDaemonInterface.h +++ /dev/null @@ -1,166 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupDaemonInterface.h -// Purpose: Interfaces for managing a BackupDaemon -// Created: 2008/12/30 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPDAEMONINTERFACE__H -#define BACKUPDAEMONINTERFACE__H - -#include <string> - -#include "BoxTime.h" - -class Archive; -class BackupClientContext; -class BackupDaemon; - -// -------------------------------------------------------------------------- -// -// Class -// Name: SysadminNotifier -// Purpose: Provides a NotifySysadmin() method to send mail to the sysadmin -// Created: 2005/11/15 -// -// -------------------------------------------------------------------------- -class SysadminNotifier -{ - public: - virtual ~SysadminNotifier() { } - - typedef enum - { - StoreFull = 0, - ReadError, - BackupError, - BackupStart, - BackupFinish, - BackupOK, - MAX - // When adding notifications, remember to add - // strings to NotifySysadmin() - } - EventCode; - - virtual void NotifySysadmin(EventCode Event) = 0; -}; - -// -------------------------------------------------------------------------- -// -// Class -// Name: ProgressNotifier -// Purpose: Provides methods for the backup library to inform the user -// interface about its progress with the backup -// Created: 2005/11/20 -// -// -------------------------------------------------------------------------- - -class BackupClientContext; -class BackupClientDirectoryRecord; - -class ProgressNotifier -{ - public: - virtual ~ProgressNotifier() { } - virtual void NotifyIDMapsSetup(BackupClientContext& rContext) = 0; - virtual void NotifyScanDirectory( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyDirStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyFileStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyDirListFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyMountPointSkipped( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileExcluded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyDirExcluded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyUnsupportedFileType( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileReadFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyFileModifiedInFuture( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileSkippedServerFull( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileUploadException( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const BoxException& rException) = 0; - virtual void NotifyFileUploadServerError( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int type, int subtype) = 0; - virtual void NotifyFileUploading( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileUploadingPatch( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t EstimatedBytesToUpload) = 0; - virtual void NotifyFileUploadingAttributes( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileUploaded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) = 0; - virtual void NotifyFileSynchronised( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize) = 0; - virtual void NotifyDirectoryCreated( - int64_t ObjectID, - const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyDirectoryDeleted( - int64_t ObjectID, - const std::string& rRemotePath) = 0; - virtual void NotifyFileDeleted( - int64_t ObjectID, - const std::string& rRemotePath) = 0; - virtual void NotifyReadProgress(int64_t readSize, int64_t offset, - int64_t length, box_time_t elapsed, box_time_t finish) = 0; - virtual void NotifyReadProgress(int64_t readSize, int64_t offset, - int64_t length) = 0; - virtual void NotifyReadProgress(int64_t readSize, int64_t offset) = 0; -}; - -// -------------------------------------------------------------------------- -// -// Class -// Name: LocationResolver -// Purpose: Interface for classes that can resolve locations to paths, -// like BackupDaemon -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -class LocationResolver -{ -public: - virtual ~LocationResolver() { } - virtual bool FindLocationPathName(const std::string &rLocationName, - std::string &rPathOut) const = 0; -}; - -#endif // BACKUPDAEMONINTERFACE__H diff --git a/bin/bbackupd/Win32BackupService.cpp b/bin/bbackupd/Win32BackupService.cpp deleted file mode 100644 index 6d027abf..00000000 --- a/bin/bbackupd/Win32BackupService.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// Win32 service functions for Box Backup, by Nick Knight - -#ifdef WIN32 - -#include "Box.h" -#include "BackupDaemon.h" -#include "MainHelper.h" -#include "BoxPortsAndFiles.h" -#include "BackupStoreException.h" - -#include "MemLeakFindOn.h" - -#include "Win32BackupService.h" - -Win32BackupService* gpDaemonService = NULL; -extern HANDLE gStopServiceEvent; -extern DWORD gServiceReturnCode; - -unsigned int WINAPI RunService(LPVOID lpParameter) -{ - DWORD retVal = gpDaemonService->WinService((const char*) lpParameter); - gServiceReturnCode = retVal; - SetEvent(gStopServiceEvent); - return retVal; -} - -void TerminateService(void) -{ - gpDaemonService->SetTerminateWanted(); -} - -DWORD Win32BackupService::WinService(const char* pConfigFileName) -{ - DWORD ret; - - if (pConfigFileName != NULL) - { - ret = this->Main(pConfigFileName); - } - else - { - ret = this->Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE); - } - - return ret; -} - -#endif // WIN32 diff --git a/bin/bbackupd/Win32BackupService.h b/bin/bbackupd/Win32BackupService.h deleted file mode 100644 index e7f077f2..00000000 --- a/bin/bbackupd/Win32BackupService.h +++ /dev/null @@ -1,21 +0,0 @@ -// Box Backup service daemon implementation by Nick Knight - -#ifndef WIN32BACKUPSERVICE_H -#define WIN32BACKUPSERVICE_H - -#ifdef WIN32 - -class Configuration; -class ConfigurationVerify; -class BackupDaemon; - -class Win32BackupService : public BackupDaemon -{ -public: - DWORD WinService(const char* pConfigFileName); -}; - -#endif // WIN32 - -#endif // WIN32BACKUPSERVICE_H - diff --git a/bin/bbackupd/Win32ServiceFunctions.cpp b/bin/bbackupd/Win32ServiceFunctions.cpp deleted file mode 100644 index 2df914a7..00000000 --- a/bin/bbackupd/Win32ServiceFunctions.cpp +++ /dev/null @@ -1,384 +0,0 @@ -//*************************************************************** -// From the book "Win32 System Services: The Heart of Windows 98 -// and Windows 2000" -// by Marshall Brain -// Published by Prentice Hall -// Copyright 1995 Prentice Hall. -// -// This code implements the Windows API Service interface -// for the Box Backup for Windows native port. -// Adapted for Box Backup by Nick Knight. -//*************************************************************** - -#ifdef WIN32 - -#include "Box.h" - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif -#ifdef HAVE_PROCESS_H - #include <process.h> -#endif - -extern void TerminateService(void); -extern unsigned int WINAPI RunService(LPVOID lpParameter); - -// Global variables - -TCHAR* gServiceName = TEXT("Box Backup Service"); -SERVICE_STATUS gServiceStatus; -SERVICE_STATUS_HANDLE gServiceStatusHandle = 0; -HANDLE gStopServiceEvent = 0; -DWORD gServiceReturnCode = 0; - -#define SERVICE_NAME "boxbackup" - -void ShowMessage(char *s) -{ - MessageBox(0, s, "Box Backup Message", - MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); -} - -void ErrorHandler(char *s, DWORD err) -{ - char buf[256]; - memset(buf, 0, sizeof(buf)); - _snprintf(buf, sizeof(buf)-1, "%s: %s", s, - GetErrorMessage(err).c_str()); - BOX_ERROR(buf); - MessageBox(0, buf, "Error", - MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); - ExitProcess(err); -} - -void WINAPI ServiceControlHandler( DWORD controlCode ) -{ - switch ( controlCode ) - { - case SERVICE_CONTROL_INTERROGATE: - break; - - case SERVICE_CONTROL_SHUTDOWN: - case SERVICE_CONTROL_STOP: - Beep(1000,100); - TerminateService(); - gServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - - SetEvent(gStopServiceEvent); - return; - - case SERVICE_CONTROL_PAUSE: - break; - - case SERVICE_CONTROL_CONTINUE: - break; - - default: - if ( controlCode >= 128 && controlCode <= 255 ) - // user defined control code - break; - else - // unrecognised control code - break; - } - - SetServiceStatus( gServiceStatusHandle, &gServiceStatus ); -} - -// ServiceMain is called when the SCM wants to -// start the service. When it returns, the service -// has stopped. It therefore waits on an event -// just before the end of the function, and -// that event gets set when it is time to stop. -// It also returns on any error because the -// service cannot start if there is an eror. - -static char* spConfigFileName; - -VOID ServiceMain(DWORD argc, LPTSTR *argv) -{ - // initialise service status - gServiceStatus.dwServiceType = SERVICE_WIN32; - gServiceStatus.dwCurrentState = SERVICE_STOPPED; - gServiceStatus.dwControlsAccepted = 0; - gServiceStatus.dwWin32ExitCode = NO_ERROR; - gServiceStatus.dwServiceSpecificExitCode = NO_ERROR; - gServiceStatus.dwCheckPoint = 0; - gServiceStatus.dwWaitHint = 0; - - gServiceStatusHandle = RegisterServiceCtrlHandler(gServiceName, - ServiceControlHandler); - - if (gServiceStatusHandle) - { - // service is starting - gServiceStatus.dwCurrentState = SERVICE_START_PENDING; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - - // do initialisation here - gStopServiceEvent = CreateEvent(0, TRUE, FALSE, 0); - if (!gStopServiceEvent) - { - gServiceStatus.dwControlsAccepted &= - ~(SERVICE_ACCEPT_STOP | - SERVICE_ACCEPT_SHUTDOWN); - gServiceStatus.dwCurrentState = SERVICE_STOPPED; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - return; - } - - HANDLE ourThread = (HANDLE)_beginthreadex( - NULL, - 0, - RunService, - spConfigFileName, - CREATE_SUSPENDED, - NULL); - - SetThreadPriority(ourThread, THREAD_PRIORITY_LOWEST); - ResumeThread(ourThread); - - // we are now running so tell the SCM - gServiceStatus.dwControlsAccepted |= - (SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); - gServiceStatus.dwCurrentState = SERVICE_RUNNING; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - - // do cleanup here - WaitForSingleObject(gStopServiceEvent, INFINITE); - CloseHandle(gStopServiceEvent); - gStopServiceEvent = 0; - - // service was stopped - gServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - - // service is now stopped - gServiceStatus.dwControlsAccepted &= - ~(SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); - - gServiceStatus.dwCurrentState = SERVICE_STOPPED; - - if (gServiceReturnCode != 0) - { - gServiceStatus.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR; - gServiceStatus.dwServiceSpecificExitCode = gServiceReturnCode; - } - - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - } -} - -int OurService(const char* pConfigFileName) -{ - spConfigFileName = strdup(pConfigFileName); - - SERVICE_TABLE_ENTRY serviceTable[] = - { - { SERVICE_NAME, (LPSERVICE_MAIN_FUNCTION) ServiceMain }, - { NULL, NULL } - }; - BOOL success; - - // Register with the SCM - success = StartServiceCtrlDispatcher(serviceTable); - - free(spConfigFileName); - spConfigFileName = NULL; - - if (!success) - { - ErrorHandler("Failed to start service. Did you start " - "Box Backup from the Service Control Manager? " - "(StartServiceCtrlDispatcher)", GetLastError()); - return 1; - } - - return 0; -} - -int InstallService(const char* pConfigFileName, const std::string& rServiceName) -{ - if (pConfigFileName != NULL) - { - EMU_STRUCT_STAT st; - - if (emu_stat(pConfigFileName, &st) != 0) - { - BOX_LOG_SYS_ERROR("Failed to open configuration file " - "'" << pConfigFileName << "'"); - return 1; - } - - if (!(st.st_mode & S_IFREG)) - { - - BOX_ERROR("Failed to open configuration file '" << - pConfigFileName << "': not a file"); - return 1; - } - } - - SC_HANDLE scm = OpenSCManager(0, 0, SC_MANAGER_CREATE_SERVICE); - - if (!scm) - { - BOX_ERROR("Failed to open service control manager: " << - GetErrorMessage(GetLastError())); - return 1; - } - - char cmd[MAX_PATH]; - GetModuleFileName(NULL, cmd, sizeof(cmd)-1); - cmd[sizeof(cmd)-1] = 0; - - std::string cmdWithArgs(cmd); - cmdWithArgs += " -s -S \"" + rServiceName + "\""; - - if (pConfigFileName != NULL) - { - cmdWithArgs += " \""; - cmdWithArgs += pConfigFileName; - cmdWithArgs += "\""; - } - - std::string serviceDesc = "Box Backup (" + rServiceName + ")"; - - SC_HANDLE newService = CreateService( - scm, - rServiceName.c_str(), - serviceDesc.c_str(), - SERVICE_ALL_ACCESS, - SERVICE_WIN32_OWN_PROCESS, - SERVICE_AUTO_START, - SERVICE_ERROR_NORMAL, - cmdWithArgs.c_str(), - 0,0,0,0,0); - - DWORD err = GetLastError(); - CloseServiceHandle(scm); - - if (!newService) - { - switch (err) - { - case ERROR_SERVICE_EXISTS: - { - BOX_ERROR("Failed to create Box Backup " - "service: it already exists"); - } - break; - - case ERROR_SERVICE_MARKED_FOR_DELETE: - { - BOX_ERROR("Failed to create Box Backup " - "service: it is waiting to be deleted"); - } - break; - - case ERROR_DUPLICATE_SERVICE_NAME: - { - BOX_ERROR("Failed to create Box Backup " - "service: a service with this name " - "already exists"); - } - break; - - default: - { - BOX_ERROR("Failed to create Box Backup " - "service: error " << - GetErrorMessage(GetLastError())); - } - } - - return 1; - } - - BOX_INFO("Created Box Backup service"); - - SERVICE_DESCRIPTION desc; - desc.lpDescription = "Backs up your data files over the Internet"; - - if (!ChangeServiceConfig2(newService, SERVICE_CONFIG_DESCRIPTION, - &desc)) - { - BOX_WARNING("Failed to set description for Box Backup " - "service: " << GetErrorMessage(GetLastError())); - } - - CloseServiceHandle(newService); - - return 0; -} - -int RemoveService(const std::string& rServiceName) -{ - SC_HANDLE scm = OpenSCManager(0,0,SC_MANAGER_CREATE_SERVICE); - - if (!scm) - { - BOX_ERROR("Failed to open service control manager: " << - GetErrorMessage(GetLastError())); - return 1; - } - - SC_HANDLE service = OpenService(scm, rServiceName.c_str(), - SERVICE_ALL_ACCESS|DELETE); - DWORD err = GetLastError(); - CloseServiceHandle(scm); - - if (!service) - { - if (err == ERROR_SERVICE_DOES_NOT_EXIST || - err == ERROR_IO_PENDING) - // hello microsoft? anyone home? - { - BOX_ERROR("Failed to open Box Backup service: " - "not installed or not found"); - } - else - { - BOX_ERROR("Failed to open Box Backup service: " << - GetErrorMessage(err)); - } - return 1; - } - - SERVICE_STATUS status; - if (!ControlService(service, SERVICE_CONTROL_STOP, &status)) - { - err = GetLastError(); - if (err != ERROR_SERVICE_NOT_ACTIVE) - { - BOX_WARNING("Failed to stop Box Backup service: " << - GetErrorMessage(err)); - } - } - - BOOL deleted = DeleteService(service); - err = GetLastError(); - CloseServiceHandle(service); - - if (deleted) - { - BOX_INFO("Box Backup service deleted"); - return 0; - } - else if (err == ERROR_SERVICE_MARKED_FOR_DELETE) - { - BOX_ERROR("Failed to remove Box Backup service: " - "it is already being deleted"); - } - else - { - BOX_ERROR("Failed to remove Box Backup service: " << - GetErrorMessage(err)); - } - - return 1; -} - -#endif // WIN32 diff --git a/bin/bbackupd/Win32ServiceFunctions.h b/bin/bbackupd/Win32ServiceFunctions.h deleted file mode 100644 index e04c368f..00000000 --- a/bin/bbackupd/Win32ServiceFunctions.h +++ /dev/null @@ -1,19 +0,0 @@ -//*************************************************************** -// From the book "Win32 System Services: The Heart of Windows 98 -// and Windows 2000" -// by Marshall Brain -// Published by Prentice Hall -// Copyright 1995 Prentice Hall. -// -// This code implements the Windows API Service interface -// for the Box Backup for Windows native port. -//*************************************************************** - -#ifndef WIN32SERVICEFUNCTIONS_H -#define WIN32SERVICEFUNCTIONS_H - -int RemoveService (const std::string& rServiceName); -int InstallService (const char* pConfigFilePath, const std::string& rServiceName); -int OurService (const char* pConfigFileName); - -#endif |