diff options
Diffstat (limited to 'bin')
22 files changed, 0 insertions, 12731 deletions
diff --git a/bin/bbackupd/BackupClientContext.cpp b/bin/bbackupd/BackupClientContext.cpp deleted file mode 100644 index 4c0b01ce..00000000 --- a/bin/bbackupd/BackupClientContext.cpp +++ /dev/null @@ -1,578 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientContext.cpp -// Purpose: Keep track of context -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#ifdef HAVE_SIGNAL_H - #include <signal.h> -#endif - -#ifdef HAVE_SYS_TIME_H - #include <sys/time.h> -#endif - -#include "BoxPortsAndFiles.h" -#include "BoxTime.h" -#include "BackupClientContext.h" -#include "SocketStreamTLS.h" -#include "Socket.h" -#include "BackupStoreConstants.h" -#include "BackupStoreException.h" -#include "BackupDaemon.h" -#include "autogen_BackupProtocol.h" -#include "BackupStoreFile.h" -#include "Logging.h" -#include "TcpNice.h" - -#include "MemLeakFindOn.h" - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::BackupClientContext(BackupDaemon &, TLSContext &, const std::string &, int32_t, bool, bool, std::string) -// Purpose: Constructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupClientContext::BackupClientContext -( - LocationResolver &rResolver, - TLSContext &rTLSContext, - const std::string &rHostname, - int Port, - uint32_t AccountNumber, - bool ExtendedLogging, - bool ExtendedLogToFile, - std::string ExtendedLogFile, - ProgressNotifier& rProgressNotifier, - bool TcpNiceMode -) -: mExperimentalSnapshotMode(false), - mrResolver(rResolver), - mrTLSContext(rTLSContext), - mHostname(rHostname), - mPort(Port), - mAccountNumber(AccountNumber), - mExtendedLogging(ExtendedLogging), - mExtendedLogToFile(ExtendedLogToFile), - mExtendedLogFile(ExtendedLogFile), - mpExtendedLogFileHandle(NULL), - mClientStoreMarker(ClientStoreMarker_NotKnown), - mpDeleteList(0), - mpCurrentIDMap(0), - mpNewIDMap(0), - mStorageLimitExceeded(false), - mpExcludeFiles(0), - mpExcludeDirs(0), - mKeepAliveTimer(0, "KeepAliveTime"), - mbIsManaged(false), - mrProgressNotifier(rProgressNotifier), - mTcpNiceMode(TcpNiceMode), - mpNice(NULL) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::~BackupClientContext() -// Purpose: Destructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupClientContext::~BackupClientContext() -{ - CloseAnyOpenConnection(); - - // Delete delete list - if(mpDeleteList != 0) - { - delete mpDeleteList; - mpDeleteList = 0; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetConnection() -// Purpose: Returns the connection, making the connection and logging into -// the backup store if necessary. -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupProtocolCallable &BackupClientContext::GetConnection() -{ - // Already got it? Just return it. - if(mapConnection.get()) - { - return *mapConnection; - } - - // Defensive. Must close connection before releasing any old socket. - mapConnection.reset(); - - std::auto_ptr<SocketStream> apSocket(new SocketStreamTLS); - - try - { - // Defensive. - mapConnection.reset(); - - // Log intention - BOX_INFO("Opening connection to server '" << mHostname << - "'..."); - - // Connect! - ((SocketStreamTLS *)(apSocket.get()))->Open(mrTLSContext, - Socket::TypeINET, mHostname, mPort); - - if(mTcpNiceMode) - { - // Pass control of apSocket to NiceSocketStream, - // which will take care of destroying it for us. - // But we need to hang onto a pointer to the nice - // socket, so we can enable and disable nice mode. - // This is scary, it could be deallocated under us. - mpNice = new NiceSocketStream(apSocket); - apSocket.reset(mpNice); - } - - // We need to call some methods that aren't defined in - // BackupProtocolCallable, so we need to hang onto a more - // strongly typed pointer (to avoid far too many casts). - BackupProtocolClient *pClient = new BackupProtocolClient(apSocket); - mapConnection.reset(pClient); - - // Set logging option - pClient->SetLogToSysLog(mExtendedLogging); - - if (mExtendedLogToFile) - { - ASSERT(mpExtendedLogFileHandle == NULL); - - mpExtendedLogFileHandle = fopen( - mExtendedLogFile.c_str(), "a+"); - - if (!mpExtendedLogFileHandle) - { - BOX_LOG_SYS_ERROR("Failed to open extended " - "log file: " << mExtendedLogFile); - } - else - { - pClient->SetLogToFile(mpExtendedLogFileHandle); - } - } - - // Handshake - pClient->Handshake(); - - // Check the version of the server - { - std::auto_ptr<BackupProtocolVersion> serverVersion( - mapConnection->QueryVersion(BACKUP_STORE_SERVER_VERSION)); - if(serverVersion->GetVersion() != BACKUP_STORE_SERVER_VERSION) - { - THROW_EXCEPTION(BackupStoreException, WrongServerVersion) - } - } - - // Login -- if this fails, the Protocol will exception - std::auto_ptr<BackupProtocolLoginConfirmed> loginConf( - mapConnection->QueryLogin(mAccountNumber, 0 /* read/write */)); - - // Check that the client store marker is the one we expect - if(mClientStoreMarker != ClientStoreMarker_NotKnown) - { - if(loginConf->GetClientStoreMarker() != mClientStoreMarker) - { - // Not good... finish the connection, abort, etc, ignoring errors - try - { - mapConnection->QueryFinished(); - } - catch(...) - { - // IGNORE - } - - // Then throw an exception about this - THROW_EXCEPTION_MESSAGE(BackupStoreException, - ClientMarkerNotAsExpected, - "Expected " << mClientStoreMarker << - " but found " << loginConf->GetClientStoreMarker() << - ": is someone else writing to the " - "same account?"); - } - } - else // mClientStoreMarker == ClientStoreMarker_NotKnown - { - // Yes, choose one, the current time will do - box_time_t marker = GetCurrentBoxTime(); - - // Set it on the store - mapConnection->QuerySetClientStoreMarker(marker); - - // Record it so that it can be picked up later. - mClientStoreMarker = marker; - } - - // Log success - BOX_INFO("Connection made, login successful"); - - // Check to see if there is any space available on the server - if(loginConf->GetBlocksUsed() >= loginConf->GetBlocksHardLimit()) - { - // no -- flag so only things like deletions happen - mStorageLimitExceeded = true; - // Log - BOX_WARNING("Exceeded storage hard-limit on server, " - "not uploading changes to files"); - } - } - catch(...) - { - // Clean up. - mapConnection.reset(); - throw; - } - - return *mapConnection; -} - -BackupProtocolCallable* BackupClientContext::GetOpenConnection() const -{ - return mapConnection.get(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::CloseAnyOpenConnection() -// Purpose: Closes a connection, if it's open -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupClientContext::CloseAnyOpenConnection() -{ - BackupProtocolCallable* pConnection(GetOpenConnection()); - if(pConnection) - { - try - { - // Quit nicely - pConnection->QueryFinished(); - } - catch(...) - { - // Ignore errors here - } - - // Delete it anyway. - mapConnection.reset(); - } - - // Delete any pending list - if(mpDeleteList != 0) - { - delete mpDeleteList; - mpDeleteList = 0; - } - - if (mpExtendedLogFileHandle != NULL) - { - fclose(mpExtendedLogFileHandle); - mpExtendedLogFileHandle = NULL; - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetTimeout() -// Purpose: Gets the current timeout time. -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -int BackupClientContext::GetTimeout() const -{ - BackupProtocolCallable* pConnection(GetOpenConnection()); - if(pConnection) - { - return pConnection->GetTimeout(); - } - - return (15*60*1000); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetDeleteList() -// Purpose: Returns the delete list, creating one if necessary -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -BackupClientDeleteList &BackupClientContext::GetDeleteList() -{ - // Already created? - if(mpDeleteList == 0) - { - mpDeleteList = new BackupClientDeleteList; - } - - // Return reference to object - return *mpDeleteList; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::PerformDeletions() -// Purpose: Perform any pending file deletions. -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientContext::PerformDeletions() -{ - // Got a list? - if(mpDeleteList == 0) - { - // Nothing to do - return; - } - - // Delegate to the delete list object - mpDeleteList->PerformDeletions(*this); - - // Delete the object - delete mpDeleteList; - mpDeleteList = 0; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetCurrentIDMap() const -// Purpose: Return a (const) reference to the current ID map -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -const BackupClientInodeToIDMap &BackupClientContext::GetCurrentIDMap() const -{ - ASSERT(mpCurrentIDMap != 0); - if(mpCurrentIDMap == 0) - { - THROW_EXCEPTION(CommonException, Internal) - } - return *mpCurrentIDMap; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetNewIDMap() const -// Purpose: Return a reference to the new ID map -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -BackupClientInodeToIDMap &BackupClientContext::GetNewIDMap() const -{ - ASSERT(mpNewIDMap != 0); - if(mpNewIDMap == 0) - { - THROW_EXCEPTION(CommonException, Internal) - } - return *mpNewIDMap; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::FindFilename(int64_t, int64_t, std::string &, bool &) const -// Purpose: Attempts to find the pathname of an object with a given ID on the server. -// Returns true if it can be found, in which case rPathOut is the local filename, -// and rIsDirectoryOut == true if the local object is a directory. -// Created: 12/11/03 -// -// -------------------------------------------------------------------------- -bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut, - bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer, box_time_t *pAttributesHashOnServer, BackupStoreFilenameClear *pLeafname) -{ - // Make a connection to the server - BackupProtocolCallable &connection(GetConnection()); - - // Request filenames from the server, in a "safe" manner to ignore errors properly - { - BackupProtocolGetObjectName send(ObjectID, ContainingDirectory); - connection.Send(send); - } - std::auto_ptr<BackupProtocolMessage> preply(connection.Receive()); - - // Is it of the right type? - if(preply->GetType() != BackupProtocolObjectName::TypeID) - { - // Was an error or something - return false; - } - - // Cast to expected type. - BackupProtocolObjectName *names = (BackupProtocolObjectName *)(preply.get()); - - // Anything found? - int32_t numElements = names->GetNumNameElements(); - if(numElements <= 0) - { - // No. - return false; - } - - // Get the stream containing all the names - std::auto_ptr<IOStream> nameStream(connection.ReceiveStream()); - - // Path - std::string path; - - // Remember this is in reverse order! - for(int l = 0; l < numElements; ++l) - { - BackupStoreFilenameClear elementName; - elementName.ReadFromStream(*nameStream, GetTimeout()); - - // Store leafname for caller? - if(l == 0 && pLeafname) - { - *pLeafname = elementName; - } - - // Is it part of the filename in the location? - if(l < (numElements - 1)) - { - // Part of filename within - path = (path.empty())?(elementName.GetClearFilename()):(elementName.GetClearFilename() + DIRECTORY_SEPARATOR_ASCHAR + path); - } - else - { - // Location name -- look up in daemon's records - std::string locPath; - if(!mrResolver.FindLocationPathName(elementName.GetClearFilename(), locPath)) - { - // Didn't find the location... so can't give the local filename - return false; - } - - // Add in location path - path = (path.empty())?(locPath):(locPath + DIRECTORY_SEPARATOR_ASCHAR + path); - } - } - - // Is it a directory? - rIsDirectoryOut = ((names->GetFlags() & BackupProtocolListDirectory::Flags_Dir) == BackupProtocolListDirectory::Flags_Dir); - - // Is it the current version? - rIsCurrentVersionOut = ((names->GetFlags() & (BackupProtocolListDirectory::Flags_OldVersion | BackupProtocolListDirectory::Flags_Deleted)) == 0); - - // And other information which may be required - if(pModTimeOnServer) *pModTimeOnServer = names->GetModificationTime(); - if(pAttributesHashOnServer) *pAttributesHashOnServer = names->GetAttributesHash(); - - // Tell caller about the pathname - rPathOut = path; - - // Found - return true; -} - -void BackupClientContext::SetMaximumDiffingTime(int iSeconds) -{ - mMaximumDiffingTime = iSeconds < 0 ? 0 : iSeconds; - BOX_TRACE("Set maximum diffing time to " << mMaximumDiffingTime << - " seconds"); -} - -void BackupClientContext::SetKeepAliveTime(int iSeconds) -{ - mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; - BOX_TRACE("Set keep-alive time to " << mKeepAliveTime << " seconds"); - mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::ManageDiffProcess() -// Purpose: Initiates a file diff control timer -// Created: 04/19/2005 -// -// -------------------------------------------------------------------------- -void BackupClientContext::ManageDiffProcess() -{ - ASSERT(!mbIsManaged); - mbIsManaged = true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::UnManageDiffProcess() -// Purpose: suspends file diff control timer -// Created: 04/19/2005 -// -// -------------------------------------------------------------------------- -void BackupClientContext::UnManageDiffProcess() -{ - // ASSERT(mbIsManaged); - mbIsManaged = false; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::DoKeepAlive() -// Purpose: Check whether it's time to send a KeepAlive -// message over the SSL link, and if so, send it. -// Created: 04/19/2005 -// -// -------------------------------------------------------------------------- -void BackupClientContext::DoKeepAlive() -{ - BackupProtocolCallable* pConnection(GetOpenConnection()); - if (!pConnection) - { - return; - } - - if (mKeepAliveTime == 0) - { - return; - } - - if (!mKeepAliveTimer.HasExpired()) - { - return; - } - - BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); - pConnection->QueryGetIsAlive(); - - mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); -} - -int BackupClientContext::GetMaximumDiffingTime() -{ - return mMaximumDiffingTime; -} diff --git a/bin/bbackupd/BackupClientContext.h b/bin/bbackupd/BackupClientContext.h deleted file mode 100644 index df43a232..00000000 --- a/bin/bbackupd/BackupClientContext.h +++ /dev/null @@ -1,252 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientContext.h -// Purpose: Keep track of context -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCLIENTCONTEXT__H -#define BACKUPCLIENTCONTEXT__H - -#include "BoxTime.h" -#include "BackupClientDeleteList.h" -#include "BackupClientDirectoryRecord.h" -#include "BackupDaemonInterface.h" -#include "BackupStoreFile.h" -#include "ExcludeList.h" -#include "TcpNice.h" -#include "Timer.h" - -class TLSContext; -class BackupProtocolClient; -class SocketStreamTLS; -class BackupClientInodeToIDMap; -class BackupDaemon; -class BackupStoreFilenameClear; - -#include <string> - - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupClientContext -// Purpose: -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -class BackupClientContext : public DiffTimer -{ -public: - BackupClientContext - ( - LocationResolver &rResolver, - TLSContext &rTLSContext, - const std::string &rHostname, - int32_t Port, - uint32_t AccountNumber, - bool ExtendedLogging, - bool ExtendedLogToFile, - std::string ExtendedLogFile, - ProgressNotifier &rProgressNotifier, - bool TcpNiceMode - ); - virtual ~BackupClientContext(); - -private: - BackupClientContext(const BackupClientContext &); - -public: - // GetConnection() will open a connection if none is currently open. - virtual BackupProtocolCallable& GetConnection(); - // GetOpenConnection() will not open a connection, just return NULL if there is - // no connection already open. - virtual BackupProtocolCallable* GetOpenConnection() const; - void CloseAnyOpenConnection(); - int GetTimeout() const; - BackupClientDeleteList &GetDeleteList(); - void PerformDeletions(); - - enum - { - ClientStoreMarker_NotKnown = 0 - }; - - void SetClientStoreMarker(int64_t ClientStoreMarker) {mClientStoreMarker = ClientStoreMarker;} - int64_t GetClientStoreMarker() const {return mClientStoreMarker;} - - bool StorageLimitExceeded() {return mStorageLimitExceeded;} - void SetStorageLimitExceeded() {mStorageLimitExceeded = true;} - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::SetIDMaps(const BackupClientInodeToIDMap *, BackupClientInodeToIDMap *) - // Purpose: Store pointers to the Current and New ID maps - // Created: 11/11/03 - // - // -------------------------------------------------------------------------- - void SetIDMaps(const BackupClientInodeToIDMap *pCurrent, BackupClientInodeToIDMap *pNew) - { - ASSERT(pCurrent != 0); - ASSERT(pNew != 0); - mpCurrentIDMap = pCurrent; - mpNewIDMap = pNew; - } - const BackupClientInodeToIDMap &GetCurrentIDMap() const; - BackupClientInodeToIDMap &GetNewIDMap() const; - - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::SetExcludeLists(ExcludeList *, ExcludeList *) - // Purpose: Sets the exclude lists for the operation. Can be 0. - // Created: 28/1/04 - // - // -------------------------------------------------------------------------- - void SetExcludeLists(ExcludeList *pExcludeFiles, ExcludeList *pExcludeDirs) - { - mpExcludeFiles = pExcludeFiles; - mpExcludeDirs = pExcludeDirs; - } - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::ExcludeFile(const std::string &) - // Purpose: Returns true is this file should be excluded from the backup - // Created: 28/1/04 - // - // -------------------------------------------------------------------------- - inline bool ExcludeFile(const std::string &rFullFilename) - { - if(mpExcludeFiles != 0) - { - return mpExcludeFiles->IsExcluded(rFullFilename); - } - // If no list, don't exclude anything - return false; - } - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::ExcludeDir(const std::string &) - // Purpose: Returns true is this directory should be excluded from the backup - // Created: 28/1/04 - // - // -------------------------------------------------------------------------- - inline bool ExcludeDir(const std::string &rFullDirName) - { - if(mpExcludeDirs != 0) - { - return mpExcludeDirs->IsExcluded(rFullDirName); - } - // If no list, don't exclude anything - return false; - } - - // Utility functions -- may do a lot of work - bool FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut, - bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer = 0, box_time_t *pAttributesHashOnServer = 0, - BackupStoreFilenameClear *pLeafname = 0); // not const as may connect to server - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::SetMaximumDiffingTime() - // Purpose: Sets the maximum time that will be spent diffing a file - // Created: 04/19/2005 - // - // -------------------------------------------------------------------------- - void SetMaximumDiffingTime(int iSeconds); - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::SetKeepAliveTime() - // Purpose: Sets the time interval for repetitive keep-alive operation - // Created: 04/19/2005 - // - // -------------------------------------------------------------------------- - virtual void SetKeepAliveTime(int iSeconds); - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::ManageDiffProcess() - // Purpose: Initiates an SSL connection/session keep-alive process - // Created: 04/19/2005 - // - // -------------------------------------------------------------------------- - void ManageDiffProcess(); - - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::UnManageDiffProcess() - // Purpose: Suspends an SSL connection/session keep-alive process - // Created: 04/19/2005 - // - // -------------------------------------------------------------------------- - void UnManageDiffProcess(); - - // ------------------------------------------------------------------- - // - // Function - // Name: BackupClientContext::DoKeepAlive() - // Purpose: Check whether it's time to send a KeepAlive - // message over the SSL link, and if so, send it. - // Created: 04/19/2005 - // - // ------------------------------------------------------------------- - virtual void DoKeepAlive(); - virtual int GetMaximumDiffingTime(); - virtual bool IsManaged() { return mbIsManaged; } - - ProgressNotifier& GetProgressNotifier() const - { - return mrProgressNotifier; - } - - void SetNiceMode(bool enabled) - { - if(mTcpNiceMode) - { - mpNice->SetEnabled(enabled); - } - } - - bool mExperimentalSnapshotMode; - -private: - LocationResolver &mrResolver; - TLSContext &mrTLSContext; - std::string mHostname; - int mPort; - uint32_t mAccountNumber; - std::auto_ptr<BackupProtocolCallable> mapConnection; - bool mExtendedLogging; - bool mExtendedLogToFile; - std::string mExtendedLogFile; - FILE* mpExtendedLogFileHandle; - int64_t mClientStoreMarker; - BackupClientDeleteList *mpDeleteList; - const BackupClientInodeToIDMap *mpCurrentIDMap; - BackupClientInodeToIDMap *mpNewIDMap; - bool mStorageLimitExceeded; - ExcludeList *mpExcludeFiles; - ExcludeList *mpExcludeDirs; - Timer mKeepAliveTimer; - bool mbIsManaged; - int mKeepAliveTime; - int mMaximumDiffingTime; - ProgressNotifier &mrProgressNotifier; - bool mTcpNiceMode; - NiceSocketStream *mpNice; -}; - -#endif // BACKUPCLIENTCONTEXT__H diff --git a/bin/bbackupd/BackupClientDeleteList.cpp b/bin/bbackupd/BackupClientDeleteList.cpp deleted file mode 100644 index ce5e6264..00000000 --- a/bin/bbackupd/BackupClientDeleteList.cpp +++ /dev/null @@ -1,229 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientDeleteList.cpp -// Purpose: List of pending deletes for backup -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <algorithm> - -#include "BackupClientDeleteList.h" -#include "BackupClientContext.h" -#include "autogen_BackupProtocol.h" - -#include "MemLeakFindOn.h" - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::BackupClientDeleteList() -// Purpose: Constructor -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -BackupClientDeleteList::BackupClientDeleteList() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::~BackupClientDeleteList() -// Purpose: Destructor -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -BackupClientDeleteList::~BackupClientDeleteList() -{ -} - -BackupClientDeleteList::FileToDelete::FileToDelete(int64_t DirectoryID, - const BackupStoreFilename& rFilename, - const std::string& rLocalPath) -: mDirectoryID(DirectoryID), - mFilename(rFilename), - mLocalPath(rLocalPath) -{ } - -BackupClientDeleteList::DirToDelete::DirToDelete(int64_t ObjectID, - const std::string& rLocalPath) -: mObjectID(ObjectID), - mLocalPath(rLocalPath) -{ } - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::AddDirectoryDelete(int64_t, -// const BackupStoreFilename&) -// Purpose: Add a directory to the list of directories to be deleted. -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID, - const std::string& rLocalPath) -{ - // Only add the delete to the list if it's not in the "no delete" set - if(mDirectoryNoDeleteList.find(ObjectID) == - mDirectoryNoDeleteList.end()) - { - // Not in the list, so should delete it - mDirectoryList.push_back(DirToDelete(ObjectID, rLocalPath)); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::AddFileDelete(int64_t, -// const BackupStoreFilename &) -// Purpose: -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, - const BackupStoreFilename &rFilename, const std::string& rLocalPath) -{ - // Try to find it in the no delete list - std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator - delEntry(mFileNoDeleteList.begin()); - while(delEntry != mFileNoDeleteList.end()) - { - if((delEntry)->first == DirectoryID - && (delEntry)->second == rFilename) - { - // Found! - break; - } - ++delEntry; - } - - // Only add it to the delete list if it wasn't in the no delete list - if(delEntry == mFileNoDeleteList.end()) - { - mFileList.push_back(FileToDelete(DirectoryID, rFilename, - rLocalPath)); - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) -// Purpose: Perform all the pending deletes -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) -{ - // Anything to do? - if(mDirectoryList.empty() && mFileList.empty()) - { - // Nothing! - return; - } - - // Get a connection - BackupProtocolCallable &connection(rContext.GetConnection()); - - // Do the deletes - for(std::vector<DirToDelete>::iterator i(mDirectoryList.begin()); - i != mDirectoryList.end(); ++i) - { - connection.QueryDeleteDirectory(i->mObjectID); - rContext.GetProgressNotifier().NotifyDirectoryDeleted( - i->mObjectID, i->mLocalPath); - } - - // Clear the directory list - mDirectoryList.clear(); - - // Delete the files - for(std::vector<FileToDelete>::iterator i(mFileList.begin()); - i != mFileList.end(); ++i) - { - connection.QueryDeleteFile(i->mDirectoryID, i->mFilename); - rContext.GetProgressNotifier().NotifyFileDeleted( - i->mDirectoryID, i->mLocalPath); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::StopDirectoryDeletion(int64_t) -// Purpose: Stop a directory being deleted -// Created: 19/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientDeleteList::StopDirectoryDeletion(int64_t ObjectID) -{ - // First of all, is it in the delete vector? - std::vector<DirToDelete>::iterator delEntry(mDirectoryList.begin()); - for(; delEntry != mDirectoryList.end(); delEntry++) - { - if(delEntry->mObjectID == ObjectID) - { - // Found! - break; - } - } - if(delEntry != mDirectoryList.end()) - { - // erase this entry - mDirectoryList.erase(delEntry); - } - else - { - // Haven't been asked to delete it yet, put it in the - // no delete list - mDirectoryNoDeleteList.insert(ObjectID); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDeleteList::StopFileDeletion(int64_t, const BackupStoreFilename &) -// Purpose: Stop a file from being deleted -// Created: 19/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, - const BackupStoreFilename &rFilename) -{ - // Find this in the delete list - std::vector<FileToDelete>::iterator delEntry(mFileList.begin()); - while(delEntry != mFileList.end()) - { - if(delEntry->mDirectoryID == DirectoryID - && delEntry->mFilename == rFilename) - { - // Found! - break; - } - ++delEntry; - } - - if(delEntry != mFileList.end()) - { - // erase this entry - mFileList.erase(delEntry); - } - else - { - // Haven't been asked to delete it yet, put it in the no delete list - mFileNoDeleteList.push_back(std::pair<int64_t, BackupStoreFilename>(DirectoryID, rFilename)); - } -} - diff --git a/bin/bbackupd/BackupClientDeleteList.h b/bin/bbackupd/BackupClientDeleteList.h deleted file mode 100644 index b0fbf51a..00000000 --- a/bin/bbackupd/BackupClientDeleteList.h +++ /dev/null @@ -1,75 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientDeleteList.h -// Purpose: List of pending deletes for backup -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCLIENTDELETELIST__H -#define BACKUPCLIENTDELETELIST__H - -#include "BackupStoreFilename.h" - -class BackupClientContext; - -#include <vector> -#include <utility> -#include <set> - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupClientDeleteList -// Purpose: List of pending deletes for backup -// Created: 10/11/03 -// -// -------------------------------------------------------------------------- -class BackupClientDeleteList -{ -private: - class FileToDelete - { - public: - int64_t mDirectoryID; - BackupStoreFilename mFilename; - std::string mLocalPath; - FileToDelete(int64_t DirectoryID, - const BackupStoreFilename& rFilename, - const std::string& rLocalPath); - }; - - class DirToDelete - { - public: - int64_t mObjectID; - std::string mLocalPath; - DirToDelete(int64_t ObjectID, const std::string& rLocalPath); - }; - -public: - BackupClientDeleteList(); - ~BackupClientDeleteList(); - - void AddDirectoryDelete(int64_t ObjectID, - const std::string& rLocalPath); - void AddFileDelete(int64_t DirectoryID, - const BackupStoreFilename &rFilename, - const std::string& rLocalPath); - - void StopDirectoryDeletion(int64_t ObjectID); - void StopFileDeletion(int64_t DirectoryID, - const BackupStoreFilename &rFilename); - - void PerformDeletions(BackupClientContext &rContext); - -private: - std::vector<DirToDelete> mDirectoryList; - std::set<int64_t> mDirectoryNoDeleteList; // note: things only get in this list if they're not present in mDirectoryList when they are 'added' - std::vector<FileToDelete> mFileList; - std::vector<std::pair<int64_t, BackupStoreFilename> > mFileNoDeleteList; -}; - -#endif // BACKUPCLIENTDELETELIST__H - diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp deleted file mode 100644 index 94cb7965..00000000 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ /dev/null @@ -1,2302 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientDirectoryRecord.cpp -// Purpose: Implementation of record about directory for -// backup client -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#ifdef HAVE_DIRENT_H - #include <dirent.h> -#endif - -#include <errno.h> -#include <string.h> - -#include "autogen_BackupProtocol.h" -#include "autogen_CipherException.h" -#include "autogen_ClientException.h" -#include "Archive.h" -#include "BackupClientContext.h" -#include "BackupClientDirectoryRecord.h" -#include "BackupClientInodeToIDMap.h" -#include "BackupDaemon.h" -#include "BackupStoreException.h" -#include "BackupStoreFile.h" -#include "BackupStoreFileEncodeStream.h" -#include "BufferedStream.h" -#include "CommonException.h" -#include "CollectInBufferStream.h" -#include "FileModificationTime.h" -#include "IOStream.h" -#include "Logging.h" -#include "MemBlockStream.h" -#include "PathUtils.h" -#include "RateLimitingStream.h" -#include "ReadLoggingStream.h" - -#include "MemLeakFindOn.h" - -typedef std::map<std::string, BackupStoreDirectory::Entry *> DecryptedEntriesMap_t; - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::BackupClientDirectoryRecord() -// Purpose: Constructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupClientDirectoryRecord::BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName) - : mObjectID(ObjectID), - mSubDirName(rSubDirName), - mInitialSyncDone(false), - mSyncDone(false), - mpPendingEntries(0) -{ - ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::~BackupClientDirectoryRecord() -// Purpose: Destructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupClientDirectoryRecord::~BackupClientDirectoryRecord() -{ - // Make deletion recursive - DeleteSubDirectories(); - - // Delete maps - if(mpPendingEntries != 0) - { - delete mpPendingEntries; - mpPendingEntries = 0; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::DeleteSubDirectories(); -// Purpose: Delete all sub directory entries -// Created: 2003/10/09 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::DeleteSubDirectories() -{ - // Delete all pointers - for(std::map<std::string, BackupClientDirectoryRecord *>::iterator i = mSubDirectories.begin(); - i != mSubDirectories.end(); ++i) - { - delete i->second; - } - - // Empty list - mSubDirectories.clear(); -} - -std::string BackupClientDirectoryRecord::ConvertVssPathToRealPath( - const std::string &rVssPath, - const Location& rBackupLocation) -{ -#ifdef ENABLE_VSS - BOX_TRACE("VSS: ConvertVssPathToRealPath: mIsSnapshotCreated = " << - rBackupLocation.mIsSnapshotCreated); - BOX_TRACE("VSS: ConvertVssPathToRealPath: File/Directory Path = " << - rVssPath.substr(0, rBackupLocation.mSnapshotPath.length())); - BOX_TRACE("VSS: ConvertVssPathToRealPath: Snapshot Path = " << - rBackupLocation.mSnapshotPath); - if (rBackupLocation.mIsSnapshotCreated && - rVssPath.substr(0, rBackupLocation.mSnapshotPath.length()) == - rBackupLocation.mSnapshotPath) - { - std::string convertedPath = rBackupLocation.mPath + - rVssPath.substr(rBackupLocation.mSnapshotPath.length()); - BOX_TRACE("VSS: ConvertVssPathToRealPath: Converted Path = " << - convertedPath); - return convertedPath; - } -#endif - - return rVssPath; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::SyncDirectory(i -// BackupClientDirectoryRecord::SyncParams &, -// int64_t, const std::string &, -// const std::string &, bool) -// Purpose: Recursively synchronise a local directory -// with the server. -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::SyncDirectory( - BackupClientDirectoryRecord::SyncParams &rParams, - int64_t ContainingDirectoryID, - const std::string &rLocalPath, - const std::string &rRemotePath, - const Location& rBackupLocation, - bool ThisDirHasJustBeenCreated) -{ - BackupClientContext& rContext(rParams.mrContext); - ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); - - // Signal received by daemon? - if(rParams.mrRunStatusProvider.StopRun()) - { - // Yes. Stop now. - THROW_EXCEPTION(BackupStoreException, SignalReceived) - } - - // Start by making some flag changes, marking this sync as not done, - // and on the immediate sub directories. - mSyncDone = false; - for(std::map<std::string, BackupClientDirectoryRecord *>::iterator - i = mSubDirectories.begin(); - i != mSubDirectories.end(); ++i) - { - i->second->mSyncDone = false; - } - - // Work out the time in the future after which the file should - // be uploaded regardless. This is a simple way to avoid having - // too many problems with file servers when they have clients - // with badly out of sync clocks. - rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + - rParams.mMaxFileTimeInFuture; - - // Build the current state checksum to compare against while - // getting info from dirs. Note checksum is used locally only, - // so byte order isn't considered. - MD5Digest currentStateChecksum; - - EMU_STRUCT_STAT dest_st; - // Stat the directory, to get attribute info - // If it's a symbolic link, we want the link target here - // (as we're about to back up the contents of the directory) - { - if(EMU_STAT(rLocalPath.c_str(), &dest_st) != 0) - { - // The directory has probably been deleted, so - // just ignore this error. In a future scan, this - // deletion will be noticed, deleted from server, - // and this object deleted. - rNotifier.NotifyDirStatFailed(this, - ConvertVssPathToRealPath(rLocalPath, rBackupLocation), - strerror(errno)); - return; - } - - BOX_TRACE("Stat dir '" << rLocalPath << "' " - "found device/inode " << - dest_st.st_dev << "/" << dest_st.st_ino); - - // Store inode number in map so directories are tracked - // in case they're renamed - { - BackupClientInodeToIDMap &idMap( - rParams.mrContext.GetNewIDMap()); - idMap.AddToMap(dest_st.st_ino, mObjectID, ContainingDirectoryID, - ConvertVssPathToRealPath(rLocalPath, rBackupLocation)); - } - // Add attributes to checksum - currentStateChecksum.Add(&dest_st.st_mode, - sizeof(dest_st.st_mode)); - currentStateChecksum.Add(&dest_st.st_uid, - sizeof(dest_st.st_uid)); - currentStateChecksum.Add(&dest_st.st_gid, - sizeof(dest_st.st_gid)); - // Inode to be paranoid about things moving around - currentStateChecksum.Add(&dest_st.st_ino, - sizeof(dest_st.st_ino)); -#ifdef HAVE_STRUCT_STAT_ST_FLAGS - currentStateChecksum.Add(&dest_st.st_flags, - sizeof(dest_st.st_flags)); -#endif - - StreamableMemBlock xattr; - BackupClientFileAttributes::FillExtendedAttr(xattr, - rLocalPath.c_str()); - currentStateChecksum.Add(xattr.GetBuffer(), xattr.GetSize()); - } - - // Read directory entries, building arrays of names - // First, need to read the contents of the directory. - std::vector<std::string> dirs; - std::vector<std::string> files; - bool downloadDirectoryRecordBecauseOfFutureFiles = false; - - // BLOCK - { - // read the contents... - DIR *dirHandle = 0; - try - { - std::string nonVssDirPath = ConvertVssPathToRealPath(rLocalPath, - rBackupLocation); - rNotifier.NotifyScanDirectory(this, nonVssDirPath); - - dirHandle = ::opendir(rLocalPath.c_str()); - if(dirHandle == 0) - { - // Report the error (logs and eventual email to administrator) - if (errno == EACCES) - { - rNotifier.NotifyDirListFailed(this, - nonVssDirPath, - "Access denied"); - } - else - { - rNotifier.NotifyDirListFailed(this, - nonVssDirPath, - strerror(errno)); - } - - // Report the error (logs and eventual email - // to administrator) - SetErrorWhenReadingFilesystemObject(rParams, - nonVssDirPath); - // Ignore this directory for now. - return; - } - - struct dirent *en = 0; - int num_entries_found = 0; - - while((en = ::readdir(dirHandle)) != 0) - { - num_entries_found++; - rParams.mrContext.DoKeepAlive(); - if(rParams.mpBackgroundTask) - { - rParams.mpBackgroundTask->RunBackgroundTask( - BackgroundTask::Scanning_Dirs, - num_entries_found, 0); - } - - if (!SyncDirectoryEntry(rParams, rNotifier, - rBackupLocation, rLocalPath, - currentStateChecksum, en, dest_st, dirs, - files, downloadDirectoryRecordBecauseOfFutureFiles)) - { - // This entry is not to be backed up. - continue; - } - } - - if(::closedir(dirHandle) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError) - } - dirHandle = 0; - } - catch(...) - { - if(dirHandle != 0) - { - ::closedir(dirHandle); - } - throw; - } - } - - // Finish off the checksum, and compare with the one currently stored - bool checksumDifferent = true; - currentStateChecksum.Finish(); - if(mInitialSyncDone && currentStateChecksum.DigestMatches(mStateChecksum)) - { - // The checksum is the same, and there was one to compare with - checksumDifferent = false; - } - - // Pointer to potentially downloaded store directory info - std::auto_ptr<BackupStoreDirectory> apDirOnStore; - - try - { - // Want to get the directory listing? - if(ThisDirHasJustBeenCreated) - { - // Avoid sending another command to the server when we know it's empty - apDirOnStore.reset(new BackupStoreDirectory(mObjectID, - ContainingDirectoryID)); - } - // Consider asking the store for it - else if(!mInitialSyncDone || checksumDifferent || - downloadDirectoryRecordBecauseOfFutureFiles) - { - apDirOnStore = FetchDirectoryListing(rParams); - } - - // Make sure the attributes are up to date -- if there's space - // on the server and this directory has not just been created - // (because it's attributes will be correct in this case) and - // the checksum is different, implying they *MIGHT* be - // different. - if((!ThisDirHasJustBeenCreated) && checksumDifferent && - !rParams.mrContext.StorageLimitExceeded()) - { - UpdateAttributes(rParams, apDirOnStore.get(), rLocalPath); - } - - // Create the list of pointers to directory entries - std::vector<BackupStoreDirectory::Entry *> entriesLeftOver; - if(apDirOnStore.get()) - { - entriesLeftOver.resize(apDirOnStore->GetNumberOfEntries(), 0); - BackupStoreDirectory::Iterator i(*apDirOnStore); - // Copy in pointers to all the entries - for(unsigned int l = 0; l < apDirOnStore->GetNumberOfEntries(); ++l) - { - entriesLeftOver[l] = i.Next(); - } - } - - // Do the directory reading - bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, - rRemotePath, rBackupLocation, apDirOnStore.get(), - entriesLeftOver, files, dirs); - - // LAST THING! (think exception safety) - // Store the new checksum -- don't fetch things unnecessarily - // in the future But... only if 1) the storage limit isn't - // exceeded -- make sure things are done again if the directory - // is modified later and 2) All the objects within the - // directory were stored successfully. - if(!rParams.mrContext.StorageLimitExceeded() && - updateCompleteSuccess) - { - currentStateChecksum.CopyDigestTo(mStateChecksum); - } - } - catch(...) - { - // Bad things have happened -- clean up - // Set things so that we get a full go at stuff later - ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); - - throw; - } - - // Flag things as having happened. - mInitialSyncDone = true; - mSyncDone = true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::SyncDirectorEntry( -// BackupClientDirectoryRecord::SyncParams &, -// int64_t, const std::string &, -// const std::string &, bool) -// Purpose: Recursively synchronise a local directory -// with the server. -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -bool BackupClientDirectoryRecord::SyncDirectoryEntry( - BackupClientDirectoryRecord::SyncParams &rParams, - ProgressNotifier& rNotifier, - const Location& rBackupLocation, - const std::string &rDirLocalPath, - MD5Digest& currentStateChecksum, - struct dirent *en, - EMU_STRUCT_STAT dir_st, - std::vector<std::string>& rDirs, - std::vector<std::string>& rFiles, - bool& rDownloadDirectoryRecordBecauseOfFutureFiles) -{ - std::string entry_name = en->d_name; - if(entry_name == "." || entry_name == "..") - { - // ignore parent directory entries - return false; - } - - // Stat file to get info - std::string filename = MakeFullPath(rDirLocalPath, entry_name); - std::string realFileName = ConvertVssPathToRealPath(filename, - rBackupLocation); - EMU_STRUCT_STAT file_st; - -#ifdef WIN32 - // Don't stat the file just yet, to ensure that users can exclude - // unreadable files to suppress warnings that they are not accessible. - // - // Our emulated readdir() abuses en->d_type, which would normally - // contain DT_REG, DT_DIR, etc, but we only use it here and prefer to - // have the full file attributes. - - int type; - if (en->d_type & FILE_ATTRIBUTE_DIRECTORY) - { - type = S_IFDIR; - } - else - { - type = S_IFREG; - } -#else // !WIN32 - if(EMU_LSTAT(filename.c_str(), &file_st) != 0) - { - // We don't know whether it's a file or a directory, so check - // both. This only affects whether a warning message is - // displayed; the file is not backed up in either case. - if(!(rParams.mrContext.ExcludeFile(filename)) && - !(rParams.mrContext.ExcludeDir(filename))) - { - // Report the error (logs and eventual email to - // administrator) - rNotifier.NotifyFileStatFailed(this, filename, - strerror(errno)); - - // FIXME move to NotifyFileStatFailed() - SetErrorWhenReadingFilesystemObject(rParams, filename); - } - - // Ignore this entry for now. - return false; - } - - BOX_TRACE("Stat entry '" << filename << "' found device/inode " << - file_st.st_dev << "/" << file_st.st_ino); - - // Workaround for apparent btrfs bug, where symlinks appear to be on - // a different filesystem than their containing directory, thanks to - // Toke Hoiland-Jorgensen. - - int type = file_st.st_mode & S_IFMT; - if(type == S_IFDIR && file_st.st_dev != dir_st.st_dev) - { - if(!(rParams.mrContext.ExcludeDir(filename))) - { - rNotifier.NotifyMountPointSkipped(this, filename); - } - return false; - } -#endif - - if(type == S_IFREG || type == S_IFLNK) - { - // File or symbolic link - - // Exclude it? - if(rParams.mrContext.ExcludeFile(realFileName)) - { - rNotifier.NotifyFileExcluded(this, realFileName); - // Next item! - return false; - } - } - else if(type == S_IFDIR) - { - // Directory - - // Exclude it? - if(rParams.mrContext.ExcludeDir(realFileName)) - { - rNotifier.NotifyDirExcluded(this, realFileName); - - // Next item! - return false; - } - - #ifdef WIN32 - // exclude reparse points, as Application Data points to the - // parent directory under Vista and later, and causes an - // infinite loop: - // http://social.msdn.microsoft.com/forums/en-US/windowscompatibility/thread/05d14368-25dd-41c8-bdba-5590bf762a68/ - if (en->d_type & FILE_ATTRIBUTE_REPARSE_POINT) - { - rNotifier.NotifyMountPointSkipped(this, realFileName); - return false; - } - #endif - } - else // not a file or directory, what is it? - { - if (type == S_IFSOCK -#ifndef WIN32 - || type == S_IFIFO -#endif - ) - { - // removed notification for these types - // see Debian bug 479145, no objections - } - else if(rParams.mrContext.ExcludeFile(realFileName)) - { - rNotifier.NotifyFileExcluded(this, realFileName); - } - else - { - rNotifier.NotifyUnsupportedFileType(this, realFileName); - SetErrorWhenReadingFilesystemObject(rParams, - realFileName); - } - - return false; - } - - // The object should be backed up (file, symlink or dir, not excluded). - // So make the information for adding to the checksum. - - #ifdef WIN32 - // We didn't stat the file before, but now we need the information. - if(emu_stat(filename.c_str(), &file_st) != 0) - { - rNotifier.NotifyFileStatFailed(this, - ConvertVssPathToRealPath(filename, rBackupLocation), - strerror(errno)); - - // Report the error (logs and eventual email to administrator) - SetErrorWhenReadingFilesystemObject(rParams, filename); - - // Ignore this entry for now. - return false; - } - - if(file_st.st_dev != dir_st.st_dev) - { - rNotifier.NotifyMountPointSkipped(this, - ConvertVssPathToRealPath(filename, rBackupLocation)); - return false; - } - #endif - - // Basic structure for checksum info - struct { - box_time_t mModificationTime; - box_time_t mAttributeModificationTime; - int64_t mSize; - // And then the name follows - } checksum_info; - - // Be paranoid about structure packing - ::memset(&checksum_info, 0, sizeof(checksum_info)); - - checksum_info.mModificationTime = FileModificationTime(file_st); - checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st); - checksum_info.mSize = file_st.st_size; - currentStateChecksum.Add(&checksum_info, sizeof(checksum_info)); - currentStateChecksum.Add(en->d_name, strlen(en->d_name)); - - // If the file has been modified madly into the future, download the - // directory record anyway to ensure that it doesn't get uploaded - // every single time the disc is scanned. - if(checksum_info.mModificationTime > rParams.mUploadAfterThisTimeInTheFuture) - { - rDownloadDirectoryRecordBecauseOfFutureFiles = true; - // Log that this has happened - if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) - { - rNotifier.NotifyFileModifiedInFuture(this, - ConvertVssPathToRealPath(filename, rBackupLocation)); - rParams.mHaveLoggedWarningAboutFutureFileTimes = true; - } - } - - // We've decided to back it up, so add to file or directory list. - if(type == S_IFREG || type == S_IFLNK) - { - rFiles.push_back(entry_name); - } - else if(type == S_IFDIR) - { - rDirs.push_back(entry_name); - } - - return true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::FetchDirectoryListing( -// BackupClientDirectoryRecord::SyncParams &) -// Purpose: Fetch the directory listing of this directory from -// the store. -// Created: 2003/10/09 -// -// -------------------------------------------------------------------------- -std::auto_ptr<BackupStoreDirectory> -BackupClientDirectoryRecord::FetchDirectoryListing( - BackupClientDirectoryRecord::SyncParams &rParams) -{ - std::auto_ptr<BackupStoreDirectory> apDir; - - // Get connection to store - BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); - - // Query the directory - std::auto_ptr<BackupProtocolSuccess> dirreply(connection.QueryListDirectory( - mObjectID, - // both files and directories - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - // exclude old/deleted stuff - BackupProtocolListDirectory::Flags_Deleted | - BackupProtocolListDirectory::Flags_OldVersion, - true /* want attributes */)); - - // Retrieve the directory from the stream following - apDir.reset(new BackupStoreDirectory(connection.ReceiveStream(), - connection.GetTimeout())); - return apDir; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::UpdateAttributes( -// BackupClientDirectoryRecord::SyncParams &, -// const std::string &) -// Purpose: Sets the attributes of the directory on the store, -// if necessary. -// Created: 2003/10/09 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::UpdateAttributes( - BackupClientDirectoryRecord::SyncParams &rParams, - BackupStoreDirectory *pDirOnStore, - const std::string &rLocalPath) -{ - // Get attributes for the directory - BackupClientFileAttributes attr; - box_time_t attrModTime = 0; - attr.ReadAttributes(rLocalPath.c_str(), true /* directories have zero mod times */, - 0 /* no modification time */, &attrModTime); - - // Assume attributes need updating, unless proved otherwise - bool updateAttr = true; - - // Got a listing to compare with? - ASSERT(pDirOnStore == 0 || (pDirOnStore != 0 && pDirOnStore->HasAttributes())); - if(pDirOnStore != 0 && pDirOnStore->HasAttributes()) - { - const StreamableMemBlock &storeAttrEnc(pDirOnStore->GetAttributes()); - // Explict decryption - BackupClientFileAttributes storeAttr(storeAttrEnc); - - // Compare the attributes - if(attr.Compare(storeAttr, true, - true /* ignore both modification times */)) - { - // No update necessary - updateAttr = false; - } - } - - // Update them? - if(updateAttr) - { - // Get connection to store - BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); - - // Exception thrown if this doesn't work - std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); - connection.QueryChangeDirAttributes(mObjectID, attrModTime, attrStream); - } -} - -std::string BackupClientDirectoryRecord::DecryptFilename( - BackupStoreDirectory::Entry *en, - const std::string& rRemoteDirectoryPath) -{ - BackupStoreFilenameClear fn(en->GetName()); - return DecryptFilename(fn, en->GetObjectID(), rRemoteDirectoryPath); -} - -std::string BackupClientDirectoryRecord::DecryptFilename( - BackupStoreFilenameClear fn, int64_t filenameObjectID, - const std::string& rRemoteDirectoryPath) -{ - std::string filenameClear; - try - { - filenameClear = fn.GetClearFilename(); - } - catch(BoxException &e) - { - BOX_ERROR("Failed to decrypt filename for object " << - BOX_FORMAT_OBJECTID(filenameObjectID) << " in " - "directory " << BOX_FORMAT_OBJECTID(mObjectID) << - " (" << rRemoteDirectoryPath << ")"); - throw; - } - return filenameClear; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &, const std::string &, BackupStoreDirectory *, std::vector<BackupStoreDirectory::Entry *> &) -// Purpose: Update the items stored on the server. The rFiles vector will be erased after it's used to save space. -// Returns true if all items were updated successfully. (If not, the failures will have been logged). -// Created: 2003/10/09 -// -// -------------------------------------------------------------------------- -bool BackupClientDirectoryRecord::UpdateItems( - BackupClientDirectoryRecord::SyncParams &rParams, - const std::string &rLocalPath, - const std::string &rRemotePath, - const Location& rBackupLocation, - BackupStoreDirectory *pDirOnStore, - std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, - std::vector<std::string> &rFiles, - const std::vector<std::string> &rDirs) -{ - BackupClientContext& rContext(rParams.mrContext); - ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); - - bool allUpdatedSuccessfully = true; - - // Decrypt all the directory entries. - // It would be nice to be able to just compare the encrypted versions, however this doesn't work - // in practise because there can be multiple encodings of the same filename using different - // methods (although each method will result in the same string for the same filename.) This - // happens when the server fixes a broken store, and gives plain text generated filenames. - // So if we didn't do things like this, then you wouldn't be able to recover from bad things - // happening with the server. - DecryptedEntriesMap_t decryptedEntries; - if(pDirOnStore != 0) - { - BackupStoreDirectory::Iterator i(*pDirOnStore); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next()) != 0) - { - std::string filenameClear; - try - { - filenameClear = DecryptFilename(en, - rRemotePath); - decryptedEntries[filenameClear] = en; - } - catch (CipherException &e) - { - BOX_ERROR("Failed to decrypt a filename, " - "pretending that the file doesn't " - "exist"); - } - } - } - - // Do files - for(std::vector<std::string>::const_iterator f = rFiles.begin(); - f != rFiles.end(); ++f) - { - // Send keep-alive message if needed - rContext.DoKeepAlive(); - - // Filename of this file - std::string filename(MakeFullPath(rLocalPath, *f)); - std::string nonVssFilePath = ConvertVssPathToRealPath(filename, - rBackupLocation); - - // Get relevant info about file - box_time_t modTime = 0; - uint64_t attributesHash = 0; - int64_t fileSize = 0; - InodeRefType inodeNum = 0; - // BLOCK - { - // Stat the file - EMU_STRUCT_STAT st; - if(EMU_LSTAT(filename.c_str(), &st) != 0) - { - rNotifier.NotifyFileStatFailed(this, nonVssFilePath, - strerror(errno)); - - // Report the error (logs and - // eventual email to administrator) - SetErrorWhenReadingFilesystemObject(rParams, nonVssFilePath); - - // Ignore this entry for now. - continue; - } - - // Extract required data - modTime = FileModificationTime(st); - fileSize = st.st_size; - inodeNum = st.st_ino; - attributesHash = BackupClientFileAttributes::GenerateAttributeHash(st, filename, *f); - } - - // See if it's in the listing (if we have one) - BackupStoreFilenameClear storeFilename(*f); - BackupStoreDirectory::Entry *en = 0; - int64_t latestObjectID = 0; - if(pDirOnStore != 0) - { - DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*f)); - if(i != decryptedEntries.end()) - { - en = i->second; - latestObjectID = en->GetObjectID(); - } - } - - // Check that the entry which might have been found is in fact a file - if((en != 0) && !(en->IsFile())) - { - // Directory exists in the place of this file -- sort it out - RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, - en, *f); - en = 0; - } - - // Check for renaming? - if(pDirOnStore != 0 && en == 0) - { - // We now know... - // 1) File has just been added - // 2) It's not in the store - - // Do we know about the inode number? - const BackupClientInodeToIDMap &idMap(rContext.GetCurrentIDMap()); - int64_t renameObjectID = 0, renameInDirectory = 0; - if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) - { - // Look up on the server to get the name, to build the local filename - std::string localPotentialOldName; - bool isDir = false; - bool isCurrentVersion = false; - box_time_t srvModTime = 0, srvAttributesHash = 0; - BackupStoreFilenameClear oldLeafname; - if(rContext.FindFilename(renameObjectID, renameInDirectory, - localPotentialOldName, isDir, isCurrentVersion, - &srvModTime, &srvAttributesHash, &oldLeafname)) - { - // Only interested if it's a file and the latest version - if(!isDir && isCurrentVersion) - { - // Check that the object we found in the ID map doesn't exist on disc - EMU_STRUCT_STAT st; - if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) - { - // Doesn't exist locally, but does exist on the server. - // Therefore we can safely rename it to this new file. - - // Get the connection to the server - BackupProtocolCallable &connection(rContext.GetConnection()); - - // Only do this step if there is room on the server. - // This step will be repeated later when there is space available - if(!rContext.StorageLimitExceeded()) - { - // Rename the existing files (ie include old versions) on the server - connection.QueryMoveObject(renameObjectID, - renameInDirectory, - mObjectID /* move to this directory */, - BackupProtocolMoveObject::Flags_MoveAllWithSameName | - BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, - storeFilename); - - // Stop the attempt to delete the file in the original location - BackupClientDeleteList &rdelList(rContext.GetDeleteList()); - rdelList.StopFileDeletion(renameInDirectory, oldLeafname); - - // Create new entry in the directory for it - // -- will be near enough what's actually on the server for the rest to work. - en = pDirOnStore->AddEntry(storeFilename, - srvModTime, renameObjectID, - 0 /* size in blocks unknown, but not needed */, - BackupStoreDirectory::Entry::Flags_File, - srvAttributesHash); - - // Store the object ID for the inode lookup map later - latestObjectID = renameObjectID; - } - } - } - } - } - } - - // Is it in the mPendingEntries list? - box_time_t pendingFirstSeenTime = 0; // ie not seen - if(mpPendingEntries != 0) - { - std::map<std::string, box_time_t>::const_iterator i(mpPendingEntries->find(*f)); - if(i != mpPendingEntries->end()) - { - // found it -- set flag - pendingFirstSeenTime = i->second; - } - } - - // If pDirOnStore == 0, then this must have been after an initial sync: - ASSERT(pDirOnStore != 0 || mInitialSyncDone); - // So, if pDirOnStore == 0, then we know that everything before syncPeriodStart - // is either on the server, or in the toupload list. If the directory had changed, - // we'd have got a directory listing. - // - // At this point, if (pDirOnStore == 0 && en == 0), we can assume it's on the server with a - // mod time < syncPeriodStart, or didn't exist before that time. - // - // But if en != 0, then we need to compare modification times to avoid uploading it again. - - // Need to update? - // - // Condition for upload: - // modification time within sync period - // if it's been seen before but not uploaded, is the time from this first sight longer than the MaxUploadWait - // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store - - bool doUpload = false; - std::string decisionReason = "unknown"; - - // Only upload a file if the mod time locally is - // different to that on the server. - - if (en == 0 || en->GetModificationTime() != modTime) - { - // Check the file modified within the acceptable time period we're checking - // If the file isn't on the server, the acceptable time starts at zero. - // Check pDirOnStore and en, because if we didn't download a directory listing, - // pDirOnStore will be zero, but we know it's on the server. - if (modTime < rParams.mSyncPeriodEnd) - { - if (pDirOnStore != 0 && en == 0) - { - doUpload = true; - decisionReason = "not on server"; - } - else if (modTime >= rParams.mSyncPeriodStart) - { - doUpload = true; - decisionReason = "modified since last sync"; - } - } - - // However, just in case things are continually - // modified, we check the first seen time. - // The two compares of syncPeriodEnd and - // pendingFirstSeenTime are because the values - // are unsigned. - - if (!doUpload && - pendingFirstSeenTime != 0 && - rParams.mSyncPeriodEnd > pendingFirstSeenTime && - (rParams.mSyncPeriodEnd - pendingFirstSeenTime) - > rParams.mMaxUploadWait) - { - doUpload = true; - decisionReason = "continually modified"; - } - - // Then make sure that if files are added with a - // time less than the sync period start - // (which can easily happen on file server), it - // gets uploaded. The directory contents checksum - // will pick up the fact it has been added, so the - // store listing will be available when this happens. - - if (!doUpload && - modTime <= rParams.mSyncPeriodStart && - en != 0 && - en->GetModificationTime() != modTime) - { - doUpload = true; - decisionReason = "mod time changed"; - } - - // And just to catch really badly off clocks in - // the future for file server clients, - // just upload the file if it's madly in the future. - - if (!doUpload && modTime > - rParams.mUploadAfterThisTimeInTheFuture) - { - doUpload = true; - decisionReason = "mod time in the future"; - } - } - - if (en != 0 && en->GetModificationTime() == modTime) - { - doUpload = false; - decisionReason = "not modified since last upload"; - } - else if (!doUpload) - { - if (modTime > rParams.mSyncPeriodEnd) - { - box_time_t now = GetCurrentBoxTime(); - int age = BoxTimeToSeconds(now - - modTime); - std::ostringstream s; - s << "modified too recently: only " << - age << " seconds ago"; - decisionReason = s.str(); - } - else - { - std::ostringstream s; - s << "mod time is " << modTime << - " which is outside sync window, " - << rParams.mSyncPeriodStart << " to " - << rParams.mSyncPeriodEnd; - decisionReason = s.str(); - } - } - - BOX_TRACE("Upload decision: " << nonVssFilePath << ": " << - (doUpload ? "will upload" : "will not upload") << - " (" << decisionReason << ")"); - - bool fileSynced = true; - - if (doUpload) - { - // Upload needed, don't mark sync success until - // we've actually done it - fileSynced = false; - - // Make sure we're connected -- must connect here so we know whether - // the storage limit has been exceeded, and hence whether or not - // to actually upload the file. - rContext.GetConnection(); - - // Only do this step if there is room on the server. - // This step will be repeated later when there is space available - if(!rContext.StorageLimitExceeded()) - { - // Upload the file to the server, recording the - // object ID it returns - bool noPreviousVersionOnServer = - ((pDirOnStore != 0) && (en == 0)); - - // Surround this in a try/catch block, to - // catch errors, but still continue - bool uploadSuccess = false; - try - { - latestObjectID = UploadFile(rParams, - filename, - nonVssFilePath, - rRemotePath + "/" + *f, - storeFilename, - fileSize, modTime, - attributesHash, - noPreviousVersionOnServer); - - if (latestObjectID == 0) - { - // storage limit exceeded - rParams.mrContext.SetStorageLimitExceeded(); - uploadSuccess = false; - allUpdatedSuccessfully = false; - } - else - { - uploadSuccess = true; - } - } - catch(ConnectionException &e) - { - // Connection errors should just be - // passed on to the main handler, - // retries would probably just cause - // more problems. - rNotifier.NotifyFileUploadException( - this, nonVssFilePath, e); - throw; - } - catch(BoxException &e) - { - if (e.GetType() == BackupStoreException::ExceptionType && - e.GetSubType() == BackupStoreException::SignalReceived) - { - // abort requested, pass the - // exception on up. - throw; - } - - // an error occured -- make return - // code false, to show error in directory - allUpdatedSuccessfully = false; - // Log it. - SetErrorWhenReadingFilesystemObject(rParams, - nonVssFilePath); - rNotifier.NotifyFileUploadException(this, - nonVssFilePath, e); - } - - // Update structures if the file was uploaded - // successfully. - if(uploadSuccess) - { - fileSynced = true; - - // delete from pending entries - if(pendingFirstSeenTime != 0 && mpPendingEntries != 0) - { - mpPendingEntries->erase(*f); - } - } - } - else - { - rNotifier.NotifyFileSkippedServerFull(this, nonVssFilePath); - } - } - else if(en != 0 && en->GetAttributesHash() != attributesHash) - { - // Attributes have probably changed, upload them again. - // If the attributes have changed enough, the directory - // hash will have changed too, and so the dir will have - // been downloaded, and the entry will be available. - - // Get connection - BackupProtocolCallable &connection(rContext.GetConnection()); - - // Only do this step if there is room on the server. - // This step will be repeated later when there is - // space available - if(!rContext.StorageLimitExceeded()) - { - try - { - rNotifier.NotifyFileUploadingAttributes(this, - nonVssFilePath); - - // Update store - BackupClientFileAttributes attr; - attr.ReadAttributes(filename, - false /* put mod times in the attributes, please */); - std::auto_ptr<IOStream> attrStream( - new MemBlockStream(attr)); - connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream); - fileSynced = true; - } - catch (BoxException &e) - { - BOX_ERROR("Failed to read or store file attributes " - "for '" << nonVssFilePath << "', will try again " - "later"); - } - } - } - - if(modTime >= rParams.mSyncPeriodEnd) - { - // Allocate? - if(mpPendingEntries == 0) - { - mpPendingEntries = new std::map<std::string, box_time_t>; - } - // Adding to mPendingEntries list - if(pendingFirstSeenTime == 0) - { - // Haven't seen this before -- add to list! - (*mpPendingEntries)[*f] = modTime; - } - } - - // Zero pointer in rEntriesLeftOver, if we have a pointer to zero - if(en != 0) - { - for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) - { - if(rEntriesLeftOver[l] == en) - { - rEntriesLeftOver[l] = 0; - break; - } - } - } - - // Does this file need an entry in the ID map? - if(fileSize >= rParams.mFileTrackingSizeThreshold) - { - // Get the map - BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap()); - - // Need to get an ID from somewhere... - if(latestObjectID == 0) - { - // Don't know it -- haven't sent anything to the store, and didn't get a listing. - // Look it up in the current map, and if it's there, use that. - const BackupClientInodeToIDMap ¤tIDMap(rContext.GetCurrentIDMap()); - int64_t objid = 0, dirid = 0; - if(currentIDMap.Lookup(inodeNum, objid, dirid)) - { - // Found - if (dirid != mObjectID) - { - BOX_WARNING("Found conflicting parent ID for " - "file ID " << inodeNum << " (" << - nonVssFilePath << "): expected " << - mObjectID << " but found " << dirid << - " (same directory used in two different " - "locations?)"); - } - - ASSERT(dirid == mObjectID); - - // NOTE: If the above assert fails, an inode number has been reused by the OS, - // or there is a problem somewhere. If this happened on a short test run, look - // into it. However, in a long running process this may happen occasionally and - // not indicate anything wrong. - // Run the release version for real life use, where this check is not made. - - latestObjectID = objid; - } - } - - if(latestObjectID != 0) - { - BOX_TRACE("Storing uploaded file ID " << - inodeNum << " (" << nonVssFilePath << ") " - "in ID map as object " << - latestObjectID << " with parent " << - mObjectID); - idMap.AddToMap(inodeNum, latestObjectID, - mObjectID /* containing directory */, - nonVssFilePath); - } - - } - - if (fileSynced) - { - rNotifier.NotifyFileSynchronised(this, nonVssFilePath, - fileSize); - } - } - - // Erase contents of files to save space when recursing - rFiles.clear(); - - // Delete the pending entries, if the map is empty - if(mpPendingEntries != 0 && mpPendingEntries->size() == 0) - { - BOX_TRACE("Deleting mpPendingEntries from dir ID " << - BOX_FORMAT_OBJECTID(mObjectID)); - delete mpPendingEntries; - mpPendingEntries = 0; - } - - // Do directories - for(std::vector<std::string>::const_iterator d = rDirs.begin(); - d != rDirs.end(); ++d) - { - // Send keep-alive message if needed - rContext.DoKeepAlive(); - - // Get the local filename - std::string dirname(MakeFullPath(rLocalPath, *d)); - std::string nonVssDirPath = ConvertVssPathToRealPath(dirname, - rBackupLocation); - - // See if it's in the listing (if we have one) - BackupStoreFilenameClear storeFilename(*d); - BackupStoreDirectory::Entry *en = 0; - if(pDirOnStore != 0) - { - DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*d)); - if(i != decryptedEntries.end()) - { - en = i->second; - } - } - - // Check that the entry which might have been found is in fact a directory - if((en != 0) && !(en->IsDir())) - { - // Entry exists, but is not a directory. Bad. - // Get rid of it. - BackupProtocolCallable &connection(rContext.GetConnection()); - connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); - - std::string filenameClear = DecryptFilename(en, - rRemotePath); - rNotifier.NotifyFileDeleted(en->GetObjectID(), - filenameClear); - - // Nothing found - en = 0; - } - - // Zero pointer in rEntriesLeftOver, if we have a pointer to zero - if(en != 0) - { - for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) - { - if(rEntriesLeftOver[l] == en) - { - rEntriesLeftOver[l] = 0; - break; - } - } - } - - // Flag for having created directory, so can optimise the - // recursive call not to read it again, because we know - // it's empty. - bool haveJustCreatedDirOnServer = false; - - // Next, see if it's in the list of sub directories - BackupClientDirectoryRecord *psubDirRecord = 0; - std::map<std::string, BackupClientDirectoryRecord *>::iterator - e(mSubDirectories.find(*d)); - - if(e != mSubDirectories.end()) - { - // In the list, just use this pointer - psubDirRecord = e->second; - } - else - { - // Note: if we have exceeded our storage limit, then - // we should not upload any more data, nor create any - // DirectoryRecord representing data that would have - // been uploaded. This step will be repeated when - // there is some space available. - bool doCreateDirectoryRecord = true; - - // Need to create the record. But do we need to create the directory on the server? - int64_t subDirObjectID = 0; - if(en != 0) - { - // No. Exists on the server, and we know about it from the listing. - subDirObjectID = en->GetObjectID(); - } - else if(rContext.StorageLimitExceeded()) - // know we've got a connection if we get this far, - // as dir will have been modified. - { - doCreateDirectoryRecord = false; - } - else - { - // Yes, creation required! - // It is known that it doesn't exist: - // - // if en == 0 and pDirOnStore == 0, then the - // directory has had an initial sync, and - // hasn't been modified (Really? then why - // are we here? TODO FIXME) - // so it has definitely been created already - // (so why create it again?) - // - // if en == 0 but pDirOnStore != 0, well... obviously it doesn't exist. - // - subDirObjectID = CreateRemoteDir(dirname, - nonVssDirPath, rRemotePath + "/" + *d, - storeFilename, &haveJustCreatedDirOnServer, - rParams); - doCreateDirectoryRecord = (subDirObjectID != 0); - } - - if (doCreateDirectoryRecord) - { - // New an object for this - psubDirRecord = new BackupClientDirectoryRecord(subDirObjectID, *d); - - // Store in list - try - { - mSubDirectories[*d] = psubDirRecord; - } - catch(...) - { - delete psubDirRecord; - psubDirRecord = 0; - throw; - } - } - } - - // ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded()); - // There's another possible reason now: the directory no longer - // existed when we finally got around to checking its - // attributes. See for example Brendon Baumgartner's reported - // error with Wordpress cache directories. - - if(psubDirRecord) - { - // Sync this sub directory too - psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, - rRemotePath + "/" + *d, rBackupLocation, - haveJustCreatedDirOnServer); - } - } - - // Delete everything which is on the store, but not on disc - for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) - { - if(rEntriesLeftOver[l] != 0) - { - BackupStoreDirectory::Entry *en = rEntriesLeftOver[l]; - - // These entries can't be deleted immediately, as it would prevent - // renaming and moving of objects working properly. So we add them - // to a list, which is actually deleted at the very end of the session. - // If there's an error during the process, it doesn't matter if things - // aren't actually deleted, as the whole state will be reset anyway. - BackupClientDeleteList &rdel(rContext.GetDeleteList()); - std::string filenameClear; - bool isCorruptFilename = false; - - try - { - filenameClear = DecryptFilename(en, - rRemotePath); - } - catch (CipherException &e) - { - BOX_ERROR("Failed to decrypt a filename, " - "scheduling that file for deletion"); - filenameClear = "<corrupt filename>"; - isCorruptFilename = true; - } - - std::string localName = MakeFullPath(rLocalPath, - filenameClear); - std::string nonVssLocalName = ConvertVssPathToRealPath(localName, - rBackupLocation); - - // Delete this entry -- file or directory? - if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) - { - // Set a pending deletion for the file - rdel.AddFileDelete(mObjectID, en->GetName(), - localName); - } - else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) - { - // Set as a pending deletion for the directory - rdel.AddDirectoryDelete(en->GetObjectID(), - localName); - - // If there's a directory record for it in - // the sub directory map, delete it now - BackupStoreFilenameClear dirname(en->GetName()); - std::map<std::string, BackupClientDirectoryRecord *>::iterator - e(mSubDirectories.find(filenameClear)); - if(e != mSubDirectories.end() && !isCorruptFilename) - { - // Carefully delete the entry from the map - BackupClientDirectoryRecord *rec = e->second; - mSubDirectories.erase(e); - delete rec; - - BOX_TRACE("Deleted directory record for " << - nonVssLocalName); - } - } - } - } - - // Return success flag (will be false if some files failed) - return allUpdatedSuccessfully; -} - -int64_t BackupClientDirectoryRecord::CreateRemoteDir(const std::string& localDirPath, - const std::string& nonVssDirPath, const std::string& remoteDirPath, - BackupStoreFilenameClear& storeFilename, bool* pHaveJustCreatedDirOnServer, - BackupClientDirectoryRecord::SyncParams &rParams) -{ - // Get attributes - box_time_t attrModTime = 0; - InodeRefType inodeNum = 0; - BackupClientFileAttributes attr; - *pHaveJustCreatedDirOnServer = false; - ProgressNotifier& rNotifier(rParams.mrContext.GetProgressNotifier()); - - try - { - attr.ReadAttributes(localDirPath, - true /* directories have zero mod times */, - 0 /* not interested in mod time */, - &attrModTime, 0 /* not file size */, - &inodeNum); - } - catch (BoxException &e) - { - // We used to try to recover from this, but we'd need an - // attributes block to upload to the server, so we have to - // skip creating the directory instead. - BOX_WARNING("Failed to read attributes of directory, " - "ignoring it for now: " << nonVssDirPath); - return 0; // no object ID - } - - // Check to see if the directory been renamed - // First, do we have a record in the ID map? - int64_t renameObjectID = 0, renameInDirectory = 0; - bool renameDir = false; - const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap()); - - if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) - { - // Look up on the server to get the name, to build the local filename - std::string localPotentialOldName; - bool isDir = false; - bool isCurrentVersion = false; - if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, - localPotentialOldName, isDir, isCurrentVersion)) - { - // Only interested if it's a directory - if(isDir && isCurrentVersion) - { - // Check that the object doesn't exist already - EMU_STRUCT_STAT st; - if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && - errno == ENOENT) - { - // Doesn't exist locally, but does exist - // on the server. Therefore we can - // safely rename it. - renameDir = true; - } - } - } - } - - // Get connection - BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); - - // Don't do a check for storage limit exceeded here, because if we get to this - // stage, a connection will have been opened, and the status known, so the check - // in the else if(...) above will be correct. - - // Build attribute stream for sending - std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); - - if(renameDir) - { - // Rename the existing directory on the server - connection.QueryMoveObject(renameObjectID, - renameInDirectory, - mObjectID /* move to this directory */, - BackupProtocolMoveObject::Flags_MoveAllWithSameName | - BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, - storeFilename); - - // Put the latest attributes on it - connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream); - - // Stop it being deleted later - BackupClientDeleteList &rdelList( - rParams.mrContext.GetDeleteList()); - rdelList.StopDirectoryDeletion(renameObjectID); - - // This is the ID for the renamed directory - return renameObjectID; - } - else - { - int64_t subDirObjectID = 0; // no object ID - - // Create a new directory - try - { - subDirObjectID = connection.QueryCreateDirectory( - mObjectID, attrModTime, storeFilename, - attrStream)->GetObjectID(); - // Flag as having done this for optimisation later - *pHaveJustCreatedDirOnServer = true; - } - catch(BoxException &e) - { - int type, subtype; - connection.GetLastError(type, subtype); - rNotifier.NotifyFileUploadServerError(this, nonVssDirPath, - type, subtype); - if(e.GetType() == ConnectionException::ExceptionType && - e.GetSubType() == ConnectionException::Protocol_UnexpectedReply && - type == BackupProtocolError::ErrorType && - subtype == BackupProtocolError::Err_StorageLimitExceeded) - { - // The hard limit was exceeded on the server, notify! - rParams.mrContext.SetStorageLimitExceeded(); - rParams.mrSysadminNotifier.NotifySysadmin( - SysadminNotifier::StoreFull); - } - else - { - throw; - } - } - - if(*pHaveJustCreatedDirOnServer) - { - rNotifier.NotifyDirectoryCreated(subDirObjectID, - nonVssDirPath, remoteDirPath); - } - - return subDirObjectID; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &, BackupStoreDirectory *, int64_t, const std::string &) -// Purpose: Called to resolve difficulties when a directory is found on the -// store where a file is to be uploaded. -// Created: 9/7/04 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile( - SyncParams &rParams, - BackupStoreDirectory* pDirOnStore, - BackupStoreDirectory::Entry* pEntry, - const std::string &rFilename) -{ - // First, delete the directory - BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); - connection.QueryDeleteDirectory(pEntry->GetObjectID()); - - BackupStoreFilenameClear clear(pEntry->GetName()); - rParams.mrContext.GetProgressNotifier().NotifyDirectoryDeleted( - pEntry->GetObjectID(), clear.GetClearFilename()); - - // Then, delete any directory record - std::map<std::string, BackupClientDirectoryRecord *>::iterator - e(mSubDirectories.find(rFilename)); - - if(e != mSubDirectories.end()) - { - // A record exists for this, remove it - BackupClientDirectoryRecord *psubDirRecord = e->second; - mSubDirectories.erase(e); - - // And delete the object - delete psubDirRecord; - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::UploadFile( -// BackupClientDirectoryRecord::SyncParams &, -// const std::string &, -// const BackupStoreFilename &, -// int64_t, box_time_t, box_time_t, bool) -// Purpose: Private. Upload a file to the server. May send -// a patch instead of the whole thing -// Created: 20/1/04 -// -// -------------------------------------------------------------------------- -int64_t BackupClientDirectoryRecord::UploadFile( - BackupClientDirectoryRecord::SyncParams &rParams, - const std::string &rLocalPath, - const std::string &rNonVssFilePath, - const std::string &rRemotePath, - const BackupStoreFilenameClear &rStoreFilename, - int64_t FileSize, - box_time_t ModificationTime, - box_time_t AttributesHash, - bool NoPreviousVersionOnServer) -{ - BackupClientContext& rContext(rParams.mrContext); - ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); - - // Get the connection - BackupProtocolCallable &connection(rContext.GetConnection()); - - // Info - int64_t objID = 0; - int64_t uploadedSize = -1; - - // Use a try block to catch store full errors - try - { - std::auto_ptr<BackupStoreFileEncodeStream> apStreamToUpload; - int64_t diffFromID = 0; - - // Might an old version be on the server, and is the file - // size over the diffing threshold? - if(!NoPreviousVersionOnServer && - FileSize >= rParams.mDiffingUploadSizeThreshold) - { - // YES -- try to do diff, if possible - // First, query the server to see if there's an old version available - std::auto_ptr<BackupProtocolSuccess> getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename)); - diffFromID = getBlockIndex->GetObjectID(); - - if(diffFromID != 0) - { - // Found an old version - - // Get the index - std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream()); - - // - // Diff the file - // - - rContext.ManageDiffProcess(); - - bool isCompletelyDifferent = false; - - apStreamToUpload = BackupStoreFile::EncodeFileDiff( - rLocalPath, - mObjectID, /* containing directory */ - rStoreFilename, diffFromID, *blockIndexStream, - connection.GetTimeout(), - &rContext, // DiffTimer implementation - 0 /* not interested in the modification time */, - &isCompletelyDifferent, - rParams.mpBackgroundTask); - - if(isCompletelyDifferent) - { - diffFromID = 0; - } - - rContext.UnManageDiffProcess(); - } - } - - if(apStreamToUpload.get()) - { - rNotifier.NotifyFileUploadingPatch(this, rNonVssFilePath, - apStreamToUpload->GetBytesToUpload()); - } - else // No patch upload, so do a normal upload - { - // below threshold or nothing to diff from, so upload whole - rNotifier.NotifyFileUploading(this, rNonVssFilePath); - - // Prepare to upload, getting a stream which will encode the file as we go along - apStreamToUpload = BackupStoreFile::EncodeFile( - rLocalPath, mObjectID, /* containing directory */ - rStoreFilename, NULL, &rParams, - &(rParams.mrRunStatusProvider), - rParams.mpBackgroundTask); - } - - rContext.SetNiceMode(true); - std::auto_ptr<IOStream> apWrappedStream; - - if(rParams.mMaxUploadRate > 0) - { - apWrappedStream.reset(new RateLimitingStream( - *apStreamToUpload, rParams.mMaxUploadRate)); - } - else - { - // Wrap the stream in *something*, so that - // QueryStoreFile() doesn't delete the original - // stream (upload object) and we can retrieve - // the byte counter. - apWrappedStream.reset(new BufferedStream( - *apStreamToUpload)); - } - - // Send to store - std::auto_ptr<BackupProtocolSuccess> stored( - connection.QueryStoreFile(mObjectID, ModificationTime, - AttributesHash, diffFromID, rStoreFilename, - apWrappedStream)); - - rContext.SetNiceMode(false); - - // Get object ID from the result - objID = stored->GetObjectID(); - uploadedSize = apStreamToUpload->GetTotalBytesSent(); - } - catch(BoxException &e) - { - rContext.UnManageDiffProcess(); - - if(e.GetType() == ConnectionException::ExceptionType && - e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) - { - // Check and see what error the protocol has, - // this is more useful to users than the exception. - int type, subtype; - if(connection.GetLastError(type, subtype)) - { - if(type == BackupProtocolError::ErrorType - && subtype == BackupProtocolError::Err_StorageLimitExceeded) - { - // The hard limit was exceeded on the server, notify! - rParams.mrSysadminNotifier.NotifySysadmin( - SysadminNotifier::StoreFull); - // return an error code instead of - // throwing an exception that we - // can't debug. - return 0; - } - rNotifier.NotifyFileUploadServerError(this, - rNonVssFilePath, type, subtype); - } - } - - // Send the error on it's way - throw; - } - - rNotifier.NotifyFileUploaded(this, rNonVssFilePath, FileSize, - uploadedSize, objID); - - // Return the new object ID of this file - return objID; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( -// SyncParams &, const char *) -// Purpose: Sets the error state when there were problems -// reading an object from the filesystem. -// Created: 29/3/04 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( - BackupClientDirectoryRecord::SyncParams &rParams, - const std::string& rFilename) -{ - // Zero hash, so it gets synced properly next time round. - ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); - - // More detailed logging was already done by the caller, but if we - // have a read error reported, we need to be able to search the logs - // to find out which file it was, so we need to log a consistent and - // clear error message. - BOX_WARNING("Failed to backup file, see above for details: " << - rFilename); - - // Mark that an error occured in the parameters object - rParams.mReadErrorsOnFilesystemObjects = true; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::SyncParams::SyncParams(BackupClientContext &) -// Purpose: Constructor -// Created: 8/3/04 -// -// -------------------------------------------------------------------------- -BackupClientDirectoryRecord::SyncParams::SyncParams( - RunStatusProvider &rRunStatusProvider, - SysadminNotifier &rSysadminNotifier, - ProgressNotifier &rProgressNotifier, - BackupClientContext &rContext, - BackgroundTask *pBackgroundTask) -: mSyncPeriodStart(0), - mSyncPeriodEnd(0), - mMaxUploadWait(0), - mMaxFileTimeInFuture(99999999999999999LL), - mFileTrackingSizeThreshold(16*1024), - mDiffingUploadSizeThreshold(16*1024), - mpBackgroundTask(pBackgroundTask), - mrRunStatusProvider(rRunStatusProvider), - mrSysadminNotifier(rSysadminNotifier), - mrProgressNotifier(rProgressNotifier), - mrContext(rContext), - mReadErrorsOnFilesystemObjects(false), - mMaxUploadRate(0), - mUploadAfterThisTimeInTheFuture(99999999999999999LL), - mHaveLoggedWarningAboutFutureFileTimes(false) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::SyncParams::~SyncParams() -// Purpose: Destructor -// Created: 8/3/04 -// -// -------------------------------------------------------------------------- -BackupClientDirectoryRecord::SyncParams::~SyncParams() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::Deserialize(Archive & rArchive) -// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. -// -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::Deserialize(Archive & rArchive) -{ - // Make deletion recursive - DeleteSubDirectories(); - - // Delete maps - if(mpPendingEntries != 0) - { - delete mpPendingEntries; - mpPendingEntries = 0; - } - - // - // - // - rArchive.Read(mObjectID); - rArchive.Read(mSubDirName); - rArchive.Read(mInitialSyncDone); - rArchive.Read(mSyncDone); - - // - // - // - int64_t iCount = 0; - rArchive.Read(iCount); - - if (iCount != sizeof(mStateChecksum)/sizeof(mStateChecksum[0])) - { - // we have some kind of internal system representation change: throw for now - THROW_EXCEPTION(CommonException, Internal) - } - - for (int v = 0; v < iCount; v++) - { - // Load each checksum entry - rArchive.Read(mStateChecksum[v]); - } - - // - // - // - iCount = 0; - rArchive.Read(iCount); - - if (iCount > 0) - { - // load each pending entry - mpPendingEntries = new std::map<std::string, box_time_t>; - if (!mpPendingEntries) - { - throw std::bad_alloc(); - } - - for (int v = 0; v < iCount; v++) - { - std::string strItem; - box_time_t btItem; - - rArchive.Read(strItem); - rArchive.Read(btItem); - (*mpPendingEntries)[strItem] = btItem; - } - } - - // - // - // - iCount = 0; - rArchive.Read(iCount); - - if (iCount > 0) - { - for (int v = 0; v < iCount; v++) - { - std::string strItem; - rArchive.Read(strItem); - - BackupClientDirectoryRecord* pSubDirRecord = - new BackupClientDirectoryRecord(0, ""); - // will be deserialized anyway, give it id 0 for now - - if (!pSubDirRecord) - { - throw std::bad_alloc(); - } - - /***** RECURSE *****/ - pSubDirRecord->Deserialize(rArchive); - mSubDirectories[strItem] = pSubDirRecord; - } - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientDirectoryRecord::Serialize(Archive & rArchive) -// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. -// -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const -{ - // - // - // - rArchive.Write(mObjectID); - rArchive.Write(mSubDirName); - rArchive.Write(mInitialSyncDone); - rArchive.Write(mSyncDone); - - // - // - // - int64_t iCount = 0; - - // when reading back the archive, we will - // need to know how many items there are. - iCount = sizeof(mStateChecksum) / sizeof(mStateChecksum[0]); - rArchive.Write(iCount); - - for (int v = 0; v < iCount; v++) - { - rArchive.Write(mStateChecksum[v]); - } - - // - // - // - if (!mpPendingEntries) - { - iCount = 0; - rArchive.Write(iCount); - } - else - { - iCount = mpPendingEntries->size(); - rArchive.Write(iCount); - - for (std::map<std::string, box_time_t>::const_iterator - i = mpPendingEntries->begin(); - i != mpPendingEntries->end(); i++) - { - rArchive.Write(i->first); - rArchive.Write(i->second); - } - } - // - // - // - iCount = mSubDirectories.size(); - rArchive.Write(iCount); - - for (std::map<std::string, BackupClientDirectoryRecord*>::const_iterator - i = mSubDirectories.begin(); - i != mSubDirectories.end(); i++) - { - const BackupClientDirectoryRecord* pSubItem = i->second; - ASSERT(pSubItem); - - rArchive.Write(i->first); - pSubItem->Serialize(rArchive); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: Location::Location() -// Purpose: Constructor -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -Location::Location() -: mIDMapIndex(0) -{ } - -// -------------------------------------------------------------------------- -// -// Function -// Name: Location::~Location() -// Purpose: Destructor -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -Location::~Location() -{ } - -// -------------------------------------------------------------------------- -// -// Function -// Name: Location::Serialize(Archive & rArchive) -// Purpose: Serializes this object instance into a stream of bytes, -// using an Archive abstraction. -// -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -void Location::Serialize(Archive & rArchive) const -{ - // - // - // - rArchive.Write(mName); - rArchive.Write(mPath); - rArchive.Write(mIDMapIndex); - - // - // - // - if(!mapDirectoryRecord.get()) - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; - rArchive.Write(aMagicMarker); - } - else - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows - rArchive.Write(aMagicMarker); - - mapDirectoryRecord->Serialize(rArchive); - } - - // - // - // - if(!mapExcludeFiles.get()) - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; - rArchive.Write(aMagicMarker); - } - else - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows - rArchive.Write(aMagicMarker); - - mapExcludeFiles->Serialize(rArchive); - } - - // - // - // - if(!mapExcludeDirs.get()) - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; - rArchive.Write(aMagicMarker); - } - else - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows - rArchive.Write(aMagicMarker); - - mapExcludeDirs->Serialize(rArchive); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: Location::Deserialize(Archive & rArchive) -// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. -// -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -void Location::Deserialize(Archive &rArchive) -{ - // - // - // - mapDirectoryRecord.reset(); - mapExcludeFiles.reset(); - mapExcludeDirs.reset(); - - // - // - // - rArchive.Read(mName); - rArchive.Read(mPath); - rArchive.Read(mIDMapIndex); - - // - // - // - int64_t aMagicMarker = 0; - rArchive.Read(aMagicMarker); - - if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) - { - // NOOP - } - else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) - { - BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, ""); - if(!pSubRecord) - { - throw std::bad_alloc(); - } - - mapDirectoryRecord.reset(pSubRecord); - mapDirectoryRecord->Deserialize(rArchive); - } - else - { - // there is something going on here - THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); - } - - // - // - // - rArchive.Read(aMagicMarker); - - if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) - { - // NOOP - } - else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) - { - mapExcludeFiles.reset(new ExcludeList); - if(!mapExcludeFiles.get()) - { - throw std::bad_alloc(); - } - - mapExcludeFiles->Deserialize(rArchive); - } - else - { - // there is something going on here - THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); - } - - // - // - // - rArchive.Read(aMagicMarker); - - if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) - { - // NOOP - } - else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) - { - mapExcludeDirs.reset(new ExcludeList); - if(!mapExcludeDirs.get()) - { - throw std::bad_alloc(); - } - - mapExcludeDirs->Deserialize(rArchive); - } - else - { - // there is something going on here - THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); - } -} diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/bin/bbackupd/BackupClientDirectoryRecord.h deleted file mode 100644 index 865fc747..00000000 --- a/bin/bbackupd/BackupClientDirectoryRecord.h +++ /dev/null @@ -1,244 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientDirectoryRecord.h -// Purpose: Implementation of record about directory for backup client -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCLIENTDIRECTORYRECORD__H -#define BACKUPCLIENTDIRECTORYRECORD__H - -#include <string> -#include <map> -#include <memory> - -#include "BackgroundTask.h" -#include "BackupClientFileAttributes.h" -#include "BackupDaemonInterface.h" -#include "BackupStoreDirectory.h" -#include "BoxTime.h" -#include "MD5Digest.h" -#include "ReadLoggingStream.h" -#include "RunStatusProvider.h" - -#ifdef ENABLE_VSS -# include <comdef.h> -# include <Vss.h> -# include <VsWriter.h> -# include <VsBackup.h> -#endif - -class Archive; -class BackupClientContext; -class BackupDaemon; -class ExcludeList; -class Location; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupClientDirectoryRecord -// Purpose: Implementation of record about directory for backup client -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -class BackupClientDirectoryRecord -{ -public: - BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName); - virtual ~BackupClientDirectoryRecord(); - - void Deserialize(Archive & rArchive); - void Serialize(Archive & rArchive) const; -private: - BackupClientDirectoryRecord(const BackupClientDirectoryRecord &); -public: - - enum - { - UnknownDirectoryID = 0 - }; - - // -------------------------------------------------------------------------- - // - // Class - // Name: BackupClientDirectoryRecord::SyncParams - // Purpose: Holds parameters etc for directory syncing. Not passed as - // const, some parameters may be modified during sync. - // Created: 8/3/04 - // - // -------------------------------------------------------------------------- - class SyncParams : public ReadLoggingStream::Logger - { - public: - SyncParams( - RunStatusProvider &rRunStatusProvider, - SysadminNotifier &rSysadminNotifier, - ProgressNotifier &rProgressNotifier, - BackupClientContext &rContext, - BackgroundTask *pBackgroundTask); - ~SyncParams(); - private: - // No copying - SyncParams(const SyncParams&); - SyncParams &operator=(const SyncParams&); - - public: - // Data members are public, as accessors are not justified here - box_time_t mSyncPeriodStart; - box_time_t mSyncPeriodEnd; - box_time_t mMaxUploadWait; - box_time_t mMaxFileTimeInFuture; - int32_t mFileTrackingSizeThreshold; - int32_t mDiffingUploadSizeThreshold; - BackgroundTask *mpBackgroundTask; - RunStatusProvider &mrRunStatusProvider; - SysadminNotifier &mrSysadminNotifier; - ProgressNotifier &mrProgressNotifier; - BackupClientContext &mrContext; - bool mReadErrorsOnFilesystemObjects; - int64_t mMaxUploadRate; - - // Member variables modified by syncing process - box_time_t mUploadAfterThisTimeInTheFuture; - bool mHaveLoggedWarningAboutFutureFileTimes; - - bool StopRun() { return mrRunStatusProvider.StopRun(); } - void NotifySysadmin(SysadminNotifier::EventCode Event) - { - mrSysadminNotifier.NotifySysadmin(Event); - } - ProgressNotifier& GetProgressNotifier() const - { - return mrProgressNotifier; - } - - /* ReadLoggingStream::Logger implementation */ - virtual void Log(int64_t readSize, int64_t offset, - int64_t length, box_time_t elapsed, box_time_t finish) - { - mrProgressNotifier.NotifyReadProgress(readSize, offset, - length, elapsed, finish); - } - virtual void Log(int64_t readSize, int64_t offset, - int64_t length) - { - mrProgressNotifier.NotifyReadProgress(readSize, offset, - length); - } - virtual void Log(int64_t readSize, int64_t offset) - { - mrProgressNotifier.NotifyReadProgress(readSize, offset); - } - }; - - void SyncDirectory(SyncParams &rParams, - int64_t ContainingDirectoryID, - const std::string &rLocalPath, - const std::string &rRemotePath, - const Location& rBackupLocation, - bool ThisDirHasJustBeenCreated = false); - - bool SyncDirectoryEntry(SyncParams &rParams, - ProgressNotifier& rNotifier, - const Location& rBackupLocation, - const std::string &rDirLocalPath, - MD5Digest& currentStateChecksum, - struct dirent *en, - EMU_STRUCT_STAT dir_st, - std::vector<std::string>& rDirs, - std::vector<std::string>& rFiles, - bool& rDownloadDirectoryRecordBecauseOfFutureFiles); - - std::string ConvertVssPathToRealPath(const std::string &rVssPath, - const Location& rBackupLocation); - - int64_t GetObjectID() const { return mObjectID; } - -private: - void DeleteSubDirectories(); - std::auto_ptr<BackupStoreDirectory> FetchDirectoryListing(SyncParams &rParams); - void UpdateAttributes(SyncParams &rParams, - BackupStoreDirectory *pDirOnStore, - const std::string &rLocalPath); -protected: // to allow tests to hook in before UpdateItems() runs - virtual bool UpdateItems(SyncParams &rParams, - const std::string &rLocalPath, - const std::string &rRemotePath, - const Location& rBackupLocation, - BackupStoreDirectory *pDirOnStore, - std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, - std::vector<std::string> &rFiles, - const std::vector<std::string> &rDirs); -private: - int64_t CreateRemoteDir(const std::string& localDirPath, - const std::string& nonVssDirPath, - const std::string& remoteDirPath, - BackupStoreFilenameClear& storeFilename, - bool* pHaveJustCreatedDirOnServer, - BackupClientDirectoryRecord::SyncParams &rParams); - int64_t UploadFile(SyncParams &rParams, - const std::string &rFilename, - const std::string &rNonVssFilePath, - const std::string &rRemotePath, - const BackupStoreFilenameClear &rStoreFilename, - int64_t FileSize, box_time_t ModificationTime, - box_time_t AttributesHash, bool NoPreviousVersionOnServer); - void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, - const std::string& rFilename); - void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, - BackupStoreDirectory* pDirOnStore, - BackupStoreDirectory::Entry* pEntry, - const std::string &rFilename); - std::string DecryptFilename(BackupStoreDirectory::Entry *en, - const std::string& rRemoteDirectoryPath); - std::string DecryptFilename(BackupStoreFilenameClear fn, - int64_t filenameObjectID, - const std::string& rRemoteDirectoryPath); - - int64_t mObjectID; - std::string mSubDirName; - bool mInitialSyncDone; - bool mSyncDone; - - // Checksum of directory contents and attributes, used to detect changes - uint8_t mStateChecksum[MD5Digest::DigestLength]; - - std::map<std::string, box_time_t> *mpPendingEntries; - std::map<std::string, BackupClientDirectoryRecord *> mSubDirectories; - // mpPendingEntries is a pointer rather than simple a member - // variable, because most of the time it'll be empty. This would - // waste a lot of memory because of STL allocation policies. -}; - -class Location -{ -public: - Location(); - ~Location(); - - void Deserialize(Archive & rArchive); - void Serialize(Archive & rArchive) const; -private: - Location(const Location &); // copy not allowed - Location &operator=(const Location &); -public: - std::string mName; - std::string mPath; - std::auto_ptr<BackupClientDirectoryRecord> mapDirectoryRecord; - std::auto_ptr<ExcludeList> mapExcludeFiles; - std::auto_ptr<ExcludeList> mapExcludeDirs; - int mIDMapIndex; - -#ifdef ENABLE_VSS - bool mIsSnapshotCreated; - VSS_ID mSnapshotVolumeId; - std::string mSnapshotPath; -#endif -}; - -#endif // BACKUPCLIENTDIRECTORYRECORD__H - - diff --git a/bin/bbackupd/BackupClientInodeToIDMap.cpp b/bin/bbackupd/BackupClientInodeToIDMap.cpp deleted file mode 100644 index 6eaf7394..00000000 --- a/bin/bbackupd/BackupClientInodeToIDMap.cpp +++ /dev/null @@ -1,320 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientInodeToIDMap.cpp -// Purpose: Map of inode numbers to file IDs on the store -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <stdlib.h> -#include <depot.h> - -#define BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION -#include "BackupClientInodeToIDMap.h" -#undef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION - -#include "Archive.h" -#include "BackupStoreException.h" -#include "CollectInBufferStream.h" -#include "MemBlockStream.h" -#include "autogen_CommonException.h" - -#include "MemLeakFindOn.h" - -#define BOX_DBM_INODE_DB_VERSION_KEY "BackupClientInodeToIDMap.Version" -#define BOX_DBM_INODE_DB_VERSION_CURRENT 2 - -#define BOX_DBM_MESSAGE(stuff) stuff << " (qdbm): " << dperrmsg(dpecode) - -#define BOX_LOG_DBM_ERROR(stuff) \ - BOX_ERROR(BOX_DBM_MESSAGE(stuff)) - -#define THROW_DBM_ERROR(message, filename, exception, subtype) \ - BOX_LOG_DBM_ERROR(message << ": " << filename); \ - THROW_EXCEPTION_MESSAGE(exception, subtype, \ - BOX_DBM_MESSAGE(message << ": " << filename)); - -#define ASSERT_DBM_OK(operation, message, filename, exception, subtype) \ - if(!(operation)) \ - { \ - THROW_DBM_ERROR(message, filename, exception, subtype); \ - } - -#define ASSERT_DBM_OPEN() \ - if(mpDepot == 0) \ - { \ - THROW_EXCEPTION_MESSAGE(BackupStoreException, InodeMapNotOpen, \ - "Inode database not open"); \ - } - -#define ASSERT_DBM_CLOSED() \ - if(mpDepot != 0) \ - { \ - THROW_EXCEPTION_MESSAGE(CommonException, Internal, \ - "Inode database already open: " << mFilename); \ - } - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::BackupClientInodeToIDMap() -// Purpose: Constructor -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -BackupClientInodeToIDMap::BackupClientInodeToIDMap() - : mReadOnly(true), - mEmpty(false), - mpDepot(0) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::~BackupClientInodeToIDMap() -// Purpose: Destructor -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -BackupClientInodeToIDMap::~BackupClientInodeToIDMap() -{ - if(mpDepot != 0) - { - Close(); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::Open(const char *, bool, bool) -// Purpose: Open the database map, creating a file on disc to store everything -// Created: 20/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientInodeToIDMap::Open(const char *Filename, bool ReadOnly, - bool CreateNew) -{ - mFilename = Filename; - - // Correct arguments? - ASSERT(!(CreateNew && ReadOnly)); - - // Correct usage? - ASSERT_DBM_CLOSED(); - ASSERT(!mEmpty); - - // Open the database file - int mode = ReadOnly ? DP_OREADER : DP_OWRITER; - if(CreateNew) - { - mode |= DP_OCREAT; - } - - mpDepot = dpopen(Filename, mode, 0); - - if(!mpDepot) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, - BOX_DBM_MESSAGE("Failed to open inode database: " << - mFilename)); - } - - const char* version_key = BOX_DBM_INODE_DB_VERSION_KEY; - int32_t version = 0; - - if(CreateNew) - { - version = BOX_DBM_INODE_DB_VERSION_CURRENT; - - int ret = dpput(mpDepot, version_key, strlen(version_key), - (char *)(&version), sizeof(version), DP_DKEEP); - - if(!ret) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, - BOX_DBM_MESSAGE("Failed to write version number to inode " - "database: " << mFilename)); - } - } - else - { - int ret = dpgetwb(mpDepot, version_key, strlen(version_key), 0, - sizeof(version), (char *)(&version)); - - if(ret == -1) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, - "Missing version number in inode database. Perhaps it " - "needs to be recreated: " << mFilename); - } - - if(ret != sizeof(version)) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, - "Wrong size version number in inode database: expected " - << sizeof(version) << " bytes but found " << ret); - } - - if(version != BOX_DBM_INODE_DB_VERSION_CURRENT) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, - "Wrong version number in inode database: expected " << - BOX_DBM_INODE_DB_VERSION_CURRENT << " but found " << - version << ". Perhaps it needs to be recreated: " << - mFilename); - } - - // By this point the version number has been checked and is OK. - } - - // Read only flag - mReadOnly = ReadOnly; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::OpenEmpty() -// Purpose: 'Open' this map. Not associated with a disc file. -// Useful for when a map is required, but is against -// an empty file on disc which shouldn't be created. -// Implies read only. -// Created: 20/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientInodeToIDMap::OpenEmpty() -{ - ASSERT_DBM_CLOSED(); - ASSERT(mpDepot == 0); - mEmpty = true; - mReadOnly = true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::Close() -// Purpose: Close the database file -// Created: 20/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientInodeToIDMap::Close() -{ - ASSERT_DBM_OPEN(); - ASSERT_DBM_OK(dpclose(mpDepot), "Failed to close inode database", - mFilename, BackupStoreException, BerkelyDBFailure); - mpDepot = 0; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::AddToMap(InodeRefType, -// int64_t, int64_t) -// Purpose: Adds an entry to the map. Overwrites any existing -// entry. -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, - int64_t InDirectory, const std::string& LocalPath) -{ - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, InodeMapIsReadOnly); - } - - if(mpDepot == 0) - { - THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); - } - - ASSERT_DBM_OPEN(); - - // Setup structures - CollectInBufferStream buf; - Archive arc(buf, IOStream::TimeOutInfinite); - arc.WriteExact((uint64_t)ObjectID); - arc.WriteExact((uint64_t)InDirectory); - arc.Write(LocalPath); - buf.SetForReading(); - - ASSERT_DBM_OK(dpput(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), - (const char *)buf.GetBuffer(), buf.GetSize(), DP_DOVER), - "Failed to add record to inode database", mFilename, - BackupStoreException, BerkelyDBFailure); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientInodeToIDMap::Lookup(InodeRefType, -// int64_t &, int64_t &) const -// Purpose: Looks up an inode in the map, returning true if it -// exists, and the object ids of it and the directory -// it's in the reference arguments. -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, - int64_t &rInDirectoryOut, std::string* pLocalPathOut) const -{ - if(mEmpty) - { - // Map is empty - return false; - } - - if(mpDepot == 0) - { - THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); - } - - ASSERT_DBM_OPEN(); - int size; - char* data = dpget(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), - 0, -1, &size); - if(data == NULL) - { - // key not in file - return false; - } - - // Free data automatically when the guard goes out of scope. - MemoryBlockGuard<char *> guard(data); - MemBlockStream stream(data, size); - Archive arc(stream, IOStream::TimeOutInfinite); - - // Return data - try - { - arc.Read(rObjectIDOut); - arc.Read(rInDirectoryOut); - if(pLocalPathOut) - { - arc.Read(*pLocalPathOut); - } - } - catch(CommonException &e) - { - if(e.GetSubType() == CommonException::ArchiveBlockIncompleteRead) - { - THROW_FILE_ERROR("Failed to lookup record in inode database: " - << InodeRef << ": not enough data in record", mFilename, - BackupStoreException, BerkelyDBFailure); - // Need to throw precisely that exception to ensure that the - // invalid database is deleted, so that we don't hit the same - // error next time. - } - - throw; - } - - // Found - return true; -} diff --git a/bin/bbackupd/BackupClientInodeToIDMap.h b/bin/bbackupd/BackupClientInodeToIDMap.h deleted file mode 100644 index 4bb1e085..00000000 --- a/bin/bbackupd/BackupClientInodeToIDMap.h +++ /dev/null @@ -1,59 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupClientInodeToIDMap.h -// Purpose: Map of inode numbers to file IDs on the store -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCLIENTINODETOIDMAP_H -#define BACKUPCLIENTINODETOIDMAP_H - -#include <sys/types.h> - -#include <map> -#include <utility> - -// avoid having to include the DB files when not necessary -#ifndef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION - class DEPOT; -#endif - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupClientInodeToIDMap -// Purpose: Map of inode numbers to file IDs on the store -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -class BackupClientInodeToIDMap -{ -public: - BackupClientInodeToIDMap(); - ~BackupClientInodeToIDMap(); -private: - BackupClientInodeToIDMap(const BackupClientInodeToIDMap &rToCopy); // not allowed -public: - - void Open(const char *Filename, bool ReadOnly, bool CreateNew); - void OpenEmpty(); - - void AddToMap(InodeRefType InodeRef, int64_t ObjectID, - int64_t InDirectory, const std::string& LocalPath); - bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, - int64_t &rInDirectoryOut, std::string* pLocalPathOut = NULL) const; - - void Close(); - -private: - bool mReadOnly; - bool mEmpty; - std::string mFilename; - DEPOT *mpDepot; -}; - -#endif // BACKUPCLIENTINODETOIDMAP_H - - diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp deleted file mode 100644 index 03ce7454..00000000 --- a/bin/bbackupd/BackupDaemon.cpp +++ /dev/null @@ -1,3654 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupDaemon.cpp -// Purpose: Backup daemon -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif -#ifdef HAVE_SIGNAL_H - #include <signal.h> -#endif -#ifdef HAVE_SYS_PARAM_H - #include <sys/param.h> -#endif -#ifdef HAVE_SYS_WAIT_H - #include <sys/wait.h> -#endif -#ifdef HAVE_SYS_MOUNT_H - #include <sys/mount.h> -#endif -#ifdef HAVE_MNTENT_H - #include <mntent.h> -#endif -#ifdef HAVE_SYS_MNTTAB_H - #include <cstdio> - #include <sys/mnttab.h> -#endif -#ifdef HAVE_PROCESS_H - #include <process.h> -#endif - -#include <iostream> -#include <set> -#include <sstream> - -#include "Configuration.h" -#include "IOStream.h" -#include "MemBlockStream.h" -#include "CommonException.h" -#include "BoxPortsAndFiles.h" - -#include "SSLLib.h" - -#include "autogen_BackupProtocol.h" -#include "autogen_ClientException.h" -#include "autogen_CommonException.h" -#include "autogen_ConversionException.h" -#include "Archive.h" -#include "BackupClientContext.h" -#include "BackupClientCryptoKeys.h" -#include "BackupClientDirectoryRecord.h" -#include "BackupClientFileAttributes.h" -#include "BackupClientInodeToIDMap.h" -#include "BackupClientMakeExcludeList.h" -#include "BackupConstants.h" -#include "BackupDaemon.h" -#include "BackupDaemonConfigVerify.h" -#include "BackupStoreConstants.h" -#include "BackupStoreDirectory.h" -#include "BackupStoreException.h" -#include "BackupStoreFile.h" -#include "BackupStoreFilenameClear.h" -#include "BannerText.h" -#include "Conversion.h" -#include "ExcludeList.h" -#include "FileStream.h" -#include "IOStreamGetLine.h" -#include "LocalProcessStream.h" -#include "Logging.h" -#include "Random.h" -#include "Timer.h" -#include "Utils.h" - -#ifdef WIN32 - #include "Win32ServiceFunctions.h" - #include "Win32BackupService.h" - - extern Win32BackupService* gpDaemonService; - -# ifdef ENABLE_VSS -# include <comdef.h> -# include <Vss.h> -# include <VsWriter.h> -# include <VsBackup.h> - - // http://www.flounder.com/cstring.htm - std::string GetMsgForHresult(HRESULT hr) - { - std::ostringstream buf; - - if(hr == VSS_S_ASYNC_CANCELLED) - { - buf << "VSS async operation cancelled"; - } - else if(hr == VSS_S_ASYNC_FINISHED) - { - buf << "VSS async operation finished"; - } - else if(hr == VSS_S_ASYNC_PENDING) - { - buf << "VSS async operation pending"; - } - else - { - buf << _com_error(hr).ErrorMessage(); - } - - buf << " (" << BOX_FORMAT_HEX32(hr) << ")"; - return buf.str(); - } - - std::string WideStringToString(WCHAR *buf) - { - if (buf == NULL) - { - return "(null)"; - } - - char* pStr = ConvertFromWideString(buf, CP_UTF8); - - if(pStr == NULL) - { - return "(conversion failed)"; - } - - std::string result(pStr); - free(pStr); - return result; - } - - std::string GuidToString(GUID guid) - { - wchar_t buf[64]; - StringFromGUID2(guid, buf, sizeof(buf)); - return WideStringToString(buf); - } - - std::string BstrToString(const BSTR arg) - { - if(arg == NULL) - { - return std::string("(null)"); - } - else - { - // Extract the *long* before where the arg points to - long len = ((long *)arg)[-1] / 2; - std::wstring wstr((WCHAR *)arg, len); - std::string str; - if(!ConvertFromWideString(wstr, &str, CP_UTF8)) - { - throw std::exception("string conversion failed"); - } - return str; - } - } -# endif - - // Mutex support by Achim: see https://www.boxbackup.org/ticket/67 - - // Creates the two mutexes checked for by the installer/uninstaller to - // see if the program is still running. One of the mutexes is created - // in the global name space (which makes it possible to access the - // mutex across user sessions in Windows XP); the other is created in - // the session name space (because versions of Windows NT prior to - // 4.0 TSE don't have a global name space and don't support the - // 'Global\' prefix). - - void CreateMutexes(const std::string& rName) - { - SECURITY_DESCRIPTOR SecurityDesc; - SECURITY_ATTRIBUTES SecurityAttr; - - /* By default on Windows NT, created mutexes are accessible only by the user - running the process. We need our mutexes to be accessible to all users, so - that the mutex detection can work across user sessions in Windows XP. To - do this we use a security descriptor with a null DACL. - */ - - InitializeSecurityDescriptor(&SecurityDesc, SECURITY_DESCRIPTOR_REVISION); - SetSecurityDescriptorDacl(&SecurityDesc, TRUE, NULL, FALSE); - SecurityAttr.nLength = sizeof(SecurityAttr); - SecurityAttr.lpSecurityDescriptor = &SecurityDesc; - SecurityAttr.bInheritHandle = FALSE; - // We don't care if this succeeds or fails. It's only used to - // ensure that an installer can detect if Box Backup is running. - CreateMutexA(&SecurityAttr, FALSE, rName.c_str()); - std::string global_name = "Global\\" + rName; - CreateMutexA(&SecurityAttr, FALSE, global_name.c_str()); - } -#endif - -#include "MemLeakFindOn.h" - -static const time_t MAX_SLEEP_TIME = 1024; - -// Make the actual sync period have a little bit of extra time, up to a 64th of the main sync period. -// This prevents repetative cycles of load on the server -#define SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY 6 - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::BackupDaemon() -// Purpose: constructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupDaemon::BackupDaemon() - : mState(BackupDaemon::State_Initialising), - mDeleteRedundantLocationsAfter(0), - mLastNotifiedEvent(SysadminNotifier::MAX), - mDeleteUnusedRootDirEntriesAfter(0), - mClientStoreMarker(BackupClientContext::ClientStoreMarker_NotKnown), - mStorageLimitExceeded(false), - mReadErrorsOnFilesystemObjects(false), - mLastSyncTime(0), - mNextSyncTime(0), - mCurrentSyncStartTime(0), - mUpdateStoreInterval(0), - mDeleteStoreObjectInfoFile(false), - mDoSyncForcedByPreviousSyncError(false), - mNumFilesUploaded(-1), - mNumDirsCreated(-1), - mMaxBandwidthFromSyncAllowScript(0), - mLogAllFileAccess(false), - mpProgressNotifier(this), - mpLocationResolver(this), - mpRunStatusProvider(this), - mpSysadminNotifier(this), - mapCommandSocketPollTimer(NULL) - #ifdef WIN32 - , mInstallService(false), - mRemoveService(false), - mRunAsService(false), - mServiceName("bbackupd") - #endif -#ifdef ENABLE_VSS - , mpVssBackupComponents(NULL) -#endif -{ - // Only ever one instance of a daemon - SSLLib::Initialise(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::~BackupDaemon() -// Purpose: Destructor -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -BackupDaemon::~BackupDaemon() -{ - DeleteAllLocations(); - DeleteAllIDMaps(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DaemonName() -// Purpose: Get name of daemon -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -const char *BackupDaemon::DaemonName() const -{ - return "bbackupd"; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DaemonBanner() -// Purpose: Daemon banner -// Created: 1/1/04 -// -// -------------------------------------------------------------------------- -std::string BackupDaemon::DaemonBanner() const -{ - return BANNER_TEXT("Backup Client"); -} - -void BackupDaemon::Usage() -{ - this->Daemon::Usage(); - -#ifdef WIN32 - std::cout << - " -s Run as a Windows Service, for internal use only\n" - " -i Install Windows Service (you may want to specify a config file)\n" - " -r Remove Windows Service\n" - " -S <name> Service name for -i and -r options\n"; -#endif -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::GetConfigVerify() -// Purpose: Get configuration specification -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -const ConfigurationVerify *BackupDaemon::GetConfigVerify() const -{ - // Defined elsewhere - return &BackupDaemonConfigVerify; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::SetupInInitialProcess() -// Purpose: Platforms with non-checkable credentials on -// local sockets only. -// Prints a warning if the command socket is used. -// Created: 25/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::SetupInInitialProcess() -{ - const Configuration& config(GetConfiguration()); - - // These keys may or may not be required, depending on the configured - // store type (e.g. not when using Amazon S3 stores), so they can't be - // verified by BackupDaemonConfigVerify. - std::vector<std::string> requiredKeys; - requiredKeys.push_back("StoreHostname"); - requiredKeys.push_back("AccountNumber"); - requiredKeys.push_back("CertificateFile"); - requiredKeys.push_back("PrivateKeyFile"); - requiredKeys.push_back("TrustedCAsFile"); - bool missingRequiredKeys = false; - - for(std::vector<std::string>::const_iterator i = requiredKeys.begin(); - i != requiredKeys.end(); i++) - { - if(!config.KeyExists(*i)) - { - BOX_ERROR("Missing required configuration key: " << *i); - missingRequiredKeys = true; - } - } - - if(missingRequiredKeys) - { - THROW_EXCEPTION_MESSAGE(CommonException, InvalidConfiguration, - "Some required configuration keys are missing in " << - GetConfigFileName()); - } - -#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET - // Print a warning on this platform if the CommandSocket is used. - if(GetConfiguration().KeyExists("CommandSocket")) - { - BOX_WARNING( - "==============================================================================\n" - "SECURITY WARNING: This platform cannot check the credentials of connections to\n" - "the command socket. This is a potential DoS security problem.\n" - "Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl\n" - "is not used.\n" - "==============================================================================\n" - ); - } -#endif -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeleteAllLocations() -// Purpose: Deletes all records stored -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupDaemon::DeleteAllLocations() -{ - // Run through, and delete everything - for(Locations::iterator i = mLocations.begin(); - i != mLocations.end(); ++i) - { - delete *i; - } - - // Clear the contents of the map, so it is empty - mLocations.clear(); - - // And delete everything from the associated mount vector - mIDMapMounts.clear(); -} - -#ifdef WIN32 -std::string BackupDaemon::GetOptionString() -{ - std::string oldOpts = this->Daemon::GetOptionString(); - ASSERT(oldOpts.find("s") == std::string::npos); - ASSERT(oldOpts.find("S") == std::string::npos); - ASSERT(oldOpts.find("i") == std::string::npos); - ASSERT(oldOpts.find("r") == std::string::npos); - return oldOpts + "sS:ir"; -} - -int BackupDaemon::ProcessOption(signed int option) -{ - switch(option) - { - case 's': - { - mRunAsService = true; - return 0; - } - - case 'S': - { - mServiceName = optarg; - Logging::SetProgramName(mServiceName); - return 0; - } - - case 'i': - { - mInstallService = true; - return 0; - } - - case 'r': - { - mRemoveService = true; - return 0; - } - - default: - { - return this->Daemon::ProcessOption(option); - } - } -} - -int BackupDaemon::Main(const std::string &rConfigFileName) -{ - if (mInstallService) - { - return InstallService(rConfigFileName.c_str(), mServiceName); - } - - if (mRemoveService) - { - return RemoveService(mServiceName); - } - -#ifdef ENABLE_VSS - HRESULT result = CoInitialize(NULL); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to initialize COM: " << - GetMsgForHresult(result)); - return 1; - } -#endif - - CreateMutexes("__boxbackup_mutex__"); - - int returnCode; - - if (mRunAsService) - { - // We will be called reentrantly by the Service Control - // Manager, and we had better not call OurService again! - mRunAsService = false; - - BOX_INFO("Box Backup service starting"); - returnCode = OurService(rConfigFileName.c_str()); - BOX_INFO("Box Backup service shut down"); - } - else - { - returnCode = this->Daemon::Main(rConfigFileName); - } - - return returnCode; -} -#endif // WIN32 - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::Run() -// Purpose: Run function for daemon -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::Run() -{ - // initialise global timer mechanism - Timers::Init(); - - mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, - "CommandSocketPollTimer")); - - #ifndef WIN32 - // Ignore SIGPIPE so that if a command connection is broken, - // the daemon doesn't terminate. - ::signal(SIGPIPE, SIG_IGN); - #endif - - // Create a command socket? - const Configuration &conf(GetConfiguration()); - if(conf.KeyExists("CommandSocket")) - { - // Yes, create a local UNIX socket - mapCommandSocketInfo.reset(new CommandSocketInfo); - const char *socketName = - conf.GetKeyValue("CommandSocket").c_str(); - #ifdef WIN32 - mapCommandSocketInfo->mListeningSocket.Listen( - socketName); - #else - ::unlink(socketName); - mapCommandSocketInfo->mListeningSocket.Listen( - Socket::TypeUNIX, socketName); - #endif - } - - // Handle things nicely on exceptions - try - { - Run2(); - } - catch(...) - { - try - { - mapCommandSocketInfo.reset(); - } - catch(std::exception &e) - { - BOX_WARNING("Internal error while closing command " - "socket after another exception, ignored: " << - e.what()); - } - catch(...) - { - BOX_WARNING("Error closing command socket after " - "exception, ignored."); - } - - mapCommandSocketPollTimer.reset(); - Timers::Cleanup(); - - throw; - } - - // Clean up - mapCommandSocketInfo.reset(); - mapCommandSocketPollTimer.reset(); - Timers::Cleanup(); -} - -void BackupDaemon::InitCrypto() -{ - // Read in the certificates creating a TLS context - const Configuration &conf(GetConfiguration()); - std::string certFile(conf.GetKeyValue("CertificateFile")); - std::string keyFile(conf.GetKeyValue("PrivateKeyFile")); - std::string caFile(conf.GetKeyValue("TrustedCAsFile")); - mTlsContext.Initialise(false /* as client */, certFile.c_str(), - keyFile.c_str(), caFile.c_str()); - - // Set up the keys for various things - BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile")); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::Run2() -// Purpose: Run function for daemon (second stage) -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupDaemon::Run2() -{ - InitCrypto(); - - const Configuration &conf(GetConfiguration()); - - // How often to connect to the store (approximate) - mUpdateStoreInterval = SecondsToBoxTime( - conf.GetKeyValueInt("UpdateStoreInterval")); - mBackupErrorDelay = conf.GetKeyValueInt("BackupErrorDelay"); - - // But are we connecting automatically? - bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup"); - - // When the next sync should take place -- which is ASAP - mNextSyncTime = 0; - - // When the last sync started (only updated if the store was not full when the sync ended) - mLastSyncTime = 0; - - // -------------------------------------------------------------------------------------------- - - mDeleteStoreObjectInfoFile = DeserializeStoreObjectInfo(mLastSyncTime, - mNextSyncTime); - - // -------------------------------------------------------------------------------------------- - - - // Set state - SetState(State_Idle); - - mDoSyncForcedByPreviousSyncError = false; - - // Loop around doing backups - do - { - // Flags used below - bool storageLimitExceeded = false; - bool doSync = false; - bool mDoSyncForcedByCommand = false; - - // Check whether we should be stopping, and if so, - // don't hang around waiting on the command socket. - if(StopRun()) - { - BOX_INFO("Skipping command socket polling " - "due to shutdown request"); - break; - } - - // Is a delay necessary? - box_time_t currentTime = GetCurrentBoxTime(); - box_time_t requiredDelay = (mNextSyncTime < currentTime) - ? (0) : (mNextSyncTime - currentTime); - mNextSyncTime = currentTime + requiredDelay; - - if (mDoSyncForcedByPreviousSyncError) - { - BOX_INFO("Last backup was not successful, " - "next one starting at " << - FormatTime(mNextSyncTime, false, true)); - } - else if (automaticBackup) - { - BOX_INFO("Automatic backups are enabled, " - "next one starting at " << - FormatTime(mNextSyncTime, false, true)); - } - else - { - BOX_INFO("No automatic backups, waiting for " - "bbackupctl snapshot command"); - requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); - } - - if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME)) - { - requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); - } - - // Only delay if necessary - if(requiredDelay == 0) - { - // No sleep necessary, so don't listen on the command - // socket at all right now. - } - else if(mapCommandSocketInfo.get() != 0) - { - // A command socket exists, so sleep by waiting for a - // connection or command on it. - WaitOnCommandSocket(requiredDelay, doSync, - mDoSyncForcedByCommand); - } - else - { - // No command socket or connection, just do a normal - // sleep. - time_t sleepSeconds = - BoxTimeToSeconds(requiredDelay); - ::sleep((sleepSeconds <= 0) - ? 1 : sleepSeconds); - } - - // We have now slept, so if automaticBackup is enabled then - // it's time for a backup now. - - if(StopRun()) - { - BOX_INFO("Stopping idle loop due to shutdown request"); - break; - } - else if(doSync) - { - BOX_INFO("Starting a backup immediately due to " - "bbackupctl sync command"); - } - else if(GetCurrentBoxTime() < mNextSyncTime) - { - BOX_TRACE("Deadline not reached, sleeping again"); - continue; - } - else if(mDoSyncForcedByPreviousSyncError) - { - BOX_INFO("Last backup was not successful, next one " - "starting now"); - } - else if(!automaticBackup) - { - BOX_TRACE("Sleeping again because automatic backups " - "are not enabled"); - continue; - } - else - { - BOX_INFO("Automatic backups are enabled, next one " - "starting now"); - } - - // If we pass this point, or exit the loop, we should have - // logged something at INFO level or higher to explain why. - - // Use a script to see if sync is allowed now? - if(mDoSyncForcedByCommand) - { - BOX_INFO("Skipping SyncAllowScript due to bbackupctl " - "force-sync command"); - } - else - { - int d = UseScriptToSeeIfSyncAllowed(); - if(d > 0) - { - // Script has asked for a delay - mNextSyncTime = GetCurrentBoxTime() + - SecondsToBoxTime(d); - BOX_INFO("Impending backup stopped by " - "SyncAllowScript, next attempt " - "scheduled for " << - FormatTime(mNextSyncTime, false)); - continue; - } - } - - mCurrentSyncStartTime = GetCurrentBoxTime(); - RunSyncNowWithExceptionHandling(); - - // Set state - SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle); - } - while(!StopRun()); - - // Make sure we have a clean start next time round (if restart) - DeleteAllLocations(); - DeleteAllIDMaps(); -} - -std::auto_ptr<BackupClientContext> BackupDaemon::RunSyncNowWithExceptionHandling() -{ - bool errorOccurred = false; - int errorCode = 0, errorSubCode = 0; - std::string errorString = "unknown"; - - try - { - OnBackupStart(); - // Do sync - RunSyncNow(); - } - catch(BoxException &e) - { - errorOccurred = true; - errorString = e.what(); - errorCode = e.GetType(); - errorSubCode = e.GetSubType(); - } - catch(std::exception &e) - { - BOX_ERROR("Internal error during backup run: " << e.what()); - errorOccurred = true; - errorString = e.what(); - } - catch(...) - { - // TODO: better handling of exceptions here... - // need to be very careful - errorOccurred = true; - } - - // do not retry immediately without a good reason - mDoSyncForcedByPreviousSyncError = false; - - // Is it a berkely db failure? - bool isBerkelyDbFailure = false; - - // Notify system administrator about the final state of the backup - if(errorOccurred) - { - if (errorCode == BackupStoreException::ExceptionType - && errorSubCode == BackupStoreException::BerkelyDBFailure) - { - isBerkelyDbFailure = true; - } - - if(isBerkelyDbFailure) - { - // Delete corrupt files - DeleteCorruptBerkelyDbFiles(); - } - - ResetCachedState(); - - // Handle restart? - if(StopRun()) - { - BOX_NOTICE("Exception (" << errorCode << "/" << - errorSubCode << ") due to signal"); - OnBackupFinish(); - return mapClientContext; // releases mapClientContext - } - - NotifySysadmin(SysadminNotifier::BackupError); - - // If the Berkely db files get corrupted, - // delete them and try again immediately. - if(isBerkelyDbFailure) - { - BOX_ERROR("Berkely db inode map files corrupted, " - "deleting and restarting scan. Renamed files " - "and directories will not be tracked until " - "after this scan."); - ::sleep(1); - } - else - { - // Not restart/terminate, pause and retry - // Notify administrator - SetState(State_Error); - BOX_ERROR("Exception caught (" << errorString << - " " << errorCode << "/" << errorSubCode << - "), reset state and waiting to retry..."); - ::sleep(10); - mNextSyncTime = GetCurrentBoxTime() + - SecondsToBoxTime(mBackupErrorDelay) + - Random::RandomInt(mUpdateStoreInterval >> - SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); - } - } - - if(mReadErrorsOnFilesystemObjects) - { - NotifySysadmin(SysadminNotifier::ReadError); - } - - if(mStorageLimitExceeded) - { - NotifySysadmin(SysadminNotifier::StoreFull); - } - - if (!errorOccurred && !mReadErrorsOnFilesystemObjects && - !mStorageLimitExceeded) - { - NotifySysadmin(SysadminNotifier::BackupOK); - } - - // If we were retrying after an error, and this backup succeeded, - // then now would be a good time to stop :-) - mDoSyncForcedByPreviousSyncError = errorOccurred && !isBerkelyDbFailure; - - OnBackupFinish(); - return mapClientContext; // releases mapClientContext -} - -void BackupDaemon::ResetCachedState() -{ - // Clear state data - // Go back to beginning of time - mLastSyncTime = 0; - mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything - DeleteAllLocations(); - DeleteAllIDMaps(); -} - -std::auto_ptr<BackupClientContext> BackupDaemon::GetNewContext -( - LocationResolver &rResolver, - TLSContext &rTLSContext, - const std::string &rHostname, - int32_t Port, - uint32_t AccountNumber, - bool ExtendedLogging, - bool ExtendedLogToFile, - std::string ExtendedLogFile, - ProgressNotifier &rProgressNotifier, - bool TcpNiceMode -) -{ - std::auto_ptr<BackupClientContext> context(new BackupClientContext( - rResolver, rTLSContext, rHostname, Port, AccountNumber, - ExtendedLogging, ExtendedLogToFile, ExtendedLogFile, - rProgressNotifier, TcpNiceMode)); - return context; -} - -// Returns the BackupClientContext so that tests can use it to hold the -// connection open and prevent housekeeping from running. Otherwise don't use -// it, let it be destroyed and close the connection. -std::auto_ptr<BackupClientContext> BackupDaemon::RunSyncNow() -{ - Timers::AssertInitialised(); - - // Delete the serialised store object file, - // so that we don't try to reload it after a - // partially completed backup - if(mDeleteStoreObjectInfoFile && !DeleteStoreObjectInfo()) - { - BOX_ERROR("Failed to delete the StoreObjectInfoFile, " - "backup cannot continue safely."); - THROW_EXCEPTION(ClientException, - FailedToDeleteStoreObjectInfoFile); - } - - // In case the backup throws an exception, - // we should not try to delete the store info - // object file again. - mDeleteStoreObjectInfoFile = false; - - const Configuration &conf(GetConfiguration()); - - std::auto_ptr<FileLogger> fileLogger; - - if (conf.KeyExists("LogFile")) - { - bool overwrite = false; - if (conf.KeyExists("LogFileOverwrite")) - { - overwrite = conf.GetKeyValueBool("LogFileOverwrite"); - } - - Log::Level level = Log::INFO; - if (conf.KeyExists("LogFileLevel")) - { - level = Logging::GetNamedLevel( - conf.GetKeyValue("LogFileLevel")); - } - - fileLogger.reset(new FileLogger(conf.GetKeyValue("LogFile"), - level, !overwrite)); - } - - std::string extendedLogFile; - if (conf.KeyExists("ExtendedLogFile")) - { - extendedLogFile = conf.GetKeyValue("ExtendedLogFile"); - } - - if (conf.KeyExists("LogAllFileAccess")) - { - mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess"); - } - - // Then create a client context object (don't - // just connect, as this may be unnecessary) - mapClientContext = GetNewContext( - *mpLocationResolver, - mTlsContext, - conf.GetKeyValue("StoreHostname"), - conf.GetKeyValueInt("StorePort"), - conf.GetKeyValueUint32("AccountNumber"), - conf.GetKeyValueBool("ExtendedLogging"), - conf.KeyExists("ExtendedLogFile"), - extendedLogFile, - *mpProgressNotifier, - conf.GetKeyValueBool("TcpNice") - ); - - // The minimum age a file needs to be before it will be - // considered for uploading - box_time_t minimumFileAge = SecondsToBoxTime( - conf.GetKeyValueInt("MinimumFileAge")); - - // The maximum time we'll wait to upload a file, regardless - // of how often it's modified - box_time_t maxUploadWait = SecondsToBoxTime( - conf.GetKeyValueInt("MaxUploadWait")); - // Adjust by subtracting the minimum file age, so is relative - // to sync period end in comparisons - if (maxUploadWait > minimumFileAge) - { - maxUploadWait -= minimumFileAge; - } - else - { - maxUploadWait = 0; - } - - // Calculate the sync period of files to examine - box_time_t syncPeriodStart = mLastSyncTime; - box_time_t syncPeriodEnd = GetCurrentBoxTime() - minimumFileAge; - - if(syncPeriodStart >= syncPeriodEnd && - syncPeriodStart - syncPeriodEnd < minimumFileAge) - { - // This can happen if we receive a force-sync command less - // than minimumFileAge after the last sync. Deal with it by - // moving back syncPeriodStart, which should not do any - // damage. - syncPeriodStart = syncPeriodEnd - SecondsToBoxTime(1); - } - - if(syncPeriodStart >= syncPeriodEnd) - { - BOX_ERROR("Invalid (negative) sync period: perhaps your clock " - "is going backwards? (" << syncPeriodStart << " to " << - syncPeriodEnd << ")"); - THROW_EXCEPTION(ClientException, ClockWentBackwards); - } - - // Check logic - ASSERT(syncPeriodEnd > syncPeriodStart); - // Paranoid check on sync times - if(syncPeriodStart >= syncPeriodEnd) - { - return mapClientContext; // releases mapClientContext - } - - // Adjust syncPeriodEnd to emulate snapshot behaviour properly - box_time_t syncPeriodEndExtended = syncPeriodEnd; - - // Using zero min file age? - if(minimumFileAge == 0) - { - // Add a year on to the end of the end time, - // to make sure we sync files which are - // modified after the scan run started. - // Of course, they may be eligible to be - // synced again the next time round, - // but this should be OK, because the changes - // only upload should upload no data. - syncPeriodEndExtended += SecondsToBoxTime( - (time_t)(356*24*3600)); - } - - // Set up the sync parameters - BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider, - *mpSysadminNotifier, *mpProgressNotifier, *mapClientContext, this); - params.mSyncPeriodStart = syncPeriodStart; - params.mSyncPeriodEnd = syncPeriodEndExtended; - // use potentially extended end time - params.mMaxUploadWait = maxUploadWait; - params.mFileTrackingSizeThreshold = - conf.GetKeyValueInt("FileTrackingSizeThreshold"); - params.mDiffingUploadSizeThreshold = - conf.GetKeyValueInt("DiffingUploadSizeThreshold"); - params.mMaxFileTimeInFuture = - SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture")); - mNumFilesUploaded = 0; - mNumDirsCreated = 0; - - if(conf.KeyExists("MaxUploadRate")) - { - params.mMaxUploadRate = conf.GetKeyValueInt("MaxUploadRate"); - } - - if(mMaxBandwidthFromSyncAllowScript != 0) - { - params.mMaxUploadRate = mMaxBandwidthFromSyncAllowScript; - } - - mDeleteRedundantLocationsAfter = - conf.GetKeyValueInt("DeleteRedundantLocationsAfter"); - mStorageLimitExceeded = false; - mReadErrorsOnFilesystemObjects = false; - - // Setup various timings - int maximumDiffingTime = 600; - int keepAliveTime = 60; - - // max diffing time, keep-alive time - if(conf.KeyExists("MaximumDiffingTime")) - { - maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime"); - } - if(conf.KeyExists("KeepAliveTime")) - { - keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); - } - - mapClientContext->SetMaximumDiffingTime(maximumDiffingTime); - mapClientContext->SetKeepAliveTime(keepAliveTime); - - // Set store marker - mapClientContext->SetClientStoreMarker(mClientStoreMarker); - - // Set up the locations, if necessary -- need to do it here so we have - // a (potential) connection to use. - { - const Configuration &locations( - conf.GetSubConfiguration( - "BackupLocations")); - - // Make sure all the directory records - // are set up - SetupLocations(*mapClientContext, locations); - } - - mpProgressNotifier->NotifyIDMapsSetup(*mapClientContext); - - // Get some ID maps going - SetupIDMapsForSync(); - - // Delete any unused directories? - DeleteUnusedRootDirEntries(*mapClientContext); - -#ifdef ENABLE_VSS - CreateVssBackupComponents(); -#endif - - // Go through the records, syncing them - for(Locations::const_iterator - i(mLocations.begin()); - i != mLocations.end(); ++i) - { - // Set current and new ID map pointers - // in the context - mapClientContext->SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], - mNewIDMaps[(*i)->mIDMapIndex]); - - // Set exclude lists (context doesn't - // take ownership) - mapClientContext->SetExcludeLists( - (*i)->mapExcludeFiles.get(), - (*i)->mapExcludeDirs.get()); - - // Sync the directory - std::string locationPath = (*i)->mPath; -#ifdef ENABLE_VSS - if((*i)->mIsSnapshotCreated) - { - locationPath = (*i)->mSnapshotPath; - } -#endif - - (*i)->mapDirectoryRecord->SyncDirectory(params, - BackupProtocolListDirectory::RootDirectory, - locationPath, std::string("/") + (*i)->mName, **i); - - // Unset exclude lists (just in case) - mapClientContext->SetExcludeLists(0, 0); - } - - // Perform any deletions required -- these are - // delayed until the end to allow renaming to - // happen neatly. - mapClientContext->PerformDeletions(); - -#ifdef ENABLE_VSS - CleanupVssBackupComponents(); -#endif - - // Get the new store marker - mClientStoreMarker = mapClientContext->GetClientStoreMarker(); - mStorageLimitExceeded = mapClientContext->StorageLimitExceeded(); - mReadErrorsOnFilesystemObjects |= - params.mReadErrorsOnFilesystemObjects; - - if(!mStorageLimitExceeded) - { - // The start time of the next run is the end time of this - // run. This is only done if the storage limit wasn't - // exceeded (as things won't have been done properly if - // it was) - mLastSyncTime = syncPeriodEnd; - } - - // Commit the ID Maps - CommitIDMapsAfterSync(); - - // Calculate when the next sync run should be - mNextSyncTime = mCurrentSyncStartTime + - mUpdateStoreInterval + - Random::RandomInt(mUpdateStoreInterval >> - SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); - - // -------------------------------------------------------------------------------------------- - - // We had a successful backup, save the store - // info. If we save successfully, we must - // delete the file next time we start a backup - - mDeleteStoreObjectInfoFile = - SerializeStoreObjectInfo(mLastSyncTime, mNextSyncTime); - - // -------------------------------------------------------------------------------------------- - - return mapClientContext; // releases mapClientContext -} - -#ifdef ENABLE_VSS -bool BackupDaemon::WaitForAsync(IVssAsync *pAsync, - const std::string& description) -{ - BOX_INFO("VSS: waiting for " << description << " to complete"); - HRESULT result; - - do - { - result = pAsync->Wait(1000); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to wait for " << description << - " to complete: " << GetMsgForHresult(result)); - break; - } - - HRESULT result2; - result = pAsync->QueryStatus(&result2, NULL); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to query " << description << - " status: " << GetMsgForHresult(result)); - break; - } - - result = result2; - BOX_INFO("VSS: " << description << " status: " << - GetMsgForHresult(result)); - } - while(result == VSS_S_ASYNC_PENDING); - - pAsync->Release(); - - return (result == VSS_S_ASYNC_FINISHED); -} - -#define CALL_MEMBER_FN(object, method) ((object).*(method)) - -bool BackupDaemon::CallAndWaitForAsync(AsyncMethod method, - const std::string& description) -{ - IVssAsync *pAsync; - HRESULT result = CALL_MEMBER_FN(*mpVssBackupComponents, method)(&pAsync); - if(result != S_OK) - { - BOX_ERROR("VSS: " << description << " failed: " << - GetMsgForHresult(result)); - return false; - } - - return WaitForAsync(pAsync, description); -} - -void BackupDaemon::CreateVssBackupComponents() -{ - std::map<char, VSS_ID> volumesIncluded; - - HRESULT result = ::CreateVssBackupComponents(&mpVssBackupComponents); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to create backup components: " << - GetMsgForHresult(result)); - return; - } - - result = mpVssBackupComponents->InitializeForBackup(NULL); - if(result != S_OK) - { - std::string message = GetMsgForHresult(result); - - if (result == VSS_E_UNEXPECTED) - { - message = "Check the Application Log for details, and ensure " - "that the Volume Shadow Copy, COM+ System Application, " - "and Distributed Transaction Coordinator services " - "are running"; - } - - BOX_ERROR("VSS: Failed to initialize for backup: " << message); - return; - } - - result = mpVssBackupComponents->SetContext(VSS_CTX_BACKUP); - if(result == E_NOTIMPL) - { - BOX_INFO("VSS: Failed to set context to VSS_CTX_BACKUP: " - "not implemented, probably Windows XP, ignored."); - } - else if(result != S_OK) - { - BOX_ERROR("VSS: Failed to set context to VSS_CTX_BACKUP: " << - GetMsgForHresult(result)); - return; - } - - result = mpVssBackupComponents->SetBackupState( - false, /* no components for now */ - true, /* might as well ask for a bootable backup */ - VSS_BT_FULL, - false /* what is Partial File Support? */); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to set backup state: " << - GetMsgForHresult(result)); - return; - } - - if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterMetadata, - "GatherWriterMetadata()")) - { - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - - UINT writerCount; - result = mpVssBackupComponents->GetWriterMetadataCount(&writerCount); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get writer count: " << - GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - - for(UINT iWriter = 0; iWriter < writerCount; iWriter++) - { - BOX_INFO("VSS: Getting metadata from writer " << iWriter); - VSS_ID writerInstance; - IVssExamineWriterMetadata* pMetadata; - result = mpVssBackupComponents->GetWriterMetadata(iWriter, - &writerInstance, &pMetadata); - if(result != S_OK) - { - BOX_ERROR("Failed to get VSS metadata from writer " << iWriter << - ": " << GetMsgForHresult(result)); - continue; - } - - UINT includeFiles, excludeFiles, numComponents; - result = pMetadata->GetFileCounts(&includeFiles, &excludeFiles, - &numComponents); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get metadata file counts from " - "writer " << iWriter << ": " << - GetMsgForHresult(result)); - pMetadata->Release(); - continue; - } - - for(UINT iComponent = 0; iComponent < numComponents; iComponent++) - { - IVssWMComponent* pComponent; - result = pMetadata->GetComponent(iComponent, &pComponent); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get metadata component " << - iComponent << " from writer " << iWriter << ": " << - GetMsgForHresult(result)); - continue; - } - - PVSSCOMPONENTINFO pComponentInfo; - result = pComponent->GetComponentInfo(&pComponentInfo); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get metadata component " << - iComponent << " info from writer " << iWriter << ": " << - GetMsgForHresult(result)); - pComponent->Release(); - continue; - } - - BOX_TRACE("VSS: writer " << iWriter << " component " << - iComponent << " info:"); - switch(pComponentInfo->type) - { - case VSS_CT_UNDEFINED: BOX_TRACE("VSS: type: undefined"); break; - case VSS_CT_DATABASE: BOX_TRACE("VSS: type: database"); break; - case VSS_CT_FILEGROUP: BOX_TRACE("VSS: type: filegroup"); break; - default: - BOX_WARNING("VSS: type: unknown (" << pComponentInfo->type << ")"); - } - - BOX_TRACE("VSS: logical path: " << - BstrToString(pComponentInfo->bstrLogicalPath)); - BOX_TRACE("VSS: component name: " << - BstrToString(pComponentInfo->bstrComponentName)); - BOX_TRACE("VSS: caption: " << - BstrToString(pComponentInfo->bstrCaption)); - BOX_TRACE("VSS: restore metadata: " << - pComponentInfo->bRestoreMetadata); - BOX_TRACE("VSS: notify on complete: " << - pComponentInfo->bRestoreMetadata); - BOX_TRACE("VSS: selectable: " << - pComponentInfo->bSelectable); - BOX_TRACE("VSS: selectable for restore: " << - pComponentInfo->bSelectableForRestore); - BOX_TRACE("VSS: component flags: " << - BOX_FORMAT_HEX32(pComponentInfo->dwComponentFlags)); - BOX_TRACE("VSS: file count: " << - pComponentInfo->cFileCount); - BOX_TRACE("VSS: databases: " << - pComponentInfo->cDatabases); - BOX_TRACE("VSS: log files: " << - pComponentInfo->cLogFiles); - BOX_TRACE("VSS: dependencies: " << - pComponentInfo->cDependencies); - - pComponent->FreeComponentInfo(pComponentInfo); - pComponent->Release(); - } - - pMetadata->Release(); - } - - VSS_ID snapshotSetId; - result = mpVssBackupComponents->StartSnapshotSet(&snapshotSetId); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to start snapshot set: " << - GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - - // Add all volumes included as backup locations to the snapshot set - for(Locations::iterator - iLocation = mLocations.begin(); - iLocation != mLocations.end(); - iLocation++) - { - Location& rLocation(**iLocation); - std::string path = rLocation.mPath; - // convert to absolute and remove Unicode prefix - path = ConvertPathToAbsoluteUnicode(path.c_str()).substr(4); - - if(path.length() >= 3 && path[1] == ':' && path[2] == '\\') - { - std::string volumeRoot = path.substr(0, 3); - - std::map<char, VSS_ID>::iterator i = - volumesIncluded.find(path[0]); - - if(i == volumesIncluded.end()) - { - std::wstring volumeRootWide; - volumeRootWide.push_back((WCHAR) path[0]); - volumeRootWide.push_back((WCHAR) ':'); - volumeRootWide.push_back((WCHAR) '\\'); - VSS_ID newVolumeId; - result = mpVssBackupComponents->AddToSnapshotSet( - (VSS_PWSZ)(volumeRootWide.c_str()), GUID_NULL, - &newVolumeId); - if(result == S_OK) - { - BOX_TRACE("VSS: Added volume " << volumeRoot << - " for backup location " << path << - " to snapshot set"); - volumesIncluded[path[0]] = newVolumeId; - rLocation.mSnapshotVolumeId = newVolumeId; - } - else - { - BOX_ERROR("VSS: Failed to add volume " << - volumeRoot << " to snapshot set: " << - GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - } - else - { - BOX_TRACE("VSS: Skipping already included volume " << - volumeRoot << " for backup location " << path); - rLocation.mSnapshotVolumeId = i->second; - } - - rLocation.mIsSnapshotCreated = true; - - // If the snapshot path starts with the volume root - // (drive letter), because the path is absolute (as it - // should be), then remove it so that the resulting - // snapshot path can be appended to the snapshot device - // object to make a real path, without a spurious drive - // letter in it. - - if (path.substr(0, volumeRoot.length()) == volumeRoot) - { - path = path.substr(volumeRoot.length()); - } - - rLocation.mSnapshotPath = path; - } - else - { - BOX_WARNING("VSS: Skipping backup location " << path << - " which does not start with a volume specification"); - } - } - - if(!CallAndWaitForAsync(&IVssBackupComponents::PrepareForBackup, - "PrepareForBackup()")) - { - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - - if(!CallAndWaitForAsync(&IVssBackupComponents::DoSnapshotSet, - "DoSnapshotSet()")) - { - goto CreateVssBackupComponents_cleanup_WriterMetadata; - } - - if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterStatus, - "GatherWriterStatus()")) - { - goto CreateVssBackupComponents_cleanup_WriterStatus; - } - - result = mpVssBackupComponents->GetWriterStatusCount(&writerCount); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get writer status count: " << - GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterStatus; - } - - for(UINT iWriter = 0; iWriter < writerCount; iWriter++) - { - VSS_ID instance, writer; - BSTR writerNameBstr; - VSS_WRITER_STATE writerState; - HRESULT writerResult; - - result = mpVssBackupComponents->GetWriterStatus(iWriter, - &instance, &writer, &writerNameBstr, &writerState, - &writerResult); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to query writer " << iWriter << - " status: " << GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterStatus; - } - - std::string writerName = BstrToString(writerNameBstr); - ::SysFreeString(writerNameBstr); - - if(writerResult != S_OK) - { - BOX_ERROR("VSS: Writer " << iWriter << " (" << - writerName << ") failed: " << - GetMsgForHresult(writerResult)); - continue; - } - - std::string stateName; - - switch(writerState) - { -#define WRITER_STATE(code) \ - case code: stateName = #code; break; - WRITER_STATE(VSS_WS_UNKNOWN); - WRITER_STATE(VSS_WS_STABLE); - WRITER_STATE(VSS_WS_WAITING_FOR_FREEZE); - WRITER_STATE(VSS_WS_WAITING_FOR_THAW); - WRITER_STATE(VSS_WS_WAITING_FOR_POST_SNAPSHOT); - WRITER_STATE(VSS_WS_WAITING_FOR_BACKUP_COMPLETE); - WRITER_STATE(VSS_WS_FAILED_AT_IDENTIFY); - WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_BACKUP); - WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_SNAPSHOT); - WRITER_STATE(VSS_WS_FAILED_AT_FREEZE); - WRITER_STATE(VSS_WS_FAILED_AT_THAW); - WRITER_STATE(VSS_WS_FAILED_AT_POST_SNAPSHOT); - WRITER_STATE(VSS_WS_FAILED_AT_BACKUP_COMPLETE); - WRITER_STATE(VSS_WS_FAILED_AT_PRE_RESTORE); - WRITER_STATE(VSS_WS_FAILED_AT_POST_RESTORE); - WRITER_STATE(VSS_WS_FAILED_AT_BACKUPSHUTDOWN); -#undef WRITER_STATE - default: - std::ostringstream o; - o << "unknown (" << writerState << ")"; - stateName = o.str(); - } - - BOX_TRACE("VSS: Writer " << iWriter << " (" << - writerName << ") is in state " << stateName); - } - - // lookup new snapshot volume for each location that has a snapshot - for(Locations::iterator - iLocation = mLocations.begin(); - iLocation != mLocations.end(); - iLocation++) - { - Location& rLocation(**iLocation); - if(rLocation.mIsSnapshotCreated) - { - VSS_SNAPSHOT_PROP prop; - result = mpVssBackupComponents->GetSnapshotProperties( - rLocation.mSnapshotVolumeId, &prop); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to get snapshot properties " - "for volume " << GuidToString(rLocation.mSnapshotVolumeId) << - " for location " << rLocation.mPath << ": " << - GetMsgForHresult(result)); - rLocation.mIsSnapshotCreated = false; - continue; - } - - rLocation.mSnapshotPath = - WideStringToString(prop.m_pwszSnapshotDeviceObject) + - DIRECTORY_SEPARATOR + rLocation.mSnapshotPath; - VssFreeSnapshotProperties(&prop); - - BOX_INFO("VSS: Location " << rLocation.mPath << " using " - "snapshot path " << rLocation.mSnapshotPath); - } - } - - IVssEnumObject *pEnum; - result = mpVssBackupComponents->Query(GUID_NULL, VSS_OBJECT_NONE, - VSS_OBJECT_SNAPSHOT, &pEnum); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to query snapshot list: " << - GetMsgForHresult(result)); - goto CreateVssBackupComponents_cleanup_WriterStatus; - } - - while(result == S_OK) - { - VSS_OBJECT_PROP rgelt; - ULONG count; - result = pEnum->Next(1, &rgelt, &count); - - if(result == S_FALSE) - { - // end of list, break out of the loop - break; - } - else if(result != S_OK) - { - BOX_ERROR("VSS: Failed to enumerate snapshot: " << - GetMsgForHresult(result)); - } - else if(count != 1) - { - BOX_ERROR("VSS: Failed to enumerate snapshot: " << - "Next() returned " << count << " objects instead of 1"); - } - else if(rgelt.Type != VSS_OBJECT_SNAPSHOT) - { - BOX_ERROR("VSS: Failed to enumerate snapshot: " << - "Next() returned a type " << rgelt.Type << " object " - "instead of VSS_OBJECT_SNAPSHOT"); - } - else - { - VSS_SNAPSHOT_PROP *pSnap = &rgelt.Obj.Snap; - BOX_TRACE("VSS: Snapshot ID: " << - GuidToString(pSnap->m_SnapshotId)); - BOX_TRACE("VSS: Snapshot set ID: " << - GuidToString(pSnap->m_SnapshotSetId)); - BOX_TRACE("VSS: Number of volumes: " << - pSnap->m_lSnapshotsCount); - BOX_TRACE("VSS: Snapshot device object: " << - WideStringToString(pSnap->m_pwszSnapshotDeviceObject)); - BOX_TRACE("VSS: Original volume name: " << - WideStringToString(pSnap->m_pwszOriginalVolumeName)); - BOX_TRACE("VSS: Originating machine: " << - WideStringToString(pSnap->m_pwszOriginatingMachine)); - BOX_TRACE("VSS: Service machine: " << - WideStringToString(pSnap->m_pwszServiceMachine)); - BOX_TRACE("VSS: Exposed name: " << - WideStringToString(pSnap->m_pwszExposedName)); - BOX_TRACE("VSS: Exposed path: " << - WideStringToString(pSnap->m_pwszExposedPath)); - BOX_TRACE("VSS: Provider ID: " << - GuidToString(pSnap->m_ProviderId)); - BOX_TRACE("VSS: Snapshot attributes: " << - BOX_FORMAT_HEX32(pSnap->m_lSnapshotAttributes)); - BOX_TRACE("VSS: Snapshot creation time: " << - BOX_FORMAT_HEX32(pSnap->m_tsCreationTimestamp)); - - std::string status; - switch(pSnap->m_eStatus) - { - case VSS_SS_UNKNOWN: status = "Unknown (error)"; break; - case VSS_SS_PREPARING: status = "Preparing"; break; - case VSS_SS_PROCESSING_PREPARE: status = "Preparing (processing)"; break; - case VSS_SS_PREPARED: status = "Prepared"; break; - case VSS_SS_PROCESSING_PRECOMMIT: status = "Precommitting"; break; - case VSS_SS_PRECOMMITTED: status = "Precommitted"; break; - case VSS_SS_PROCESSING_COMMIT: status = "Commiting"; break; - case VSS_SS_COMMITTED: status = "Committed"; break; - case VSS_SS_PROCESSING_POSTCOMMIT: status = "Postcommitting"; break; - case VSS_SS_PROCESSING_PREFINALCOMMIT: status = "Pre final committing"; break; - case VSS_SS_PREFINALCOMMITTED: status = "Pre final committed"; break; - case VSS_SS_PROCESSING_POSTFINALCOMMIT: status = "Post final committing"; break; - case VSS_SS_CREATED: status = "Created"; break; - case VSS_SS_ABORTED: status = "Aborted"; break; - case VSS_SS_DELETED: status = "Deleted"; break; - case VSS_SS_POSTCOMMITTED: status = "Postcommitted"; break; - default: - std::ostringstream buf; - buf << "Unknown code: " << pSnap->m_eStatus; - status = buf.str(); - } - - BOX_TRACE("VSS: Snapshot status: " << status); - VssFreeSnapshotProperties(pSnap); - } - } - - pEnum->Release(); - -CreateVssBackupComponents_cleanup_WriterStatus: - result = mpVssBackupComponents->FreeWriterStatus(); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to free writer status: " << - GetMsgForHresult(result)); - } - -CreateVssBackupComponents_cleanup_WriterMetadata: - result = mpVssBackupComponents->FreeWriterMetadata(); - if(result != S_OK) - { - BOX_ERROR("VSS: Failed to free writer metadata: " << - GetMsgForHresult(result)); - } -} - -void BackupDaemon::CleanupVssBackupComponents() -{ - if(mpVssBackupComponents == NULL) - { - return; - } - - CallAndWaitForAsync(&IVssBackupComponents::BackupComplete, - "BackupComplete()"); - - mpVssBackupComponents->Release(); - mpVssBackupComponents = NULL; -} -#endif - -void BackupDaemon::OnBackupStart() -{ - ResetLogFile(); - - // Touch a file to record times in filesystem - TouchFileInWorkingDir("last_sync_start"); - - // Reset statistics on uploads - BackupStoreFile::ResetStats(); - - // Tell anything connected to the command socket - SendSyncStartOrFinish(true /* start */); - - // Notify administrator - NotifySysadmin(SysadminNotifier::BackupStart); - - // Setup timer for polling the command socket - mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, - "CommandSocketPollTimer")); - - // Set state and log start - SetState(State_Connected); - BOX_NOTICE("Beginning scan of local files"); -} - -void BackupDaemon::OnBackupFinish() -{ - try - { - // Log - BOX_NOTICE("Finished scan of local files"); - - // Log the stats - BOX_NOTICE("File statistics: total file size uploaded " - << BackupStoreFile::msStats.mBytesInEncodedFiles - << ", bytes already on server " - << BackupStoreFile::msStats.mBytesAlreadyOnServer - << ", encoded size " - << BackupStoreFile::msStats.mTotalFileStreamSize - << ", " << mNumFilesUploaded << " files uploaded, " - << mNumDirsCreated << " dirs created"); - - // Reset statistics again - BackupStoreFile::ResetStats(); - - // Notify administrator - NotifySysadmin(SysadminNotifier::BackupFinish); - - // Stop the timer for polling the command socket, - // to prevent needless alarms while sleeping. - mapCommandSocketPollTimer.reset(); - - // Tell anything connected to the command socket - SendSyncStartOrFinish(false /* finish */); - - // Touch a file to record times in filesystem - TouchFileInWorkingDir("last_sync_finish"); - } - catch (std::exception &e) - { - BOX_ERROR("Failed to perform backup finish actions: " << e.what()); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::UseScriptToSeeIfSyncAllowed() -// Purpose: Private. Use a script to see if the sync should be -// allowed now (if configured). Returns -1 if it's -// allowed, time in seconds to wait otherwise. -// Created: 21/6/04 -// -// -------------------------------------------------------------------------- -int BackupDaemon::UseScriptToSeeIfSyncAllowed() -{ - const Configuration &conf(GetConfiguration()); - - // Got a script to run? - if(!conf.KeyExists("SyncAllowScript")) - { - // No. Do sync. - return -1; - } - - // If there's no result, try again in five minutes - int waitInSeconds = (60*5); - - std::string script(conf.GetKeyValue("SyncAllowScript") + - " \"" + GetConfigFileName() + "\""); - - // Run it? - pid_t pid = 0; - try - { - std::auto_ptr<IOStream> pscript(LocalProcessStream(script, - pid)); - - // Read in the result - IOStreamGetLine getLine(*pscript); - std::string line; - if(getLine.GetLine(line, true, 30000)) // 30 seconds should be enough - { - waitInSeconds = BackupDaemon::ParseSyncAllowScriptOutput(script, line); - } - else - { - BOX_ERROR("SyncAllowScript output nothing within " - "30 seconds, waiting 5 minutes to try again" - " (" << script << ")"); - } - } - catch(std::exception &e) - { - BOX_ERROR("Internal error running SyncAllowScript: " - << e.what() << " (" << script << ")"); - } - catch(...) - { - // Ignore any exceptions - // Log that something bad happened - BOX_ERROR("Unknown error running SyncAllowScript (" << - script << ")"); - } - - // Wait and then cleanup child process, if any - if(pid != 0) - { - int status = 0; - ::waitpid(pid, &status, 0); - } - - return waitInSeconds; -} - -int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, - const std::string& output) -{ - int waitInSeconds = (60*5); - std::istringstream iss(output); - - std::string delay; - iss >> delay; - - if(delay == "") - { - BOX_ERROR("SyncAllowScript output an empty line, sleeping for " - << waitInSeconds << " seconds (" << script << ")"); - return waitInSeconds; - } - - // Got a string, interpret - if(delay == "now") - { - // Script says do it now. Obey. - waitInSeconds = -1; - - BOX_NOTICE("SyncAllowScript requested a backup now " - "(" << script << ")"); - } - else - { - try - { - // How many seconds to wait? - waitInSeconds = BoxConvert::Convert<int32_t, const std::string&>(delay); - } - catch(ConversionException &e) - { - BOX_ERROR("SyncAllowScript output an invalid " - "number: '" << output << "' (" << - script << ")"); - throw; - } - - BOX_NOTICE("SyncAllowScript requested a delay of " << - waitInSeconds << " seconds (" << script << ")"); - } - - if(iss.eof()) - { - // No bandwidth limit requested - mMaxBandwidthFromSyncAllowScript = 0; - BOX_NOTICE("SyncAllowScript did not set a maximum bandwidth " - "(" << script << ")"); - } - else - { - std::string maxBandwidth; - iss >> maxBandwidth; - - try - { - // How many seconds to wait? - mMaxBandwidthFromSyncAllowScript = - BoxConvert::Convert<int32_t, const std::string&>(maxBandwidth); - } - catch(ConversionException &e) - { - BOX_ERROR("Invalid maximum bandwidth from " - "SyncAllowScript: '" << - output << "' (" << script << ")"); - throw; - } - - BOX_NOTICE("SyncAllowScript set maximum bandwidth to " << - mMaxBandwidthFromSyncAllowScript << " kB/s (" << - script << ")"); - } - - return waitInSeconds; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::RunBackgroundTask() -// Purpose: Checks for connections or commands on the command -// socket and handles them with minimal delay. Polled -// during lengthy operations such as file uploads. -// Created: 07/04/14 -// -// -------------------------------------------------------------------------- -bool BackupDaemon::RunBackgroundTask(State state, uint64_t progress, - uint64_t maximum) -{ - BOX_TRACE("BackupDaemon::RunBackgroundTask: state = " << state << - ", progress = " << progress << "/" << maximum); - - if(!mapCommandSocketPollTimer.get()) - { - return true; // no background task - } - - if(mapCommandSocketPollTimer->HasExpired()) - { - mapCommandSocketPollTimer->Reset(COMMAND_SOCKET_POLL_INTERVAL); - } - else - { - // Do no more work right now - return true; - } - - if(mapCommandSocketInfo.get()) - { - BOX_TRACE("BackupDaemon::RunBackgroundTask: polling command socket"); - - bool sync_flag_out, sync_is_forced_out; - - WaitOnCommandSocket(0, // RequiredDelay - sync_flag_out, sync_is_forced_out); - - if(sync_flag_out) - { - BOX_WARNING("Ignoring request to sync while " - "already syncing."); - } - } - - return true; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::WaitOnCommandSocket(box_time_t, bool &, bool &) -// Purpose: Waits on a the command socket for a time of UP TO -// the required time but may be much less, and handles -// a command if necessary. -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut) -{ - DoSyncFlagOut = false; - SyncIsForcedOut = false; - - ASSERT(mapCommandSocketInfo.get()); - if(!mapCommandSocketInfo.get()) - { - // failure case isn't too bad - ::sleep(1); - return; - } - - BOX_TRACE("Wait on command socket, delay = " << - BOX_FORMAT_MICROSECONDS(RequiredDelay)); - - try - { - // Timeout value for connections and things - int timeout = ((int)BoxTimeToMilliSeconds(RequiredDelay)) + 1; - // Handle bad boundary cases - if(timeout <= 0) timeout = 1; - if(timeout == INFTIM) timeout = 100000; - - // Wait for socket connection, or handle a command? - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - // No connection, listen for a new one - mapCommandSocketInfo->mpConnectedSocket.reset(mapCommandSocketInfo->mListeningSocket.Accept(timeout).release()); - - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - // If a connection didn't arrive, there was a timeout, which means we've - // waited long enough and it's time to go. - return; - } - else - { -#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET - bool uidOK = true; - BOX_WARNING("On this platform, no security check can be made on the credentials of peers connecting to the command socket. (bbackupctl)"); -#else - // Security check -- does the process connecting to this socket have - // the same UID as this process? - bool uidOK = false; - // BLOCK - { - uid_t remoteEUID = 0xffff; - gid_t remoteEGID = 0xffff; - if(mapCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID)) - { - // Credentials are available -- check UID - if(remoteEUID == ::getuid()) - { - // Acceptable - uidOK = true; - } - } - } -#endif // PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET - - // Is this an acceptable connection? - if(!uidOK) - { - // Dump the connection - BOX_ERROR("Incoming command connection from peer had different user ID than this process, or security check could not be completed."); - mapCommandSocketInfo->mpConnectedSocket.reset(); - return; - } - else - { - // Log - BOX_INFO("Connection from command socket"); - - // Send a header line summarising the configuration and current state - const Configuration &conf(GetConfiguration()); - std::ostringstream hello; - hello << "bbackupd: " << - (conf.GetKeyValueBool("AutomaticBackup") ? 1 : 0) - << " " << - conf.GetKeyValueInt("UpdateStoreInterval") - << " " << - conf.GetKeyValueInt("MinimumFileAge") - << " " << - conf.GetKeyValueInt("MaxUploadWait") - << "\nstate " << mState << "\n"; - mapCommandSocketInfo->mpConnectedSocket->Write( - hello.str(), timeout); - - // Set the timeout to something very small, so we don't wait too long on waiting - // for any incoming data - timeout = 10; // milliseconds - } - } - } - - // So there must be a connection now. - ASSERT(mapCommandSocketInfo->mpConnectedSocket.get() != 0); - - // Is there a getline object ready? - if(mapCommandSocketInfo->mpGetLine == 0) - { - // Create a new one - mapCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mapCommandSocketInfo->mpConnectedSocket.get())); - } - - // Ping the remote side, to provide errors which will mean the socket gets closed - mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5, - timeout); - - // Wait for a command or something on the socket - std::string command; - while(mapCommandSocketInfo->mpGetLine != 0 && !mapCommandSocketInfo->mpGetLine->IsEOF() - && mapCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) - { - BOX_TRACE("Receiving command '" << command - << "' over command socket"); - - bool sendOK = false; - bool sendResponse = true; - - // Command to process! - if(command == "quit" || command == "") - { - // Close the socket. - CloseCommandConnection(); - sendResponse = false; - } - else if(command == "sync") - { - // Sync now! - DoSyncFlagOut = true; - SyncIsForcedOut = false; - sendOK = true; - } - else if(command == "force-sync") - { - // Sync now (forced -- overrides any SyncAllowScript) - DoSyncFlagOut = true; - SyncIsForcedOut = true; - sendOK = true; - } - else if(command == "reload") - { - // Reload the configuration - SetReloadConfigWanted(); - sendOK = true; - } - else if(command == "terminate") - { - // Terminate the daemon cleanly - SetTerminateWanted(); - sendOK = true; - } - - // Send a response back? - if(sendResponse) - { - std::string response = sendOK ? "ok\n" : "error\n"; - mapCommandSocketInfo->mpConnectedSocket->Write( - response, timeout); - } - - // Set timeout to something very small, so this just checks for data which is waiting - timeout = 1; - } - - // Close on EOF? - if(mapCommandSocketInfo->mpGetLine != 0 && mapCommandSocketInfo->mpGetLine->IsEOF()) - { - CloseCommandConnection(); - } - } - catch(ConnectionException &ce) - { - BOX_NOTICE("Failed to write to command socket: " << ce.what()); - - // If an error occurs, and there is a connection active, - // just close that connection and continue. Otherwise, - // let the error propagate. - - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - throw; // thread will die - } - else - { - // Close socket and ignore error - CloseCommandConnection(); - } - } - catch(std::exception &e) - { - BOX_ERROR("Failed to write to command socket: " << - e.what()); - - // If an error occurs, and there is a connection active, - // just close that connection and continue. Otherwise, - // let the error propagate. - - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - throw; // thread will die - } - else - { - // Close socket and ignore error - CloseCommandConnection(); - } - } - catch(...) - { - BOX_ERROR("Failed to write to command socket: unknown error"); - - // If an error occurs, and there is a connection active, - // just close that connection and continue. Otherwise, - // let the error propagate. - - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - throw; // thread will die - } - else - { - // Close socket and ignore error - CloseCommandConnection(); - } - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::CloseCommandConnection() -// Purpose: Close the command connection, ignoring any errors -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::CloseCommandConnection() -{ - try - { - BOX_TRACE("Closing command connection"); - - if(mapCommandSocketInfo->mpGetLine) - { - delete mapCommandSocketInfo->mpGetLine; - mapCommandSocketInfo->mpGetLine = 0; - } - mapCommandSocketInfo->mpConnectedSocket.reset(); - } - catch(std::exception &e) - { - BOX_ERROR("Internal error while closing command " - "socket: " << e.what()); - } - catch(...) - { - // Ignore any errors - } -} - - -// -------------------------------------------------------------------------- -// -// File -// Name: BackupDaemon.cpp -// Purpose: Send a start or finish sync message to the command socket, if it's connected. -// -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::SendSyncStartOrFinish(bool SendStart) -{ - // The bbackupctl program can't rely on a state change, because it - // may never change if the server doesn't need to be contacted. - - if(mapCommandSocketInfo.get() && - mapCommandSocketInfo->mpConnectedSocket.get() != 0) - { - std::string message = SendStart ? "start-sync" : "finish-sync"; - try - { - message += "\n"; - mapCommandSocketInfo->mpConnectedSocket->Write(message, - 1); // short timeout, it's overlapped - } - catch(std::exception &e) - { - BOX_ERROR("Internal error while sending to " - "command socket client: " << e.what()); - CloseCommandConnection(); - } - catch(...) - { - CloseCommandConnection(); - } - } -} - - - - -#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_NMTONNAME) - // string comparison ordering for when mount points are handled - // by code, rather than the OS. - typedef struct - { - bool operator()(const std::string &s1, const std::string &s2) - { - if(s1.size() == s2.size()) - { - // Equal size, sort according to natural sort order - return s1 < s2; - } - else - { - // Make sure longer strings go first - return s1.size() > s2.size(); - } - } - } mntLenCompare; -#endif - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::SetupLocations(BackupClientContext &, const Configuration &) -// Purpose: Makes sure that the list of directories records is correctly set up -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf) -{ - // Going to need a copy of the root directory. Get a connection, - // and fetch it. - BackupProtocolCallable& connection(rClientContext.GetConnection()); - - // Ask server for a list of everything in the root directory, - // which is a directory itself - std::auto_ptr<BackupProtocolSuccess> dirreply( - connection.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - // only directories - BackupProtocolListDirectory::Flags_Dir, - // exclude old/deleted stuff - BackupProtocolListDirectory::Flags_Deleted | - BackupProtocolListDirectory::Flags_OldVersion, - false /* no attributes */)); - - // Retrieve the directory from the stream following - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(connection.ReceiveStream()); - dir.ReadFromStream(*dirstream, connection.GetTimeout()); - - // Map of mount names to ID map index - std::map<std::string, int> mounts; - int numIDMaps = 0; - -#ifdef HAVE_MOUNTS -#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_MNTONNAME) - // Linux and others can't tell you where a directory is mounted. So we - // have to read the mount entries from /etc/mtab! Bizarre that the OS - // itself can't tell you, but there you go. - std::set<std::string, mntLenCompare> mountPoints; - // BLOCK - FILE *mountPointsFile = 0; - -#ifdef HAVE_STRUCT_MNTENT_MNT_DIR - // Open mounts file - mountPointsFile = ::setmntent("/proc/mounts", "r"); - if(mountPointsFile == 0) - { - mountPointsFile = ::setmntent("/etc/mtab", "r"); - } - if(mountPointsFile == 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } - - try - { - // Read all the entries, and put them in the set - struct mntent *entry = 0; - while((entry = ::getmntent(mountPointsFile)) != 0) - { - BOX_TRACE("Found mount point at " << entry->mnt_dir); - mountPoints.insert(std::string(entry->mnt_dir)); - } - - // Close mounts file - ::endmntent(mountPointsFile); - } - catch(...) - { - ::endmntent(mountPointsFile); - throw; - } -#else // ! HAVE_STRUCT_MNTENT_MNT_DIR - // Open mounts file - mountPointsFile = ::fopen("/etc/mnttab", "r"); - if(mountPointsFile == 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } - - try - { - // Read all the entries, and put them in the set - struct mnttab entry; - while(getmntent(mountPointsFile, &entry) == 0) - { - BOX_TRACE("Found mount point at " << entry.mnt_mountp); - mountPoints.insert(std::string(entry.mnt_mountp)); - } - - // Close mounts file - ::fclose(mountPointsFile); - } - catch(...) - { - ::fclose(mountPointsFile); - throw; - } -#endif // HAVE_STRUCT_MNTENT_MNT_DIR - // Check sorting and that things are as we expect - ASSERT(mountPoints.size() > 0); -#ifndef BOX_RELEASE_BUILD - { - std::set<std::string, mntLenCompare>::reverse_iterator i(mountPoints.rbegin()); - ASSERT(*i == "/"); - } -#endif // n BOX_RELEASE_BUILD -#endif // n HAVE_STRUCT_STATFS_F_MNTONNAME || n HAVE_STRUCT_STATVFS_F_MNTONNAME -#endif // HAVE_MOUNTS - - // Then... go through each of the entries in the configuration, - // making sure there's a directory created for it. - std::vector<std::string> locNames = - rLocationsConf.GetSubConfigurationNames(); - - // We only want completely configured locations to be in the list - // when this function exits, so move them all to a temporary list. - // Entries matching a properly configured location will be moved - // back to mLocations. Anything left in this list after the loop - // finishes will be deleted. - Locations tmpLocations = mLocations; - mLocations.clear(); - - // The ID map list will be repopulated automatically by this loop - mIDMapMounts.clear(); - - for(std::vector<std::string>::iterator - pLocName = locNames.begin(); - pLocName != locNames.end(); - pLocName++) - { - Location* pLoc = NULL; - - // Try to find and reuse an existing Location object - for(Locations::const_iterator - i = tmpLocations.begin(); - i != tmpLocations.end(); i++) - { - if ((*i)->mName == *pLocName) - { - BOX_TRACE("Location already configured: " << *pLocName); - pLoc = *i; - break; - } - } - - const Configuration& rConfig( - rLocationsConf.GetSubConfiguration(*pLocName)); - std::auto_ptr<Location> apLoc; - - try - { - if(pLoc == NULL) - { - // Create a record for it - BOX_TRACE("New location: " << *pLocName); - pLoc = new Location; - - // ensure deletion if setup fails - apLoc.reset(pLoc); - - // Setup names in the location record - pLoc->mName = *pLocName; - pLoc->mPath = rConfig.GetKeyValue("Path"); - } - - // Read the exclude lists from the Configuration - pLoc->mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rConfig)); - pLoc->mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rConfig)); - - // Does this exist on the server? - // Remove from dir object early, so that if we fail - // to stat the local directory, we still don't - // consider to remote one for deletion. - BackupStoreDirectory::Iterator iter(dir); - BackupStoreFilenameClear dirname(pLoc->mName); // generate the filename - BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname); - int64_t oid = 0; - if(en != 0) - { - oid = en->GetObjectID(); - - // Delete the entry from the directory, so we get a list of - // unused root directories at the end of this. - dir.DeleteEntry(oid); - } - - // Do a fsstat on the pathname to find out which mount it's on - { - -#if defined HAVE_STRUCT_STATFS_F_MNTONNAME || defined HAVE_STRUCT_STATVFS_F_MNTONNAME || defined WIN32 - - // BSD style statfs -- includes mount point, which is nice. -#ifdef HAVE_STRUCT_STATVFS_F_MNTONNAME - struct statvfs s; - if(::statvfs(pLoc->mPath.c_str(), &s) != 0) -#else // HAVE_STRUCT_STATVFS_F_MNTONNAME - struct statfs s; - if(::statfs(pLoc->mPath.c_str(), &s) != 0) -#endif // HAVE_STRUCT_STATVFS_F_MNTONNAME - { - THROW_SYS_ERROR("Failed to stat path " - "'" << pLoc->mPath << "' " - "for location " - "'" << pLoc->mName << "'", - CommonException, OSFileError); - } - - // Where the filesystem is mounted - std::string mountName(s.f_mntonname); - -#else // !HAVE_STRUCT_STATFS_F_MNTONNAME && !WIN32 - - // Warn in logs if the directory isn't absolute - if(pLoc->mPath[0] != '/') - { - BOX_WARNING("Location path '" - << pLoc->mPath - << "' is not absolute"); - } - // Go through the mount points found, and find a suitable one - std::string mountName("/"); - { - std::set<std::string, mntLenCompare>::const_iterator i(mountPoints.begin()); - BOX_TRACE(mountPoints.size() - << " potential mount points"); - for(; i != mountPoints.end(); ++i) - { - // Compare first n characters with the filename - // If it matches, the file belongs in that mount point - // (sorting order ensures this) - BOX_TRACE("checking against mount point " << *i); - if(::strncmp(i->c_str(), pLoc->mPath.c_str(), i->size()) == 0) - { - // Match - mountName = *i; - break; - } - } - BOX_TRACE("mount point chosen for " - << pLoc->mPath << " is " - << mountName); - } - -#endif - - // Got it? - std::map<std::string, int>::iterator f(mounts.find(mountName)); - if(f != mounts.end()) - { - // Yes -- store the index - pLoc->mIDMapIndex = f->second; - } - else - { - // No -- new index - pLoc->mIDMapIndex = numIDMaps; - mounts[mountName] = numIDMaps; - - // Store the mount name - mIDMapMounts.push_back(mountName); - - // Increment number of maps - ++numIDMaps; - } - } - - // Does this exist on the server? - if(en == 0) - { - // Doesn't exist, so it has to be created on the server. Let's go! - // First, get the directory's attributes and modification time - box_time_t attrModTime = 0; - BackupClientFileAttributes attr; - try - { - attr.ReadAttributes(pLoc->mPath.c_str(), - true /* directories have zero mod times */, - 0 /* not interested in mod time */, - &attrModTime /* get the attribute modification time */); - } - catch (BoxException &e) - { - BOX_ERROR("Failed to get attributes " - "for path '" << pLoc->mPath - << "', skipping location '" << - pLoc->mName << "'"); - throw; - } - - // Execute create directory command - try - { - std::auto_ptr<IOStream> attrStream( - new MemBlockStream(attr)); - std::auto_ptr<BackupProtocolSuccess> - dirCreate(connection.QueryCreateDirectory( - BACKUPSTORE_ROOT_DIRECTORY_ID, // containing directory - attrModTime, dirname, attrStream)); - - // Object ID for later creation - oid = dirCreate->GetObjectID(); - } - catch (BoxException &e) - { - BOX_ERROR("Failed to create remote " - "directory '/" << pLoc->mName << - "', skipping location '" << - pLoc->mName << "'"); - throw; - } - - } - - // Create and store the directory object for the root of this location - ASSERT(oid != 0); - if(pLoc->mapDirectoryRecord.get() == NULL) - { - pLoc->mapDirectoryRecord.reset( - new BackupClientDirectoryRecord(oid, *pLocName)); - } - - // Remove it from the temporary list to avoid deletion - tmpLocations.remove(pLoc); - - // Push it back on the vector of locations - mLocations.push_back(pLoc); - - if(apLoc.get() != NULL) - { - // Don't delete it now! - apLoc.release(); - } - } - catch (std::exception &e) - { - BOX_ERROR("Failed to configure location '" - << pLoc->mName << "' path '" - << pLoc->mPath << "': " << e.what() << - ": please check for previous errors"); - mReadErrorsOnFilesystemObjects = true; - } - catch(...) - { - BOX_ERROR("Failed to configure location '" - << pLoc->mName << "' path '" - << pLoc->mPath << "': please check for " - "previous errors"); - mReadErrorsOnFilesystemObjects = true; - } - } - - // Now remove any leftovers - for(BackupDaemon::Locations::iterator - i = tmpLocations.begin(); - i != tmpLocations.end(); i++) - { - BOX_INFO("Removing obsolete location from memory: " << - (*i)->mName); - delete *i; - } - - tmpLocations.clear(); - - // Any entries in the root directory which need deleting? - if(dir.GetNumberOfEntries() > 0 && - mDeleteRedundantLocationsAfter == 0) - { - BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " - "in root directory found, but will not delete because " - "DeleteRedundantLocationsAfter = 0"); - } - else if(dir.GetNumberOfEntries() > 0) - { - box_time_t now = GetCurrentBoxTime(); - - // This should reset the timer if the list of unused - // locations changes, but it will not if the number of - // unused locations does not change, but the locations - // do change, e.g. one mysteriously appears and another - // mysteriously appears. (FIXME) - if (dir.GetNumberOfEntries() != mUnusedRootDirEntries.size() || - mDeleteUnusedRootDirEntriesAfter == 0) - { - mDeleteUnusedRootDirEntriesAfter = now + - SecondsToBoxTime(mDeleteRedundantLocationsAfter); - } - - int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter - - now); - - BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " - "in root directory found, will delete from store " - "after " << secs << " seconds."); - - // Store directories in list of things to delete - mUnusedRootDirEntries.clear(); - BackupStoreDirectory::Iterator iter(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = iter.Next()) != 0) - { - // Add name to list - BackupStoreFilenameClear clear(en->GetName()); - const std::string &name(clear.GetClearFilename()); - mUnusedRootDirEntries.push_back( - std::pair<int64_t,std::string> - (en->GetObjectID(), name)); - // Log this - BOX_INFO("Unused location in root: " << name); - } - ASSERT(mUnusedRootDirEntries.size() > 0); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::SetupIDMapsForSync() -// Purpose: Sets up ID maps for the sync process -- make sure they're all there -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::SetupIDMapsForSync() -{ - // Make sure we have some blank, empty ID maps - DeleteIDMapVector(mNewIDMaps); - FillIDMapVector(mNewIDMaps, true /* new maps */); - DeleteIDMapVector(mCurrentIDMaps); - FillIDMapVector(mCurrentIDMaps, false /* new maps */); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &) -// Purpose: Fills the vector with the right number of empty ID maps -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps) -{ - ASSERT(rVector.size() == 0); - rVector.reserve(mIDMapMounts.size()); - - for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) - { - // Create the object - BackupClientInodeToIDMap *pmap = new BackupClientInodeToIDMap(); - try - { - // Get the base filename of this map - std::string filename; - MakeMapBaseName(l, filename); - - // If it's a new one, add a suffix - if(NewMaps) - { - filename += ".n"; - } - - // The new map file should not exist yet. If there's - // one left over from a previous failed run, it's not - // useful to us because we never read from it and will - // overwrite the entries of all files that still - // exist, so we should just delete it and start afresh. - if(NewMaps && FileExists(filename.c_str())) - { - BOX_NOTICE("Found an incomplete ID map " - "database, deleting it to start " - "afresh: " << filename); - if(unlink(filename.c_str()) != 0) - { - BOX_LOG_NATIVE_ERROR(BOX_FILE_MESSAGE( - filename, "Failed to delete " - "incomplete ID map database")); - } - } - - // If it's not a new map, it may not exist in which case an empty map should be created - if(!NewMaps && !FileExists(filename.c_str())) - { - pmap->OpenEmpty(); - } - else - { - // Open the map - pmap->Open(filename.c_str(), !NewMaps /* read only */, NewMaps /* create new */); - } - - // Store on vector - rVector.push_back(pmap); - } - catch(...) - { - delete pmap; - throw; - } - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeleteCorruptBerkelyDbFiles() -// Purpose: Delete the Berkely db files from disc after they have been corrupted. -// Created: 14/9/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::DeleteCorruptBerkelyDbFiles() -{ - for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) - { - // Get the base filename of this map - std::string filename; - MakeMapBaseName(l, filename); - - // Delete the file - BOX_TRACE("Deleting " << filename); - ::unlink(filename.c_str()); - - // Add a suffix for the new map - filename += ".n"; - - // Delete that too - BOX_TRACE("Deleting " << filename); - ::unlink(filename.c_str()); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: MakeMapBaseName(unsigned int, std::string &) -// Purpose: Makes the base name for a inode map -// Created: 20/11/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const -{ - // Get the directory for the maps - const Configuration &config(GetConfiguration()); - std::string dir(config.GetKeyValue("DataDirectory")); - - // Make a leafname - std::string leaf(mIDMapMounts[MountNumber]); - for(unsigned int z = 0; z < leaf.size(); ++z) - { - if(leaf[z] == DIRECTORY_SEPARATOR_ASCHAR) - { - leaf[z] = '_'; - } - } - - // Build the final filename - rNameOut = dir + DIRECTORY_SEPARATOR "mnt" + leaf; -} - - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::CommitIDMapsAfterSync() -// Purpose: Commits the new ID maps, so the 'new' maps are now the 'current' maps. -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::CommitIDMapsAfterSync() -{ - // Get rid of the maps in memory (leaving them on disc of course) - DeleteIDMapVector(mCurrentIDMaps); - DeleteIDMapVector(mNewIDMaps); - - // Then move the old maps into the new places - for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) - { - std::string target; - MakeMapBaseName(l, target); - std::string newmap(target + ".n"); - - // Try to rename -#ifdef WIN32 - // win32 rename doesn't overwrite existing files - ::remove(target.c_str()); -#endif - if(::rename(newmap.c_str(), target.c_str()) != 0) - { - BOX_LOG_SYS_ERROR("Failed to rename ID map: " << - newmap << " to " << target); - THROW_EXCEPTION(CommonException, OSFileError) - } - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &) -// Purpose: Deletes the contents of a vector of ID maps -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector) -{ - while(!rVector.empty()) - { - // Pop off list - BackupClientInodeToIDMap *toDel = rVector.back(); - rVector.pop_back(); - - // Close and delete - delete toDel; - } - ASSERT(rVector.size() == 0); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::FindLocationPathName(const std::string &, std::string &) const -// Purpose: Tries to find the path of the root of a backup location. Returns true (and path in rPathOut) -// if it can be found, false otherwise. -// Created: 12/11/03 -// -// -------------------------------------------------------------------------- -bool BackupDaemon::FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const -{ - // Search for the location - for(Locations::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i) - { - if((*i)->mName == rLocationName) - { - rPathOut = (*i)->mPath; - return true; - } - } - - // Didn't find it - return false; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::SetState(int) -// Purpose: Record current action of daemon, and update process title to reflect this -// Created: 11/12/03 -// -// -------------------------------------------------------------------------- -void BackupDaemon::SetState(int State) -{ - // Two little checks - if(State == mState) return; - if(State < 0) return; - - // Update - mState = State; - - // Set process title - const static char *stateText[] = {"idle", "connected", "error -- waiting for retry", "over limit on server -- not backing up"}; - SetProcessTitle(stateText[State]); - - // If there's a command socket connected, then inform it -- disconnecting from the - // command socket if there's an error - - std::ostringstream msg; - msg << "state " << State << "\n"; - - if(!mapCommandSocketInfo.get()) - { - return; - } - - if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) - { - return; - } - - // Something connected to the command socket, tell it about the new state - try - { - mapCommandSocketInfo->mpConnectedSocket->Write(msg.str(), - 1); // very short timeout, it's overlapped anyway - } - catch(ConnectionException &ce) - { - BOX_NOTICE("Failed to write state to command socket: " << - ce.what()); - CloseCommandConnection(); - } - catch(std::exception &e) - { - BOX_ERROR("Failed to write state to command socket: " << - e.what()); - CloseCommandConnection(); - } - catch(...) - { - BOX_ERROR("Failed to write state to command socket: " - "unknown error"); - CloseCommandConnection(); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::TouchFileInWorkingDir(const char *) -// Purpose: Make sure a zero length file of the name exists in the working directory. -// Use for marking times of events in the filesystem. -// Created: 21/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::TouchFileInWorkingDir(const char *Filename) -{ - // Filename - const Configuration &config(GetConfiguration()); - std::string fn(config.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); - fn += Filename; - - // Open and close it to update the timestamp - try - { - FileStream touch(fn, O_WRONLY | O_CREAT | O_TRUNC, - S_IRUSR | S_IWUSR); - } - catch (std::exception &e) - { - BOX_ERROR("Failed to write to timestamp file: " << fn << ": " << - e.what()); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::NotifySysadmin(int) -// Purpose: Run the script to tell the sysadmin about events -// which need attention. -// Created: 25/2/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::NotifySysadmin(SysadminNotifier::EventCode Event) -{ - static const char *sEventNames[] = - { - "store-full", - "read-error", - "backup-error", - "backup-start", - "backup-finish", - "backup-ok", - 0 - }; - - // BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames)); - // BOX_TRACE("sizeof(*sEventNames) == " << sizeof(*sEventNames)); - // BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX); - ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == SysadminNotifier::MAX + 1); - - if(Event < 0 || Event >= SysadminNotifier::MAX) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, - BadNotifySysadminEventCode, "NotifySysadmin() called " - "for unknown event code " << Event); - } - - BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << - sEventNames[Event]); - - if(!GetConfiguration().KeyExists("NotifyAlways") || - !GetConfiguration().GetKeyValueBool("NotifyAlways")) - { - // Don't send lots of repeated messages - // Note: backup-start and backup-finish will always be - // logged, because mLastNotifiedEvent is never set to - // these values and therefore they are never "duplicates". - if(mLastNotifiedEvent == Event) - { - if(Event == SysadminNotifier::BackupOK) - { - BOX_INFO("Suppressing duplicate notification " - "about " << sEventNames[Event]); - } - else - { - BOX_WARNING("Suppressing duplicate notification " - "about " << sEventNames[Event]); - } - return; - } - } - - // Is there a notification script? - const Configuration &conf(GetConfiguration()); - if(!conf.KeyExists("NotifyScript")) - { - // Log, and then return - if(Event != SysadminNotifier::BackupStart && - Event != SysadminNotifier::BackupFinish) - { - BOX_INFO("Not notifying administrator about event " - << sEventNames[Event] << ", set NotifyScript " - "to do this in future"); - } - return; - } - - // Script to run - std::string script(conf.GetKeyValue("NotifyScript") + " " + - sEventNames[Event] + " \"" + GetConfigFileName() + "\""); - - // Log what we're about to do - BOX_INFO("About to notify administrator about event " - << sEventNames[Event] << ", running script '" << script << "'"); - - // Then do it - int returnCode = ::system(script.c_str()); - if(returnCode != 0) - { - BOX_WARNING("Notify script returned error code: " << - returnCode << " (" << script << ")"); - } - else if(Event != SysadminNotifier::BackupStart && - Event != SysadminNotifier::BackupFinish) - { - mLastNotifiedEvent = Event; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &) -// Purpose: Deletes any unused entries in the root directory, if they're scheduled to be deleted. -// Created: 13/5/04 -// -// -------------------------------------------------------------------------- -void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) -{ - if(mUnusedRootDirEntries.empty()) - { - BOX_INFO("Not deleting unused entries - none in list"); - return; - } - - if(mDeleteUnusedRootDirEntriesAfter == 0) - { - BOX_INFO("Not deleting unused entries - " - "zero delete time (bad)"); - return; - } - - // Check time - box_time_t now = GetCurrentBoxTime(); - if(now < mDeleteUnusedRootDirEntriesAfter) - { - int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter - - now); - BOX_INFO("Not deleting unused entries - too early (" - << secs << " seconds remaining)"); - return; - } - - // Entries to delete, and it's the right time to do so... - BOX_NOTICE("Deleting unused locations from store root..."); - BackupProtocolCallable &connection(rContext.GetConnection()); - for(std::vector<std::pair<int64_t,std::string> >::iterator - i(mUnusedRootDirEntries.begin()); - i != mUnusedRootDirEntries.end(); ++i) - { - connection.QueryDeleteDirectory(i->first); - rContext.GetProgressNotifier().NotifyFileDeleted( - i->first, i->second); - } - - // Reset state - mDeleteUnusedRootDirEntriesAfter = 0; - mUnusedRootDirEntries.clear(); -} - -// -------------------------------------------------------------------------- - -typedef struct -{ - int32_t mMagicValue; // also the version number - int32_t mNumEntries; - int64_t mObjectID; // this object ID - int64_t mContainerID; // ID of container - uint64_t mAttributesModTime; - int32_t mOptionsPresent; // bit mask of optional sections / features present - -} loc_StreamFormat; - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::CommandSocketInfo::CommandSocketInfo() -// Purpose: Constructor -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -BackupDaemon::CommandSocketInfo::CommandSocketInfo() - : mpGetLine(0) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::CommandSocketInfo::~CommandSocketInfo() -// Purpose: Destructor -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -BackupDaemon::CommandSocketInfo::~CommandSocketInfo() -{ - if(mpGetLine) - { - delete mpGetLine; - mpGetLine = 0; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::SerializeStoreObjectInfo( -// box_time_t theLastSyncTime, -// box_time_t theNextSyncTime) -// Purpose: Serializes remote directory and file information -// into a stream of bytes, using an Archive -// abstraction. -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- - -static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; -static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; -static const int STOREOBJECTINFO_VERSION = 2; - -bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime, - box_time_t theNextSyncTime) const -{ - if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) - { - return false; - } - - std::string StoreObjectInfoFile = - GetConfiguration().GetKeyValue("StoreObjectInfoFile"); - - if(StoreObjectInfoFile.size() <= 0) - { - return false; - } - - bool created = false; - - try - { - FileStream aFile(StoreObjectInfoFile.c_str(), - O_WRONLY | O_CREAT | O_TRUNC); - created = true; - - Archive anArchive(aFile, 0); - - anArchive.Write(STOREOBJECTINFO_MAGIC_ID_VALUE); - anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); - anArchive.Write(STOREOBJECTINFO_VERSION); - anArchive.Write(GetLoadedConfigModifiedTime()); - anArchive.Write(mClientStoreMarker); - anArchive.Write(theLastSyncTime); - anArchive.Write(theNextSyncTime); - - // - // - // - int64_t iCount = mLocations.size(); - anArchive.Write(iCount); - - for(Locations::const_iterator i = mLocations.begin(); - i != mLocations.end(); i++) - { - ASSERT(*i); - (*i)->Serialize(anArchive); - } - - // - // - // - iCount = mIDMapMounts.size(); - anArchive.Write(iCount); - - for(int v = 0; v < iCount; v++) - anArchive.Write(mIDMapMounts[v]); - - // - // - // - iCount = mUnusedRootDirEntries.size(); - anArchive.Write(iCount); - - for(int v = 0; v < iCount; v++) - { - anArchive.Write(mUnusedRootDirEntries[v].first); - anArchive.Write(mUnusedRootDirEntries[v].second); - } - - if (iCount > 0) - { - anArchive.Write(mDeleteUnusedRootDirEntriesAfter); - } - - // - // - // - aFile.Close(); - BOX_INFO("Saved store object info file version " << - STOREOBJECTINFO_VERSION << " (" << - StoreObjectInfoFile << ")"); - } - catch(std::exception &e) - { - BOX_ERROR("Failed to write StoreObjectInfoFile: " << - StoreObjectInfoFile << ": " << e.what()); - } - catch(...) - { - BOX_ERROR("Failed to write StoreObjectInfoFile: " << - StoreObjectInfoFile << ": unknown error"); - } - - return created; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeserializeStoreObjectInfo( -// box_time_t & theLastSyncTime, -// box_time_t & theNextSyncTime) -// Purpose: Deserializes remote directory and file information -// from a stream of bytes, using an Archive -// abstraction. -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, - box_time_t & theNextSyncTime) -{ - // - // - // - DeleteAllLocations(); - - // - // - // - if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) - { - BOX_NOTICE("Store object info file is not enabled. Will " - "download directory listings from store."); - return false; - } - - std::string StoreObjectInfoFile = - GetConfiguration().GetKeyValue("StoreObjectInfoFile"); - - if(StoreObjectInfoFile.size() <= 0) - { - return false; - } - - int64_t fileSize; - if (!FileExists(StoreObjectInfoFile, &fileSize) || fileSize == 0) - { - BOX_NOTICE(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Store object info file does not exist or is empty")); - } - else - { - try - { - FileStream aFile(StoreObjectInfoFile, O_RDONLY); - Archive anArchive(aFile, 0); - - // - // see if the content looks like a valid serialised archive - // - int iMagicValue = 0; - anArchive.Read(iMagicValue); - - if(iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE) - { - BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Store object info file is not a valid " - "or compatible serialised archive")); - return false; - } - - // - // get a bit optimistic and read in a string identifier - // - std::string strMagicValue; - anArchive.Read(strMagicValue); - - if(strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING) - { - BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Store object info file is not a valid " - "or compatible serialised archive")); - return false; - } - - // - // check if we are loading some future format - // version by mistake - // - int iVersion = 0; - anArchive.Read(iVersion); - - if(iVersion != STOREOBJECTINFO_VERSION) - { - BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Store object info file version " << - iVersion << " is not supported")); - return false; - } - - // - // check if this state file is even valid - // for the loaded bbackupd.conf file - // - box_time_t lastKnownConfigModTime; - anArchive.Read(lastKnownConfigModTime); - - if(lastKnownConfigModTime != GetLoadedConfigModifiedTime()) - { - BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Store object info file is older than " - "configuration file")); - return false; - } - - // - // this is it, go at it - // - anArchive.Read(mClientStoreMarker); - anArchive.Read(theLastSyncTime); - anArchive.Read(theNextSyncTime); - - // - // - // - int64_t iCount = 0; - anArchive.Read(iCount); - - for(int v = 0; v < iCount; v++) - { - Location* pLocation = new Location; - if(!pLocation) - { - throw std::bad_alloc(); - } - - pLocation->Deserialize(anArchive); - mLocations.push_back(pLocation); - } - - // - // - // - iCount = 0; - anArchive.Read(iCount); - - for(int v = 0; v < iCount; v++) - { - std::string strItem; - anArchive.Read(strItem); - - mIDMapMounts.push_back(strItem); - } - - // - // - // - iCount = 0; - anArchive.Read(iCount); - - for(int v = 0; v < iCount; v++) - { - int64_t anId; - anArchive.Read(anId); - - std::string aName; - anArchive.Read(aName); - - mUnusedRootDirEntries.push_back(std::pair<int64_t, std::string>(anId, aName)); - } - - if (iCount > 0) - anArchive.Read(mDeleteUnusedRootDirEntriesAfter); - - // - // - // - aFile.Close(); - - BOX_INFO(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Loaded store object info file version " << iVersion)); - return true; - } - catch(std::exception &e) - { - BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Internal error reading store object info " - "file: " << e.what())); - } - catch(...) - { - BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, - "Internal error reading store object info " - "file: unknown error")); - } - } - - BOX_NOTICE("No usable cache, will download directory listings from " - "server."); - - DeleteAllLocations(); - - mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; - theLastSyncTime = 0; - theNextSyncTime = 0; - - return false; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::DeleteStoreObjectInfo() -// Purpose: Deletes the serialised state file, to prevent us -// from using it again if a backup is interrupted. -// -// Created: 2006/02/12 -// -// -------------------------------------------------------------------------- - -bool BackupDaemon::DeleteStoreObjectInfo() const -{ - if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) - { - return false; - } - - std::string storeObjectInfoFile(GetConfiguration().GetKeyValue("StoreObjectInfoFile")); - - // Check to see if the file exists - if(!FileExists(storeObjectInfoFile.c_str())) - { - // File doesn't exist -- so can't be deleted. But something - // isn't quite right, so log a message - BOX_WARNING("StoreObjectInfoFile did not exist when it " - "was supposed to: " << storeObjectInfoFile); - - // Return true to stop things going around in a loop - return true; - } - - // Actually delete it - if(::unlink(storeObjectInfoFile.c_str()) != 0) - { - BOX_LOG_SYS_ERROR("Failed to delete the old " - "StoreObjectInfoFile: " << storeObjectInfoFile); - return false; - } - - return true; -} diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h deleted file mode 100644 index ffe31247..00000000 --- a/bin/bbackupd/BackupDaemon.h +++ /dev/null @@ -1,539 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupDaemon.h -// Purpose: Backup daemon -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPDAEMON__H -#define BACKUPDAEMON__H - -#include <vector> -#include <string> -#include <memory> - -#include "BackupClientContext.h" -#include "BackupClientDirectoryRecord.h" -#include "BoxTime.h" -#include "Daemon.h" -#include "Logging.h" -#include "Socket.h" -#include "SocketListen.h" -#include "SocketStream.h" -#include "TLSContext.h" - -#include "autogen_BackupProtocol.h" -#include "autogen_BackupStoreException.h" - -#ifdef WIN32 - #include "WinNamedPipeListener.h" - #include "WinNamedPipeStream.h" -#endif - -#ifdef ENABLE_VSS -# include <comdef.h> -# include <Vss.h> -# include <VsWriter.h> -# include <VsBackup.h> -#endif - -#define COMMAND_SOCKET_POLL_INTERVAL 1000 - -class BackupClientDirectoryRecord; -class BackupClientContext; -class Configuration; -class BackupClientInodeToIDMap; -class ExcludeList; -class IOStreamGetLine; -class Archive; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupDaemon -// Purpose: Backup daemon -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -class BackupDaemon : public Daemon, public ProgressNotifier, public LocationResolver, -public RunStatusProvider, public SysadminNotifier, public BackgroundTask -{ -public: - BackupDaemon(); - ~BackupDaemon(); - -private: - // methods below do partial (specialized) serialization of - // client state only - bool SerializeStoreObjectInfo(box_time_t theLastSyncTime, - box_time_t theNextSyncTime) const; - bool DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, - box_time_t & theNextSyncTime); - bool DeleteStoreObjectInfo() const; - BackupDaemon(const BackupDaemon &); - -public: - #ifdef WIN32 - // add command-line options to handle Windows services - std::string GetOptionString(); - int ProcessOption(signed int option); - int Main(const std::string &rConfigFileName); - - // This shouldn't be here, but apparently gcc on - // Windows has no idea about inherited methods... - virtual int Main(const char *DefaultConfigFile, int argc, - const char *argv[]) - { - return Daemon::Main(DefaultConfigFile, argc, argv); - } - #endif - - void Run(); - virtual const char *DaemonName() const; - virtual std::string DaemonBanner() const; - virtual void Usage(); - const ConfigurationVerify *GetConfigVerify() const; - - bool FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const; - - enum - { - // Add stuff to this, make sure the textual equivalents in SetState() are changed too. - State_Initialising = -1, - State_Idle = 0, - State_Connected = 1, - State_Error = 2, - State_StorageLimitExceeded = 3 - }; - - int GetState() {return mState;} - static std::string GetStateName(int state) - { - std::string stateName; - - #define STATE(x) case BackupDaemon::State_ ## x: stateName = #x; break; - switch (state) - { - STATE(Initialising); - STATE(Idle); - STATE(Connected); - STATE(Error); - STATE(StorageLimitExceeded); - default: - stateName = "unknown"; - } - #undef STATE - - return stateName; - } - - // Allow other classes to call this too - void NotifySysadmin(SysadminNotifier::EventCode Event); - -private: - void Run2(); - -public: - void InitCrypto(); - std::auto_ptr<BackupClientContext> RunSyncNowWithExceptionHandling(); - std::auto_ptr<BackupClientContext> RunSyncNow(); - void ResetCachedState(); - void OnBackupStart(); - void OnBackupFinish(); - // TouchFileInWorkingDir is only here for use by Boxi. - // This does NOT constitute an API! - void TouchFileInWorkingDir(const char *Filename); - -protected: - virtual std::auto_ptr<BackupClientContext> GetNewContext - ( - LocationResolver &rResolver, - TLSContext &rTLSContext, - const std::string &rHostname, - int32_t Port, - uint32_t AccountNumber, - bool ExtendedLogging, - bool ExtendedLogToFile, - std::string ExtendedLogFile, - ProgressNotifier &rProgressNotifier, - bool TcpNiceMode - ); - -private: - void DeleteAllLocations(); - void SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf); - - void DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector); - void DeleteAllIDMaps() - { - DeleteIDMapVector(mCurrentIDMaps); - DeleteIDMapVector(mNewIDMaps); - } - void FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps); - - void SetupIDMapsForSync(); - void CommitIDMapsAfterSync(); - void DeleteCorruptBerkelyDbFiles(); - - void MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const; - - void SetState(int State); - - void WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut); - void CloseCommandConnection(); - void SendSyncStartOrFinish(bool SendStart); - - void DeleteUnusedRootDirEntries(BackupClientContext &rContext); - - // For warning user about potential security hole - virtual void SetupInInitialProcess(); - - int UseScriptToSeeIfSyncAllowed(); - -public: - int ParseSyncAllowScriptOutput(const std::string& script, - const std::string& output); - typedef std::list<Location *> Locations; - Locations GetLocations() { return mLocations; } - -private: - int mState; // what the daemon is currently doing - - Locations mLocations; - - std::vector<std::string> mIDMapMounts; - std::vector<BackupClientInodeToIDMap *> mCurrentIDMaps; - std::vector<BackupClientInodeToIDMap *> mNewIDMaps; - - int mDeleteRedundantLocationsAfter; - - // For the command socket - class CommandSocketInfo - { - public: - CommandSocketInfo(); - ~CommandSocketInfo(); - private: - CommandSocketInfo(const CommandSocketInfo &); // no copying - CommandSocketInfo &operator=(const CommandSocketInfo &); - public: -#ifdef WIN32 - WinNamedPipeListener<1 /* listen backlog */> mListeningSocket; - std::auto_ptr<WinNamedPipeStream> mpConnectedSocket; -#else - SocketListen<SocketStream, 1 /* listen backlog */> mListeningSocket; - std::auto_ptr<SocketStream> mpConnectedSocket; -#endif - IOStreamGetLine *mpGetLine; - }; - - // Using a socket? - std::auto_ptr<CommandSocketInfo> mapCommandSocketInfo; - - // Stop notifications being repeated. - SysadminNotifier::EventCode mLastNotifiedEvent; - - // Unused entries in the root directory wait a while before being deleted - box_time_t mDeleteUnusedRootDirEntriesAfter; // time to delete them - std::vector<std::pair<int64_t,std::string> > mUnusedRootDirEntries; - - int64_t mClientStoreMarker; - bool mStorageLimitExceeded; - bool mReadErrorsOnFilesystemObjects; - box_time_t mLastSyncTime, mNextSyncTime; - box_time_t mCurrentSyncStartTime, mUpdateStoreInterval, - mBackupErrorDelay; - TLSContext mTlsContext; - bool mDeleteStoreObjectInfoFile; - bool mDoSyncForcedByPreviousSyncError; - int64_t mNumFilesUploaded, mNumDirsCreated; - int mMaxBandwidthFromSyncAllowScript; - -public: - int GetMaxBandwidthFromSyncAllowScript() { return mMaxBandwidthFromSyncAllowScript; } - bool StopRun() { return this->Daemon::StopRun(); } - bool StorageLimitExceeded() { return mStorageLimitExceeded; } - -private: - bool mLogAllFileAccess; - -public: - ProgressNotifier* GetProgressNotifier() { return mpProgressNotifier; } - LocationResolver* GetLocationResolver() { return mpLocationResolver; } - RunStatusProvider* GetRunStatusProvider() { return mpRunStatusProvider; } - SysadminNotifier* GetSysadminNotifier() { return mpSysadminNotifier; } - void SetProgressNotifier (ProgressNotifier* p) { mpProgressNotifier = p; } - void SetLocationResolver (LocationResolver* p) { mpLocationResolver = p; } - void SetRunStatusProvider(RunStatusProvider* p) { mpRunStatusProvider = p; } - void SetSysadminNotifier (SysadminNotifier* p) { mpSysadminNotifier = p; } - virtual bool RunBackgroundTask(State state, uint64_t progress, - uint64_t maximum); - -private: - ProgressNotifier* mpProgressNotifier; - LocationResolver* mpLocationResolver; - RunStatusProvider* mpRunStatusProvider; - SysadminNotifier* mpSysadminNotifier; - std::auto_ptr<Timer> mapCommandSocketPollTimer; - std::auto_ptr<BackupClientContext> mapClientContext; - - /* ProgressNotifier implementation */ -public: - virtual void NotifyIDMapsSetup(BackupClientContext& rContext) { } - - virtual void NotifyScanDirectory( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - if (mLogAllFileAccess) - { - BOX_INFO("Scanning directory: " << rLocalPath); - } - - if (!RunBackgroundTask(BackgroundTask::Scanning_Dirs, 0, 0)) - { - THROW_EXCEPTION(BackupStoreException, - CancelledByBackgroundTask); - } - } - virtual void NotifyDirStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { - BOX_WARNING("Failed to access directory: " << rLocalPath - << ": " << rErrorMsg); - } - virtual void NotifyFileStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { - BOX_WARNING("Failed to access file: " << rLocalPath - << ": " << rErrorMsg); - } - virtual void NotifyDirListFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { - BOX_WARNING("Failed to list directory: " << rLocalPath - << ": " << rErrorMsg); - } - virtual void NotifyMountPointSkipped( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - #ifdef WIN32 - BOX_WARNING("Ignored directory: " << rLocalPath << - ": is an NTFS junction/reparse point; create " - "a new location if you want to back it up"); - #else - BOX_WARNING("Ignored directory: " << rLocalPath << - ": is a mount point; create a new location " - "if you want to back it up"); - #endif - } - virtual void NotifyFileExcluded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - if (mLogAllFileAccess) - { - BOX_INFO("Skipping excluded file: " << rLocalPath); - } - } - virtual void NotifyDirExcluded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - if (mLogAllFileAccess) - { - BOX_INFO("Skipping excluded directory: " << rLocalPath); - } - } - virtual void NotifyUnsupportedFileType( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - BOX_WARNING("Ignoring file of unknown type: " << rLocalPath); - } - virtual void NotifyFileReadFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { - BOX_WARNING("Error reading file: " << rLocalPath - << ": " << rErrorMsg); - } - virtual void NotifyFileModifiedInFuture( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - BOX_WARNING("Some files have modification times excessively " - "in the future. Check clock synchronisation. " - "Example file (only one shown): " << rLocalPath); - } - virtual void NotifyFileSkippedServerFull( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - BOX_WARNING("Skipped file: server is full: " << rLocalPath); - } - virtual void NotifyFileUploadException( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const BoxException& rException) - { - if (rException.GetType() == CommonException::ExceptionType && - rException.GetSubType() == CommonException::AccessDenied) - { - BOX_ERROR("Failed to upload file: " << rLocalPath - << ": Access denied"); - } - else - { - BOX_ERROR("Failed to upload file: " << rLocalPath - << ": caught exception: " << rException.what() - << " (" << rException.GetType() - << "/" << rException.GetSubType() << ")"); - } - } - virtual void NotifyFileUploadServerError( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int type, int subtype) - { - BOX_ERROR("Failed to upload file: " << rLocalPath << - ": server error: " << - BackupProtocolError::GetMessage(subtype)); - } - virtual void NotifyFileUploading( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Uploading complete file: " << rLocalPath); - } - } - virtual void NotifyFileUploadingPatch( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t EstimatedBytesToUpload) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Uploading patch to file: " << rLocalPath << - ", estimated upload size = " << - EstimatedBytesToUpload); - } - } - virtual void NotifyFileUploadingAttributes( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Uploading new file attributes: " << - rLocalPath); - } - } - virtual void NotifyFileUploaded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Uploaded file: " << rLocalPath << - " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << - "): total size = " << FileSize << ", " - "uploaded size = " << UploadedSize); - } - mNumFilesUploaded++; - } - virtual void NotifyFileSynchronised( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize) - { - if (mLogAllFileAccess) - { - BOX_INFO("Synchronised file: " << rLocalPath); - } - } - virtual void NotifyDirectoryCreated( - int64_t ObjectID, - const std::string& rLocalPath, - const std::string& rRemotePath) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Created directory: " << rRemotePath << - " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << - ")"); - } - mNumDirsCreated++; - } - virtual void NotifyDirectoryDeleted( - int64_t ObjectID, - const std::string& rRemotePath) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Deleted directory: " << rRemotePath << - " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << - ")"); - } - } - virtual void NotifyFileDeleted( - int64_t ObjectID, - const std::string& rRemotePath) - { - if (mLogAllFileAccess) - { - BOX_NOTICE("Deleted file: " << rRemotePath << - " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << - ")"); - } - } - virtual void NotifyReadProgress(int64_t readSize, int64_t offset, - int64_t length, box_time_t elapsed, box_time_t finish) - { - BOX_TRACE("Read " << readSize << " bytes at " << offset << - ", " << (length - offset) << " remain, eta " << - BoxTimeToSeconds(finish - elapsed) << "s"); - } - virtual void NotifyReadProgress(int64_t readSize, int64_t offset, - int64_t length) - { - BOX_TRACE("Read " << readSize << " bytes at " << offset << - ", " << (length - offset) << " remain"); - } - virtual void NotifyReadProgress(int64_t readSize, int64_t offset) - { - BOX_TRACE("Read " << readSize << " bytes at " << offset << - ", unknown bytes remaining"); - } - -#ifdef WIN32 - private: - bool mInstallService, mRemoveService, mRunAsService; - std::string mServiceName; -#endif - -#ifdef ENABLE_VSS - IVssBackupComponents* mpVssBackupComponents; - void CreateVssBackupComponents(); - bool WaitForAsync(IVssAsync *pAsync, const std::string& description); - typedef HRESULT (__stdcall IVssBackupComponents::*AsyncMethod)(IVssAsync**); - bool CallAndWaitForAsync(AsyncMethod method, - const std::string& description); - void CleanupVssBackupComponents(); -#endif -}; - -#endif // BACKUPDAEMON__H diff --git a/bin/bbackupd/BackupDaemonInterface.h b/bin/bbackupd/BackupDaemonInterface.h deleted file mode 100644 index d5c47c85..00000000 --- a/bin/bbackupd/BackupDaemonInterface.h +++ /dev/null @@ -1,166 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupDaemonInterface.h -// Purpose: Interfaces for managing a BackupDaemon -// Created: 2008/12/30 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPDAEMONINTERFACE__H -#define BACKUPDAEMONINTERFACE__H - -#include <string> - -#include "BoxTime.h" - -class Archive; -class BackupClientContext; -class BackupDaemon; - -// -------------------------------------------------------------------------- -// -// Class -// Name: SysadminNotifier -// Purpose: Provides a NotifySysadmin() method to send mail to the sysadmin -// Created: 2005/11/15 -// -// -------------------------------------------------------------------------- -class SysadminNotifier -{ - public: - virtual ~SysadminNotifier() { } - - typedef enum - { - StoreFull = 0, - ReadError, - BackupError, - BackupStart, - BackupFinish, - BackupOK, - MAX - // When adding notifications, remember to add - // strings to NotifySysadmin() - } - EventCode; - - virtual void NotifySysadmin(EventCode Event) = 0; -}; - -// -------------------------------------------------------------------------- -// -// Class -// Name: ProgressNotifier -// Purpose: Provides methods for the backup library to inform the user -// interface about its progress with the backup -// Created: 2005/11/20 -// -// -------------------------------------------------------------------------- - -class BackupClientContext; -class BackupClientDirectoryRecord; - -class ProgressNotifier -{ - public: - virtual ~ProgressNotifier() { } - virtual void NotifyIDMapsSetup(BackupClientContext& rContext) = 0; - virtual void NotifyScanDirectory( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyDirStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyFileStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyDirListFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyMountPointSkipped( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileExcluded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyDirExcluded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyUnsupportedFileType( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileReadFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyFileModifiedInFuture( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileSkippedServerFull( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileUploadException( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const BoxException& rException) = 0; - virtual void NotifyFileUploadServerError( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int type, int subtype) = 0; - virtual void NotifyFileUploading( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileUploadingPatch( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t EstimatedBytesToUpload) = 0; - virtual void NotifyFileUploadingAttributes( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileUploaded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) = 0; - virtual void NotifyFileSynchronised( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize) = 0; - virtual void NotifyDirectoryCreated( - int64_t ObjectID, - const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyDirectoryDeleted( - int64_t ObjectID, - const std::string& rRemotePath) = 0; - virtual void NotifyFileDeleted( - int64_t ObjectID, - const std::string& rRemotePath) = 0; - virtual void NotifyReadProgress(int64_t readSize, int64_t offset, - int64_t length, box_time_t elapsed, box_time_t finish) = 0; - virtual void NotifyReadProgress(int64_t readSize, int64_t offset, - int64_t length) = 0; - virtual void NotifyReadProgress(int64_t readSize, int64_t offset) = 0; -}; - -// -------------------------------------------------------------------------- -// -// Class -// Name: LocationResolver -// Purpose: Interface for classes that can resolve locations to paths, -// like BackupDaemon -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -class LocationResolver -{ -public: - virtual ~LocationResolver() { } - virtual bool FindLocationPathName(const std::string &rLocationName, - std::string &rPathOut) const = 0; -}; - -#endif // BACKUPDAEMONINTERFACE__H diff --git a/bin/bbackupd/Win32BackupService.cpp b/bin/bbackupd/Win32BackupService.cpp deleted file mode 100644 index 6d027abf..00000000 --- a/bin/bbackupd/Win32BackupService.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// Win32 service functions for Box Backup, by Nick Knight - -#ifdef WIN32 - -#include "Box.h" -#include "BackupDaemon.h" -#include "MainHelper.h" -#include "BoxPortsAndFiles.h" -#include "BackupStoreException.h" - -#include "MemLeakFindOn.h" - -#include "Win32BackupService.h" - -Win32BackupService* gpDaemonService = NULL; -extern HANDLE gStopServiceEvent; -extern DWORD gServiceReturnCode; - -unsigned int WINAPI RunService(LPVOID lpParameter) -{ - DWORD retVal = gpDaemonService->WinService((const char*) lpParameter); - gServiceReturnCode = retVal; - SetEvent(gStopServiceEvent); - return retVal; -} - -void TerminateService(void) -{ - gpDaemonService->SetTerminateWanted(); -} - -DWORD Win32BackupService::WinService(const char* pConfigFileName) -{ - DWORD ret; - - if (pConfigFileName != NULL) - { - ret = this->Main(pConfigFileName); - } - else - { - ret = this->Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE); - } - - return ret; -} - -#endif // WIN32 diff --git a/bin/bbackupd/Win32BackupService.h b/bin/bbackupd/Win32BackupService.h deleted file mode 100644 index e7f077f2..00000000 --- a/bin/bbackupd/Win32BackupService.h +++ /dev/null @@ -1,21 +0,0 @@ -// Box Backup service daemon implementation by Nick Knight - -#ifndef WIN32BACKUPSERVICE_H -#define WIN32BACKUPSERVICE_H - -#ifdef WIN32 - -class Configuration; -class ConfigurationVerify; -class BackupDaemon; - -class Win32BackupService : public BackupDaemon -{ -public: - DWORD WinService(const char* pConfigFileName); -}; - -#endif // WIN32 - -#endif // WIN32BACKUPSERVICE_H - diff --git a/bin/bbackupd/Win32ServiceFunctions.cpp b/bin/bbackupd/Win32ServiceFunctions.cpp deleted file mode 100644 index 2df914a7..00000000 --- a/bin/bbackupd/Win32ServiceFunctions.cpp +++ /dev/null @@ -1,384 +0,0 @@ -//*************************************************************** -// From the book "Win32 System Services: The Heart of Windows 98 -// and Windows 2000" -// by Marshall Brain -// Published by Prentice Hall -// Copyright 1995 Prentice Hall. -// -// This code implements the Windows API Service interface -// for the Box Backup for Windows native port. -// Adapted for Box Backup by Nick Knight. -//*************************************************************** - -#ifdef WIN32 - -#include "Box.h" - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif -#ifdef HAVE_PROCESS_H - #include <process.h> -#endif - -extern void TerminateService(void); -extern unsigned int WINAPI RunService(LPVOID lpParameter); - -// Global variables - -TCHAR* gServiceName = TEXT("Box Backup Service"); -SERVICE_STATUS gServiceStatus; -SERVICE_STATUS_HANDLE gServiceStatusHandle = 0; -HANDLE gStopServiceEvent = 0; -DWORD gServiceReturnCode = 0; - -#define SERVICE_NAME "boxbackup" - -void ShowMessage(char *s) -{ - MessageBox(0, s, "Box Backup Message", - MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); -} - -void ErrorHandler(char *s, DWORD err) -{ - char buf[256]; - memset(buf, 0, sizeof(buf)); - _snprintf(buf, sizeof(buf)-1, "%s: %s", s, - GetErrorMessage(err).c_str()); - BOX_ERROR(buf); - MessageBox(0, buf, "Error", - MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); - ExitProcess(err); -} - -void WINAPI ServiceControlHandler( DWORD controlCode ) -{ - switch ( controlCode ) - { - case SERVICE_CONTROL_INTERROGATE: - break; - - case SERVICE_CONTROL_SHUTDOWN: - case SERVICE_CONTROL_STOP: - Beep(1000,100); - TerminateService(); - gServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - - SetEvent(gStopServiceEvent); - return; - - case SERVICE_CONTROL_PAUSE: - break; - - case SERVICE_CONTROL_CONTINUE: - break; - - default: - if ( controlCode >= 128 && controlCode <= 255 ) - // user defined control code - break; - else - // unrecognised control code - break; - } - - SetServiceStatus( gServiceStatusHandle, &gServiceStatus ); -} - -// ServiceMain is called when the SCM wants to -// start the service. When it returns, the service -// has stopped. It therefore waits on an event -// just before the end of the function, and -// that event gets set when it is time to stop. -// It also returns on any error because the -// service cannot start if there is an eror. - -static char* spConfigFileName; - -VOID ServiceMain(DWORD argc, LPTSTR *argv) -{ - // initialise service status - gServiceStatus.dwServiceType = SERVICE_WIN32; - gServiceStatus.dwCurrentState = SERVICE_STOPPED; - gServiceStatus.dwControlsAccepted = 0; - gServiceStatus.dwWin32ExitCode = NO_ERROR; - gServiceStatus.dwServiceSpecificExitCode = NO_ERROR; - gServiceStatus.dwCheckPoint = 0; - gServiceStatus.dwWaitHint = 0; - - gServiceStatusHandle = RegisterServiceCtrlHandler(gServiceName, - ServiceControlHandler); - - if (gServiceStatusHandle) - { - // service is starting - gServiceStatus.dwCurrentState = SERVICE_START_PENDING; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - - // do initialisation here - gStopServiceEvent = CreateEvent(0, TRUE, FALSE, 0); - if (!gStopServiceEvent) - { - gServiceStatus.dwControlsAccepted &= - ~(SERVICE_ACCEPT_STOP | - SERVICE_ACCEPT_SHUTDOWN); - gServiceStatus.dwCurrentState = SERVICE_STOPPED; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - return; - } - - HANDLE ourThread = (HANDLE)_beginthreadex( - NULL, - 0, - RunService, - spConfigFileName, - CREATE_SUSPENDED, - NULL); - - SetThreadPriority(ourThread, THREAD_PRIORITY_LOWEST); - ResumeThread(ourThread); - - // we are now running so tell the SCM - gServiceStatus.dwControlsAccepted |= - (SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); - gServiceStatus.dwCurrentState = SERVICE_RUNNING; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - - // do cleanup here - WaitForSingleObject(gStopServiceEvent, INFINITE); - CloseHandle(gStopServiceEvent); - gStopServiceEvent = 0; - - // service was stopped - gServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - - // service is now stopped - gServiceStatus.dwControlsAccepted &= - ~(SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); - - gServiceStatus.dwCurrentState = SERVICE_STOPPED; - - if (gServiceReturnCode != 0) - { - gServiceStatus.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR; - gServiceStatus.dwServiceSpecificExitCode = gServiceReturnCode; - } - - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - } -} - -int OurService(const char* pConfigFileName) -{ - spConfigFileName = strdup(pConfigFileName); - - SERVICE_TABLE_ENTRY serviceTable[] = - { - { SERVICE_NAME, (LPSERVICE_MAIN_FUNCTION) ServiceMain }, - { NULL, NULL } - }; - BOOL success; - - // Register with the SCM - success = StartServiceCtrlDispatcher(serviceTable); - - free(spConfigFileName); - spConfigFileName = NULL; - - if (!success) - { - ErrorHandler("Failed to start service. Did you start " - "Box Backup from the Service Control Manager? " - "(StartServiceCtrlDispatcher)", GetLastError()); - return 1; - } - - return 0; -} - -int InstallService(const char* pConfigFileName, const std::string& rServiceName) -{ - if (pConfigFileName != NULL) - { - EMU_STRUCT_STAT st; - - if (emu_stat(pConfigFileName, &st) != 0) - { - BOX_LOG_SYS_ERROR("Failed to open configuration file " - "'" << pConfigFileName << "'"); - return 1; - } - - if (!(st.st_mode & S_IFREG)) - { - - BOX_ERROR("Failed to open configuration file '" << - pConfigFileName << "': not a file"); - return 1; - } - } - - SC_HANDLE scm = OpenSCManager(0, 0, SC_MANAGER_CREATE_SERVICE); - - if (!scm) - { - BOX_ERROR("Failed to open service control manager: " << - GetErrorMessage(GetLastError())); - return 1; - } - - char cmd[MAX_PATH]; - GetModuleFileName(NULL, cmd, sizeof(cmd)-1); - cmd[sizeof(cmd)-1] = 0; - - std::string cmdWithArgs(cmd); - cmdWithArgs += " -s -S \"" + rServiceName + "\""; - - if (pConfigFileName != NULL) - { - cmdWithArgs += " \""; - cmdWithArgs += pConfigFileName; - cmdWithArgs += "\""; - } - - std::string serviceDesc = "Box Backup (" + rServiceName + ")"; - - SC_HANDLE newService = CreateService( - scm, - rServiceName.c_str(), - serviceDesc.c_str(), - SERVICE_ALL_ACCESS, - SERVICE_WIN32_OWN_PROCESS, - SERVICE_AUTO_START, - SERVICE_ERROR_NORMAL, - cmdWithArgs.c_str(), - 0,0,0,0,0); - - DWORD err = GetLastError(); - CloseServiceHandle(scm); - - if (!newService) - { - switch (err) - { - case ERROR_SERVICE_EXISTS: - { - BOX_ERROR("Failed to create Box Backup " - "service: it already exists"); - } - break; - - case ERROR_SERVICE_MARKED_FOR_DELETE: - { - BOX_ERROR("Failed to create Box Backup " - "service: it is waiting to be deleted"); - } - break; - - case ERROR_DUPLICATE_SERVICE_NAME: - { - BOX_ERROR("Failed to create Box Backup " - "service: a service with this name " - "already exists"); - } - break; - - default: - { - BOX_ERROR("Failed to create Box Backup " - "service: error " << - GetErrorMessage(GetLastError())); - } - } - - return 1; - } - - BOX_INFO("Created Box Backup service"); - - SERVICE_DESCRIPTION desc; - desc.lpDescription = "Backs up your data files over the Internet"; - - if (!ChangeServiceConfig2(newService, SERVICE_CONFIG_DESCRIPTION, - &desc)) - { - BOX_WARNING("Failed to set description for Box Backup " - "service: " << GetErrorMessage(GetLastError())); - } - - CloseServiceHandle(newService); - - return 0; -} - -int RemoveService(const std::string& rServiceName) -{ - SC_HANDLE scm = OpenSCManager(0,0,SC_MANAGER_CREATE_SERVICE); - - if (!scm) - { - BOX_ERROR("Failed to open service control manager: " << - GetErrorMessage(GetLastError())); - return 1; - } - - SC_HANDLE service = OpenService(scm, rServiceName.c_str(), - SERVICE_ALL_ACCESS|DELETE); - DWORD err = GetLastError(); - CloseServiceHandle(scm); - - if (!service) - { - if (err == ERROR_SERVICE_DOES_NOT_EXIST || - err == ERROR_IO_PENDING) - // hello microsoft? anyone home? - { - BOX_ERROR("Failed to open Box Backup service: " - "not installed or not found"); - } - else - { - BOX_ERROR("Failed to open Box Backup service: " << - GetErrorMessage(err)); - } - return 1; - } - - SERVICE_STATUS status; - if (!ControlService(service, SERVICE_CONTROL_STOP, &status)) - { - err = GetLastError(); - if (err != ERROR_SERVICE_NOT_ACTIVE) - { - BOX_WARNING("Failed to stop Box Backup service: " << - GetErrorMessage(err)); - } - } - - BOOL deleted = DeleteService(service); - err = GetLastError(); - CloseServiceHandle(service); - - if (deleted) - { - BOX_INFO("Box Backup service deleted"); - return 0; - } - else if (err == ERROR_SERVICE_MARKED_FOR_DELETE) - { - BOX_ERROR("Failed to remove Box Backup service: " - "it is already being deleted"); - } - else - { - BOX_ERROR("Failed to remove Box Backup service: " << - GetErrorMessage(err)); - } - - return 1; -} - -#endif // WIN32 diff --git a/bin/bbackupd/Win32ServiceFunctions.h b/bin/bbackupd/Win32ServiceFunctions.h deleted file mode 100644 index e04c368f..00000000 --- a/bin/bbackupd/Win32ServiceFunctions.h +++ /dev/null @@ -1,19 +0,0 @@ -//*************************************************************** -// From the book "Win32 System Services: The Heart of Windows 98 -// and Windows 2000" -// by Marshall Brain -// Published by Prentice Hall -// Copyright 1995 Prentice Hall. -// -// This code implements the Windows API Service interface -// for the Box Backup for Windows native port. -//*************************************************************** - -#ifndef WIN32SERVICEFUNCTIONS_H -#define WIN32SERVICEFUNCTIONS_H - -int RemoveService (const std::string& rServiceName); -int InstallService (const char* pConfigFilePath, const std::string& rServiceName); -int OurService (const char* pConfigFileName); - -#endif diff --git a/bin/bbackupquery/BackupQueries.cpp b/bin/bbackupquery/BackupQueries.cpp deleted file mode 100644 index bcb1827e..00000000 --- a/bin/bbackupquery/BackupQueries.cpp +++ /dev/null @@ -1,2410 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupQueries.cpp -// Purpose: Perform various queries on the backup store server. -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - -#include <stdio.h> -#include <errno.h> -#include <stdlib.h> -#include <limits.h> -#include <sys/types.h> -#include <sys/stat.h> - -#ifdef HAVE_DIRENT_H - #include <dirent.h> -#endif - -#include <algorithm> -#include <cstring> -#include <limits> -#include <iostream> -#include <ostream> -#include <set> - -#include "BackupClientFileAttributes.h" -#include "BackupClientMakeExcludeList.h" -#include "BackupClientRestore.h" -#include "BackupQueries.h" -#include "BackupStoreDirectory.h" -#include "BackupStoreException.h" -#include "BackupStoreFile.h" -#include "BackupStoreFilenameClear.h" -#include "BoxTimeToText.h" -#include "CommonException.h" -#include "Configuration.h" -#include "ExcludeList.h" -#include "FileModificationTime.h" -#include "FileStream.h" -#include "IOStream.h" -#include "Logging.h" -#include "PathUtils.h" -#include "SelfFlushingStream.h" -#include "Utils.h" -#include "autogen_BackupProtocol.h" -#include "autogen_CipherException.h" - -#include "MemLeakFindOn.h" - -// min() and max() macros from stdlib.h break numeric_limits<>::min(), etc. -#undef min -#undef max - -#define COMPARE_RETURN_SAME 1 -#define COMPARE_RETURN_DIFFERENT 2 -#define COMPARE_RETURN_ERROR 3 -#define COMMAND_RETURN_ERROR 4 - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::BackupQueries() -// Purpose: Constructor -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -BackupQueries::BackupQueries(BackupProtocolCallable &rConnection, - const Configuration &rConfiguration, bool readWrite) - : mReadWrite(readWrite), - mrConnection(rConnection), - mrConfiguration(rConfiguration), - mQuitNow(false), - mRunningAsRoot(false), - mWarnedAboutOwnerAttributes(false), - mReturnCode(0) // default return code -{ - #ifdef WIN32 - mRunningAsRoot = TRUE; - #else - mRunningAsRoot = (::geteuid() == 0); - #endif -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::~BackupQueries() -// Purpose: Destructor -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -BackupQueries::~BackupQueries() -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::DoCommand(const char *, bool) -// Purpose: Perform a command -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -void BackupQueries::DoCommand(ParsedCommand& rCommand) -{ - // Check... - - if(rCommand.mFailed) - { - BOX_ERROR("Parse failed: unknown command '" << - rCommand.mCmdElements[0] << "' or failed to convert " - "encoding of arguments"); - return; - } - - if(rCommand.mCmdElements.size() < 1) - { - // blank command - return; - } - - if(rCommand.pSpec->type == Command_sh && - rCommand.mCmdElements.size() == 2) - { - // Yes, run shell command - int result = ::system(rCommand.mCmdElements[1].c_str()); - if(result != 0) - { - BOX_WARNING("System command returned error code " << - result); - SetReturnCode(ReturnCode::Command_Error); - } - return; - } - - if(rCommand.pSpec->type == Command_Unknown) - { - // No such command - BOX_ERROR("Unrecognised command: " << rCommand.mCmdElements[0]); - return; - } - - // Arguments - std::vector<std::string> args(rCommand.mCmdElements.begin() + 1, - rCommand.mCmdElements.end()); - - // Set up options - bool opts[256]; - for(int o = 0; o < 256; ++o) opts[o] = false; - // BLOCK - { - // options - const char *c = rCommand.mOptions.c_str(); - while(*c != 0) - { - // Valid option? - if(::strchr(rCommand.pSpec->opts, *c) == NULL) - { - BOX_ERROR("Invalid option '" << *c << "' for " - "command " << rCommand.pSpec->name); - return; - } - opts[(int)*c] = true; - ++c; - } - } - - if(rCommand.pSpec->type != Command_Quit) - { - // If not a quit command, set the return code to zero - SetReturnCode(ReturnCode::Command_OK); - } - - // Handle command - switch(rCommand.pSpec->type) - { - case Command_Quit: - mQuitNow = true; - break; - - case Command_List: - CommandList(args, opts); - break; - - case Command_pwd: - { - // Simple implementation, so do it here - BOX_NOTICE(GetCurrentDirectoryName() << " (" << - BOX_FORMAT_OBJECTID(GetCurrentDirectoryID()) << - ")"); - } - break; - - case Command_cd: - CommandChangeDir(args, opts); - break; - - case Command_lcd: - CommandChangeLocalDir(args); - break; - - case Command_sh: - BOX_ERROR("The command to run must be specified as an argument."); - break; - - case Command_GetObject: - CommandGetObject(args, opts); - break; - - case Command_Get: - CommandGet(args, opts); - break; - - case Command_Compare: - CommandCompare(args, opts); - break; - - case Command_Restore: - CommandRestore(args, opts); - break; - - case Command_Usage: - CommandUsage(opts); - break; - - case Command_Help: - CommandHelp(args); - break; - - case Command_Undelete: - CommandUndelete(args, opts); - break; - - case Command_Delete: - CommandDelete(args, opts); - break; - - default: - BOX_ERROR("Unknown command: " << rCommand.mCmdElements[0]); - break; - } -} - -#define LIST_OPTION_TIMES_ATTRIBS 'a' -#define LIST_OPTION_SORT_NO_DIRS_FIRST 'D' -#define LIST_OPTION_NOFLAGS 'F' -#define LIST_OPTION_DISPLAY_HASH 'h' -#define LIST_OPTION_SORT_ID 'i' -#define LIST_OPTION_NOOBJECTID 'I' -#define LIST_OPTION_SORT_REVERSE 'r' -#define LIST_OPTION_RECURSIVE 'R' -#define LIST_OPTION_SIZEINBLOCKS 's' -#define LIST_OPTION_SORT_SIZE 'S' -#define LIST_OPTION_TIMES_LOCAL 't' -#define LIST_OPTION_TIMES_UTC 'T' -#define LIST_OPTION_SORT_NONE 'U' - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandList(const std::vector<std::string> &, const bool *) -// Purpose: List directories (optionally recursive) -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandList(const std::vector<std::string> &args, const bool *opts) -{ - // default to using the current directory - int64_t rootDir = GetCurrentDirectoryID(); - - // name of base directory - std::string listRoot; // blank - - // Got a directory in the arguments? - if(args.size() > 0) - { -#ifdef WIN32 - std::string storeDirEncoded; - if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) - return; -#else - const std::string& storeDirEncoded(args[0]); -#endif - - // Attempt to find the directory - rootDir = FindDirectoryObjectID(storeDirEncoded, - opts[LIST_OPTION_ALLOWOLD], - opts[LIST_OPTION_ALLOWDELETED]); - - if(rootDir == 0) - { - BOX_ERROR("Directory '" << args[0] << "' not found " - "on store."); - SetReturnCode(ReturnCode::Command_Error); - return; - } - } - - // List it - List(rootDir, listRoot, opts, true /* first level to list */); -} - -static std::string GetTimeString(BackupStoreDirectory::Entry& en, - bool useLocalTime, bool showAttrModificationTimes) -{ - std::ostringstream out; - box_time_t originalTime, newAttributesTime; - - // there is no attribute modification time in the directory - // entry, unfortunately, so we can't display it. - originalTime = en.GetModificationTime(); - out << BoxTimeToISO8601String(originalTime, useLocalTime); - - if(en.HasAttributes()) - { - const StreamableMemBlock &storeAttr(en.GetAttributes()); - BackupClientFileAttributes attr(storeAttr); - - box_time_t NewModificationTime, NewAttrModificationTime; - attr.GetModificationTimes(&NewModificationTime, - &NewAttrModificationTime); - - if (showAttrModificationTimes) - { - newAttributesTime = NewAttrModificationTime; - } - else - { - newAttributesTime = NewModificationTime; - } - - if (newAttributesTime == originalTime) - { - out << "*"; - } - else - { - out << "~" << BoxTimeToISO8601String(newAttributesTime, - useLocalTime); - } - } - else - { - out << " "; - } - - return out.str(); -} - -/* We need a way to pass options to sort functions for sorting. The algorithm - * doesn't seem to provide a way to do this, so I'm using a global variable. - * Which is not thread safe, but we don't currently use threads so that should - * be OK. Do not use threads without checking! - */ -const bool *gThreadUnsafeOptions; - -int DirsFirst(BackupStoreDirectory::Entry* a, - BackupStoreDirectory::Entry* b) -{ - if (a->IsDir() && !b->IsDir()) - { - return -1; // a < b - } - else if (!a->IsDir() && b->IsDir()) - { - return 1; // b > a - } - else - { - return 0; // continue comparison - } -} - -#define MAYBE_DIRS_FIRST(a, b) \ - if (!gThreadUnsafeOptions[LIST_OPTION_SORT_NO_DIRS_FIRST]) \ - { \ - int result = DirsFirst(a, b); \ - if (result < 0) return true; /* a < b */ \ - else if (result > 0) return false; /* a > b */ \ - /* else: fall through */ \ - } - -#define MAYBE_REVERSE(result) \ - (result != gThreadUnsafeOptions[LIST_OPTION_SORT_REVERSE]) -// result is false, opts[reverse] is false => return false -// result is false, opts[reverse] is true => return true -// result is true, opts[reverse] is false => return true -// result is true, opts[reverse] is true => return false -// this is logical XOR, for which the boolean operator is !=. - -bool SortById(BackupStoreDirectory::Entry* a, - BackupStoreDirectory::Entry* b) -{ - MAYBE_DIRS_FIRST(a, b); - bool result = (a->GetObjectID() < b->GetObjectID()); - return MAYBE_REVERSE(result); -} - -bool SortBySize(BackupStoreDirectory::Entry* a, - BackupStoreDirectory::Entry* b) -{ - MAYBE_DIRS_FIRST(a, b); - bool result = (a->GetSizeInBlocks() < b->GetSizeInBlocks()); - return MAYBE_REVERSE(result); -} - -bool SortByName(BackupStoreDirectory::Entry* a, - BackupStoreDirectory::Entry* b) -{ - MAYBE_DIRS_FIRST(a, b); - BackupStoreFilenameClear afc(a->GetName()); - BackupStoreFilenameClear bfc(b->GetName()); - std::string an = afc.GetClearFilename(); - std::string bn = bfc.GetClearFilename(); - bool result = (an < bn); - return MAYBE_REVERSE(result); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::List(int64_t, const std::string &, const bool *, bool) -// Purpose: Do the actual listing of directories and files -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -void BackupQueries::List(int64_t DirID, const std::string &rListRoot, - const bool *opts, bool FirstLevel, std::ostream* pOut) -{ -#ifdef WIN32 - DWORD n_chars; - HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE); -#endif - - // Generate exclude flags - int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; - if(!opts[LIST_OPTION_ALLOWOLD]) excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; - if(!opts[LIST_OPTION_ALLOWDELETED]) excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; - - // Do communication - try - { - mrConnection.QueryListDirectory( - DirID, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - // both files and directories - excludeFlags, - true /* want attributes */); - } - catch (std::exception &e) - { - BOX_ERROR("Failed to list directory: " << e.what()); - SetReturnCode(ReturnCode::Command_Error); - return; - } - catch (...) - { - BOX_ERROR("Failed to list directory: unknown error"); - SetReturnCode(ReturnCode::Command_Error); - return; - } - - // Retrieve the directory from the stream following - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); - dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); - - // Store entry pointers in a std::vector for sorting - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - std::vector<BackupStoreDirectory::Entry*> sorted_entries; - while((en = i.Next()) != 0) - { - sorted_entries.push_back(en); - } - - // Typedef to avoid mind-bending while dealing with pointers to functions. - typedef bool (EntryComparator_t)(BackupStoreDirectory::Entry* a, - BackupStoreDirectory::Entry* b); - // Default is no comparator, i.e. no sorting. - EntryComparator_t* pComparator = NULL; - - if (opts[LIST_OPTION_SORT_ID]) - { - pComparator = &SortById; - } - else if (opts[LIST_OPTION_SORT_SIZE]) - { - pComparator = &SortBySize; - } - else if (opts[LIST_OPTION_SORT_NONE]) - { - // do nothing - } - else // sort by name - { - pComparator = &SortByName; - } - - if (pComparator != NULL) - { - gThreadUnsafeOptions = opts; - sort(sorted_entries.begin(), sorted_entries.end(), - pComparator); - gThreadUnsafeOptions = NULL; - } - - for (std::vector<BackupStoreDirectory::Entry*>::const_iterator - i = sorted_entries.begin(); - i != sorted_entries.end(); i++) - { - en = *i; - std::ostringstream buf; - - // Display this entry - BackupStoreFilenameClear clear(en->GetName()); - - // Object ID? - if(!opts[LIST_OPTION_NOOBJECTID]) - { - // add object ID to line - buf << std::hex << std::internal << std::setw(8) << - std::setfill('0') << en->GetObjectID() << - std::dec << " "; - } - - // Flags? - if(!opts[LIST_OPTION_NOFLAGS]) - { - static const char *flags = BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES; - char displayflags[16]; - // make sure f is big enough - ASSERT(sizeof(displayflags) >= sizeof(BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES) + 3); - // Insert flags - char *f = displayflags; - const char *t = flags; - int16_t en_flags = en->GetFlags(); - while(*t != 0) - { - *f = ((en_flags&1) == 0)?'-':*t; - en_flags >>= 1; - f++; - t++; - } - // attributes flags - *(f++) = (en->HasAttributes())?'a':'-'; - - // terminate - *(f++) = ' '; - *(f++) = '\0'; - buf << displayflags; - - if(en_flags != 0) - { - buf << "[ERROR: Entry has additional flags set] "; - } - } - - if(opts[LIST_OPTION_TIMES_UTC]) - { - // Show UTC times... - buf << GetTimeString(*en, false, - opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; - } - - if(opts[LIST_OPTION_TIMES_LOCAL]) - { - // Show local times... - buf << GetTimeString(*en, true, - opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; - } - - if(opts[LIST_OPTION_DISPLAY_HASH]) - { - buf << std::hex << std::internal << std::setw(16) << - std::setfill('0') << en->GetAttributesHash() << - std::dec; - } - - if(opts[LIST_OPTION_SIZEINBLOCKS]) - { - buf << std::internal << std::setw(5) << - std::setfill('0') << en->GetSizeInBlocks() << - " "; - } - - // add name - if(!FirstLevel) - { -#ifdef WIN32 - std::string listRootDecoded; - if(!ConvertUtf8ToConsole(rListRoot.c_str(), - listRootDecoded)) return; - listRootDecoded += "/"; - buf << listRootDecoded; - WriteConsole(hOut, listRootDecoded.c_str(), - strlen(listRootDecoded.c_str()), &n_chars, NULL); -#else - buf << rListRoot << "/"; -#endif - } - - std::string fileName; - try - { - fileName = clear.GetClearFilename(); - } - catch(CipherException &e) - { - fileName = "<decrypt failed>"; - } - -#ifdef WIN32 - std::string fileNameUtf8 = fileName; - if(!ConvertUtf8ToConsole(fileNameUtf8, fileName)) - { - fileName = fileNameUtf8 + " [convert encoding failed]"; - } -#endif - - buf << fileName; - - if(!en->GetName().IsEncrypted()) - { - buf << " [FILENAME NOT ENCRYPTED]"; - } - - buf << std::endl; - - if(pOut) - { - (*pOut) << buf.str(); - } - else - { -#ifdef WIN32 - std::string line = buf.str(); - if (!WriteConsole(hOut, line.c_str(), line.size(), - &n_chars, NULL)) - { - // WriteConsole failed, try standard method - std::cout << buf.str(); - } -#else - std::cout << buf.str(); -#endif - } - - // Directory? - if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) - { - // Recurse? - if(opts[LIST_OPTION_RECURSIVE]) - { - std::string subroot(rListRoot); - if(!FirstLevel) subroot += '/'; - subroot += clear.GetClearFilename(); - List(en->GetObjectID(), subroot, opts, - false /* not the first level to list */, - pOut); - } - } - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::FindDirectoryObjectID(const -// std::string &) -// Purpose: Find the object ID of a directory on the store, -// or return 0 for not found. If pStack != 0, the -// object is set to the stack of directories. -// Will start from the current directory stack. -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, - bool AllowOldVersion, bool AllowDeletedDirs, - std::vector<std::pair<std::string, int64_t> > *pStack) -{ - // Split up string into elements - std::vector<std::string> dirElements; - SplitString(rDirName, '/', dirElements); - - // Start from current stack, or root, whichever is required - std::vector<std::pair<std::string, int64_t> > stack; - int64_t dirID = BackupProtocolListDirectory::RootDirectory; - if(rDirName.size() > 0 && rDirName[0] == '/') - { - // Root, do nothing - } - else - { - // Copy existing stack - stack = mDirStack; - if(stack.size() > 0) - { - dirID = stack[stack.size() - 1].second; - } - } - - // Generate exclude flags - int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; - if(!AllowOldVersion) excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; - if(!AllowDeletedDirs) excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; - - // Read directories - for(unsigned int e = 0; e < dirElements.size(); ++e) - { - if(dirElements[e].size() > 0) - { - if(dirElements[e] == ".") - { - // Ignore. - } - else if(dirElements[e] == "..") - { - // Up one! - if(stack.size() > 0) - { - // Remove top element - stack.pop_back(); - - // New dir ID - dirID = (stack.size() > 0)?(stack[stack.size() - 1].second):BackupProtocolListDirectory::RootDirectory; - } - else - { - // At root anyway - dirID = BackupProtocolListDirectory::RootDirectory; - } - } - else - { - // Not blank element. Read current directory. - std::auto_ptr<BackupProtocolSuccess> dirreply(mrConnection.QueryListDirectory( - dirID, - BackupProtocolListDirectory::Flags_Dir, // just directories - excludeFlags, - true /* want attributes */)); - - // Retrieve the directory from the stream following - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); - dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); - - // Then... find the directory within it - BackupStoreDirectory::Iterator i(dir); - BackupStoreFilenameClear dirname(dirElements[e]); - BackupStoreDirectory::Entry *en = i.FindMatchingClearName(dirname); - if(en == 0) - { - // Not found - return 0; - } - - // Object ID for next round of searching - dirID = en->GetObjectID(); - - // Push onto stack - stack.push_back(std::pair<std::string, int64_t>(dirElements[e], dirID)); - } - } - } - - // If required, copy the new stack to the caller - if(pStack) - { - *pStack = stack; - } - - return dirID; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::GetCurrentDirectoryID() -// Purpose: Returns the ID of the current directory -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -int64_t BackupQueries::GetCurrentDirectoryID() -{ - // Special case for root - if(mDirStack.size() == 0) - { - return BackupProtocolListDirectory::RootDirectory; - } - - // Otherwise, get from the last entry on the stack - return mDirStack[mDirStack.size() - 1].second; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::GetCurrentDirectoryName() -// Purpose: Gets the name of the current directory -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -std::string BackupQueries::GetCurrentDirectoryName() -{ - // Special case for root - if(mDirStack.size() == 0) - { - return std::string("/"); - } - - // Build path - std::string r; - for(unsigned int l = 0; l < mDirStack.size(); ++l) - { - r += "/"; -#ifdef WIN32 - std::string dirName; - if(!ConvertUtf8ToConsole(mDirStack[l].first.c_str(), dirName)) - return "error"; - r += dirName; -#else - r += mDirStack[l].first; -#endif - } - - return r; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandChangeDir(const std::vector<std::string> &) -// Purpose: Change directory command -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandChangeDir(const std::vector<std::string> &args, const bool *opts) -{ - if(args.size() != 1 || args[0].size() == 0) - { - BOX_ERROR("Incorrect usage. cd [-o] [-d] <directory>"); - SetReturnCode(ReturnCode::Command_Error); - return; - } - -#ifdef WIN32 - std::string dirName; - if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) return; -#else - const std::string& dirName(args[0]); -#endif - - std::vector<std::pair<std::string, int64_t> > newStack; - int64_t id = FindDirectoryObjectID(dirName, opts['o'], opts['d'], - &newStack); - - if(id == 0) - { - BOX_ERROR("Directory '" << args[0] << "' not found."); - SetReturnCode(ReturnCode::Command_Error); - return; - } - - // Store new stack - mDirStack = newStack; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandChangeLocalDir(const std::vector<std::string> &) -// Purpose: Change local directory command -// Created: 2003/10/11 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandChangeLocalDir(const std::vector<std::string> &args) -{ - if(args.size() != 1 || args[0].size() == 0) - { - BOX_ERROR("Incorrect usage. lcd <local-directory>"); - SetReturnCode(ReturnCode::Command_Error); - return; - } - - // Try changing directory -#ifdef WIN32 - std::string dirName; - if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) - { - BOX_ERROR("Failed to convert path from console encoding."); - SetReturnCode(ReturnCode::Command_Error); - return; - } - int result = ::chdir(dirName.c_str()); -#else - int result = ::chdir(args[0].c_str()); -#endif - if(result != 0) - { - if(errno == ENOENT || errno == ENOTDIR) - { - BOX_ERROR("Directory '" << args[0] << "' does not exist."); - } - else - { - BOX_LOG_SYS_ERROR("Failed to change to directory " - "'" << args[0] << "'"); - } - - SetReturnCode(ReturnCode::Command_Error); - return; - } - - // Report current dir - char wd[PATH_MAX]; - if(::getcwd(wd, PATH_MAX) == 0) - { - BOX_LOG_SYS_ERROR("Error getting current directory"); - SetReturnCode(ReturnCode::Command_Error); - return; - } - -#ifdef WIN32 - if(!ConvertUtf8ToConsole(wd, dirName)) - { - BOX_ERROR("Failed to convert new path from console encoding."); - SetReturnCode(ReturnCode::Command_Error); - return; - } - BOX_INFO("Local current directory is now '" << dirName << "'."); -#else - BOX_INFO("Local current directory is now '" << wd << "'."); -#endif -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandGetObject(const std::vector<std::string> &, const bool *) -// Purpose: Gets an object without any translation. -// Created: 2003/10/11 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandGetObject(const std::vector<std::string> &args, const bool *opts) -{ - // Check args - if(args.size() != 2) - { - BOX_ERROR("Incorrect usage. getobject <object-id> " - "<local-filename>"); - return; - } - - int64_t id = ::strtoll(args[0].c_str(), 0, 16); - if(id == std::numeric_limits<long long>::min() || id == std::numeric_limits<long long>::max() || id == 0) - { - BOX_ERROR("Not a valid object ID (specified in hex): " << - args[0]); - return; - } - - // Does file exist? - EMU_STRUCT_STAT st; - if(EMU_STAT(args[1].c_str(), &st) == 0 || errno != ENOENT) - { - BOX_ERROR("The local file '" << args[1] << " already exists."); - return; - } - - // Open file - FileStream out(args[1].c_str(), O_WRONLY | O_CREAT | O_EXCL); - - // Request that object - try - { - // Request object - std::auto_ptr<BackupProtocolSuccess> getobj(mrConnection.QueryGetObject(id)); - - // Stream that object out to the file - std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); - objectStream->CopyStreamTo(out); - - BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(id) << - " fetched successfully."); - } - catch(ConnectionException &e) - { - if(mrConnection.GetLastErrorType() == BackupProtocolError::Err_DoesNotExist) - { - BOX_ERROR("Object ID " << BOX_FORMAT_OBJECTID(id) << - " does not exist on store."); - ::unlink(args[1].c_str()); - } - else - { - BOX_ERROR("Error occured fetching object."); - ::unlink(args[1].c_str()); - } - } - catch(...) - { - ::unlink(args[1].c_str()); - BOX_ERROR("Error occured fetching object."); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::FindFileID(const std::string& -// rNameOrIdString, const bool *options, -// int64_t *pDirIdOut, std::string* pFileNameOut) -// Purpose: Locate a file on the store (either by name or by -// object ID, depending on opts['i'], where name can -// include a path) and return the file ID, placing the -// directory ID in *pDirIdOut and the filename part -// of the path in *pFileNameOut (if not NULL). -// Created: 2008-09-12 -// -// -------------------------------------------------------------------------- -int64_t BackupQueries::FindFileID(const std::string& rNameOrIdString, - const bool *opts, int64_t *pDirIdOut, std::string* pFileNameOut, - int16_t flagsInclude, int16_t flagsExclude, int16_t* pFlagsOut) -{ - // Find object ID somehow - int64_t fileId; - int64_t dirId = GetCurrentDirectoryID(); - std::string fileName = rNameOrIdString; - - if(!opts['i']) - { - // does this remote filename include a path? - std::string::size_type index = fileName.rfind('/'); - if(index != std::string::npos) - { - std::string dirName(fileName.substr(0, index)); - fileName = fileName.substr(index + 1); - - dirId = FindDirectoryObjectID(dirName); - if(dirId == 0) - { - BOX_ERROR("Directory '" << dirName << - "' not found."); - return 0; - } - } - } - - BackupStoreFilenameClear fn(fileName); - - // Need to look it up in the current directory - mrConnection.QueryListDirectory( - dirId, flagsInclude, flagsExclude, - true /* do want attributes */); - - // Retrieve the directory from the stream following - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); - dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); - BackupStoreDirectory::Entry *en; - - if(opts['i']) - { - // Specified as ID. - fileId = ::strtoll(rNameOrIdString.c_str(), 0, 16); - if(fileId == std::numeric_limits<long long>::min() || - fileId == std::numeric_limits<long long>::max() || - fileId == 0) - { - BOX_ERROR("Not a valid object ID (specified in hex): " - << rNameOrIdString); - return 0; - } - - // Check that the item is actually in the directory - en = dir.FindEntryByID(fileId); - if(en == 0) - { - BOX_ERROR("File ID " << - BOX_FORMAT_OBJECTID(fileId) << - " not found in current directory on store.\n" - "(You can only access files by ID from the " - "current directory.)"); - return 0; - } - } - else - { - // Specified by name, find the object in the directory to get the ID - BackupStoreDirectory::Iterator i(dir); - en = i.FindMatchingClearName(fn); - if(en == 0) - { - BOX_ERROR("Filename '" << rNameOrIdString << "' " - "not found in current directory on store.\n" - "(Subdirectories in path not searched.)"); - return 0; - } - - fileId = en->GetObjectID(); - } - - *pDirIdOut = dirId; - - if(pFlagsOut) - { - *pFlagsOut = en->GetFlags(); - } - - if(pFileNameOut) - { - BackupStoreFilenameClear entryName(en->GetName()); - *pFileNameOut = entryName.GetClearFilename(); - } - - return fileId; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandGet(const std::vector<std::string> &, const bool *) -// Purpose: Command to get a file from the store -// Created: 2003/10/12 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) -{ - // At least one argument? - // Check args - if(args.size() < 1 || (opts['i'] && args.size() != 2) || args.size() > 2) - { - BOX_ERROR("Incorrect usage.\n" - "get <remote-filename> [<local-filename>] or\n" - "get -i <object-id> <local-filename>"); - return; - } - - // Find object ID somehow - int64_t fileId, dirId; - std::string localName; - -#ifdef WIN32 - for (std::vector<std::string>::iterator - i = args.begin(); i != args.end(); i++) - { - std::string out; - if(!ConvertConsoleToUtf8(i->c_str(), out)) - { - BOX_ERROR("Failed to convert encoding."); - return; - } - *i = out; - } -#endif - - int16_t flagsExclude; - - if(opts['i']) - { - // can retrieve anything by ID - flagsExclude = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; - } - else - { - // only current versions by name - flagsExclude = - BackupProtocolListDirectory::Flags_OldVersion | - BackupProtocolListDirectory::Flags_Deleted; - } - - - fileId = FindFileID(args[0], opts, &dirId, &localName, - BackupProtocolListDirectory::Flags_File, // just files - flagsExclude, NULL /* don't care about flags found */); - - if (fileId == 0) - { - // error already reported - return; - } - - if(opts['i']) - { - // Specified as ID. Must have a local name in the arguments - // (check at beginning of function ensures this) - localName = args[1]; - } - else - { - // Specified by name. Local name already set by FindFileID, - // but may be overridden by user supplying a second argument. - if(args.size() == 2) - { - localName = args[1]; - } - } - - // Does local file already exist? (don't want to overwrite) - EMU_STRUCT_STAT st; - if(EMU_STAT(localName.c_str(), &st) == 0 || errno != ENOENT) - { - BOX_ERROR("The local file " << localName << " already exists, " - "will not overwrite it."); - SetReturnCode(ReturnCode::Command_Error); - return; - } - - // Request it from the store - try - { - // Request object - mrConnection.QueryGetFile(dirId, fileId); - - // Stream containing encoded file - std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); - - // Decode it - BackupStoreFile::DecodeFile(*objectStream, localName.c_str(), mrConnection.GetTimeout()); - - // Done. - BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(fileId) << - " fetched successfully."); - } - catch (BoxException &e) - { - BOX_ERROR("Failed to fetch file: " << - e.what()); - ::unlink(localName.c_str()); - } - catch(std::exception &e) - { - BOX_ERROR("Failed to fetch file: " << - e.what()); - ::unlink(localName.c_str()); - } - catch(...) - { - BOX_ERROR("Failed to fetch file: unknown error"); - ::unlink(localName.c_str()); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CompareParams::CompareParams() -// Purpose: Constructor -// Created: 29/1/04 -// -// -------------------------------------------------------------------------- -BackupQueries::CompareParams::CompareParams(bool QuickCompare, - bool IgnoreExcludes, bool IgnoreAttributes, - box_time_t LatestFileUploadTime) -: BoxBackupCompareParams(QuickCompare, IgnoreExcludes, IgnoreAttributes, - LatestFileUploadTime), - mDifferences(0), - mDifferencesExplainedByModTime(0), - mUncheckedFiles(0), - mExcludedDirs(0), - mExcludedFiles(0) -{ } - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandCompare(const std::vector<std::string> &, const bool *) -// Purpose: Command to compare data on the store with local data -// Created: 2003/10/12 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandCompare(const std::vector<std::string> &args, const bool *opts) -{ - box_time_t LatestFileUploadTime = GetCurrentBoxTime(); - - // Try and work out the time before which all files should be on the server - { - std::string syncTimeFilename(mrConfiguration.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); - syncTimeFilename += "last_sync_start"; - // Stat it to get file time - EMU_STRUCT_STAT st; - if(EMU_STAT(syncTimeFilename.c_str(), &st) == 0) - { - // Files modified after this time shouldn't be on the server, so report errors slightly differently - LatestFileUploadTime = FileModificationTime(st) - - SecondsToBoxTime(mrConfiguration.GetKeyValueInt("MinimumFileAge")); - } - else - { - BOX_WARNING("Failed to determine the time of the last " - "synchronisation -- checks not performed."); - } - } - - // Parameters, including count of differences - BackupQueries::CompareParams params(opts['q'], // quick compare? - opts['E'], // ignore excludes - opts['A'], // ignore attributes - LatestFileUploadTime); - - params.mQuietCompare = opts['Q']; - - // Quick compare? - if(params.QuickCompare()) - { - BOX_WARNING("Quick compare used -- file attributes are not " - "checked."); - } - - if(!opts['l'] && opts['a'] && args.size() == 0) - { - // Compare all locations - const Configuration &rLocations( - mrConfiguration.GetSubConfiguration("BackupLocations")); - std::vector<std::string> locNames = - rLocations.GetSubConfigurationNames(); - for(std::vector<std::string>::iterator - pLocName = locNames.begin(); - pLocName != locNames.end(); - pLocName++) - { - CompareLocation(*pLocName, params); - } - } - else if(opts['l'] && !opts['a'] && args.size() == 1) - { - // Compare one location - CompareLocation(args[0], params); - } - else if(!opts['l'] && !opts['a'] && args.size() == 2) - { - // Compare directory to directory - - // Can't be bothered to do all the hard work to work out which location it's on, and hence which exclude list - if(!params.IgnoreExcludes()) - { - BOX_ERROR("Cannot use excludes on directory to directory comparison -- use -E flag to specify ignored excludes."); - return; - } - else - { - // Do compare - Compare(args[0], args[1], params); - } - } - else - { - BOX_ERROR("Incorrect usage.\ncompare -a\n or compare -l <location-name>\n or compare <store-dir-name> <local-dir-name>"); - return; - } - - if (!params.mQuietCompare) - { - BOX_INFO("[ " << - params.mDifferencesExplainedByModTime << " (of " << - params.mDifferences << ") differences probably " - "due to file modifications after the last upload ]"); - } - - BOX_INFO("Differences: " << params.mDifferences << " (" << - params.mExcludedDirs << " dirs excluded, " << - params.mExcludedFiles << " files excluded, " << - params.mUncheckedFiles << " files not checked)"); - - // Set return code? - if(opts['c']) - { - if (params.mUncheckedFiles != 0) - { - SetReturnCode(ReturnCode::Compare_Error); - } - else if (params.mDifferences != 0) - { - SetReturnCode(ReturnCode::Compare_Different); - } - else - { - SetReturnCode(ReturnCode::Compare_Same); - } - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CompareLocation(const std::string &, BackupQueries::CompareParams &) -// Purpose: Compare a location -// Created: 2003/10/13 -// -// -------------------------------------------------------------------------- -void BackupQueries::CompareLocation(const std::string &rLocation, - BoxBackupCompareParams &rParams) -{ - // Find the location's sub configuration - const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations")); - if(!locations.SubConfigurationExists(rLocation.c_str())) - { - BOX_ERROR("Location " << rLocation << " does not exist."); - return; - } - const Configuration &loc(locations.GetSubConfiguration(rLocation.c_str())); - - #ifdef WIN32 - { - std::string path = loc.GetKeyValue("Path"); - if (path.size() > 0 && path[path.size()-1] == - DIRECTORY_SEPARATOR_ASCHAR) - { - BOX_WARNING("Location '" << rLocation << "' path ends " - "with '" DIRECTORY_SEPARATOR "', " - "compare may fail!"); - } - } - #endif - - // Generate the exclude lists - if(!rParams.IgnoreExcludes()) - { - rParams.LoadExcludeLists(loc); - } - - // Then get it compared - Compare(std::string("/") + rLocation, loc.GetKeyValue("Path"), rParams); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::Compare(const std::string &, -// const std::string &, BackupQueries::CompareParams &) -// Purpose: Compare a store directory against a local directory -// Created: 2003/10/13 -// -// -------------------------------------------------------------------------- -void BackupQueries::Compare(const std::string &rStoreDir, - const std::string &rLocalDir, BoxBackupCompareParams &rParams) -{ -#ifdef WIN32 - std::string localDirEncoded; - std::string storeDirEncoded; - if(!ConvertConsoleToUtf8(rLocalDir.c_str(), localDirEncoded)) return; - if(!ConvertConsoleToUtf8(rStoreDir.c_str(), storeDirEncoded)) return; -#else - const std::string& localDirEncoded(rLocalDir); - const std::string& storeDirEncoded(rStoreDir); -#endif - - // Get the directory ID of the directory -- only use current data - int64_t dirID = FindDirectoryObjectID(storeDirEncoded); - - // Found? - if(dirID == 0) - { - bool modifiedAfterLastSync = false; - - EMU_STRUCT_STAT st; - if(EMU_STAT(rLocalDir.c_str(), &st) == 0) - { - if(FileAttrModificationTime(st) > - rParams.LatestFileUploadTime()) - { - modifiedAfterLastSync = true; - } - } - - rParams.NotifyRemoteFileMissing(localDirEncoded, - storeDirEncoded, modifiedAfterLastSync); - return; - } - - // Go! - Compare(dirID, storeDirEncoded, localDirEncoded, rParams); -} - -void BackupQueries::CompareOneFile(int64_t DirID, - BackupStoreDirectory::Entry *pEntry, - const std::string& rLocalPath, - const std::string& rStorePath, - BoxBackupCompareParams &rParams) -{ - int64_t fileId = pEntry->GetObjectID(); - int64_t fileSize = 0; - - EMU_STRUCT_STAT st; - if(EMU_STAT(rLocalPath.c_str(), &st) == 0) - { - fileSize = st.st_size; - } - - try - { - // Files the same flag? - bool equal = true; - - // File modified after last sync flag - bool modifiedAfterLastSync = false; - - bool hasDifferentAttribs = false; - - bool alreadyReported = false; - - if(rParams.QuickCompare()) - { - // Compare file -- fetch it - mrConnection.QueryGetBlockIndexByID(fileId); - - // Stream containing block index - std::auto_ptr<IOStream> blockIndexStream(mrConnection.ReceiveStream()); - - // Compare - equal = BackupStoreFile::CompareFileContentsAgainstBlockIndex( - rLocalPath.c_str(), *blockIndexStream, - mrConnection.GetTimeout()); - } - else - { - // Compare file -- fetch it - mrConnection.QueryGetFile(DirID, pEntry->GetObjectID()); - - // Stream containing encoded file - std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); - - // Decode it - std::auto_ptr<BackupStoreFile::DecodedStream> fileOnServerStream; - - // Got additional attributes? - if(pEntry->HasAttributes()) - { - // Use these attributes - const StreamableMemBlock &storeAttr(pEntry->GetAttributes()); - BackupClientFileAttributes attr(storeAttr); - fileOnServerStream.reset( - BackupStoreFile::DecodeFileStream( - *objectStream, - mrConnection.GetTimeout(), - &attr).release()); - } - else - { - // Use attributes stored in file - fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout()).release()); - } - - // Should always be something in the auto_ptr, it's how the interface is defined. But be paranoid. - if(!fileOnServerStream.get()) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Compare attributes - BackupClientFileAttributes localAttr; - box_time_t fileModTime = 0; - localAttr.ReadAttributes(rLocalPath.c_str(), false /* don't zero mod times */, &fileModTime); - modifiedAfterLastSync = (fileModTime > rParams.LatestFileUploadTime()); - bool ignoreAttrModTime = true; - - #ifdef WIN32 - // attr mod time is really - // creation time, so check it - ignoreAttrModTime = false; - #endif - - if(!rParams.IgnoreAttributes() && - #ifdef PLATFORM_DISABLE_SYMLINK_ATTRIB_COMPARE - !fileOnServerStream->IsSymLink() && - #endif - !localAttr.Compare(fileOnServerStream->GetAttributes(), - ignoreAttrModTime, - fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */)) - { - hasDifferentAttribs = true; - } - - // Compare contents, if it's a regular file not a link - // Remember, we MUST read the entire stream from the server. - SelfFlushingStream flushObject(*objectStream); - - if(!fileOnServerStream->IsSymLink()) - { - SelfFlushingStream flushFile(*fileOnServerStream); - // Open the local file - std::auto_ptr<FileStream> apLocalFile; - - try - { - apLocalFile.reset(new FileStream(rLocalPath.c_str())); - } - catch(std::exception &e) - { - rParams.NotifyLocalFileReadFailed(rLocalPath, - rStorePath, fileSize, e); - alreadyReported = true; - } - catch(...) - { - rParams.NotifyLocalFileReadFailed(rLocalPath, - rStorePath, fileSize); - alreadyReported = true; - } - - if(apLocalFile.get()) - { - equal = apLocalFile->CompareWith(*fileOnServerStream, - mrConnection.GetTimeout()); - } - } - } - - rParams.NotifyFileCompared(rLocalPath, rStorePath, fileSize, - hasDifferentAttribs, !equal, modifiedAfterLastSync, - pEntry->HasAttributes()); - } - catch(BoxException &e) - { - rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize, - e); - } - catch(std::exception &e) - { - rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize, - e); - } - catch(...) - { - rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::Compare(int64_t, const std::string &, -// const std::string &, BackupQueries::CompareParams &) -// Purpose: Compare a store directory against a local directory -// Created: 2003/10/13 -// -// -------------------------------------------------------------------------- -void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, - const std::string &rLocalDir, BoxBackupCompareParams &rParams) -{ - rParams.NotifyDirComparing(rLocalDir, rStoreDir); - - // Get info on the local directory - EMU_STRUCT_STAT st; - if(EMU_LSTAT(rLocalDir.c_str(), &st) != 0) - { - // What kind of error? - if(errno == ENOTDIR || errno == ENOENT) - { - rParams.NotifyLocalDirMissing(rLocalDir, rStoreDir); - } - else - { - rParams.NotifyLocalDirAccessFailed(rLocalDir, rStoreDir); - } - return; - } - - // Get the directory listing from the store - mrConnection.QueryListDirectory( - DirID, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - // get everything - BackupProtocolListDirectory::Flags_OldVersion | - BackupProtocolListDirectory::Flags_Deleted, - // except for old versions and deleted files - true /* want attributes */); - - // Retrieve the directory from the stream following - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); - dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); - - // Test out the attributes - if(!dir.HasAttributes()) - { - rParams.NotifyStoreDirMissingAttributes(rLocalDir, rStoreDir); - } - else - { - // Fetch the attributes - const StreamableMemBlock &storeAttr(dir.GetAttributes()); - BackupClientFileAttributes attr(storeAttr); - - // Get attributes of local directory - BackupClientFileAttributes localAttr; - localAttr.ReadAttributes(rLocalDir.c_str(), - true /* directories have zero mod times */); - - if(attr.Compare(localAttr, true, true /* ignore modification times */)) - { - rParams.NotifyDirCompared(rLocalDir, rStoreDir, - false, false /* actually we didn't check :) */); - } - else - { - bool modifiedAfterLastSync = false; - - EMU_STRUCT_STAT st; - if(EMU_STAT(rLocalDir.c_str(), &st) == 0) - { - if(FileAttrModificationTime(st) > - rParams.LatestFileUploadTime()) - { - modifiedAfterLastSync = true; - } - } - - rParams.NotifyDirCompared(rLocalDir, rStoreDir, - true, modifiedAfterLastSync); - } - } - - // Open the local directory - DIR *dirhandle = ::opendir(rLocalDir.c_str()); - if(dirhandle == 0) - { - rParams.NotifyLocalDirAccessFailed(rLocalDir, rStoreDir); - return; - } - - try - { - // Read the files and directories into sets - std::set<std::string> localFiles; - std::set<std::string> localDirs; - struct dirent *localDirEn = 0; - while((localDirEn = readdir(dirhandle)) != 0) - { - // Not . and ..! - if(localDirEn->d_name[0] == '.' && - (localDirEn->d_name[1] == '\0' || (localDirEn->d_name[1] == '.' && localDirEn->d_name[2] == '\0'))) - { - // ignore, it's . or .. - -#ifdef HAVE_VALID_DIRENT_D_TYPE - if (localDirEn->d_type != DT_DIR) - { - BOX_ERROR("d_type does not really " - "work on your platform. " - "Reconfigure Box!"); - return; - } -#endif - - continue; - } - - std::string localDirPath(MakeFullPath(rLocalDir, - localDirEn->d_name)); - std::string storeDirPath(rStoreDir + "/" + - localDirEn->d_name); - -#ifndef HAVE_VALID_DIRENT_D_TYPE - EMU_STRUCT_STAT st; - if(EMU_LSTAT(localDirPath.c_str(), &st) != 0) - { - // Check whether dir is excluded before trying - // to stat it, to fix problems with .gvfs - // directories that are not readable by root - // causing compare to crash: - // http://lists.boxbackup.org/pipermail/boxbackup/2010-January/000013.html - if(rParams.IsExcludedDir(localDirPath)) - { - rParams.NotifyExcludedDir(localDirPath, - storeDirPath); - continue; - } - else - { - THROW_EXCEPTION_MESSAGE(CommonException, - OSFileError, localDirPath); - } - } - - // Entry -- file or dir? - if(S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) - { - // File or symbolic link - localFiles.insert(std::string(localDirEn->d_name)); - } - else if(S_ISDIR(st.st_mode)) - { - // Directory - localDirs.insert(std::string(localDirEn->d_name)); - } -#else - // Entry -- file or dir? - if(localDirEn->d_type == DT_REG || localDirEn->d_type == DT_LNK) - { - // File or symbolic link - localFiles.insert(std::string(localDirEn->d_name)); - } - else if(localDirEn->d_type == DT_DIR) - { - // Directory - localDirs.insert(std::string(localDirEn->d_name)); - } -#endif - } - // Close directory - if(::closedir(dirhandle) != 0) - { - BOX_LOG_SYS_ERROR("Failed to close local directory " - "'" << rLocalDir << "'"); - } - dirhandle = 0; - - // Do the same for the store directories - std::set<std::pair<std::string, BackupStoreDirectory::Entry *> > storeFiles; - std::set<std::pair<std::string, BackupStoreDirectory::Entry *> > storeDirs; - - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *storeDirEn = 0; - while((storeDirEn = i.Next()) != 0) - { - // Decrypt filename - BackupStoreFilenameClear name(storeDirEn->GetName()); - - // What is it? - if((storeDirEn->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == BackupStoreDirectory::Entry::Flags_File) - { - // File - storeFiles.insert(std::pair<std::string, BackupStoreDirectory::Entry *>(name.GetClearFilename(), storeDirEn)); - } - else - { - // Dir - storeDirs.insert(std::pair<std::string, BackupStoreDirectory::Entry *>(name.GetClearFilename(), storeDirEn)); - } - } - -#ifdef _MSC_VER - typedef std::set<std::string>::iterator string_set_iter_t; -#else - typedef std::set<std::string>::const_iterator string_set_iter_t; -#endif - - // Now compare files. - for(std::set<std::pair<std::string, BackupStoreDirectory::Entry *> >::const_iterator i = storeFiles.begin(); i != storeFiles.end(); ++i) - { - const std::string& fileName(i->first); - - std::string localPath(MakeFullPath(rLocalDir, fileName)); - std::string storePath(rStoreDir + "/" + fileName); - - rParams.NotifyFileComparing(localPath, storePath); - - // Does the file exist locally? - string_set_iter_t local(localFiles.find(fileName)); - if(local == localFiles.end()) - { - // Not found -- report - rParams.NotifyLocalFileMissing(localPath, - storePath); - } - else - { - CompareOneFile(DirID, i->second, localPath, - storePath, rParams); - - // Remove from set so that we know it's been compared - localFiles.erase(local); - } - } - - // Report any files which exist locally, but not on the store - for(string_set_iter_t i = localFiles.begin(); i != localFiles.end(); ++i) - { - std::string localPath(MakeFullPath(rLocalDir, *i)); - std::string storePath(rStoreDir + "/" + *i); - - // Should this be ignored (ie is excluded)? - if(!rParams.IsExcludedFile(localPath)) - { - bool modifiedAfterLastSync = false; - - EMU_STRUCT_STAT st; - if(EMU_STAT(localPath.c_str(), &st) == 0) - { - if(FileModificationTime(st) > - rParams.LatestFileUploadTime()) - { - modifiedAfterLastSync = true; - } - } - - rParams.NotifyRemoteFileMissing(localPath, - storePath, modifiedAfterLastSync); - } - else - { - rParams.NotifyExcludedFile(localPath, - storePath); - } - } - - // Finished with the files, clear the sets to reduce memory usage slightly - localFiles.clear(); - storeFiles.clear(); - - // Now do the directories, recursively to check subdirectories - for(std::set<std::pair<std::string, BackupStoreDirectory::Entry *> >::const_iterator i = storeDirs.begin(); i != storeDirs.end(); ++i) - { - std::string localPath(MakeFullPath(rLocalDir, i->first)); - std::string storePath(rStoreDir + "/" + i->first); - - // Does the directory exist locally? - string_set_iter_t local(localDirs.find(i->first)); - if(local == localDirs.end() && - rParams.IsExcludedDir(localPath)) - { - rParams.NotifyExcludedFileNotDeleted(localPath, - storePath); - } - else if(local == localDirs.end()) - { - // Not found -- report - rParams.NotifyLocalFileMissing(localPath, - storePath); - } - else if(rParams.IsExcludedDir(localPath)) - { - // don't recurse into excluded directories - } - else - { - // Compare directory - Compare(i->second->GetObjectID(), - storePath, localPath, rParams); - - // Remove from set so that we know it's been compared - localDirs.erase(local); - } - } - - // Report any directories which exist locally, but not on the store - for(std::set<std::string>::const_iterator - i = localDirs.begin(); - i != localDirs.end(); ++i) - { - std::string localPath(MakeFullPath(rLocalDir, *i)); - std::string storePath(rStoreDir + "/" + *i); - - // Should this be ignored (ie is excluded)? - if(!rParams.IsExcludedDir(localPath)) - { - bool modifiedAfterLastSync = false; - - // Check the dir modification time - EMU_STRUCT_STAT st; - if(EMU_STAT(localPath.c_str(), &st) == 0 && - FileModificationTime(st) > - rParams.LatestFileUploadTime()) - { - modifiedAfterLastSync = true; - } - - rParams.NotifyRemoteFileMissing(localPath, - storePath, modifiedAfterLastSync); - } - else - { - rParams.NotifyExcludedDir(localPath, storePath); - } - } - } - catch(...) - { - if(dirhandle != 0) - { - ::closedir(dirhandle); - } - throw; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandRestore(const std::vector<std::string> &, const bool *) -// Purpose: Restore a directory -// Created: 23/11/03 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandRestore(const std::vector<std::string> &args, const bool *opts) -{ - // Check arguments - if(args.size() < 1 || args.size() > 2) - { - BOX_ERROR("Incorrect usage. restore [-drif] <remote-name> " - "[<local-name>]"); - return; - } - - // Restoring deleted things? - bool restoreDeleted = opts['d']; - - std::string storeDirEncoded; - - // Get directory ID - int64_t dirID = 0; - if(opts['i']) - { - // Specified as ID. - dirID = ::strtoll(args[0].c_str(), 0, 16); - if(dirID == std::numeric_limits<long long>::min() || dirID == std::numeric_limits<long long>::max() || dirID == 0) - { - BOX_ERROR("Not a valid object ID (specified in hex): " - << args[0]); - return; - } - std::ostringstream oss; - oss << BOX_FORMAT_OBJECTID(args[0]); - storeDirEncoded = oss.str(); - } - else - { -#ifdef WIN32 - if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) - return; -#else - storeDirEncoded = args[0]; -#endif - - // Look up directory ID - dirID = FindDirectoryObjectID(storeDirEncoded, - false /* no old versions */, - restoreDeleted /* find deleted dirs */); - } - - // Allowable? - if(dirID == 0) - { - BOX_ERROR("Directory '" << args[0] << "' not found on server"); - return; - } - - if(dirID == BackupProtocolListDirectory::RootDirectory) - { - BOX_ERROR("Cannot restore the root directory -- restore locations individually."); - return; - } - - std::string localName; - - if(args.size() == 2) - { - #ifdef WIN32 - if(!ConvertConsoleToUtf8(args[1].c_str(), localName)) - { - return; - } - #else - localName = args[1]; - #endif - } - else - { - localName = args[0]; - } - - // Go and restore... - int result; - - try - { - // At TRACE level, we print a line for each file and - // directory, so we don't need dots. - - result = BackupClientRestore(mrConnection, dirID, - storeDirEncoded.c_str(), localName.c_str(), - true /* print progress dots */, restoreDeleted, - false /* don't undelete after restore! */, - opts['r'] /* resume? */, - opts['f'] /* force continue after errors */); - } - catch(std::exception &e) - { - BOX_ERROR("Failed to restore: " << e.what()); - SetReturnCode(ReturnCode::Command_Error); - return; - } - catch(...) - { - BOX_ERROR("Failed to restore: unknown exception"); - SetReturnCode(ReturnCode::Command_Error); - return; - } - - switch(result) - { - case Restore_Complete: - BOX_INFO("Restore complete."); - break; - - case Restore_CompleteWithErrors: - BOX_WARNING("Restore complete, but some files could not be " - "restored."); - break; - - case Restore_ResumePossible: - BOX_ERROR("Resume possible -- repeat command with -r flag " - "to resume."); - SetReturnCode(ReturnCode::Command_Error); - break; - - case Restore_TargetExists: - BOX_ERROR("The target directory exists. You cannot restore " - "over an existing directory."); - SetReturnCode(ReturnCode::Command_Error); - break; - - case Restore_TargetPathNotFound: - BOX_ERROR("The target directory path does not exist.\n" - "To restore to a directory whose parent " - "does not exist, create the parent first."); - SetReturnCode(ReturnCode::Command_Error); - break; - - case Restore_UnknownError: - BOX_ERROR("Unknown error during restore."); - SetReturnCode(ReturnCode::Command_Error); - break; - - default: - BOX_ERROR("Unknown restore result " << result << "."); - SetReturnCode(ReturnCode::Command_Error); - break; - } -} - - - -// These are autogenerated by a script. -extern const char *help_commands[]; -extern const char *help_text[]; - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandHelp(const std::vector<std::string> &args) -// Purpose: Display help on commands -// Created: 15/2/04 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandHelp(const std::vector<std::string> &args) -{ - if(args.size() == 0) - { - // Display a list of all commands - printf("Available commands are:\n"); - for(int c = 0; help_commands[c] != 0; ++c) - { - printf(" %s\n", help_commands[c]); - } - printf("Type \"help <command>\" for more information on a command.\n\n"); - } - else - { - // Display help on a particular command - int c; - for(c = 0; help_commands[c] != 0; ++c) - { - if(::strcmp(help_commands[c], args[0].c_str()) == 0) - { - // Found the command, print help - printf("\n%s\n", help_text[c]); - break; - } - } - if(help_commands[c] == 0) - { - printf("No help found for command '%s'\n", args[0].c_str()); - } - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandUsage() -// Purpose: Display storage space used on server -// Created: 19/4/04 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandUsage(const bool *opts) -{ - bool MachineReadable = opts['m']; - - // Request full details from the server - std::auto_ptr<BackupProtocolAccountUsage> usage(mrConnection.QueryGetAccountUsage()); - - // Display each entry in turn - int64_t hardLimit = usage->GetBlocksHardLimit(); - int32_t blockSize = usage->GetBlockSize(); - CommandUsageDisplayEntry("Used", usage->GetBlocksUsed(), hardLimit, - blockSize, MachineReadable); - CommandUsageDisplayEntry("Old files", usage->GetBlocksInOldFiles(), - hardLimit, blockSize, MachineReadable); - CommandUsageDisplayEntry("Deleted files", usage->GetBlocksInDeletedFiles(), - hardLimit, blockSize, MachineReadable); - CommandUsageDisplayEntry("Directories", usage->GetBlocksInDirectories(), - hardLimit, blockSize, MachineReadable); - CommandUsageDisplayEntry("Soft limit", usage->GetBlocksSoftLimit(), - hardLimit, blockSize, MachineReadable); - CommandUsageDisplayEntry("Hard limit", hardLimit, hardLimit, blockSize, - MachineReadable); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandUsageDisplayEntry(const char *, -// int64_t, int64_t, int32_t, bool) -// Purpose: Display an entry in the usage table -// Created: 19/4/04 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandUsageDisplayEntry(const char *Name, int64_t Size, -int64_t HardLimit, int32_t BlockSize, bool MachineReadable) -{ - std::cout << FormatUsageLineStart(Name, MachineReadable) << - FormatUsageBar(Size, Size * BlockSize, HardLimit * BlockSize, - MachineReadable) << std::endl; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandUndelete(const std::vector<std::string> &, const bool *) -// Purpose: Undelete a directory -// Created: 23/11/03 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandUndelete(const std::vector<std::string> &args, const bool *opts) -{ - if (!mReadWrite) - { - BOX_ERROR("This command requires a read-write connection. " - "Please reconnect with the -w option."); - return; - } - - // Check arguments - if(args.size() != 1) - { - BOX_ERROR("Incorrect usage. undelete <name> or undelete -i <object-id>"); - return; - } - -#ifdef WIN32 - std::string storeDirEncoded; - if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) return; -#else - const std::string& storeDirEncoded(args[0]); -#endif - - // Find object ID somehow - int64_t fileId, parentId; - std::string fileName; - int16_t flagsOut; - - fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, - /* include files and directories */ - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, - /* include old and deleted files */ - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, - &flagsOut); - - if (fileId == 0) - { - // error already reported - return; - } - - // Undelete it on the store - try - { - // Undelete object - if(flagsOut & BackupProtocolListDirectory::Flags_File) - { - mrConnection.QueryUndeleteFile(parentId, fileId); - } - else - { - mrConnection.QueryUndeleteDirectory(fileId); - } - } - catch (BoxException &e) - { - BOX_ERROR("Failed to undelete object: " << - e.what()); - } - catch(std::exception &e) - { - BOX_ERROR("Failed to undelete object: " << - e.what()); - } - catch(...) - { - BOX_ERROR("Failed to undelete object: unknown error"); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CommandDelete(const -// std::vector<std::string> &, const bool *) -// Purpose: Deletes a file -// Created: 23/11/03 -// -// -------------------------------------------------------------------------- -void BackupQueries::CommandDelete(const std::vector<std::string> &args, - const bool *opts) -{ - if (!mReadWrite) - { - BOX_ERROR("This command requires a read-write connection. " - "Please reconnect with the -w option."); - return; - } - - // Check arguments - if(args.size() != 1) - { - BOX_ERROR("Incorrect usage. delete <name>"); - return; - } - -#ifdef WIN32 - std::string storeDirEncoded; - if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) return; -#else - const std::string& storeDirEncoded(args[0]); -#endif - - // Find object ID somehow - int64_t fileId, parentId; - std::string fileName; - int16_t flagsOut; - - fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, - /* include files and directories */ - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, - /* exclude old and deleted files */ - BackupProtocolListDirectory::Flags_OldVersion | - BackupProtocolListDirectory::Flags_Deleted, - &flagsOut); - - if (fileId == 0) - { - // error already reported - return; - } - - BackupStoreFilenameClear fn(fileName); - - // Delete it on the store - try - { - // Delete object - if(flagsOut & BackupProtocolListDirectory::Flags_File) - { - mrConnection.QueryDeleteFile(parentId, fn); - } - else - { - mrConnection.QueryDeleteDirectory(fileId); - } - } - catch (BoxException &e) - { - BOX_ERROR("Failed to delete object: " << - e.what()); - } - catch(std::exception &e) - { - BOX_ERROR("Failed to delete object: " << - e.what()); - } - catch(...) - { - BOX_ERROR("Failed to delete object: unknown error"); - } -} diff --git a/bin/bbackupquery/BackupQueries.h b/bin/bbackupquery/BackupQueries.h deleted file mode 100644 index 96df34f5..00000000 --- a/bin/bbackupquery/BackupQueries.h +++ /dev/null @@ -1,440 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupQueries.h -// Purpose: Perform various queries on the backup store server. -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPQUERIES__H -#define BACKUPQUERIES__H - -#include <iostream> -#include <string> -#include <vector> - -#include "BoxTime.h" -#include "BoxBackupCompareParams.h" -#include "BackupStoreDirectory.h" - -class BackupProtocolCallable; -class Configuration; -class ExcludeList; - -typedef enum -{ - Command_Unknown = 0, - Command_Quit, - Command_List, - Command_pwd, - Command_cd, - Command_lcd, - Command_sh, - Command_GetObject, - Command_Get, - Command_Compare, - Command_Restore, - Command_Help, - Command_Usage, - Command_Undelete, - Command_Delete, -} -CommandType; - -struct QueryCommandSpecification; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupQueries -// Purpose: Perform various queries on the backup store server. -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -class BackupQueries -{ -public: - BackupQueries(BackupProtocolCallable &rConnection, - const Configuration &rConfiguration, - bool readWrite); - ~BackupQueries(); -private: - BackupQueries(const BackupQueries &); -public: - struct ParsedCommand - { - std::vector<std::string> mCmdElements; - std::string mOptions; - std::string mCompleteCommand; - bool mInOptions, mFailed; - QueryCommandSpecification* pSpec; - // mArgCount is the same as mCmdElements.size() for a complete - // command, but if the command line ends in a space, - // e.g. during readline parsing, it can be one greater, - // to indicate that we should complete the next item instead. - size_t mCompleteArgCount; - ParsedCommand(const std::string& Command, - bool isFromCommandLine); - bool IsEmpty() { return mCmdElements.empty(); } - bool IsFailed() { return mFailed; } - }; - - void DoCommand(ParsedCommand& rCommand); - - // Ready to stop? - bool Stop() {return mQuitNow;} - - // Return code? - int GetReturnCode() {return mReturnCode;} - - void List(int64_t DirID, const std::string &rListRoot, const bool *opts, - bool FirstLevel, std::ostream* pOut = NULL); - void CommandList(const std::vector<std::string> &args, const bool *opts); - - // Commands - void CommandChangeDir(const std::vector<std::string> &args, const bool *opts); - void CommandChangeLocalDir(const std::vector<std::string> &args); - void CommandGetObject(const std::vector<std::string> &args, const bool *opts); - void CommandGet(std::vector<std::string> args, const bool *opts); - void CommandCompare(const std::vector<std::string> &args, const bool *opts); - void CommandRestore(const std::vector<std::string> &args, const bool *opts); - void CommandUndelete(const std::vector<std::string> &args, const bool *opts); - void CommandDelete(const std::vector<std::string> &args, - const bool *opts); - void CommandUsage(const bool *opts); - void CommandUsageDisplayEntry(const char *Name, int64_t Size, - int64_t HardLimit, int32_t BlockSize, bool MachineReadable); - void CommandHelp(const std::vector<std::string> &args); - - class CompareParams : public BoxBackupCompareParams - { - public: - CompareParams(bool QuickCompare, bool IgnoreExcludes, - bool IgnoreAttributes, box_time_t LatestFileUploadTime); - - bool mQuietCompare; - int mDifferences; - int mDifferencesExplainedByModTime; - int mUncheckedFiles; - int mExcludedDirs; - int mExcludedFiles; - - std::string ConvertForConsole(const std::string& rUtf8String) - { - #ifdef WIN32 - std::string output; - - if(!ConvertUtf8ToConsole(rUtf8String.c_str(), output)) - { - BOX_WARNING("Character set conversion failed " - "on string: " << rUtf8String); - return rUtf8String; - } - - return output; - #else - return rUtf8String; - #endif - } - - virtual void NotifyLocalDirMissing(const std::string& rLocalPath, - const std::string& rRemotePath) - { - BOX_WARNING("Local directory '" << - ConvertForConsole(rLocalPath) << "' " - "does not exist, but remote directory does."); - mDifferences ++; - } - - virtual void NotifyLocalDirAccessFailed( - const std::string& rLocalPath, - const std::string& rRemotePath) - { - BOX_LOG_SYS_WARNING("Failed to access local directory " - "'" << ConvertForConsole(rLocalPath) << "'"); - mUncheckedFiles ++; - } - - virtual void NotifyStoreDirMissingAttributes( - const std::string& rLocalPath, - const std::string& rRemotePath) - { - BOX_WARNING("Store directory '" << - ConvertForConsole(rRemotePath) << "' " - "doesn't have attributes."); - } - - virtual void NotifyRemoteFileMissing( - const std::string& rLocalPath, - const std::string& rRemotePath, - bool modifiedAfterLastSync) - { - BOX_WARNING("Local file '" << - ConvertForConsole(rLocalPath) << "' " - "exists, but remote file '" << - ConvertForConsole(rRemotePath) << "' " - "does not."); - mDifferences ++; - - if(modifiedAfterLastSync) - { - mDifferencesExplainedByModTime ++; - BOX_INFO("(the file above was modified after " - "the last sync time -- might be " - "reason for difference)"); - } - } - - virtual void NotifyLocalFileMissing( - const std::string& rLocalPath, - const std::string& rRemotePath) - { - BOX_WARNING("Remote file '" << - ConvertForConsole(rRemotePath) << "' " - "exists, but local file '" << - ConvertForConsole(rLocalPath) << "' does not."); - mDifferences ++; - } - - virtual void NotifyExcludedFileNotDeleted( - const std::string& rLocalPath, - const std::string& rRemotePath) - { - BOX_WARNING("Local file '" << - ConvertForConsole(rLocalPath) << "' " - "is excluded, but remote file '" << - ConvertForConsole(rRemotePath) << "' " - "still exists."); - mDifferences ++; - } - - virtual void NotifyDownloadFailed(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes, - BoxException& rException) - { - BOX_ERROR("Failed to download remote file '" << - ConvertForConsole(rRemotePath) << "': " << - rException.what() << " (" << - rException.GetType() << "/" << - rException.GetSubType() << ")"); - mUncheckedFiles ++; - } - - virtual void NotifyDownloadFailed(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes, - std::exception& rException) - { - BOX_ERROR("Failed to download remote file '" << - ConvertForConsole(rRemotePath) << "': " << - rException.what()); - mUncheckedFiles ++; - } - - virtual void NotifyDownloadFailed(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes) - { - BOX_ERROR("Failed to download remote file '" << - ConvertForConsole(rRemotePath)); - mUncheckedFiles ++; - } - - virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes, - std::exception& rException) - { - BOX_ERROR("Failed to read local file '" << - ConvertForConsole(rLocalPath) << "': " << - rException.what()); - mUncheckedFiles ++; - } - - virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes) - { - BOX_ERROR("Failed to read local file '" << - ConvertForConsole(rLocalPath)); - mUncheckedFiles ++; - } - - virtual void NotifyExcludedFile(const std::string& rLocalPath, - const std::string& rRemotePath) - { - mExcludedFiles ++; - } - - virtual void NotifyExcludedDir(const std::string& rLocalPath, - const std::string& rRemotePath) - { - mExcludedDirs ++; - } - - virtual void NotifyDirComparing(const std::string& rLocalPath, - const std::string& rRemotePath) - { - BOX_INFO("Comparing directory: " << rLocalPath); - } - - virtual void NotifyDirCompared( - const std::string& rLocalPath, - const std::string& rRemotePath, - bool HasDifferentAttributes, - bool modifiedAfterLastSync) - { - if(HasDifferentAttributes) - { - BOX_WARNING("Local directory '" << - ConvertForConsole(rLocalPath) << "' " - "has different attributes to " - "store directory '" << - ConvertForConsole(rRemotePath) << "'."); - mDifferences ++; - - if(modifiedAfterLastSync) - { - mDifferencesExplainedByModTime ++; - BOX_INFO("(the directory above was " - "modified after the last sync " - "time -- might be reason for " - "difference)"); - } - } - } - - virtual void NotifyFileComparing(const std::string& rLocalPath, - const std::string& rRemotePath) - { - BOX_TRACE("Comparing file: " << rLocalPath); - } - - virtual void NotifyFileCompared(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes, - bool HasDifferentAttributes, bool HasDifferentContents, - bool ModifiedAfterLastSync, bool NewAttributesApplied) - { - int NewDifferences = 0; - - if(HasDifferentAttributes) - { - BOX_WARNING("Local file '" << - ConvertForConsole(rLocalPath) << "' " - "has different attributes to " - "store file '" << - ConvertForConsole(rRemotePath) << "'."); - NewDifferences ++; - } - - if(HasDifferentContents) - { - BOX_WARNING("Local file '" << - ConvertForConsole(rLocalPath) << "' " - "has different contents to " - "store file '" << - ConvertForConsole(rRemotePath) << "'."); - NewDifferences ++; - } - - if(HasDifferentAttributes || HasDifferentContents) - { - if(ModifiedAfterLastSync) - { - mDifferencesExplainedByModTime += - NewDifferences; - BOX_INFO("(the file above was modified " - "after the last sync time -- " - "might be reason for difference)"); - } - else if(NewAttributesApplied) - { - BOX_INFO("(the file above has had new " - "attributes applied)\n"); - } - } - - mDifferences += NewDifferences; - } - }; - void CompareLocation(const std::string &rLocation, - BoxBackupCompareParams &rParams); - void Compare(const std::string &rStoreDir, - const std::string &rLocalDir, BoxBackupCompareParams &rParams); - void Compare(int64_t DirID, const std::string &rStoreDir, - const std::string &rLocalDir, BoxBackupCompareParams &rParams); - void CompareOneFile(int64_t DirID, BackupStoreDirectory::Entry *pEntry, - const std::string& rLocalPath, const std::string& rStorePath, - BoxBackupCompareParams &rParams); - -public: - - class ReturnCode - { - public: - typedef enum { - Command_OK = 0, - Compare_Same = 1, - Compare_Different, - Compare_Error, - Command_Error, - } Type; - }; - - // Were private, but needed by completion functions: - int64_t GetCurrentDirectoryID(); - int64_t FindDirectoryObjectID(const std::string &rDirName, - bool AllowOldVersion = false, bool AllowDeletedDirs = false, - std::vector<std::pair<std::string, int64_t> > *pStack = 0); - -private: - - // Utility functions - int64_t FindFileID(const std::string& rNameOrIdString, - const bool *opts, int64_t *pDirIdOut, - std::string* pFileNameOut, int16_t flagsInclude, - int16_t flagsExclude, int16_t* pFlagsOut); - std::string GetCurrentDirectoryName(); - void SetReturnCode(int code) {mReturnCode = code;} - -private: - bool mReadWrite; - BackupProtocolCallable &mrConnection; - const Configuration &mrConfiguration; - bool mQuitNow; - std::vector<std::pair<std::string, int64_t> > mDirStack; - bool mRunningAsRoot; - bool mWarnedAboutOwnerAttributes; - int mReturnCode; -}; - -typedef std::vector<std::string> (*CompletionHandler) - (BackupQueries::ParsedCommand& rCommand, const std::string& prefix, - BackupProtocolCallable& rProtocol, const Configuration& rConfig, - BackupQueries& rQueries); - -std::vector<std::string> CompleteCommand(BackupQueries::ParsedCommand& rCommand, - const std::string& prefix, BackupProtocolCallable& rProtocol, - const Configuration& rConfig, BackupQueries& rQueries); -std::vector<std::string> CompleteOptions(BackupQueries::ParsedCommand& rCommand, - const std::string& prefix, BackupProtocolCallable& rProtocol, - const Configuration& rConfig, BackupQueries& rQueries); - -#define MAX_COMPLETION_HANDLERS 4 - -struct QueryCommandSpecification -{ - const char* name; - const char* opts; - CommandType type; - CompletionHandler complete[MAX_COMPLETION_HANDLERS]; -}; - -// Data about commands -extern QueryCommandSpecification commands[]; - -extern const char *alias[]; -extern const int aliasIs[]; - -#define LIST_OPTION_ALLOWOLD 'o' -#define LIST_OPTION_ALLOWDELETED 'd' - -#endif // BACKUPQUERIES__H - diff --git a/bin/bbackupquery/BoxBackupCompareParams.h b/bin/bbackupquery/BoxBackupCompareParams.h deleted file mode 100644 index 655df947..00000000 --- a/bin/bbackupquery/BoxBackupCompareParams.h +++ /dev/null @@ -1,112 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BoxBackupCompareParams.h -// Purpose: Parameters and notifiers for a compare operation -// Created: 2008/12/30 -// -// -------------------------------------------------------------------------- - -#ifndef BOXBACKUPCOMPAREPARAMS__H -#define BOXBACKUPCOMPAREPARAMS__H - -#include <memory> -#include <string> - -#include "BoxTime.h" -#include "ExcludeList.h" -#include "BackupClientMakeExcludeList.h" - -// -------------------------------------------------------------------------- -// -// Class -// Name: BoxBackupCompareParams -// Purpose: Parameters and notifiers for a compare operation -// Created: 2003/10/10 -// -// -------------------------------------------------------------------------- -class BoxBackupCompareParams -{ -private: - std::auto_ptr<const ExcludeList> mapExcludeFiles, mapExcludeDirs; - bool mQuickCompare; - bool mIgnoreExcludes; - bool mIgnoreAttributes; - box_time_t mLatestFileUploadTime; - -public: - BoxBackupCompareParams(bool QuickCompare, bool IgnoreExcludes, - bool IgnoreAttributes, box_time_t LatestFileUploadTime) - : mQuickCompare(QuickCompare), - mIgnoreExcludes(IgnoreExcludes), - mIgnoreAttributes(IgnoreAttributes), - mLatestFileUploadTime(LatestFileUploadTime) - { } - - virtual ~BoxBackupCompareParams() { } - - bool QuickCompare() { return mQuickCompare; } - bool IgnoreExcludes() { return mIgnoreExcludes; } - bool IgnoreAttributes() { return mIgnoreAttributes; } - box_time_t LatestFileUploadTime() { return mLatestFileUploadTime; } - - void LoadExcludeLists(const Configuration& rLoc) - { - mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rLoc)); - mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rLoc)); - } - bool IsExcludedFile(const std::string& rLocalPath) - { - if (!mapExcludeFiles.get()) return false; - return mapExcludeFiles->IsExcluded(rLocalPath); - } - bool IsExcludedDir(const std::string& rLocalPath) - { - if (!mapExcludeDirs.get()) return false; - return mapExcludeDirs->IsExcluded(rLocalPath); - } - - virtual void NotifyLocalDirMissing(const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyLocalDirAccessFailed(const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyStoreDirMissingAttributes(const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyRemoteFileMissing(const std::string& rLocalPath, - const std::string& rRemotePath, - bool modifiedAfterLastSync) = 0; - virtual void NotifyLocalFileMissing(const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyExcludedFileNotDeleted(const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyDownloadFailed(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes, - BoxException& rException) = 0; - virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes, - std::exception& rException) = 0; - virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes) = 0; - virtual void NotifyDownloadFailed(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes, - std::exception& rException) = 0; - virtual void NotifyDownloadFailed(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes) = 0; - virtual void NotifyExcludedFile(const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyExcludedDir(const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyDirComparing(const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyDirCompared(const std::string& rLocalPath, - const std::string& rRemotePath, bool HasDifferentAttributes, - bool modifiedAfterLastSync) = 0; - virtual void NotifyFileComparing(const std::string& rLocalPath, - const std::string& rRemotePath) = 0; - virtual void NotifyFileCompared(const std::string& rLocalPath, - const std::string& rRemotePath, int64_t NumBytes, - bool HasDifferentAttributes, bool HasDifferentContents, - bool modifiedAfterLastSync, bool newAttributesApplied) = 0; -}; - -#endif // BOXBACKUPCOMPAREPARAMS__H diff --git a/bin/bbackupquery/CommandCompletion.cpp b/bin/bbackupquery/CommandCompletion.cpp deleted file mode 100644 index 761fc97e..00000000 --- a/bin/bbackupquery/CommandCompletion.cpp +++ /dev/null @@ -1,604 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: CommandCompletion.cpp -// Purpose: Parts of BackupQueries that depend on readline -// Created: 2011/01/21 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#ifdef HAVE_LIBREADLINE - #ifdef HAVE_READLINE_READLINE_H - #include <readline/readline.h> - #elif defined(HAVE_EDITLINE_READLINE_H) - #include <editline/readline.h> - #elif defined(HAVE_READLINE_H) - #include <readline.h> - #endif -#endif - -#ifdef HAVE_READLINE_HISTORY - #ifdef HAVE_READLINE_HISTORY_H - #include <readline/history.h> - #elif defined(HAVE_HISTORY_H) - #include <history.h> - #endif -#endif - -#include <cstring> - -#include "BackupQueries.h" -#include "Configuration.h" - -#include "autogen_BackupProtocol.h" - -#include "MemLeakFindOn.h" - -#define COMPARE_RETURN_SAME 1 -#define COMPARE_RETURN_DIFFERENT 2 -#define COMPARE_RETURN_ERROR 3 -#define COMMAND_RETURN_ERROR 4 - -#define COMPLETION_FUNCTION(name, code) \ -std::vector<std::string> Complete ## name( \ - BackupQueries::ParsedCommand& rCommand, \ - const std::string& prefix, \ - BackupProtocolCallable& rProtocol, const Configuration& rConfig, \ - BackupQueries& rQueries) \ -{ \ - std::vector<std::string> completions; \ - \ - try \ - { \ - code \ - } \ - catch(std::exception &e) \ - { \ - BOX_TRACE("Failed to complete " << prefix << ": " << e.what()); \ - } \ - catch(...) \ - { \ - BOX_TRACE("Failed to complete " << prefix << ": " \ - "unknown error"); \ - } \ - \ - return completions; \ -} - -#define DELEGATE_COMPLETION(name) \ - completions = Complete ## name(rCommand, prefix, rProtocol, rConfig, \ - rQueries); - -COMPLETION_FUNCTION(None,) - -#ifdef HAVE_RL_FILENAME_COMPLETION_FUNCTION - #define RL_FILENAME_COMPLETION_FUNCTION rl_filename_completion_function - #define HAVE_A_FILENAME_COMPLETION_FUNCTION 1 -#elif defined HAVE_FILENAME_COMPLETION_FUNCTION - #define RL_FILENAME_COMPLETION_FUNCTION filename_completion_function - #define HAVE_A_FILENAME_COMPLETION_FUNCTION 1 -#endif - -#ifdef HAVE_A_FILENAME_COMPLETION_FUNCTION -COMPLETION_FUNCTION(Default, - int i = 0; - - while (const char *match = RL_FILENAME_COMPLETION_FUNCTION(prefix.c_str(), i)) - { - completions.push_back(match); - ++i; - } -) -#else // !HAVE_A_FILENAME_COMPLETION_FUNCTION -COMPLETION_FUNCTION(Default,) -#endif // HAVE_A_FILENAME_COMPLETION_FUNCTION - -COMPLETION_FUNCTION(Command, - int len = prefix.length(); - - for(int i = 0; commands[i].name != NULL; i++) - { - if(::strncmp(commands[i].name, prefix.c_str(), len) == 0) - { - completions.push_back(commands[i].name); - } - } -) - -void CompleteOptionsInternal(const std::string& prefix, - BackupQueries::ParsedCommand& rCommand, - std::vector<std::string>& completions) -{ - std::string availableOptions = rCommand.pSpec->opts; - - for(std::string::iterator - opt = availableOptions.begin(); - opt != availableOptions.end(); opt++) - { - if(rCommand.mOptions.find(*opt) == std::string::npos) - { - if(prefix == "") - { - // complete with possible option strings - completions.push_back(std::string("-") + *opt); - } - else - { - // complete with possible additional options - completions.push_back(prefix + *opt); - } - } - } -} - -COMPLETION_FUNCTION(Options, - CompleteOptionsInternal(prefix, rCommand, completions); -) - -std::string EncodeFileName(const std::string &rUnEncodedName) -{ -#ifdef WIN32 - std::string encodedName; - if(!ConvertConsoleToUtf8(rUnEncodedName, encodedName)) - { - return std::string(); - } - return encodedName; -#else - return rUnEncodedName; -#endif -} - -int16_t GetExcludeFlags(BackupQueries::ParsedCommand& rCommand) -{ - int16_t excludeFlags = 0; - - if (rCommand.mOptions.find(LIST_OPTION_ALLOWOLD) == std::string::npos) - { - excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; - } - - if (rCommand.mOptions.find(LIST_OPTION_ALLOWDELETED) == std::string::npos) - { - excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; - } - - return excludeFlags; -} - -std::vector<std::string> CompleteRemoteFileOrDirectory( - BackupQueries::ParsedCommand& rCommand, - const std::string& prefix, BackupProtocolCallable& rProtocol, - BackupQueries& rQueries, int16_t includeFlags) -{ - std::vector<std::string> completions; - - // default to using the current directory - int64_t listDirId = rQueries.GetCurrentDirectoryID(); - std::string searchPrefix; - std::string listDir = prefix; - - if(rCommand.mCompleteArgCount == rCommand.mCmdElements.size()) - { - // completing an empty name, from the current directory - // nothing to change - } - else - { - // completing a partially-completed subdirectory name - searchPrefix = prefix; - listDir = ""; - - // do we need to list a subdirectory to complete? - size_t lastSlash = searchPrefix.rfind('/'); - if(lastSlash == std::string::npos) - { - // no slashes, so the whole name is the prefix - // nothing to change - } - else - { - // listing a partially-completed subdirectory name - listDir = searchPrefix.substr(0, lastSlash); - - listDirId = rQueries.FindDirectoryObjectID(listDir, - rCommand.mOptions.find(LIST_OPTION_ALLOWOLD) - != std::string::npos, - rCommand.mOptions.find(LIST_OPTION_ALLOWDELETED) - != std::string::npos); - - if(listDirId == 0) - { - // no matches for subdir to list, - // return empty-handed. - return completions; - } - - // matched, and updated listDir and listDirId already - searchPrefix = searchPrefix.substr(lastSlash + 1); - } - } - - // Always include directories, because they contain files. - // We will append a slash later for each directory if we're - // actually looking for files. - // - // If we're looking for directories, then only list directories. - - bool completeFiles = includeFlags & - BackupProtocolListDirectory::Flags_File; - bool completeDirs = includeFlags & - BackupProtocolListDirectory::Flags_Dir; - int16_t listFlags = 0; - - if(completeFiles) - { - listFlags = BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING; - } - else if(completeDirs) - { - listFlags = BackupProtocolListDirectory::Flags_Dir; - } - - rProtocol.QueryListDirectory(listDirId, - listFlags, GetExcludeFlags(rCommand), - false /* no attributes */); - - // Retrieve the directory from the stream following - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, rProtocol.GetTimeout()); - - // Then... display everything - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next()) != 0) - { - BackupStoreFilenameClear clear(en->GetName()); - std::string name = clear.GetClearFilename().c_str(); - if(name.compare(0, searchPrefix.length(), searchPrefix) == 0) - { - bool dir_added = false; - - if(en->IsDir() && - (includeFlags & BackupProtocolListDirectory::Flags_Dir) == 0) - { - // Was looking for a file, but this is a - // directory, so append a slash to the name - name += "/"; - } - - #ifdef HAVE_LIBREADLINE - if(strchr(name.c_str(), ' ')) - { - int n_quote = 0; - - for(int k = strlen(rl_line_buffer); k >= 0; k--) - { - if (rl_line_buffer[k] == '\"') { - ++n_quote; - } - } - - dir_added = false; - - if (!(n_quote % 2)) - { - name = "\"" + (listDir == "" ? name : listDir + "/" + name); - dir_added = true; - } - - name = name + "\""; - } - #endif - - if(listDir == "" || dir_added) - { - completions.push_back(name); - } - else - { - completions.push_back(listDir + "/" + name); - } - } - } - - return completions; -} - -COMPLETION_FUNCTION(RemoteDir, - completions = CompleteRemoteFileOrDirectory(rCommand, prefix, - rProtocol, rQueries, - BackupProtocolListDirectory::Flags_Dir); -) - -COMPLETION_FUNCTION(RemoteFile, - completions = CompleteRemoteFileOrDirectory(rCommand, prefix, - rProtocol, rQueries, - BackupProtocolListDirectory::Flags_File); -) - -COMPLETION_FUNCTION(LocalDir, - DELEGATE_COMPLETION(Default); -) - -COMPLETION_FUNCTION(LocalFile, - DELEGATE_COMPLETION(Default); -) - -COMPLETION_FUNCTION(LocationName, - const Configuration &locations(rConfig.GetSubConfiguration( - "BackupLocations")); - - std::vector<std::string> locNames = - locations.GetSubConfigurationNames(); - - for(std::vector<std::string>::iterator - pLocName = locNames.begin(); - pLocName != locNames.end(); - pLocName++) - { - if(pLocName->compare(0, pLocName->length(), prefix) == 0) - { - completions.push_back(*pLocName); - } - } -) - -COMPLETION_FUNCTION(RemoteFileIdInCurrentDir, - int64_t listDirId = rQueries.GetCurrentDirectoryID(); - int16_t excludeFlags = GetExcludeFlags(rCommand); - - rProtocol.QueryListDirectory( - listDirId, - BackupProtocolListDirectory::Flags_File, - excludeFlags, false /* no attributes */); - - // Retrieve the directory from the stream following - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, rProtocol.GetTimeout()); - - // Then... compare each item - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next()) != 0) - { - std::ostringstream hexId; - hexId << std::hex << en->GetObjectID(); - if(hexId.str().compare(0, prefix.length(), prefix) == 0) - { - completions.push_back(hexId.str()); - } - } -) - -// TODO implement completion of hex IDs up to the maximum according to Usage -COMPLETION_FUNCTION(RemoteId,) - -COMPLETION_FUNCTION(GetFileOrId, - if(rCommand.mOptions.find('i') != std::string::npos) - { - DELEGATE_COMPLETION(RemoteFileIdInCurrentDir); - } - else - { - DELEGATE_COMPLETION(RemoteFile); - } -) - -COMPLETION_FUNCTION(CompareLocationOrRemoteDir, - if(rCommand.mOptions.find('l') != std::string::npos) - { - DELEGATE_COMPLETION(LocationName); - } - else - { - DELEGATE_COMPLETION(RemoteDir); - } -) - -COMPLETION_FUNCTION(CompareNoneOrLocalDir, - if(rCommand.mOptions.find('l') != std::string::npos) - { - // no completions - DELEGATE_COMPLETION(None); - } - else - { - DELEGATE_COMPLETION(LocalDir); - } -) - -COMPLETION_FUNCTION(RestoreRemoteDirOrId, - if(rCommand.mOptions.find('i') != std::string::npos) - { - DELEGATE_COMPLETION(RemoteId); - } - else - { - DELEGATE_COMPLETION(RemoteDir); - } -) - -// Data about commands -QueryCommandSpecification commands[] = -{ - { "quit", "", Command_Quit, {} }, - { "exit", "", Command_Quit, {} }, - { "list", "adDFhiIorRsStTU", Command_List, {CompleteRemoteDir} }, - { "pwd", "", Command_pwd, {} }, - { "cd", "od", Command_cd, {CompleteRemoteDir} }, - { "lcd", "", Command_lcd, {CompleteLocalDir} }, - { "sh", "", Command_sh, {CompleteDefault} }, - { "getobject", "", Command_GetObject, - {CompleteRemoteId, CompleteLocalDir} }, - { "get", "i", Command_Get, - {CompleteGetFileOrId, CompleteLocalDir} }, - { "compare", "alcqAEQ", Command_Compare, - {CompleteCompareLocationOrRemoteDir, CompleteCompareNoneOrLocalDir} }, - { "restore", "drif", Command_Restore, - {CompleteRestoreRemoteDirOrId, CompleteLocalDir} }, - { "help", "", Command_Help, {} }, - { "usage", "m", Command_Usage, {} }, - { "undelete", "i", Command_Undelete, - {CompleteGetFileOrId} }, - { "delete", "i", Command_Delete, {CompleteGetFileOrId} }, - { NULL, NULL, Command_Unknown, {} } -}; - -const char *alias[] = {"ls", 0}; -const int aliasIs[] = {Command_List, 0}; - -BackupQueries::ParsedCommand::ParsedCommand(const std::string& Command, - bool isFromCommandLine) -: mInOptions(false), - mFailed(false), - pSpec(NULL), - mCompleteArgCount(0) -{ - mCompleteCommand = Command; - - // is the command a shell command? - if(Command[0] == 's' && Command[1] == 'h' && Command[2] == ' ' && Command[3] != '\0') - { - // Yes, run shell command - for(int i = 0; commands[i].type != Command_Unknown; i++) - { - if(commands[i].type == Command_sh) - { - pSpec = &(commands[i]); - break; - } - } - - mCmdElements[0] = "sh"; - mCmdElements[1] = Command.c_str() + 3; - return; - } - - // split command into components - bool inQuoted = false; - mInOptions = false; - - std::string currentArg; - for (std::string::const_iterator c = Command.begin(); - c != Command.end(); c++) - { - // Terminating char? - if(*c == ((inQuoted)?'"':' ')) - { - if(!currentArg.empty()) - { - mCmdElements.push_back(currentArg); - - // Because we just found a space, and the last - // word was not options (otherwise currentArg - // would be empty), we've received a complete - // command or non-option argument. - mCompleteArgCount++; - } - - currentArg.resize(0); - inQuoted = false; - mInOptions = false; - } - // Start of quoted parameter? - else if(currentArg.empty() && *c == '"') - { - inQuoted = true; - } - // Start of options? You can't have options if there's no - // command before them, so treat the options as a command (which - // doesn't exist, so it will fail to parse) in that case. - else if(currentArg.empty() && *c == '-' && !mCmdElements.empty()) - { - mInOptions = true; - } - else if(mInOptions) - { - // Option char - mOptions += *c; - } - else - { - // Normal string char, part of current arg - currentArg += *c; - } - } - - if(!currentArg.empty()) - { - mCmdElements.push_back(currentArg); - } - - // If there are no commands then there's nothing to do except return - if(mCmdElements.empty()) - { - return; - } - - // Work out which command it is... - int cmd = 0; - while(commands[cmd].name != 0 && - mCmdElements[0] != commands[cmd].name) - { - cmd++; - } - - if(commands[cmd].name == 0) - { - // Check for aliases - int a; - for(a = 0; alias[a] != 0; ++a) - { - if(mCmdElements[0] == alias[a]) - { - // Found an alias - cmd = aliasIs[a]; - break; - } - } - } - - if(commands[cmd].name == 0) - { - mFailed = true; - return; - } - - pSpec = &(commands[cmd]); - - #ifdef WIN32 - if(isFromCommandLine) - { - std::string converted; - - if(!ConvertEncoding(mCompleteCommand, CP_ACP, converted, - GetConsoleCP())) - { - BOX_ERROR("Failed to convert encoding"); - mFailed = true; - } - - mCompleteCommand = converted; - - for(std::vector<std::string>::iterator - i = mCmdElements.begin(); - i != mCmdElements.end(); i++) - { - if(!ConvertEncoding(*i, CP_ACP, converted, - GetConsoleCP())) - { - BOX_ERROR("Failed to convert encoding"); - mFailed = true; - } - - *i = converted; - } - } - #endif -} - diff --git a/bin/bbackupquery/Makefile.extra b/bin/bbackupquery/Makefile.extra deleted file mode 100644 index e1049b6d..00000000 --- a/bin/bbackupquery/Makefile.extra +++ /dev/null @@ -1,6 +0,0 @@ - -# AUTOGEN SEEDING -autogen_Documentation.cpp: makedocumentation.pl documentation.txt - $(_PERL) makedocumentation.pl - - diff --git a/bin/bbackupquery/documentation.txt b/bin/bbackupquery/documentation.txt deleted file mode 100644 index b16a6f7c..00000000 --- a/bin/bbackupquery/documentation.txt +++ /dev/null @@ -1,194 +0,0 @@ - -bbackupquery utility -- examine store, compare files, restore, etc. - -This file has markers for automatic help generation script -- '>' marks a start of a command/help topic, -and '<' marks the end of a section. - -Command line: -============= - -> bbackupquery [-q] [-c configfile] [commands ...] - - -q -- quiet, no information prompts - -c -- specify another bbackupd configuation file - -The commands following the options are executed, then (if there was no quit -command) an interactive mode is entered. - -If a command contains a space, enclose it in quotes. Example - - bbackupquery "list testdir1" quit - -to list the contents of testdir1, and then exit without interactive mode. -< - -Commands: -========= - -All directory names relative to a "current" directory, or from root if they -start with '/'. The initial directory is always the root directory. - - -> ls [options] [directory-name] - - List contents of current directory, or specified directory. - - -R -- recursively list all files - -d -- list deleted files/directories - -o -- list old versions of files/directories - -I -- don't display object ID - -F -- don't display flags - -t -- show file modification time in local time - (and attr mod time if has the object has attributes, ~ separated) - -T -- show file modification time in GMT - -a -- show updated attribute instead of file modification time - -s -- show file size in blocks used on server - (only very approximate indication of size locally) - -h -- show file attributes hash - -D -- sort directories together with files (not dirs first) - -i -- sort by object ID (the old default) - -S -- sort by object size in blocks - -U -- don't sort the results (new default is to sort by name) - -list can be used as an alias. -< - -> list - - Alias for 'ls'. Type 'help ls' for options. -< - -> cd [options] <directory-name> - - Change directory - - -d -- consider deleted directories for traversal - -o -- consider old versions of directories for traversal - (this option should never be useful in a correctly formed store) -< - -> pwd - - Print current directory, always root relative. -< - -> lcd <local-directory-name> - - Change local directory. - - Type "sh ls" to list the contents. -< - -> sh <shell command> - - All of the parameters after the "sh" are run as a shell command. - - For example, to list the contents of the location directory, type "sh ls" -< - -> get <object-filename> [<local-filename>] -get -i <object-id> <local-filename> - - Gets a file from the store. Object is specified as the filename within - the current directory, and local filename is optional. Ignores old and - deleted files when searching the directory for the file to retrieve. - - To get an old or deleted file, use the -i option and select the object - as a hex object ID (first column in listing). The local filename must - be specified. -< - -> compare -a -compare -l <location-name> -compare <store-dir-name> <local-dir-name> - - Compares the (current) data on the store with the data on the disc. - All the data will be downloaded -- this is potentially a very long - operation. - - -a -- compare all locations - -l -- compare one backup location as specified in the configuration file. - -c -- set return code - -q -- quick compare. Only checks file contents against checksums, - doesn't do a full download - -A -- ignore attribute differences - -E -- ignore exclusion settings - - Comparing with the root directory is an error, use -a option instead. - - If -c is set, then the return code (if quit is the next command) will be - 1 Comparison was exact - 2 Differences were found - 3 An error occured - This can be used for automated tests. -< - -> restore [-drif] <directory-name> [<local-directory-name>] - - Restores a directory to the local disc. The local directory specified - must not exist (unless a previous restore is being restarted). If the - local directory is omitted, the default is to restore to the same - directory name and path, relative to the current local directory, - as set with the "lcd" command. - - The root cannot be restored -- restore locations individually. - - -d -- restore a deleted directory or deleted files inside - -r -- resume an interrupted restoration - -i -- directory name is actually an ID - -f -- force restore to continue if errors are encountered - - If a restore operation is interrupted for any reason, it can be restarted - using the -r switch. Restore progress information is saved in a file at - regular intervals during the restore operation to allow restarts. -< - -> getobject <object-id> <local-filename> - - Gets the object specified by the object id (in hex) and stores the raw - contents in the local file specified. - - This is only useful for debugging as it does not decode files from the - stored format, which is encrypted and compressed. -< - -> usage [-m] - - Show space used on the server for this account. - - -m -- display the output in machine-readable form - - Used: Total amount of space used on the server. - Old files: Space used by old files - Deleted files: Space used by deleted files - Directories: Space used by the directory structure. - - When Used exceeds the soft limit, the server will start to remove old and - deleted files until the usage drops below the soft limit. - - After a while, you would expect to see the usage stay at just below the - soft limit. You only need more space if the space used by old and deleted - files is near zero. -< - -> undelete <directory-name> -undelete -i <object-id> - - Removes the deleted flag from the specified directory name (in the - current directory) or hex object ID. Be careful not to use this - command where a directory already exists with the same name which is - not marked as deleted. -< - -> delete <file-name> - - Sets the deleted flag on the specified file name (in the current - directory, or with a relative path). -< - -> quit - - End session and exit. -< - - diff --git a/bin/bbackupquery/makedocumentation.pl.in b/bin/bbackupquery/makedocumentation.pl.in deleted file mode 100755 index 530c4ff6..00000000 --- a/bin/bbackupquery/makedocumentation.pl.in +++ /dev/null @@ -1,75 +0,0 @@ -#!@PERL@ -use strict; - -print "Creating built-in documentation for bbackupquery...\n"; - -open DOC,"documentation.txt" or die "Can't open documentation.txt file"; -my $section; -my %help; -my @in_order; - -while(<DOC>) -{ - if(m/\A>\s+(\w+)/) - { - $section = $1; - m/\A>\s+(.+)\Z/; - $help{$section} = $1."\n"; - push @in_order,$section; - } - elsif(m/\A</) - { - $section = ''; - } - elsif($section ne '') - { - $help{$section} .= $_; - } -} - -close DOC; - -open OUT,">autogen_Documentation.cpp" or die "Can't open output file for writing"; - -print OUT <<__E; -// -// Automatically generated file, do not edit. -// - -#include "Box.h" - -#include "MemLeakFindOn.h" - -const char *help_commands[] = -{ -__E - -for(@in_order) -{ - print OUT qq:\t"$_",\n:; -} - -print OUT <<__E; - 0 -}; - -const char *help_text[] = -{ -__E - -for(@in_order) -{ - my $t = $help{$_}; - $t =~ s/\t/ /g; - $t =~ s/\n/\\n/g; - $t =~ s/"/\\"/g; - print OUT qq:\t"$t",\n:; -} - -print OUT <<__E; - 0 -}; - -__E - -close OUT; |