diff options
author | Chris Wilson <chris+github@qwirx.com> | 2015-12-14 19:39:24 +0000 |
---|---|---|
committer | Chris Wilson <chris+github@qwirx.com> | 2015-12-14 19:39:24 +0000 |
commit | d16cc7f012fbe3eca75a7c711604a4f3f4c0846f (patch) | |
tree | bc19e0bf1de8f28a4d199d5f3f80d18c59ea30de /lib | |
parent | c459a8b5c88d82c44874edc6933c53a2b2a422d1 (diff) | |
parent | 25155a7021a5145c3c2c38b0a2511f94a9243bf3 (diff) |
Merge branch 'appveyor' of github.com:boxbackup/boxbackup into appveyor
Diffstat (limited to 'lib')
25 files changed, 12788 insertions, 3 deletions
diff --git a/lib/bbackupd/BackupClientContext.cpp b/lib/bbackupd/BackupClientContext.cpp new file mode 100644 index 00000000..4c0b01ce --- /dev/null +++ b/lib/bbackupd/BackupClientContext.cpp @@ -0,0 +1,578 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientContext.cpp +// Purpose: Keep track of context +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_SIGNAL_H + #include <signal.h> +#endif + +#ifdef HAVE_SYS_TIME_H + #include <sys/time.h> +#endif + +#include "BoxPortsAndFiles.h" +#include "BoxTime.h" +#include "BackupClientContext.h" +#include "SocketStreamTLS.h" +#include "Socket.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" +#include "BackupDaemon.h" +#include "autogen_BackupProtocol.h" +#include "BackupStoreFile.h" +#include "Logging.h" +#include "TcpNice.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::BackupClientContext(BackupDaemon &, TLSContext &, const std::string &, int32_t, bool, bool, std::string) +// Purpose: Constructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientContext::BackupClientContext +( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier& rProgressNotifier, + bool TcpNiceMode +) +: mExperimentalSnapshotMode(false), + mrResolver(rResolver), + mrTLSContext(rTLSContext), + mHostname(rHostname), + mPort(Port), + mAccountNumber(AccountNumber), + mExtendedLogging(ExtendedLogging), + mExtendedLogToFile(ExtendedLogToFile), + mExtendedLogFile(ExtendedLogFile), + mpExtendedLogFileHandle(NULL), + mClientStoreMarker(ClientStoreMarker_NotKnown), + mpDeleteList(0), + mpCurrentIDMap(0), + mpNewIDMap(0), + mStorageLimitExceeded(false), + mpExcludeFiles(0), + mpExcludeDirs(0), + mKeepAliveTimer(0, "KeepAliveTime"), + mbIsManaged(false), + mrProgressNotifier(rProgressNotifier), + mTcpNiceMode(TcpNiceMode), + mpNice(NULL) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::~BackupClientContext() +// Purpose: Destructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientContext::~BackupClientContext() +{ + CloseAnyOpenConnection(); + + // Delete delete list + if(mpDeleteList != 0) + { + delete mpDeleteList; + mpDeleteList = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetConnection() +// Purpose: Returns the connection, making the connection and logging into +// the backup store if necessary. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupProtocolCallable &BackupClientContext::GetConnection() +{ + // Already got it? Just return it. + if(mapConnection.get()) + { + return *mapConnection; + } + + // Defensive. Must close connection before releasing any old socket. + mapConnection.reset(); + + std::auto_ptr<SocketStream> apSocket(new SocketStreamTLS); + + try + { + // Defensive. + mapConnection.reset(); + + // Log intention + BOX_INFO("Opening connection to server '" << mHostname << + "'..."); + + // Connect! + ((SocketStreamTLS *)(apSocket.get()))->Open(mrTLSContext, + Socket::TypeINET, mHostname, mPort); + + if(mTcpNiceMode) + { + // Pass control of apSocket to NiceSocketStream, + // which will take care of destroying it for us. + // But we need to hang onto a pointer to the nice + // socket, so we can enable and disable nice mode. + // This is scary, it could be deallocated under us. + mpNice = new NiceSocketStream(apSocket); + apSocket.reset(mpNice); + } + + // We need to call some methods that aren't defined in + // BackupProtocolCallable, so we need to hang onto a more + // strongly typed pointer (to avoid far too many casts). + BackupProtocolClient *pClient = new BackupProtocolClient(apSocket); + mapConnection.reset(pClient); + + // Set logging option + pClient->SetLogToSysLog(mExtendedLogging); + + if (mExtendedLogToFile) + { + ASSERT(mpExtendedLogFileHandle == NULL); + + mpExtendedLogFileHandle = fopen( + mExtendedLogFile.c_str(), "a+"); + + if (!mpExtendedLogFileHandle) + { + BOX_LOG_SYS_ERROR("Failed to open extended " + "log file: " << mExtendedLogFile); + } + else + { + pClient->SetLogToFile(mpExtendedLogFileHandle); + } + } + + // Handshake + pClient->Handshake(); + + // Check the version of the server + { + std::auto_ptr<BackupProtocolVersion> serverVersion( + mapConnection->QueryVersion(BACKUP_STORE_SERVER_VERSION)); + if(serverVersion->GetVersion() != BACKUP_STORE_SERVER_VERSION) + { + THROW_EXCEPTION(BackupStoreException, WrongServerVersion) + } + } + + // Login -- if this fails, the Protocol will exception + std::auto_ptr<BackupProtocolLoginConfirmed> loginConf( + mapConnection->QueryLogin(mAccountNumber, 0 /* read/write */)); + + // Check that the client store marker is the one we expect + if(mClientStoreMarker != ClientStoreMarker_NotKnown) + { + if(loginConf->GetClientStoreMarker() != mClientStoreMarker) + { + // Not good... finish the connection, abort, etc, ignoring errors + try + { + mapConnection->QueryFinished(); + } + catch(...) + { + // IGNORE + } + + // Then throw an exception about this + THROW_EXCEPTION_MESSAGE(BackupStoreException, + ClientMarkerNotAsExpected, + "Expected " << mClientStoreMarker << + " but found " << loginConf->GetClientStoreMarker() << + ": is someone else writing to the " + "same account?"); + } + } + else // mClientStoreMarker == ClientStoreMarker_NotKnown + { + // Yes, choose one, the current time will do + box_time_t marker = GetCurrentBoxTime(); + + // Set it on the store + mapConnection->QuerySetClientStoreMarker(marker); + + // Record it so that it can be picked up later. + mClientStoreMarker = marker; + } + + // Log success + BOX_INFO("Connection made, login successful"); + + // Check to see if there is any space available on the server + if(loginConf->GetBlocksUsed() >= loginConf->GetBlocksHardLimit()) + { + // no -- flag so only things like deletions happen + mStorageLimitExceeded = true; + // Log + BOX_WARNING("Exceeded storage hard-limit on server, " + "not uploading changes to files"); + } + } + catch(...) + { + // Clean up. + mapConnection.reset(); + throw; + } + + return *mapConnection; +} + +BackupProtocolCallable* BackupClientContext::GetOpenConnection() const +{ + return mapConnection.get(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::CloseAnyOpenConnection() +// Purpose: Closes a connection, if it's open +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupClientContext::CloseAnyOpenConnection() +{ + BackupProtocolCallable* pConnection(GetOpenConnection()); + if(pConnection) + { + try + { + // Quit nicely + pConnection->QueryFinished(); + } + catch(...) + { + // Ignore errors here + } + + // Delete it anyway. + mapConnection.reset(); + } + + // Delete any pending list + if(mpDeleteList != 0) + { + delete mpDeleteList; + mpDeleteList = 0; + } + + if (mpExtendedLogFileHandle != NULL) + { + fclose(mpExtendedLogFileHandle); + mpExtendedLogFileHandle = NULL; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetTimeout() +// Purpose: Gets the current timeout time. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +int BackupClientContext::GetTimeout() const +{ + BackupProtocolCallable* pConnection(GetOpenConnection()); + if(pConnection) + { + return pConnection->GetTimeout(); + } + + return (15*60*1000); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetDeleteList() +// Purpose: Returns the delete list, creating one if necessary +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +BackupClientDeleteList &BackupClientContext::GetDeleteList() +{ + // Already created? + if(mpDeleteList == 0) + { + mpDeleteList = new BackupClientDeleteList; + } + + // Return reference to object + return *mpDeleteList; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::PerformDeletions() +// Purpose: Perform any pending file deletions. +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientContext::PerformDeletions() +{ + // Got a list? + if(mpDeleteList == 0) + { + // Nothing to do + return; + } + + // Delegate to the delete list object + mpDeleteList->PerformDeletions(*this); + + // Delete the object + delete mpDeleteList; + mpDeleteList = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetCurrentIDMap() const +// Purpose: Return a (const) reference to the current ID map +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +const BackupClientInodeToIDMap &BackupClientContext::GetCurrentIDMap() const +{ + ASSERT(mpCurrentIDMap != 0); + if(mpCurrentIDMap == 0) + { + THROW_EXCEPTION(CommonException, Internal) + } + return *mpCurrentIDMap; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetNewIDMap() const +// Purpose: Return a reference to the new ID map +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +BackupClientInodeToIDMap &BackupClientContext::GetNewIDMap() const +{ + ASSERT(mpNewIDMap != 0); + if(mpNewIDMap == 0) + { + THROW_EXCEPTION(CommonException, Internal) + } + return *mpNewIDMap; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::FindFilename(int64_t, int64_t, std::string &, bool &) const +// Purpose: Attempts to find the pathname of an object with a given ID on the server. +// Returns true if it can be found, in which case rPathOut is the local filename, +// and rIsDirectoryOut == true if the local object is a directory. +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut, + bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer, box_time_t *pAttributesHashOnServer, BackupStoreFilenameClear *pLeafname) +{ + // Make a connection to the server + BackupProtocolCallable &connection(GetConnection()); + + // Request filenames from the server, in a "safe" manner to ignore errors properly + { + BackupProtocolGetObjectName send(ObjectID, ContainingDirectory); + connection.Send(send); + } + std::auto_ptr<BackupProtocolMessage> preply(connection.Receive()); + + // Is it of the right type? + if(preply->GetType() != BackupProtocolObjectName::TypeID) + { + // Was an error or something + return false; + } + + // Cast to expected type. + BackupProtocolObjectName *names = (BackupProtocolObjectName *)(preply.get()); + + // Anything found? + int32_t numElements = names->GetNumNameElements(); + if(numElements <= 0) + { + // No. + return false; + } + + // Get the stream containing all the names + std::auto_ptr<IOStream> nameStream(connection.ReceiveStream()); + + // Path + std::string path; + + // Remember this is in reverse order! + for(int l = 0; l < numElements; ++l) + { + BackupStoreFilenameClear elementName; + elementName.ReadFromStream(*nameStream, GetTimeout()); + + // Store leafname for caller? + if(l == 0 && pLeafname) + { + *pLeafname = elementName; + } + + // Is it part of the filename in the location? + if(l < (numElements - 1)) + { + // Part of filename within + path = (path.empty())?(elementName.GetClearFilename()):(elementName.GetClearFilename() + DIRECTORY_SEPARATOR_ASCHAR + path); + } + else + { + // Location name -- look up in daemon's records + std::string locPath; + if(!mrResolver.FindLocationPathName(elementName.GetClearFilename(), locPath)) + { + // Didn't find the location... so can't give the local filename + return false; + } + + // Add in location path + path = (path.empty())?(locPath):(locPath + DIRECTORY_SEPARATOR_ASCHAR + path); + } + } + + // Is it a directory? + rIsDirectoryOut = ((names->GetFlags() & BackupProtocolListDirectory::Flags_Dir) == BackupProtocolListDirectory::Flags_Dir); + + // Is it the current version? + rIsCurrentVersionOut = ((names->GetFlags() & (BackupProtocolListDirectory::Flags_OldVersion | BackupProtocolListDirectory::Flags_Deleted)) == 0); + + // And other information which may be required + if(pModTimeOnServer) *pModTimeOnServer = names->GetModificationTime(); + if(pAttributesHashOnServer) *pAttributesHashOnServer = names->GetAttributesHash(); + + // Tell caller about the pathname + rPathOut = path; + + // Found + return true; +} + +void BackupClientContext::SetMaximumDiffingTime(int iSeconds) +{ + mMaximumDiffingTime = iSeconds < 0 ? 0 : iSeconds; + BOX_TRACE("Set maximum diffing time to " << mMaximumDiffingTime << + " seconds"); +} + +void BackupClientContext::SetKeepAliveTime(int iSeconds) +{ + mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; + BOX_TRACE("Set keep-alive time to " << mKeepAliveTime << " seconds"); + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::ManageDiffProcess() +// Purpose: Initiates a file diff control timer +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::ManageDiffProcess() +{ + ASSERT(!mbIsManaged); + mbIsManaged = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::UnManageDiffProcess() +// Purpose: suspends file diff control timer +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::UnManageDiffProcess() +{ + // ASSERT(mbIsManaged); + mbIsManaged = false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::DoKeepAlive() +// Purpose: Check whether it's time to send a KeepAlive +// message over the SSL link, and if so, send it. +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::DoKeepAlive() +{ + BackupProtocolCallable* pConnection(GetOpenConnection()); + if (!pConnection) + { + return; + } + + if (mKeepAliveTime == 0) + { + return; + } + + if (!mKeepAliveTimer.HasExpired()) + { + return; + } + + BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); + pConnection->QueryGetIsAlive(); + + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); +} + +int BackupClientContext::GetMaximumDiffingTime() +{ + return mMaximumDiffingTime; +} diff --git a/lib/bbackupd/BackupClientContext.h b/lib/bbackupd/BackupClientContext.h new file mode 100644 index 00000000..df43a232 --- /dev/null +++ b/lib/bbackupd/BackupClientContext.h @@ -0,0 +1,252 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientContext.h +// Purpose: Keep track of context +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTCONTEXT__H +#define BACKUPCLIENTCONTEXT__H + +#include "BoxTime.h" +#include "BackupClientDeleteList.h" +#include "BackupClientDirectoryRecord.h" +#include "BackupDaemonInterface.h" +#include "BackupStoreFile.h" +#include "ExcludeList.h" +#include "TcpNice.h" +#include "Timer.h" + +class TLSContext; +class BackupProtocolClient; +class SocketStreamTLS; +class BackupClientInodeToIDMap; +class BackupDaemon; +class BackupStoreFilenameClear; + +#include <string> + + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientContext +// Purpose: +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class BackupClientContext : public DiffTimer +{ +public: + BackupClientContext + ( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode + ); + virtual ~BackupClientContext(); + +private: + BackupClientContext(const BackupClientContext &); + +public: + // GetConnection() will open a connection if none is currently open. + virtual BackupProtocolCallable& GetConnection(); + // GetOpenConnection() will not open a connection, just return NULL if there is + // no connection already open. + virtual BackupProtocolCallable* GetOpenConnection() const; + void CloseAnyOpenConnection(); + int GetTimeout() const; + BackupClientDeleteList &GetDeleteList(); + void PerformDeletions(); + + enum + { + ClientStoreMarker_NotKnown = 0 + }; + + void SetClientStoreMarker(int64_t ClientStoreMarker) {mClientStoreMarker = ClientStoreMarker;} + int64_t GetClientStoreMarker() const {return mClientStoreMarker;} + + bool StorageLimitExceeded() {return mStorageLimitExceeded;} + void SetStorageLimitExceeded() {mStorageLimitExceeded = true;} + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetIDMaps(const BackupClientInodeToIDMap *, BackupClientInodeToIDMap *) + // Purpose: Store pointers to the Current and New ID maps + // Created: 11/11/03 + // + // -------------------------------------------------------------------------- + void SetIDMaps(const BackupClientInodeToIDMap *pCurrent, BackupClientInodeToIDMap *pNew) + { + ASSERT(pCurrent != 0); + ASSERT(pNew != 0); + mpCurrentIDMap = pCurrent; + mpNewIDMap = pNew; + } + const BackupClientInodeToIDMap &GetCurrentIDMap() const; + BackupClientInodeToIDMap &GetNewIDMap() const; + + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetExcludeLists(ExcludeList *, ExcludeList *) + // Purpose: Sets the exclude lists for the operation. Can be 0. + // Created: 28/1/04 + // + // -------------------------------------------------------------------------- + void SetExcludeLists(ExcludeList *pExcludeFiles, ExcludeList *pExcludeDirs) + { + mpExcludeFiles = pExcludeFiles; + mpExcludeDirs = pExcludeDirs; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::ExcludeFile(const std::string &) + // Purpose: Returns true is this file should be excluded from the backup + // Created: 28/1/04 + // + // -------------------------------------------------------------------------- + inline bool ExcludeFile(const std::string &rFullFilename) + { + if(mpExcludeFiles != 0) + { + return mpExcludeFiles->IsExcluded(rFullFilename); + } + // If no list, don't exclude anything + return false; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::ExcludeDir(const std::string &) + // Purpose: Returns true is this directory should be excluded from the backup + // Created: 28/1/04 + // + // -------------------------------------------------------------------------- + inline bool ExcludeDir(const std::string &rFullDirName) + { + if(mpExcludeDirs != 0) + { + return mpExcludeDirs->IsExcluded(rFullDirName); + } + // If no list, don't exclude anything + return false; + } + + // Utility functions -- may do a lot of work + bool FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut, + bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer = 0, box_time_t *pAttributesHashOnServer = 0, + BackupStoreFilenameClear *pLeafname = 0); // not const as may connect to server + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetMaximumDiffingTime() + // Purpose: Sets the maximum time that will be spent diffing a file + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void SetMaximumDiffingTime(int iSeconds); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetKeepAliveTime() + // Purpose: Sets the time interval for repetitive keep-alive operation + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + virtual void SetKeepAliveTime(int iSeconds); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::ManageDiffProcess() + // Purpose: Initiates an SSL connection/session keep-alive process + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void ManageDiffProcess(); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::UnManageDiffProcess() + // Purpose: Suspends an SSL connection/session keep-alive process + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void UnManageDiffProcess(); + + // ------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::DoKeepAlive() + // Purpose: Check whether it's time to send a KeepAlive + // message over the SSL link, and if so, send it. + // Created: 04/19/2005 + // + // ------------------------------------------------------------------- + virtual void DoKeepAlive(); + virtual int GetMaximumDiffingTime(); + virtual bool IsManaged() { return mbIsManaged; } + + ProgressNotifier& GetProgressNotifier() const + { + return mrProgressNotifier; + } + + void SetNiceMode(bool enabled) + { + if(mTcpNiceMode) + { + mpNice->SetEnabled(enabled); + } + } + + bool mExperimentalSnapshotMode; + +private: + LocationResolver &mrResolver; + TLSContext &mrTLSContext; + std::string mHostname; + int mPort; + uint32_t mAccountNumber; + std::auto_ptr<BackupProtocolCallable> mapConnection; + bool mExtendedLogging; + bool mExtendedLogToFile; + std::string mExtendedLogFile; + FILE* mpExtendedLogFileHandle; + int64_t mClientStoreMarker; + BackupClientDeleteList *mpDeleteList; + const BackupClientInodeToIDMap *mpCurrentIDMap; + BackupClientInodeToIDMap *mpNewIDMap; + bool mStorageLimitExceeded; + ExcludeList *mpExcludeFiles; + ExcludeList *mpExcludeDirs; + Timer mKeepAliveTimer; + bool mbIsManaged; + int mKeepAliveTime; + int mMaximumDiffingTime; + ProgressNotifier &mrProgressNotifier; + bool mTcpNiceMode; + NiceSocketStream *mpNice; +}; + +#endif // BACKUPCLIENTCONTEXT__H diff --git a/lib/bbackupd/BackupClientDeleteList.cpp b/lib/bbackupd/BackupClientDeleteList.cpp new file mode 100644 index 00000000..ce5e6264 --- /dev/null +++ b/lib/bbackupd/BackupClientDeleteList.cpp @@ -0,0 +1,229 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDeleteList.cpp +// Purpose: List of pending deletes for backup +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <algorithm> + +#include "BackupClientDeleteList.h" +#include "BackupClientContext.h" +#include "autogen_BackupProtocol.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::BackupClientDeleteList() +// Purpose: Constructor +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +BackupClientDeleteList::BackupClientDeleteList() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::~BackupClientDeleteList() +// Purpose: Destructor +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +BackupClientDeleteList::~BackupClientDeleteList() +{ +} + +BackupClientDeleteList::FileToDelete::FileToDelete(int64_t DirectoryID, + const BackupStoreFilename& rFilename, + const std::string& rLocalPath) +: mDirectoryID(DirectoryID), + mFilename(rFilename), + mLocalPath(rLocalPath) +{ } + +BackupClientDeleteList::DirToDelete::DirToDelete(int64_t ObjectID, + const std::string& rLocalPath) +: mObjectID(ObjectID), + mLocalPath(rLocalPath) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::AddDirectoryDelete(int64_t, +// const BackupStoreFilename&) +// Purpose: Add a directory to the list of directories to be deleted. +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID, + const std::string& rLocalPath) +{ + // Only add the delete to the list if it's not in the "no delete" set + if(mDirectoryNoDeleteList.find(ObjectID) == + mDirectoryNoDeleteList.end()) + { + // Not in the list, so should delete it + mDirectoryList.push_back(DirToDelete(ObjectID, rLocalPath)); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::AddFileDelete(int64_t, +// const BackupStoreFilename &) +// Purpose: +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, + const BackupStoreFilename &rFilename, const std::string& rLocalPath) +{ + // Try to find it in the no delete list + std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator + delEntry(mFileNoDeleteList.begin()); + while(delEntry != mFileNoDeleteList.end()) + { + if((delEntry)->first == DirectoryID + && (delEntry)->second == rFilename) + { + // Found! + break; + } + ++delEntry; + } + + // Only add it to the delete list if it wasn't in the no delete list + if(delEntry == mFileNoDeleteList.end()) + { + mFileList.push_back(FileToDelete(DirectoryID, rFilename, + rLocalPath)); + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) +// Purpose: Perform all the pending deletes +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) +{ + // Anything to do? + if(mDirectoryList.empty() && mFileList.empty()) + { + // Nothing! + return; + } + + // Get a connection + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Do the deletes + for(std::vector<DirToDelete>::iterator i(mDirectoryList.begin()); + i != mDirectoryList.end(); ++i) + { + connection.QueryDeleteDirectory(i->mObjectID); + rContext.GetProgressNotifier().NotifyDirectoryDeleted( + i->mObjectID, i->mLocalPath); + } + + // Clear the directory list + mDirectoryList.clear(); + + // Delete the files + for(std::vector<FileToDelete>::iterator i(mFileList.begin()); + i != mFileList.end(); ++i) + { + connection.QueryDeleteFile(i->mDirectoryID, i->mFilename); + rContext.GetProgressNotifier().NotifyFileDeleted( + i->mDirectoryID, i->mLocalPath); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::StopDirectoryDeletion(int64_t) +// Purpose: Stop a directory being deleted +// Created: 19/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::StopDirectoryDeletion(int64_t ObjectID) +{ + // First of all, is it in the delete vector? + std::vector<DirToDelete>::iterator delEntry(mDirectoryList.begin()); + for(; delEntry != mDirectoryList.end(); delEntry++) + { + if(delEntry->mObjectID == ObjectID) + { + // Found! + break; + } + } + if(delEntry != mDirectoryList.end()) + { + // erase this entry + mDirectoryList.erase(delEntry); + } + else + { + // Haven't been asked to delete it yet, put it in the + // no delete list + mDirectoryNoDeleteList.insert(ObjectID); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::StopFileDeletion(int64_t, const BackupStoreFilename &) +// Purpose: Stop a file from being deleted +// Created: 19/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, + const BackupStoreFilename &rFilename) +{ + // Find this in the delete list + std::vector<FileToDelete>::iterator delEntry(mFileList.begin()); + while(delEntry != mFileList.end()) + { + if(delEntry->mDirectoryID == DirectoryID + && delEntry->mFilename == rFilename) + { + // Found! + break; + } + ++delEntry; + } + + if(delEntry != mFileList.end()) + { + // erase this entry + mFileList.erase(delEntry); + } + else + { + // Haven't been asked to delete it yet, put it in the no delete list + mFileNoDeleteList.push_back(std::pair<int64_t, BackupStoreFilename>(DirectoryID, rFilename)); + } +} + diff --git a/lib/bbackupd/BackupClientDeleteList.h b/lib/bbackupd/BackupClientDeleteList.h new file mode 100644 index 00000000..b0fbf51a --- /dev/null +++ b/lib/bbackupd/BackupClientDeleteList.h @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDeleteList.h +// Purpose: List of pending deletes for backup +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTDELETELIST__H +#define BACKUPCLIENTDELETELIST__H + +#include "BackupStoreFilename.h" + +class BackupClientContext; + +#include <vector> +#include <utility> +#include <set> + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientDeleteList +// Purpose: List of pending deletes for backup +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +class BackupClientDeleteList +{ +private: + class FileToDelete + { + public: + int64_t mDirectoryID; + BackupStoreFilename mFilename; + std::string mLocalPath; + FileToDelete(int64_t DirectoryID, + const BackupStoreFilename& rFilename, + const std::string& rLocalPath); + }; + + class DirToDelete + { + public: + int64_t mObjectID; + std::string mLocalPath; + DirToDelete(int64_t ObjectID, const std::string& rLocalPath); + }; + +public: + BackupClientDeleteList(); + ~BackupClientDeleteList(); + + void AddDirectoryDelete(int64_t ObjectID, + const std::string& rLocalPath); + void AddFileDelete(int64_t DirectoryID, + const BackupStoreFilename &rFilename, + const std::string& rLocalPath); + + void StopDirectoryDeletion(int64_t ObjectID); + void StopFileDeletion(int64_t DirectoryID, + const BackupStoreFilename &rFilename); + + void PerformDeletions(BackupClientContext &rContext); + +private: + std::vector<DirToDelete> mDirectoryList; + std::set<int64_t> mDirectoryNoDeleteList; // note: things only get in this list if they're not present in mDirectoryList when they are 'added' + std::vector<FileToDelete> mFileList; + std::vector<std::pair<int64_t, BackupStoreFilename> > mFileNoDeleteList; +}; + +#endif // BACKUPCLIENTDELETELIST__H + diff --git a/lib/bbackupd/BackupClientDirectoryRecord.cpp b/lib/bbackupd/BackupClientDirectoryRecord.cpp new file mode 100644 index 00000000..94cb7965 --- /dev/null +++ b/lib/bbackupd/BackupClientDirectoryRecord.cpp @@ -0,0 +1,2302 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDirectoryRecord.cpp +// Purpose: Implementation of record about directory for +// backup client +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_DIRENT_H + #include <dirent.h> +#endif + +#include <errno.h> +#include <string.h> + +#include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" +#include "autogen_ClientException.h" +#include "Archive.h" +#include "BackupClientContext.h" +#include "BackupClientDirectoryRecord.h" +#include "BackupClientInodeToIDMap.h" +#include "BackupDaemon.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileEncodeStream.h" +#include "BufferedStream.h" +#include "CommonException.h" +#include "CollectInBufferStream.h" +#include "FileModificationTime.h" +#include "IOStream.h" +#include "Logging.h" +#include "MemBlockStream.h" +#include "PathUtils.h" +#include "RateLimitingStream.h" +#include "ReadLoggingStream.h" + +#include "MemLeakFindOn.h" + +typedef std::map<std::string, BackupStoreDirectory::Entry *> DecryptedEntriesMap_t; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::BackupClientDirectoryRecord() +// Purpose: Constructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName) + : mObjectID(ObjectID), + mSubDirName(rSubDirName), + mInitialSyncDone(false), + mSyncDone(false), + mpPendingEntries(0) +{ + ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::~BackupClientDirectoryRecord() +// Purpose: Destructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::~BackupClientDirectoryRecord() +{ + // Make deletion recursive + DeleteSubDirectories(); + + // Delete maps + if(mpPendingEntries != 0) + { + delete mpPendingEntries; + mpPendingEntries = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::DeleteSubDirectories(); +// Purpose: Delete all sub directory entries +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::DeleteSubDirectories() +{ + // Delete all pointers + for(std::map<std::string, BackupClientDirectoryRecord *>::iterator i = mSubDirectories.begin(); + i != mSubDirectories.end(); ++i) + { + delete i->second; + } + + // Empty list + mSubDirectories.clear(); +} + +std::string BackupClientDirectoryRecord::ConvertVssPathToRealPath( + const std::string &rVssPath, + const Location& rBackupLocation) +{ +#ifdef ENABLE_VSS + BOX_TRACE("VSS: ConvertVssPathToRealPath: mIsSnapshotCreated = " << + rBackupLocation.mIsSnapshotCreated); + BOX_TRACE("VSS: ConvertVssPathToRealPath: File/Directory Path = " << + rVssPath.substr(0, rBackupLocation.mSnapshotPath.length())); + BOX_TRACE("VSS: ConvertVssPathToRealPath: Snapshot Path = " << + rBackupLocation.mSnapshotPath); + if (rBackupLocation.mIsSnapshotCreated && + rVssPath.substr(0, rBackupLocation.mSnapshotPath.length()) == + rBackupLocation.mSnapshotPath) + { + std::string convertedPath = rBackupLocation.mPath + + rVssPath.substr(rBackupLocation.mSnapshotPath.length()); + BOX_TRACE("VSS: ConvertVssPathToRealPath: Converted Path = " << + convertedPath); + return convertedPath; + } +#endif + + return rVssPath; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncDirectory(i +// BackupClientDirectoryRecord::SyncParams &, +// int64_t, const std::string &, +// const std::string &, bool) +// Purpose: Recursively synchronise a local directory +// with the server. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::SyncDirectory( + BackupClientDirectoryRecord::SyncParams &rParams, + int64_t ContainingDirectoryID, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + bool ThisDirHasJustBeenCreated) +{ + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + + // Signal received by daemon? + if(rParams.mrRunStatusProvider.StopRun()) + { + // Yes. Stop now. + THROW_EXCEPTION(BackupStoreException, SignalReceived) + } + + // Start by making some flag changes, marking this sync as not done, + // and on the immediate sub directories. + mSyncDone = false; + for(std::map<std::string, BackupClientDirectoryRecord *>::iterator + i = mSubDirectories.begin(); + i != mSubDirectories.end(); ++i) + { + i->second->mSyncDone = false; + } + + // Work out the time in the future after which the file should + // be uploaded regardless. This is a simple way to avoid having + // too many problems with file servers when they have clients + // with badly out of sync clocks. + rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + + rParams.mMaxFileTimeInFuture; + + // Build the current state checksum to compare against while + // getting info from dirs. Note checksum is used locally only, + // so byte order isn't considered. + MD5Digest currentStateChecksum; + + EMU_STRUCT_STAT dest_st; + // Stat the directory, to get attribute info + // If it's a symbolic link, we want the link target here + // (as we're about to back up the contents of the directory) + { + if(EMU_STAT(rLocalPath.c_str(), &dest_st) != 0) + { + // The directory has probably been deleted, so + // just ignore this error. In a future scan, this + // deletion will be noticed, deleted from server, + // and this object deleted. + rNotifier.NotifyDirStatFailed(this, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation), + strerror(errno)); + return; + } + + BOX_TRACE("Stat dir '" << rLocalPath << "' " + "found device/inode " << + dest_st.st_dev << "/" << dest_st.st_ino); + + // Store inode number in map so directories are tracked + // in case they're renamed + { + BackupClientInodeToIDMap &idMap( + rParams.mrContext.GetNewIDMap()); + idMap.AddToMap(dest_st.st_ino, mObjectID, ContainingDirectoryID, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation)); + } + // Add attributes to checksum + currentStateChecksum.Add(&dest_st.st_mode, + sizeof(dest_st.st_mode)); + currentStateChecksum.Add(&dest_st.st_uid, + sizeof(dest_st.st_uid)); + currentStateChecksum.Add(&dest_st.st_gid, + sizeof(dest_st.st_gid)); + // Inode to be paranoid about things moving around + currentStateChecksum.Add(&dest_st.st_ino, + sizeof(dest_st.st_ino)); +#ifdef HAVE_STRUCT_STAT_ST_FLAGS + currentStateChecksum.Add(&dest_st.st_flags, + sizeof(dest_st.st_flags)); +#endif + + StreamableMemBlock xattr; + BackupClientFileAttributes::FillExtendedAttr(xattr, + rLocalPath.c_str()); + currentStateChecksum.Add(xattr.GetBuffer(), xattr.GetSize()); + } + + // Read directory entries, building arrays of names + // First, need to read the contents of the directory. + std::vector<std::string> dirs; + std::vector<std::string> files; + bool downloadDirectoryRecordBecauseOfFutureFiles = false; + + // BLOCK + { + // read the contents... + DIR *dirHandle = 0; + try + { + std::string nonVssDirPath = ConvertVssPathToRealPath(rLocalPath, + rBackupLocation); + rNotifier.NotifyScanDirectory(this, nonVssDirPath); + + dirHandle = ::opendir(rLocalPath.c_str()); + if(dirHandle == 0) + { + // Report the error (logs and eventual email to administrator) + if (errno == EACCES) + { + rNotifier.NotifyDirListFailed(this, + nonVssDirPath, + "Access denied"); + } + else + { + rNotifier.NotifyDirListFailed(this, + nonVssDirPath, + strerror(errno)); + } + + // Report the error (logs and eventual email + // to administrator) + SetErrorWhenReadingFilesystemObject(rParams, + nonVssDirPath); + // Ignore this directory for now. + return; + } + + struct dirent *en = 0; + int num_entries_found = 0; + + while((en = ::readdir(dirHandle)) != 0) + { + num_entries_found++; + rParams.mrContext.DoKeepAlive(); + if(rParams.mpBackgroundTask) + { + rParams.mpBackgroundTask->RunBackgroundTask( + BackgroundTask::Scanning_Dirs, + num_entries_found, 0); + } + + if (!SyncDirectoryEntry(rParams, rNotifier, + rBackupLocation, rLocalPath, + currentStateChecksum, en, dest_st, dirs, + files, downloadDirectoryRecordBecauseOfFutureFiles)) + { + // This entry is not to be backed up. + continue; + } + } + + if(::closedir(dirHandle) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + dirHandle = 0; + } + catch(...) + { + if(dirHandle != 0) + { + ::closedir(dirHandle); + } + throw; + } + } + + // Finish off the checksum, and compare with the one currently stored + bool checksumDifferent = true; + currentStateChecksum.Finish(); + if(mInitialSyncDone && currentStateChecksum.DigestMatches(mStateChecksum)) + { + // The checksum is the same, and there was one to compare with + checksumDifferent = false; + } + + // Pointer to potentially downloaded store directory info + std::auto_ptr<BackupStoreDirectory> apDirOnStore; + + try + { + // Want to get the directory listing? + if(ThisDirHasJustBeenCreated) + { + // Avoid sending another command to the server when we know it's empty + apDirOnStore.reset(new BackupStoreDirectory(mObjectID, + ContainingDirectoryID)); + } + // Consider asking the store for it + else if(!mInitialSyncDone || checksumDifferent || + downloadDirectoryRecordBecauseOfFutureFiles) + { + apDirOnStore = FetchDirectoryListing(rParams); + } + + // Make sure the attributes are up to date -- if there's space + // on the server and this directory has not just been created + // (because it's attributes will be correct in this case) and + // the checksum is different, implying they *MIGHT* be + // different. + if((!ThisDirHasJustBeenCreated) && checksumDifferent && + !rParams.mrContext.StorageLimitExceeded()) + { + UpdateAttributes(rParams, apDirOnStore.get(), rLocalPath); + } + + // Create the list of pointers to directory entries + std::vector<BackupStoreDirectory::Entry *> entriesLeftOver; + if(apDirOnStore.get()) + { + entriesLeftOver.resize(apDirOnStore->GetNumberOfEntries(), 0); + BackupStoreDirectory::Iterator i(*apDirOnStore); + // Copy in pointers to all the entries + for(unsigned int l = 0; l < apDirOnStore->GetNumberOfEntries(); ++l) + { + entriesLeftOver[l] = i.Next(); + } + } + + // Do the directory reading + bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, + rRemotePath, rBackupLocation, apDirOnStore.get(), + entriesLeftOver, files, dirs); + + // LAST THING! (think exception safety) + // Store the new checksum -- don't fetch things unnecessarily + // in the future But... only if 1) the storage limit isn't + // exceeded -- make sure things are done again if the directory + // is modified later and 2) All the objects within the + // directory were stored successfully. + if(!rParams.mrContext.StorageLimitExceeded() && + updateCompleteSuccess) + { + currentStateChecksum.CopyDigestTo(mStateChecksum); + } + } + catch(...) + { + // Bad things have happened -- clean up + // Set things so that we get a full go at stuff later + ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); + + throw; + } + + // Flag things as having happened. + mInitialSyncDone = true; + mSyncDone = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncDirectorEntry( +// BackupClientDirectoryRecord::SyncParams &, +// int64_t, const std::string &, +// const std::string &, bool) +// Purpose: Recursively synchronise a local directory +// with the server. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +bool BackupClientDirectoryRecord::SyncDirectoryEntry( + BackupClientDirectoryRecord::SyncParams &rParams, + ProgressNotifier& rNotifier, + const Location& rBackupLocation, + const std::string &rDirLocalPath, + MD5Digest& currentStateChecksum, + struct dirent *en, + EMU_STRUCT_STAT dir_st, + std::vector<std::string>& rDirs, + std::vector<std::string>& rFiles, + bool& rDownloadDirectoryRecordBecauseOfFutureFiles) +{ + std::string entry_name = en->d_name; + if(entry_name == "." || entry_name == "..") + { + // ignore parent directory entries + return false; + } + + // Stat file to get info + std::string filename = MakeFullPath(rDirLocalPath, entry_name); + std::string realFileName = ConvertVssPathToRealPath(filename, + rBackupLocation); + EMU_STRUCT_STAT file_st; + +#ifdef WIN32 + // Don't stat the file just yet, to ensure that users can exclude + // unreadable files to suppress warnings that they are not accessible. + // + // Our emulated readdir() abuses en->d_type, which would normally + // contain DT_REG, DT_DIR, etc, but we only use it here and prefer to + // have the full file attributes. + + int type; + if (en->d_type & FILE_ATTRIBUTE_DIRECTORY) + { + type = S_IFDIR; + } + else + { + type = S_IFREG; + } +#else // !WIN32 + if(EMU_LSTAT(filename.c_str(), &file_st) != 0) + { + // We don't know whether it's a file or a directory, so check + // both. This only affects whether a warning message is + // displayed; the file is not backed up in either case. + if(!(rParams.mrContext.ExcludeFile(filename)) && + !(rParams.mrContext.ExcludeDir(filename))) + { + // Report the error (logs and eventual email to + // administrator) + rNotifier.NotifyFileStatFailed(this, filename, + strerror(errno)); + + // FIXME move to NotifyFileStatFailed() + SetErrorWhenReadingFilesystemObject(rParams, filename); + } + + // Ignore this entry for now. + return false; + } + + BOX_TRACE("Stat entry '" << filename << "' found device/inode " << + file_st.st_dev << "/" << file_st.st_ino); + + // Workaround for apparent btrfs bug, where symlinks appear to be on + // a different filesystem than their containing directory, thanks to + // Toke Hoiland-Jorgensen. + + int type = file_st.st_mode & S_IFMT; + if(type == S_IFDIR && file_st.st_dev != dir_st.st_dev) + { + if(!(rParams.mrContext.ExcludeDir(filename))) + { + rNotifier.NotifyMountPointSkipped(this, filename); + } + return false; + } +#endif + + if(type == S_IFREG || type == S_IFLNK) + { + // File or symbolic link + + // Exclude it? + if(rParams.mrContext.ExcludeFile(realFileName)) + { + rNotifier.NotifyFileExcluded(this, realFileName); + // Next item! + return false; + } + } + else if(type == S_IFDIR) + { + // Directory + + // Exclude it? + if(rParams.mrContext.ExcludeDir(realFileName)) + { + rNotifier.NotifyDirExcluded(this, realFileName); + + // Next item! + return false; + } + + #ifdef WIN32 + // exclude reparse points, as Application Data points to the + // parent directory under Vista and later, and causes an + // infinite loop: + // http://social.msdn.microsoft.com/forums/en-US/windowscompatibility/thread/05d14368-25dd-41c8-bdba-5590bf762a68/ + if (en->d_type & FILE_ATTRIBUTE_REPARSE_POINT) + { + rNotifier.NotifyMountPointSkipped(this, realFileName); + return false; + } + #endif + } + else // not a file or directory, what is it? + { + if (type == S_IFSOCK +#ifndef WIN32 + || type == S_IFIFO +#endif + ) + { + // removed notification for these types + // see Debian bug 479145, no objections + } + else if(rParams.mrContext.ExcludeFile(realFileName)) + { + rNotifier.NotifyFileExcluded(this, realFileName); + } + else + { + rNotifier.NotifyUnsupportedFileType(this, realFileName); + SetErrorWhenReadingFilesystemObject(rParams, + realFileName); + } + + return false; + } + + // The object should be backed up (file, symlink or dir, not excluded). + // So make the information for adding to the checksum. + + #ifdef WIN32 + // We didn't stat the file before, but now we need the information. + if(emu_stat(filename.c_str(), &file_st) != 0) + { + rNotifier.NotifyFileStatFailed(this, + ConvertVssPathToRealPath(filename, rBackupLocation), + strerror(errno)); + + // Report the error (logs and eventual email to administrator) + SetErrorWhenReadingFilesystemObject(rParams, filename); + + // Ignore this entry for now. + return false; + } + + if(file_st.st_dev != dir_st.st_dev) + { + rNotifier.NotifyMountPointSkipped(this, + ConvertVssPathToRealPath(filename, rBackupLocation)); + return false; + } + #endif + + // Basic structure for checksum info + struct { + box_time_t mModificationTime; + box_time_t mAttributeModificationTime; + int64_t mSize; + // And then the name follows + } checksum_info; + + // Be paranoid about structure packing + ::memset(&checksum_info, 0, sizeof(checksum_info)); + + checksum_info.mModificationTime = FileModificationTime(file_st); + checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st); + checksum_info.mSize = file_st.st_size; + currentStateChecksum.Add(&checksum_info, sizeof(checksum_info)); + currentStateChecksum.Add(en->d_name, strlen(en->d_name)); + + // If the file has been modified madly into the future, download the + // directory record anyway to ensure that it doesn't get uploaded + // every single time the disc is scanned. + if(checksum_info.mModificationTime > rParams.mUploadAfterThisTimeInTheFuture) + { + rDownloadDirectoryRecordBecauseOfFutureFiles = true; + // Log that this has happened + if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) + { + rNotifier.NotifyFileModifiedInFuture(this, + ConvertVssPathToRealPath(filename, rBackupLocation)); + rParams.mHaveLoggedWarningAboutFutureFileTimes = true; + } + } + + // We've decided to back it up, so add to file or directory list. + if(type == S_IFREG || type == S_IFLNK) + { + rFiles.push_back(entry_name); + } + else if(type == S_IFDIR) + { + rDirs.push_back(entry_name); + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::FetchDirectoryListing( +// BackupClientDirectoryRecord::SyncParams &) +// Purpose: Fetch the directory listing of this directory from +// the store. +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreDirectory> +BackupClientDirectoryRecord::FetchDirectoryListing( + BackupClientDirectoryRecord::SyncParams &rParams) +{ + std::auto_ptr<BackupStoreDirectory> apDir; + + // Get connection to store + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Query the directory + std::auto_ptr<BackupProtocolSuccess> dirreply(connection.QueryListDirectory( + mObjectID, + // both files and directories + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // exclude old/deleted stuff + BackupProtocolListDirectory::Flags_Deleted | + BackupProtocolListDirectory::Flags_OldVersion, + true /* want attributes */)); + + // Retrieve the directory from the stream following + apDir.reset(new BackupStoreDirectory(connection.ReceiveStream(), + connection.GetTimeout())); + return apDir; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::UpdateAttributes( +// BackupClientDirectoryRecord::SyncParams &, +// const std::string &) +// Purpose: Sets the attributes of the directory on the store, +// if necessary. +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::UpdateAttributes( + BackupClientDirectoryRecord::SyncParams &rParams, + BackupStoreDirectory *pDirOnStore, + const std::string &rLocalPath) +{ + // Get attributes for the directory + BackupClientFileAttributes attr; + box_time_t attrModTime = 0; + attr.ReadAttributes(rLocalPath.c_str(), true /* directories have zero mod times */, + 0 /* no modification time */, &attrModTime); + + // Assume attributes need updating, unless proved otherwise + bool updateAttr = true; + + // Got a listing to compare with? + ASSERT(pDirOnStore == 0 || (pDirOnStore != 0 && pDirOnStore->HasAttributes())); + if(pDirOnStore != 0 && pDirOnStore->HasAttributes()) + { + const StreamableMemBlock &storeAttrEnc(pDirOnStore->GetAttributes()); + // Explict decryption + BackupClientFileAttributes storeAttr(storeAttrEnc); + + // Compare the attributes + if(attr.Compare(storeAttr, true, + true /* ignore both modification times */)) + { + // No update necessary + updateAttr = false; + } + } + + // Update them? + if(updateAttr) + { + // Get connection to store + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Exception thrown if this doesn't work + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); + connection.QueryChangeDirAttributes(mObjectID, attrModTime, attrStream); + } +} + +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath) +{ + BackupStoreFilenameClear fn(en->GetName()); + return DecryptFilename(fn, en->GetObjectID(), rRemoteDirectoryPath); +} + +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreFilenameClear fn, int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath) +{ + std::string filenameClear; + try + { + filenameClear = fn.GetClearFilename(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to decrypt filename for object " << + BOX_FORMAT_OBJECTID(filenameObjectID) << " in " + "directory " << BOX_FORMAT_OBJECTID(mObjectID) << + " (" << rRemoteDirectoryPath << ")"); + throw; + } + return filenameClear; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &, const std::string &, BackupStoreDirectory *, std::vector<BackupStoreDirectory::Entry *> &) +// Purpose: Update the items stored on the server. The rFiles vector will be erased after it's used to save space. +// Returns true if all items were updated successfully. (If not, the failures will have been logged). +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +bool BackupClientDirectoryRecord::UpdateItems( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + BackupStoreDirectory *pDirOnStore, + std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, + std::vector<std::string> &rFiles, + const std::vector<std::string> &rDirs) +{ + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + + bool allUpdatedSuccessfully = true; + + // Decrypt all the directory entries. + // It would be nice to be able to just compare the encrypted versions, however this doesn't work + // in practise because there can be multiple encodings of the same filename using different + // methods (although each method will result in the same string for the same filename.) This + // happens when the server fixes a broken store, and gives plain text generated filenames. + // So if we didn't do things like this, then you wouldn't be able to recover from bad things + // happening with the server. + DecryptedEntriesMap_t decryptedEntries; + if(pDirOnStore != 0) + { + BackupStoreDirectory::Iterator i(*pDirOnStore); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + std::string filenameClear; + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + decryptedEntries[filenameClear] = en; + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "pretending that the file doesn't " + "exist"); + } + } + } + + // Do files + for(std::vector<std::string>::const_iterator f = rFiles.begin(); + f != rFiles.end(); ++f) + { + // Send keep-alive message if needed + rContext.DoKeepAlive(); + + // Filename of this file + std::string filename(MakeFullPath(rLocalPath, *f)); + std::string nonVssFilePath = ConvertVssPathToRealPath(filename, + rBackupLocation); + + // Get relevant info about file + box_time_t modTime = 0; + uint64_t attributesHash = 0; + int64_t fileSize = 0; + InodeRefType inodeNum = 0; + // BLOCK + { + // Stat the file + EMU_STRUCT_STAT st; + if(EMU_LSTAT(filename.c_str(), &st) != 0) + { + rNotifier.NotifyFileStatFailed(this, nonVssFilePath, + strerror(errno)); + + // Report the error (logs and + // eventual email to administrator) + SetErrorWhenReadingFilesystemObject(rParams, nonVssFilePath); + + // Ignore this entry for now. + continue; + } + + // Extract required data + modTime = FileModificationTime(st); + fileSize = st.st_size; + inodeNum = st.st_ino; + attributesHash = BackupClientFileAttributes::GenerateAttributeHash(st, filename, *f); + } + + // See if it's in the listing (if we have one) + BackupStoreFilenameClear storeFilename(*f); + BackupStoreDirectory::Entry *en = 0; + int64_t latestObjectID = 0; + if(pDirOnStore != 0) + { + DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*f)); + if(i != decryptedEntries.end()) + { + en = i->second; + latestObjectID = en->GetObjectID(); + } + } + + // Check that the entry which might have been found is in fact a file + if((en != 0) && !(en->IsFile())) + { + // Directory exists in the place of this file -- sort it out + RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, + en, *f); + en = 0; + } + + // Check for renaming? + if(pDirOnStore != 0 && en == 0) + { + // We now know... + // 1) File has just been added + // 2) It's not in the store + + // Do we know about the inode number? + const BackupClientInodeToIDMap &idMap(rContext.GetCurrentIDMap()); + int64_t renameObjectID = 0, renameInDirectory = 0; + if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) + { + // Look up on the server to get the name, to build the local filename + std::string localPotentialOldName; + bool isDir = false; + bool isCurrentVersion = false; + box_time_t srvModTime = 0, srvAttributesHash = 0; + BackupStoreFilenameClear oldLeafname; + if(rContext.FindFilename(renameObjectID, renameInDirectory, + localPotentialOldName, isDir, isCurrentVersion, + &srvModTime, &srvAttributesHash, &oldLeafname)) + { + // Only interested if it's a file and the latest version + if(!isDir && isCurrentVersion) + { + // Check that the object we found in the ID map doesn't exist on disc + EMU_STRUCT_STAT st; + if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) + { + // Doesn't exist locally, but does exist on the server. + // Therefore we can safely rename it to this new file. + + // Get the connection to the server + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Only do this step if there is room on the server. + // This step will be repeated later when there is space available + if(!rContext.StorageLimitExceeded()) + { + // Rename the existing files (ie include old versions) on the server + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, + storeFilename); + + // Stop the attempt to delete the file in the original location + BackupClientDeleteList &rdelList(rContext.GetDeleteList()); + rdelList.StopFileDeletion(renameInDirectory, oldLeafname); + + // Create new entry in the directory for it + // -- will be near enough what's actually on the server for the rest to work. + en = pDirOnStore->AddEntry(storeFilename, + srvModTime, renameObjectID, + 0 /* size in blocks unknown, but not needed */, + BackupStoreDirectory::Entry::Flags_File, + srvAttributesHash); + + // Store the object ID for the inode lookup map later + latestObjectID = renameObjectID; + } + } + } + } + } + } + + // Is it in the mPendingEntries list? + box_time_t pendingFirstSeenTime = 0; // ie not seen + if(mpPendingEntries != 0) + { + std::map<std::string, box_time_t>::const_iterator i(mpPendingEntries->find(*f)); + if(i != mpPendingEntries->end()) + { + // found it -- set flag + pendingFirstSeenTime = i->second; + } + } + + // If pDirOnStore == 0, then this must have been after an initial sync: + ASSERT(pDirOnStore != 0 || mInitialSyncDone); + // So, if pDirOnStore == 0, then we know that everything before syncPeriodStart + // is either on the server, or in the toupload list. If the directory had changed, + // we'd have got a directory listing. + // + // At this point, if (pDirOnStore == 0 && en == 0), we can assume it's on the server with a + // mod time < syncPeriodStart, or didn't exist before that time. + // + // But if en != 0, then we need to compare modification times to avoid uploading it again. + + // Need to update? + // + // Condition for upload: + // modification time within sync period + // if it's been seen before but not uploaded, is the time from this first sight longer than the MaxUploadWait + // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store + + bool doUpload = false; + std::string decisionReason = "unknown"; + + // Only upload a file if the mod time locally is + // different to that on the server. + + if (en == 0 || en->GetModificationTime() != modTime) + { + // Check the file modified within the acceptable time period we're checking + // If the file isn't on the server, the acceptable time starts at zero. + // Check pDirOnStore and en, because if we didn't download a directory listing, + // pDirOnStore will be zero, but we know it's on the server. + if (modTime < rParams.mSyncPeriodEnd) + { + if (pDirOnStore != 0 && en == 0) + { + doUpload = true; + decisionReason = "not on server"; + } + else if (modTime >= rParams.mSyncPeriodStart) + { + doUpload = true; + decisionReason = "modified since last sync"; + } + } + + // However, just in case things are continually + // modified, we check the first seen time. + // The two compares of syncPeriodEnd and + // pendingFirstSeenTime are because the values + // are unsigned. + + if (!doUpload && + pendingFirstSeenTime != 0 && + rParams.mSyncPeriodEnd > pendingFirstSeenTime && + (rParams.mSyncPeriodEnd - pendingFirstSeenTime) + > rParams.mMaxUploadWait) + { + doUpload = true; + decisionReason = "continually modified"; + } + + // Then make sure that if files are added with a + // time less than the sync period start + // (which can easily happen on file server), it + // gets uploaded. The directory contents checksum + // will pick up the fact it has been added, so the + // store listing will be available when this happens. + + if (!doUpload && + modTime <= rParams.mSyncPeriodStart && + en != 0 && + en->GetModificationTime() != modTime) + { + doUpload = true; + decisionReason = "mod time changed"; + } + + // And just to catch really badly off clocks in + // the future for file server clients, + // just upload the file if it's madly in the future. + + if (!doUpload && modTime > + rParams.mUploadAfterThisTimeInTheFuture) + { + doUpload = true; + decisionReason = "mod time in the future"; + } + } + + if (en != 0 && en->GetModificationTime() == modTime) + { + doUpload = false; + decisionReason = "not modified since last upload"; + } + else if (!doUpload) + { + if (modTime > rParams.mSyncPeriodEnd) + { + box_time_t now = GetCurrentBoxTime(); + int age = BoxTimeToSeconds(now - + modTime); + std::ostringstream s; + s << "modified too recently: only " << + age << " seconds ago"; + decisionReason = s.str(); + } + else + { + std::ostringstream s; + s << "mod time is " << modTime << + " which is outside sync window, " + << rParams.mSyncPeriodStart << " to " + << rParams.mSyncPeriodEnd; + decisionReason = s.str(); + } + } + + BOX_TRACE("Upload decision: " << nonVssFilePath << ": " << + (doUpload ? "will upload" : "will not upload") << + " (" << decisionReason << ")"); + + bool fileSynced = true; + + if (doUpload) + { + // Upload needed, don't mark sync success until + // we've actually done it + fileSynced = false; + + // Make sure we're connected -- must connect here so we know whether + // the storage limit has been exceeded, and hence whether or not + // to actually upload the file. + rContext.GetConnection(); + + // Only do this step if there is room on the server. + // This step will be repeated later when there is space available + if(!rContext.StorageLimitExceeded()) + { + // Upload the file to the server, recording the + // object ID it returns + bool noPreviousVersionOnServer = + ((pDirOnStore != 0) && (en == 0)); + + // Surround this in a try/catch block, to + // catch errors, but still continue + bool uploadSuccess = false; + try + { + latestObjectID = UploadFile(rParams, + filename, + nonVssFilePath, + rRemotePath + "/" + *f, + storeFilename, + fileSize, modTime, + attributesHash, + noPreviousVersionOnServer); + + if (latestObjectID == 0) + { + // storage limit exceeded + rParams.mrContext.SetStorageLimitExceeded(); + uploadSuccess = false; + allUpdatedSuccessfully = false; + } + else + { + uploadSuccess = true; + } + } + catch(ConnectionException &e) + { + // Connection errors should just be + // passed on to the main handler, + // retries would probably just cause + // more problems. + rNotifier.NotifyFileUploadException( + this, nonVssFilePath, e); + throw; + } + catch(BoxException &e) + { + if (e.GetType() == BackupStoreException::ExceptionType && + e.GetSubType() == BackupStoreException::SignalReceived) + { + // abort requested, pass the + // exception on up. + throw; + } + + // an error occured -- make return + // code false, to show error in directory + allUpdatedSuccessfully = false; + // Log it. + SetErrorWhenReadingFilesystemObject(rParams, + nonVssFilePath); + rNotifier.NotifyFileUploadException(this, + nonVssFilePath, e); + } + + // Update structures if the file was uploaded + // successfully. + if(uploadSuccess) + { + fileSynced = true; + + // delete from pending entries + if(pendingFirstSeenTime != 0 && mpPendingEntries != 0) + { + mpPendingEntries->erase(*f); + } + } + } + else + { + rNotifier.NotifyFileSkippedServerFull(this, nonVssFilePath); + } + } + else if(en != 0 && en->GetAttributesHash() != attributesHash) + { + // Attributes have probably changed, upload them again. + // If the attributes have changed enough, the directory + // hash will have changed too, and so the dir will have + // been downloaded, and the entry will be available. + + // Get connection + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Only do this step if there is room on the server. + // This step will be repeated later when there is + // space available + if(!rContext.StorageLimitExceeded()) + { + try + { + rNotifier.NotifyFileUploadingAttributes(this, + nonVssFilePath); + + // Update store + BackupClientFileAttributes attr; + attr.ReadAttributes(filename, + false /* put mod times in the attributes, please */); + std::auto_ptr<IOStream> attrStream( + new MemBlockStream(attr)); + connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream); + fileSynced = true; + } + catch (BoxException &e) + { + BOX_ERROR("Failed to read or store file attributes " + "for '" << nonVssFilePath << "', will try again " + "later"); + } + } + } + + if(modTime >= rParams.mSyncPeriodEnd) + { + // Allocate? + if(mpPendingEntries == 0) + { + mpPendingEntries = new std::map<std::string, box_time_t>; + } + // Adding to mPendingEntries list + if(pendingFirstSeenTime == 0) + { + // Haven't seen this before -- add to list! + (*mpPendingEntries)[*f] = modTime; + } + } + + // Zero pointer in rEntriesLeftOver, if we have a pointer to zero + if(en != 0) + { + for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) + { + if(rEntriesLeftOver[l] == en) + { + rEntriesLeftOver[l] = 0; + break; + } + } + } + + // Does this file need an entry in the ID map? + if(fileSize >= rParams.mFileTrackingSizeThreshold) + { + // Get the map + BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap()); + + // Need to get an ID from somewhere... + if(latestObjectID == 0) + { + // Don't know it -- haven't sent anything to the store, and didn't get a listing. + // Look it up in the current map, and if it's there, use that. + const BackupClientInodeToIDMap ¤tIDMap(rContext.GetCurrentIDMap()); + int64_t objid = 0, dirid = 0; + if(currentIDMap.Lookup(inodeNum, objid, dirid)) + { + // Found + if (dirid != mObjectID) + { + BOX_WARNING("Found conflicting parent ID for " + "file ID " << inodeNum << " (" << + nonVssFilePath << "): expected " << + mObjectID << " but found " << dirid << + " (same directory used in two different " + "locations?)"); + } + + ASSERT(dirid == mObjectID); + + // NOTE: If the above assert fails, an inode number has been reused by the OS, + // or there is a problem somewhere. If this happened on a short test run, look + // into it. However, in a long running process this may happen occasionally and + // not indicate anything wrong. + // Run the release version for real life use, where this check is not made. + + latestObjectID = objid; + } + } + + if(latestObjectID != 0) + { + BOX_TRACE("Storing uploaded file ID " << + inodeNum << " (" << nonVssFilePath << ") " + "in ID map as object " << + latestObjectID << " with parent " << + mObjectID); + idMap.AddToMap(inodeNum, latestObjectID, + mObjectID /* containing directory */, + nonVssFilePath); + } + + } + + if (fileSynced) + { + rNotifier.NotifyFileSynchronised(this, nonVssFilePath, + fileSize); + } + } + + // Erase contents of files to save space when recursing + rFiles.clear(); + + // Delete the pending entries, if the map is empty + if(mpPendingEntries != 0 && mpPendingEntries->size() == 0) + { + BOX_TRACE("Deleting mpPendingEntries from dir ID " << + BOX_FORMAT_OBJECTID(mObjectID)); + delete mpPendingEntries; + mpPendingEntries = 0; + } + + // Do directories + for(std::vector<std::string>::const_iterator d = rDirs.begin(); + d != rDirs.end(); ++d) + { + // Send keep-alive message if needed + rContext.DoKeepAlive(); + + // Get the local filename + std::string dirname(MakeFullPath(rLocalPath, *d)); + std::string nonVssDirPath = ConvertVssPathToRealPath(dirname, + rBackupLocation); + + // See if it's in the listing (if we have one) + BackupStoreFilenameClear storeFilename(*d); + BackupStoreDirectory::Entry *en = 0; + if(pDirOnStore != 0) + { + DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*d)); + if(i != decryptedEntries.end()) + { + en = i->second; + } + } + + // Check that the entry which might have been found is in fact a directory + if((en != 0) && !(en->IsDir())) + { + // Entry exists, but is not a directory. Bad. + // Get rid of it. + BackupProtocolCallable &connection(rContext.GetConnection()); + connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); + + std::string filenameClear = DecryptFilename(en, + rRemotePath); + rNotifier.NotifyFileDeleted(en->GetObjectID(), + filenameClear); + + // Nothing found + en = 0; + } + + // Zero pointer in rEntriesLeftOver, if we have a pointer to zero + if(en != 0) + { + for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) + { + if(rEntriesLeftOver[l] == en) + { + rEntriesLeftOver[l] = 0; + break; + } + } + } + + // Flag for having created directory, so can optimise the + // recursive call not to read it again, because we know + // it's empty. + bool haveJustCreatedDirOnServer = false; + + // Next, see if it's in the list of sub directories + BackupClientDirectoryRecord *psubDirRecord = 0; + std::map<std::string, BackupClientDirectoryRecord *>::iterator + e(mSubDirectories.find(*d)); + + if(e != mSubDirectories.end()) + { + // In the list, just use this pointer + psubDirRecord = e->second; + } + else + { + // Note: if we have exceeded our storage limit, then + // we should not upload any more data, nor create any + // DirectoryRecord representing data that would have + // been uploaded. This step will be repeated when + // there is some space available. + bool doCreateDirectoryRecord = true; + + // Need to create the record. But do we need to create the directory on the server? + int64_t subDirObjectID = 0; + if(en != 0) + { + // No. Exists on the server, and we know about it from the listing. + subDirObjectID = en->GetObjectID(); + } + else if(rContext.StorageLimitExceeded()) + // know we've got a connection if we get this far, + // as dir will have been modified. + { + doCreateDirectoryRecord = false; + } + else + { + // Yes, creation required! + // It is known that it doesn't exist: + // + // if en == 0 and pDirOnStore == 0, then the + // directory has had an initial sync, and + // hasn't been modified (Really? then why + // are we here? TODO FIXME) + // so it has definitely been created already + // (so why create it again?) + // + // if en == 0 but pDirOnStore != 0, well... obviously it doesn't exist. + // + subDirObjectID = CreateRemoteDir(dirname, + nonVssDirPath, rRemotePath + "/" + *d, + storeFilename, &haveJustCreatedDirOnServer, + rParams); + doCreateDirectoryRecord = (subDirObjectID != 0); + } + + if (doCreateDirectoryRecord) + { + // New an object for this + psubDirRecord = new BackupClientDirectoryRecord(subDirObjectID, *d); + + // Store in list + try + { + mSubDirectories[*d] = psubDirRecord; + } + catch(...) + { + delete psubDirRecord; + psubDirRecord = 0; + throw; + } + } + } + + // ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded()); + // There's another possible reason now: the directory no longer + // existed when we finally got around to checking its + // attributes. See for example Brendon Baumgartner's reported + // error with Wordpress cache directories. + + if(psubDirRecord) + { + // Sync this sub directory too + psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, + rRemotePath + "/" + *d, rBackupLocation, + haveJustCreatedDirOnServer); + } + } + + // Delete everything which is on the store, but not on disc + for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) + { + if(rEntriesLeftOver[l] != 0) + { + BackupStoreDirectory::Entry *en = rEntriesLeftOver[l]; + + // These entries can't be deleted immediately, as it would prevent + // renaming and moving of objects working properly. So we add them + // to a list, which is actually deleted at the very end of the session. + // If there's an error during the process, it doesn't matter if things + // aren't actually deleted, as the whole state will be reset anyway. + BackupClientDeleteList &rdel(rContext.GetDeleteList()); + std::string filenameClear; + bool isCorruptFilename = false; + + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "scheduling that file for deletion"); + filenameClear = "<corrupt filename>"; + isCorruptFilename = true; + } + + std::string localName = MakeFullPath(rLocalPath, + filenameClear); + std::string nonVssLocalName = ConvertVssPathToRealPath(localName, + rBackupLocation); + + // Delete this entry -- file or directory? + if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) + { + // Set a pending deletion for the file + rdel.AddFileDelete(mObjectID, en->GetName(), + localName); + } + else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) + { + // Set as a pending deletion for the directory + rdel.AddDirectoryDelete(en->GetObjectID(), + localName); + + // If there's a directory record for it in + // the sub directory map, delete it now + BackupStoreFilenameClear dirname(en->GetName()); + std::map<std::string, BackupClientDirectoryRecord *>::iterator + e(mSubDirectories.find(filenameClear)); + if(e != mSubDirectories.end() && !isCorruptFilename) + { + // Carefully delete the entry from the map + BackupClientDirectoryRecord *rec = e->second; + mSubDirectories.erase(e); + delete rec; + + BOX_TRACE("Deleted directory record for " << + nonVssLocalName); + } + } + } + } + + // Return success flag (will be false if some files failed) + return allUpdatedSuccessfully; +} + +int64_t BackupClientDirectoryRecord::CreateRemoteDir(const std::string& localDirPath, + const std::string& nonVssDirPath, const std::string& remoteDirPath, + BackupStoreFilenameClear& storeFilename, bool* pHaveJustCreatedDirOnServer, + BackupClientDirectoryRecord::SyncParams &rParams) +{ + // Get attributes + box_time_t attrModTime = 0; + InodeRefType inodeNum = 0; + BackupClientFileAttributes attr; + *pHaveJustCreatedDirOnServer = false; + ProgressNotifier& rNotifier(rParams.mrContext.GetProgressNotifier()); + + try + { + attr.ReadAttributes(localDirPath, + true /* directories have zero mod times */, + 0 /* not interested in mod time */, + &attrModTime, 0 /* not file size */, + &inodeNum); + } + catch (BoxException &e) + { + // We used to try to recover from this, but we'd need an + // attributes block to upload to the server, so we have to + // skip creating the directory instead. + BOX_WARNING("Failed to read attributes of directory, " + "ignoring it for now: " << nonVssDirPath); + return 0; // no object ID + } + + // Check to see if the directory been renamed + // First, do we have a record in the ID map? + int64_t renameObjectID = 0, renameInDirectory = 0; + bool renameDir = false; + const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap()); + + if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) + { + // Look up on the server to get the name, to build the local filename + std::string localPotentialOldName; + bool isDir = false; + bool isCurrentVersion = false; + if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, + localPotentialOldName, isDir, isCurrentVersion)) + { + // Only interested if it's a directory + if(isDir && isCurrentVersion) + { + // Check that the object doesn't exist already + EMU_STRUCT_STAT st; + if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && + errno == ENOENT) + { + // Doesn't exist locally, but does exist + // on the server. Therefore we can + // safely rename it. + renameDir = true; + } + } + } + } + + // Get connection + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Don't do a check for storage limit exceeded here, because if we get to this + // stage, a connection will have been opened, and the status known, so the check + // in the else if(...) above will be correct. + + // Build attribute stream for sending + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); + + if(renameDir) + { + // Rename the existing directory on the server + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, + storeFilename); + + // Put the latest attributes on it + connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream); + + // Stop it being deleted later + BackupClientDeleteList &rdelList( + rParams.mrContext.GetDeleteList()); + rdelList.StopDirectoryDeletion(renameObjectID); + + // This is the ID for the renamed directory + return renameObjectID; + } + else + { + int64_t subDirObjectID = 0; // no object ID + + // Create a new directory + try + { + subDirObjectID = connection.QueryCreateDirectory( + mObjectID, attrModTime, storeFilename, + attrStream)->GetObjectID(); + // Flag as having done this for optimisation later + *pHaveJustCreatedDirOnServer = true; + } + catch(BoxException &e) + { + int type, subtype; + connection.GetLastError(type, subtype); + rNotifier.NotifyFileUploadServerError(this, nonVssDirPath, + type, subtype); + if(e.GetType() == ConnectionException::ExceptionType && + e.GetSubType() == ConnectionException::Protocol_UnexpectedReply && + type == BackupProtocolError::ErrorType && + subtype == BackupProtocolError::Err_StorageLimitExceeded) + { + // The hard limit was exceeded on the server, notify! + rParams.mrContext.SetStorageLimitExceeded(); + rParams.mrSysadminNotifier.NotifySysadmin( + SysadminNotifier::StoreFull); + } + else + { + throw; + } + } + + if(*pHaveJustCreatedDirOnServer) + { + rNotifier.NotifyDirectoryCreated(subDirObjectID, + nonVssDirPath, remoteDirPath); + } + + return subDirObjectID; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &, BackupStoreDirectory *, int64_t, const std::string &) +// Purpose: Called to resolve difficulties when a directory is found on the +// store where a file is to be uploaded. +// Created: 9/7/04 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile( + SyncParams &rParams, + BackupStoreDirectory* pDirOnStore, + BackupStoreDirectory::Entry* pEntry, + const std::string &rFilename) +{ + // First, delete the directory + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + connection.QueryDeleteDirectory(pEntry->GetObjectID()); + + BackupStoreFilenameClear clear(pEntry->GetName()); + rParams.mrContext.GetProgressNotifier().NotifyDirectoryDeleted( + pEntry->GetObjectID(), clear.GetClearFilename()); + + // Then, delete any directory record + std::map<std::string, BackupClientDirectoryRecord *>::iterator + e(mSubDirectories.find(rFilename)); + + if(e != mSubDirectories.end()) + { + // A record exists for this, remove it + BackupClientDirectoryRecord *psubDirRecord = e->second; + mSubDirectories.erase(e); + + // And delete the object + delete psubDirRecord; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::UploadFile( +// BackupClientDirectoryRecord::SyncParams &, +// const std::string &, +// const BackupStoreFilename &, +// int64_t, box_time_t, box_time_t, bool) +// Purpose: Private. Upload a file to the server. May send +// a patch instead of the whole thing +// Created: 20/1/04 +// +// -------------------------------------------------------------------------- +int64_t BackupClientDirectoryRecord::UploadFile( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string &rLocalPath, + const std::string &rNonVssFilePath, + const std::string &rRemotePath, + const BackupStoreFilenameClear &rStoreFilename, + int64_t FileSize, + box_time_t ModificationTime, + box_time_t AttributesHash, + bool NoPreviousVersionOnServer) +{ + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + + // Get the connection + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Info + int64_t objID = 0; + int64_t uploadedSize = -1; + + // Use a try block to catch store full errors + try + { + std::auto_ptr<BackupStoreFileEncodeStream> apStreamToUpload; + int64_t diffFromID = 0; + + // Might an old version be on the server, and is the file + // size over the diffing threshold? + if(!NoPreviousVersionOnServer && + FileSize >= rParams.mDiffingUploadSizeThreshold) + { + // YES -- try to do diff, if possible + // First, query the server to see if there's an old version available + std::auto_ptr<BackupProtocolSuccess> getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename)); + diffFromID = getBlockIndex->GetObjectID(); + + if(diffFromID != 0) + { + // Found an old version + + // Get the index + std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream()); + + // + // Diff the file + // + + rContext.ManageDiffProcess(); + + bool isCompletelyDifferent = false; + + apStreamToUpload = BackupStoreFile::EncodeFileDiff( + rLocalPath, + mObjectID, /* containing directory */ + rStoreFilename, diffFromID, *blockIndexStream, + connection.GetTimeout(), + &rContext, // DiffTimer implementation + 0 /* not interested in the modification time */, + &isCompletelyDifferent, + rParams.mpBackgroundTask); + + if(isCompletelyDifferent) + { + diffFromID = 0; + } + + rContext.UnManageDiffProcess(); + } + } + + if(apStreamToUpload.get()) + { + rNotifier.NotifyFileUploadingPatch(this, rNonVssFilePath, + apStreamToUpload->GetBytesToUpload()); + } + else // No patch upload, so do a normal upload + { + // below threshold or nothing to diff from, so upload whole + rNotifier.NotifyFileUploading(this, rNonVssFilePath); + + // Prepare to upload, getting a stream which will encode the file as we go along + apStreamToUpload = BackupStoreFile::EncodeFile( + rLocalPath, mObjectID, /* containing directory */ + rStoreFilename, NULL, &rParams, + &(rParams.mrRunStatusProvider), + rParams.mpBackgroundTask); + } + + rContext.SetNiceMode(true); + std::auto_ptr<IOStream> apWrappedStream; + + if(rParams.mMaxUploadRate > 0) + { + apWrappedStream.reset(new RateLimitingStream( + *apStreamToUpload, rParams.mMaxUploadRate)); + } + else + { + // Wrap the stream in *something*, so that + // QueryStoreFile() doesn't delete the original + // stream (upload object) and we can retrieve + // the byte counter. + apWrappedStream.reset(new BufferedStream( + *apStreamToUpload)); + } + + // Send to store + std::auto_ptr<BackupProtocolSuccess> stored( + connection.QueryStoreFile(mObjectID, ModificationTime, + AttributesHash, diffFromID, rStoreFilename, + apWrappedStream)); + + rContext.SetNiceMode(false); + + // Get object ID from the result + objID = stored->GetObjectID(); + uploadedSize = apStreamToUpload->GetTotalBytesSent(); + } + catch(BoxException &e) + { + rContext.UnManageDiffProcess(); + + if(e.GetType() == ConnectionException::ExceptionType && + e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) + { + // Check and see what error the protocol has, + // this is more useful to users than the exception. + int type, subtype; + if(connection.GetLastError(type, subtype)) + { + if(type == BackupProtocolError::ErrorType + && subtype == BackupProtocolError::Err_StorageLimitExceeded) + { + // The hard limit was exceeded on the server, notify! + rParams.mrSysadminNotifier.NotifySysadmin( + SysadminNotifier::StoreFull); + // return an error code instead of + // throwing an exception that we + // can't debug. + return 0; + } + rNotifier.NotifyFileUploadServerError(this, + rNonVssFilePath, type, subtype); + } + } + + // Send the error on it's way + throw; + } + + rNotifier.NotifyFileUploaded(this, rNonVssFilePath, FileSize, + uploadedSize, objID); + + // Return the new object ID of this file + return objID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( +// SyncParams &, const char *) +// Purpose: Sets the error state when there were problems +// reading an object from the filesystem. +// Created: 29/3/04 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string& rFilename) +{ + // Zero hash, so it gets synced properly next time round. + ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); + + // More detailed logging was already done by the caller, but if we + // have a read error reported, we need to be able to search the logs + // to find out which file it was, so we need to log a consistent and + // clear error message. + BOX_WARNING("Failed to backup file, see above for details: " << + rFilename); + + // Mark that an error occured in the parameters object + rParams.mReadErrorsOnFilesystemObjects = true; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncParams::SyncParams(BackupClientContext &) +// Purpose: Constructor +// Created: 8/3/04 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::SyncParams::SyncParams( + RunStatusProvider &rRunStatusProvider, + SysadminNotifier &rSysadminNotifier, + ProgressNotifier &rProgressNotifier, + BackupClientContext &rContext, + BackgroundTask *pBackgroundTask) +: mSyncPeriodStart(0), + mSyncPeriodEnd(0), + mMaxUploadWait(0), + mMaxFileTimeInFuture(99999999999999999LL), + mFileTrackingSizeThreshold(16*1024), + mDiffingUploadSizeThreshold(16*1024), + mpBackgroundTask(pBackgroundTask), + mrRunStatusProvider(rRunStatusProvider), + mrSysadminNotifier(rSysadminNotifier), + mrProgressNotifier(rProgressNotifier), + mrContext(rContext), + mReadErrorsOnFilesystemObjects(false), + mMaxUploadRate(0), + mUploadAfterThisTimeInTheFuture(99999999999999999LL), + mHaveLoggedWarningAboutFutureFileTimes(false) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncParams::~SyncParams() +// Purpose: Destructor +// Created: 8/3/04 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::SyncParams::~SyncParams() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::Deserialize(Archive & rArchive) +{ + // Make deletion recursive + DeleteSubDirectories(); + + // Delete maps + if(mpPendingEntries != 0) + { + delete mpPendingEntries; + mpPendingEntries = 0; + } + + // + // + // + rArchive.Read(mObjectID); + rArchive.Read(mSubDirName); + rArchive.Read(mInitialSyncDone); + rArchive.Read(mSyncDone); + + // + // + // + int64_t iCount = 0; + rArchive.Read(iCount); + + if (iCount != sizeof(mStateChecksum)/sizeof(mStateChecksum[0])) + { + // we have some kind of internal system representation change: throw for now + THROW_EXCEPTION(CommonException, Internal) + } + + for (int v = 0; v < iCount; v++) + { + // Load each checksum entry + rArchive.Read(mStateChecksum[v]); + } + + // + // + // + iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + // load each pending entry + mpPendingEntries = new std::map<std::string, box_time_t>; + if (!mpPendingEntries) + { + throw std::bad_alloc(); + } + + for (int v = 0; v < iCount; v++) + { + std::string strItem; + box_time_t btItem; + + rArchive.Read(strItem); + rArchive.Read(btItem); + (*mpPendingEntries)[strItem] = btItem; + } + } + + // + // + // + iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + for (int v = 0; v < iCount; v++) + { + std::string strItem; + rArchive.Read(strItem); + + BackupClientDirectoryRecord* pSubDirRecord = + new BackupClientDirectoryRecord(0, ""); + // will be deserialized anyway, give it id 0 for now + + if (!pSubDirRecord) + { + throw std::bad_alloc(); + } + + /***** RECURSE *****/ + pSubDirRecord->Deserialize(rArchive); + mSubDirectories[strItem] = pSubDirRecord; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mObjectID); + rArchive.Write(mSubDirName); + rArchive.Write(mInitialSyncDone); + rArchive.Write(mSyncDone); + + // + // + // + int64_t iCount = 0; + + // when reading back the archive, we will + // need to know how many items there are. + iCount = sizeof(mStateChecksum) / sizeof(mStateChecksum[0]); + rArchive.Write(iCount); + + for (int v = 0; v < iCount; v++) + { + rArchive.Write(mStateChecksum[v]); + } + + // + // + // + if (!mpPendingEntries) + { + iCount = 0; + rArchive.Write(iCount); + } + else + { + iCount = mpPendingEntries->size(); + rArchive.Write(iCount); + + for (std::map<std::string, box_time_t>::const_iterator + i = mpPendingEntries->begin(); + i != mpPendingEntries->end(); i++) + { + rArchive.Write(i->first); + rArchive.Write(i->second); + } + } + // + // + // + iCount = mSubDirectories.size(); + rArchive.Write(iCount); + + for (std::map<std::string, BackupClientDirectoryRecord*>::const_iterator + i = mSubDirectories.begin(); + i != mSubDirectories.end(); i++) + { + const BackupClientDirectoryRecord* pSubItem = i->second; + ASSERT(pSubItem); + + rArchive.Write(i->first); + pSubItem->Serialize(rArchive); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Location() +// Purpose: Constructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +Location::Location() +: mIDMapIndex(0) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::~Location() +// Purpose: Destructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +Location::~Location() +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, +// using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void Location::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mName); + rArchive.Write(mPath); + rArchive.Write(mIDMapIndex); + + // + // + // + if(!mapDirectoryRecord.get()) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mapDirectoryRecord->Serialize(rArchive); + } + + // + // + // + if(!mapExcludeFiles.get()) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mapExcludeFiles->Serialize(rArchive); + } + + // + // + // + if(!mapExcludeDirs.get()) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mapExcludeDirs->Serialize(rArchive); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void Location::Deserialize(Archive &rArchive) +{ + // + // + // + mapDirectoryRecord.reset(); + mapExcludeFiles.reset(); + mapExcludeDirs.reset(); + + // + // + // + rArchive.Read(mName); + rArchive.Read(mPath); + rArchive.Read(mIDMapIndex); + + // + // + // + int64_t aMagicMarker = 0; + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, ""); + if(!pSubRecord) + { + throw std::bad_alloc(); + } + + mapDirectoryRecord.reset(pSubRecord); + mapDirectoryRecord->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } + + // + // + // + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mapExcludeFiles.reset(new ExcludeList); + if(!mapExcludeFiles.get()) + { + throw std::bad_alloc(); + } + + mapExcludeFiles->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } + + // + // + // + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mapExcludeDirs.reset(new ExcludeList); + if(!mapExcludeDirs.get()) + { + throw std::bad_alloc(); + } + + mapExcludeDirs->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } +} diff --git a/lib/bbackupd/BackupClientDirectoryRecord.h b/lib/bbackupd/BackupClientDirectoryRecord.h new file mode 100644 index 00000000..865fc747 --- /dev/null +++ b/lib/bbackupd/BackupClientDirectoryRecord.h @@ -0,0 +1,244 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDirectoryRecord.h +// Purpose: Implementation of record about directory for backup client +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTDIRECTORYRECORD__H +#define BACKUPCLIENTDIRECTORYRECORD__H + +#include <string> +#include <map> +#include <memory> + +#include "BackgroundTask.h" +#include "BackupClientFileAttributes.h" +#include "BackupDaemonInterface.h" +#include "BackupStoreDirectory.h" +#include "BoxTime.h" +#include "MD5Digest.h" +#include "ReadLoggingStream.h" +#include "RunStatusProvider.h" + +#ifdef ENABLE_VSS +# include <comdef.h> +# include <Vss.h> +# include <VsWriter.h> +# include <VsBackup.h> +#endif + +class Archive; +class BackupClientContext; +class BackupDaemon; +class ExcludeList; +class Location; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientDirectoryRecord +// Purpose: Implementation of record about directory for backup client +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class BackupClientDirectoryRecord +{ +public: + BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName); + virtual ~BackupClientDirectoryRecord(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; +private: + BackupClientDirectoryRecord(const BackupClientDirectoryRecord &); +public: + + enum + { + UnknownDirectoryID = 0 + }; + + // -------------------------------------------------------------------------- + // + // Class + // Name: BackupClientDirectoryRecord::SyncParams + // Purpose: Holds parameters etc for directory syncing. Not passed as + // const, some parameters may be modified during sync. + // Created: 8/3/04 + // + // -------------------------------------------------------------------------- + class SyncParams : public ReadLoggingStream::Logger + { + public: + SyncParams( + RunStatusProvider &rRunStatusProvider, + SysadminNotifier &rSysadminNotifier, + ProgressNotifier &rProgressNotifier, + BackupClientContext &rContext, + BackgroundTask *pBackgroundTask); + ~SyncParams(); + private: + // No copying + SyncParams(const SyncParams&); + SyncParams &operator=(const SyncParams&); + + public: + // Data members are public, as accessors are not justified here + box_time_t mSyncPeriodStart; + box_time_t mSyncPeriodEnd; + box_time_t mMaxUploadWait; + box_time_t mMaxFileTimeInFuture; + int32_t mFileTrackingSizeThreshold; + int32_t mDiffingUploadSizeThreshold; + BackgroundTask *mpBackgroundTask; + RunStatusProvider &mrRunStatusProvider; + SysadminNotifier &mrSysadminNotifier; + ProgressNotifier &mrProgressNotifier; + BackupClientContext &mrContext; + bool mReadErrorsOnFilesystemObjects; + int64_t mMaxUploadRate; + + // Member variables modified by syncing process + box_time_t mUploadAfterThisTimeInTheFuture; + bool mHaveLoggedWarningAboutFutureFileTimes; + + bool StopRun() { return mrRunStatusProvider.StopRun(); } + void NotifySysadmin(SysadminNotifier::EventCode Event) + { + mrSysadminNotifier.NotifySysadmin(Event); + } + ProgressNotifier& GetProgressNotifier() const + { + return mrProgressNotifier; + } + + /* ReadLoggingStream::Logger implementation */ + virtual void Log(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset, + length, elapsed, finish); + } + virtual void Log(int64_t readSize, int64_t offset, + int64_t length) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset, + length); + } + virtual void Log(int64_t readSize, int64_t offset) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset); + } + }; + + void SyncDirectory(SyncParams &rParams, + int64_t ContainingDirectoryID, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + bool ThisDirHasJustBeenCreated = false); + + bool SyncDirectoryEntry(SyncParams &rParams, + ProgressNotifier& rNotifier, + const Location& rBackupLocation, + const std::string &rDirLocalPath, + MD5Digest& currentStateChecksum, + struct dirent *en, + EMU_STRUCT_STAT dir_st, + std::vector<std::string>& rDirs, + std::vector<std::string>& rFiles, + bool& rDownloadDirectoryRecordBecauseOfFutureFiles); + + std::string ConvertVssPathToRealPath(const std::string &rVssPath, + const Location& rBackupLocation); + + int64_t GetObjectID() const { return mObjectID; } + +private: + void DeleteSubDirectories(); + std::auto_ptr<BackupStoreDirectory> FetchDirectoryListing(SyncParams &rParams); + void UpdateAttributes(SyncParams &rParams, + BackupStoreDirectory *pDirOnStore, + const std::string &rLocalPath); +protected: // to allow tests to hook in before UpdateItems() runs + virtual bool UpdateItems(SyncParams &rParams, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + BackupStoreDirectory *pDirOnStore, + std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, + std::vector<std::string> &rFiles, + const std::vector<std::string> &rDirs); +private: + int64_t CreateRemoteDir(const std::string& localDirPath, + const std::string& nonVssDirPath, + const std::string& remoteDirPath, + BackupStoreFilenameClear& storeFilename, + bool* pHaveJustCreatedDirOnServer, + BackupClientDirectoryRecord::SyncParams &rParams); + int64_t UploadFile(SyncParams &rParams, + const std::string &rFilename, + const std::string &rNonVssFilePath, + const std::string &rRemotePath, + const BackupStoreFilenameClear &rStoreFilename, + int64_t FileSize, box_time_t ModificationTime, + box_time_t AttributesHash, bool NoPreviousVersionOnServer); + void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, + const std::string& rFilename); + void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, + BackupStoreDirectory* pDirOnStore, + BackupStoreDirectory::Entry* pEntry, + const std::string &rFilename); + std::string DecryptFilename(BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath); + std::string DecryptFilename(BackupStoreFilenameClear fn, + int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath); + + int64_t mObjectID; + std::string mSubDirName; + bool mInitialSyncDone; + bool mSyncDone; + + // Checksum of directory contents and attributes, used to detect changes + uint8_t mStateChecksum[MD5Digest::DigestLength]; + + std::map<std::string, box_time_t> *mpPendingEntries; + std::map<std::string, BackupClientDirectoryRecord *> mSubDirectories; + // mpPendingEntries is a pointer rather than simple a member + // variable, because most of the time it'll be empty. This would + // waste a lot of memory because of STL allocation policies. +}; + +class Location +{ +public: + Location(); + ~Location(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; +private: + Location(const Location &); // copy not allowed + Location &operator=(const Location &); +public: + std::string mName; + std::string mPath; + std::auto_ptr<BackupClientDirectoryRecord> mapDirectoryRecord; + std::auto_ptr<ExcludeList> mapExcludeFiles; + std::auto_ptr<ExcludeList> mapExcludeDirs; + int mIDMapIndex; + +#ifdef ENABLE_VSS + bool mIsSnapshotCreated; + VSS_ID mSnapshotVolumeId; + std::string mSnapshotPath; +#endif +}; + +#endif // BACKUPCLIENTDIRECTORYRECORD__H + + diff --git a/lib/bbackupd/BackupClientInodeToIDMap.cpp b/lib/bbackupd/BackupClientInodeToIDMap.cpp new file mode 100644 index 00000000..6eaf7394 --- /dev/null +++ b/lib/bbackupd/BackupClientInodeToIDMap.cpp @@ -0,0 +1,320 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientInodeToIDMap.cpp +// Purpose: Map of inode numbers to file IDs on the store +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> +#include <depot.h> + +#define BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION +#include "BackupClientInodeToIDMap.h" +#undef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION + +#include "Archive.h" +#include "BackupStoreException.h" +#include "CollectInBufferStream.h" +#include "MemBlockStream.h" +#include "autogen_CommonException.h" + +#include "MemLeakFindOn.h" + +#define BOX_DBM_INODE_DB_VERSION_KEY "BackupClientInodeToIDMap.Version" +#define BOX_DBM_INODE_DB_VERSION_CURRENT 2 + +#define BOX_DBM_MESSAGE(stuff) stuff << " (qdbm): " << dperrmsg(dpecode) + +#define BOX_LOG_DBM_ERROR(stuff) \ + BOX_ERROR(BOX_DBM_MESSAGE(stuff)) + +#define THROW_DBM_ERROR(message, filename, exception, subtype) \ + BOX_LOG_DBM_ERROR(message << ": " << filename); \ + THROW_EXCEPTION_MESSAGE(exception, subtype, \ + BOX_DBM_MESSAGE(message << ": " << filename)); + +#define ASSERT_DBM_OK(operation, message, filename, exception, subtype) \ + if(!(operation)) \ + { \ + THROW_DBM_ERROR(message, filename, exception, subtype); \ + } + +#define ASSERT_DBM_OPEN() \ + if(mpDepot == 0) \ + { \ + THROW_EXCEPTION_MESSAGE(BackupStoreException, InodeMapNotOpen, \ + "Inode database not open"); \ + } + +#define ASSERT_DBM_CLOSED() \ + if(mpDepot != 0) \ + { \ + THROW_EXCEPTION_MESSAGE(CommonException, Internal, \ + "Inode database already open: " << mFilename); \ + } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::BackupClientInodeToIDMap() +// Purpose: Constructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +BackupClientInodeToIDMap::BackupClientInodeToIDMap() + : mReadOnly(true), + mEmpty(false), + mpDepot(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::~BackupClientInodeToIDMap() +// Purpose: Destructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +BackupClientInodeToIDMap::~BackupClientInodeToIDMap() +{ + if(mpDepot != 0) + { + Close(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::Open(const char *, bool, bool) +// Purpose: Open the database map, creating a file on disc to store everything +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::Open(const char *Filename, bool ReadOnly, + bool CreateNew) +{ + mFilename = Filename; + + // Correct arguments? + ASSERT(!(CreateNew && ReadOnly)); + + // Correct usage? + ASSERT_DBM_CLOSED(); + ASSERT(!mEmpty); + + // Open the database file + int mode = ReadOnly ? DP_OREADER : DP_OWRITER; + if(CreateNew) + { + mode |= DP_OCREAT; + } + + mpDepot = dpopen(Filename, mode, 0); + + if(!mpDepot) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + BOX_DBM_MESSAGE("Failed to open inode database: " << + mFilename)); + } + + const char* version_key = BOX_DBM_INODE_DB_VERSION_KEY; + int32_t version = 0; + + if(CreateNew) + { + version = BOX_DBM_INODE_DB_VERSION_CURRENT; + + int ret = dpput(mpDepot, version_key, strlen(version_key), + (char *)(&version), sizeof(version), DP_DKEEP); + + if(!ret) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + BOX_DBM_MESSAGE("Failed to write version number to inode " + "database: " << mFilename)); + } + } + else + { + int ret = dpgetwb(mpDepot, version_key, strlen(version_key), 0, + sizeof(version), (char *)(&version)); + + if(ret == -1) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Missing version number in inode database. Perhaps it " + "needs to be recreated: " << mFilename); + } + + if(ret != sizeof(version)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Wrong size version number in inode database: expected " + << sizeof(version) << " bytes but found " << ret); + } + + if(version != BOX_DBM_INODE_DB_VERSION_CURRENT) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Wrong version number in inode database: expected " << + BOX_DBM_INODE_DB_VERSION_CURRENT << " but found " << + version << ". Perhaps it needs to be recreated: " << + mFilename); + } + + // By this point the version number has been checked and is OK. + } + + // Read only flag + mReadOnly = ReadOnly; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::OpenEmpty() +// Purpose: 'Open' this map. Not associated with a disc file. +// Useful for when a map is required, but is against +// an empty file on disc which shouldn't be created. +// Implies read only. +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::OpenEmpty() +{ + ASSERT_DBM_CLOSED(); + ASSERT(mpDepot == 0); + mEmpty = true; + mReadOnly = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::Close() +// Purpose: Close the database file +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::Close() +{ + ASSERT_DBM_OPEN(); + ASSERT_DBM_OK(dpclose(mpDepot), "Failed to close inode database", + mFilename, BackupStoreException, BerkelyDBFailure); + mpDepot = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::AddToMap(InodeRefType, +// int64_t, int64_t) +// Purpose: Adds an entry to the map. Overwrites any existing +// entry. +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, + int64_t InDirectory, const std::string& LocalPath) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, InodeMapIsReadOnly); + } + + if(mpDepot == 0) + { + THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); + } + + ASSERT_DBM_OPEN(); + + // Setup structures + CollectInBufferStream buf; + Archive arc(buf, IOStream::TimeOutInfinite); + arc.WriteExact((uint64_t)ObjectID); + arc.WriteExact((uint64_t)InDirectory); + arc.Write(LocalPath); + buf.SetForReading(); + + ASSERT_DBM_OK(dpput(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), + (const char *)buf.GetBuffer(), buf.GetSize(), DP_DOVER), + "Failed to add record to inode database", mFilename, + BackupStoreException, BerkelyDBFailure); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::Lookup(InodeRefType, +// int64_t &, int64_t &) const +// Purpose: Looks up an inode in the map, returning true if it +// exists, and the object ids of it and the directory +// it's in the reference arguments. +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, + int64_t &rInDirectoryOut, std::string* pLocalPathOut) const +{ + if(mEmpty) + { + // Map is empty + return false; + } + + if(mpDepot == 0) + { + THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); + } + + ASSERT_DBM_OPEN(); + int size; + char* data = dpget(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), + 0, -1, &size); + if(data == NULL) + { + // key not in file + return false; + } + + // Free data automatically when the guard goes out of scope. + MemoryBlockGuard<char *> guard(data); + MemBlockStream stream(data, size); + Archive arc(stream, IOStream::TimeOutInfinite); + + // Return data + try + { + arc.Read(rObjectIDOut); + arc.Read(rInDirectoryOut); + if(pLocalPathOut) + { + arc.Read(*pLocalPathOut); + } + } + catch(CommonException &e) + { + if(e.GetSubType() == CommonException::ArchiveBlockIncompleteRead) + { + THROW_FILE_ERROR("Failed to lookup record in inode database: " + << InodeRef << ": not enough data in record", mFilename, + BackupStoreException, BerkelyDBFailure); + // Need to throw precisely that exception to ensure that the + // invalid database is deleted, so that we don't hit the same + // error next time. + } + + throw; + } + + // Found + return true; +} diff --git a/lib/bbackupd/BackupClientInodeToIDMap.h b/lib/bbackupd/BackupClientInodeToIDMap.h new file mode 100644 index 00000000..4bb1e085 --- /dev/null +++ b/lib/bbackupd/BackupClientInodeToIDMap.h @@ -0,0 +1,59 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientInodeToIDMap.h +// Purpose: Map of inode numbers to file IDs on the store +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTINODETOIDMAP_H +#define BACKUPCLIENTINODETOIDMAP_H + +#include <sys/types.h> + +#include <map> +#include <utility> + +// avoid having to include the DB files when not necessary +#ifndef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION + class DEPOT; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientInodeToIDMap +// Purpose: Map of inode numbers to file IDs on the store +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +class BackupClientInodeToIDMap +{ +public: + BackupClientInodeToIDMap(); + ~BackupClientInodeToIDMap(); +private: + BackupClientInodeToIDMap(const BackupClientInodeToIDMap &rToCopy); // not allowed +public: + + void Open(const char *Filename, bool ReadOnly, bool CreateNew); + void OpenEmpty(); + + void AddToMap(InodeRefType InodeRef, int64_t ObjectID, + int64_t InDirectory, const std::string& LocalPath); + bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, + int64_t &rInDirectoryOut, std::string* pLocalPathOut = NULL) const; + + void Close(); + +private: + bool mReadOnly; + bool mEmpty; + std::string mFilename; + DEPOT *mpDepot; +}; + +#endif // BACKUPCLIENTINODETOIDMAP_H + + diff --git a/lib/bbackupd/BackupDaemon.cpp b/lib/bbackupd/BackupDaemon.cpp new file mode 100644 index 00000000..03ce7454 --- /dev/null +++ b/lib/bbackupd/BackupDaemon.cpp @@ -0,0 +1,3654 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemon.cpp +// Purpose: Backup daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif +#ifdef HAVE_SIGNAL_H + #include <signal.h> +#endif +#ifdef HAVE_SYS_PARAM_H + #include <sys/param.h> +#endif +#ifdef HAVE_SYS_WAIT_H + #include <sys/wait.h> +#endif +#ifdef HAVE_SYS_MOUNT_H + #include <sys/mount.h> +#endif +#ifdef HAVE_MNTENT_H + #include <mntent.h> +#endif +#ifdef HAVE_SYS_MNTTAB_H + #include <cstdio> + #include <sys/mnttab.h> +#endif +#ifdef HAVE_PROCESS_H + #include <process.h> +#endif + +#include <iostream> +#include <set> +#include <sstream> + +#include "Configuration.h" +#include "IOStream.h" +#include "MemBlockStream.h" +#include "CommonException.h" +#include "BoxPortsAndFiles.h" + +#include "SSLLib.h" + +#include "autogen_BackupProtocol.h" +#include "autogen_ClientException.h" +#include "autogen_CommonException.h" +#include "autogen_ConversionException.h" +#include "Archive.h" +#include "BackupClientContext.h" +#include "BackupClientCryptoKeys.h" +#include "BackupClientDirectoryRecord.h" +#include "BackupClientFileAttributes.h" +#include "BackupClientInodeToIDMap.h" +#include "BackupClientMakeExcludeList.h" +#include "BackupConstants.h" +#include "BackupDaemon.h" +#include "BackupDaemonConfigVerify.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFilenameClear.h" +#include "BannerText.h" +#include "Conversion.h" +#include "ExcludeList.h" +#include "FileStream.h" +#include "IOStreamGetLine.h" +#include "LocalProcessStream.h" +#include "Logging.h" +#include "Random.h" +#include "Timer.h" +#include "Utils.h" + +#ifdef WIN32 + #include "Win32ServiceFunctions.h" + #include "Win32BackupService.h" + + extern Win32BackupService* gpDaemonService; + +# ifdef ENABLE_VSS +# include <comdef.h> +# include <Vss.h> +# include <VsWriter.h> +# include <VsBackup.h> + + // http://www.flounder.com/cstring.htm + std::string GetMsgForHresult(HRESULT hr) + { + std::ostringstream buf; + + if(hr == VSS_S_ASYNC_CANCELLED) + { + buf << "VSS async operation cancelled"; + } + else if(hr == VSS_S_ASYNC_FINISHED) + { + buf << "VSS async operation finished"; + } + else if(hr == VSS_S_ASYNC_PENDING) + { + buf << "VSS async operation pending"; + } + else + { + buf << _com_error(hr).ErrorMessage(); + } + + buf << " (" << BOX_FORMAT_HEX32(hr) << ")"; + return buf.str(); + } + + std::string WideStringToString(WCHAR *buf) + { + if (buf == NULL) + { + return "(null)"; + } + + char* pStr = ConvertFromWideString(buf, CP_UTF8); + + if(pStr == NULL) + { + return "(conversion failed)"; + } + + std::string result(pStr); + free(pStr); + return result; + } + + std::string GuidToString(GUID guid) + { + wchar_t buf[64]; + StringFromGUID2(guid, buf, sizeof(buf)); + return WideStringToString(buf); + } + + std::string BstrToString(const BSTR arg) + { + if(arg == NULL) + { + return std::string("(null)"); + } + else + { + // Extract the *long* before where the arg points to + long len = ((long *)arg)[-1] / 2; + std::wstring wstr((WCHAR *)arg, len); + std::string str; + if(!ConvertFromWideString(wstr, &str, CP_UTF8)) + { + throw std::exception("string conversion failed"); + } + return str; + } + } +# endif + + // Mutex support by Achim: see https://www.boxbackup.org/ticket/67 + + // Creates the two mutexes checked for by the installer/uninstaller to + // see if the program is still running. One of the mutexes is created + // in the global name space (which makes it possible to access the + // mutex across user sessions in Windows XP); the other is created in + // the session name space (because versions of Windows NT prior to + // 4.0 TSE don't have a global name space and don't support the + // 'Global\' prefix). + + void CreateMutexes(const std::string& rName) + { + SECURITY_DESCRIPTOR SecurityDesc; + SECURITY_ATTRIBUTES SecurityAttr; + + /* By default on Windows NT, created mutexes are accessible only by the user + running the process. We need our mutexes to be accessible to all users, so + that the mutex detection can work across user sessions in Windows XP. To + do this we use a security descriptor with a null DACL. + */ + + InitializeSecurityDescriptor(&SecurityDesc, SECURITY_DESCRIPTOR_REVISION); + SetSecurityDescriptorDacl(&SecurityDesc, TRUE, NULL, FALSE); + SecurityAttr.nLength = sizeof(SecurityAttr); + SecurityAttr.lpSecurityDescriptor = &SecurityDesc; + SecurityAttr.bInheritHandle = FALSE; + // We don't care if this succeeds or fails. It's only used to + // ensure that an installer can detect if Box Backup is running. + CreateMutexA(&SecurityAttr, FALSE, rName.c_str()); + std::string global_name = "Global\\" + rName; + CreateMutexA(&SecurityAttr, FALSE, global_name.c_str()); + } +#endif + +#include "MemLeakFindOn.h" + +static const time_t MAX_SLEEP_TIME = 1024; + +// Make the actual sync period have a little bit of extra time, up to a 64th of the main sync period. +// This prevents repetative cycles of load on the server +#define SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY 6 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::BackupDaemon() +// Purpose: constructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupDaemon::BackupDaemon() + : mState(BackupDaemon::State_Initialising), + mDeleteRedundantLocationsAfter(0), + mLastNotifiedEvent(SysadminNotifier::MAX), + mDeleteUnusedRootDirEntriesAfter(0), + mClientStoreMarker(BackupClientContext::ClientStoreMarker_NotKnown), + mStorageLimitExceeded(false), + mReadErrorsOnFilesystemObjects(false), + mLastSyncTime(0), + mNextSyncTime(0), + mCurrentSyncStartTime(0), + mUpdateStoreInterval(0), + mDeleteStoreObjectInfoFile(false), + mDoSyncForcedByPreviousSyncError(false), + mNumFilesUploaded(-1), + mNumDirsCreated(-1), + mMaxBandwidthFromSyncAllowScript(0), + mLogAllFileAccess(false), + mpProgressNotifier(this), + mpLocationResolver(this), + mpRunStatusProvider(this), + mpSysadminNotifier(this), + mapCommandSocketPollTimer(NULL) + #ifdef WIN32 + , mInstallService(false), + mRemoveService(false), + mRunAsService(false), + mServiceName("bbackupd") + #endif +#ifdef ENABLE_VSS + , mpVssBackupComponents(NULL) +#endif +{ + // Only ever one instance of a daemon + SSLLib::Initialise(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::~BackupDaemon() +// Purpose: Destructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupDaemon::~BackupDaemon() +{ + DeleteAllLocations(); + DeleteAllIDMaps(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DaemonName() +// Purpose: Get name of daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +const char *BackupDaemon::DaemonName() const +{ + return "bbackupd"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DaemonBanner() +// Purpose: Daemon banner +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +std::string BackupDaemon::DaemonBanner() const +{ + return BANNER_TEXT("Backup Client"); +} + +void BackupDaemon::Usage() +{ + this->Daemon::Usage(); + +#ifdef WIN32 + std::cout << + " -s Run as a Windows Service, for internal use only\n" + " -i Install Windows Service (you may want to specify a config file)\n" + " -r Remove Windows Service\n" + " -S <name> Service name for -i and -r options\n"; +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::GetConfigVerify() +// Purpose: Get configuration specification +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify *BackupDaemon::GetConfigVerify() const +{ + // Defined elsewhere + return &BackupDaemonConfigVerify; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetupInInitialProcess() +// Purpose: Platforms with non-checkable credentials on +// local sockets only. +// Prints a warning if the command socket is used. +// Created: 25/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetupInInitialProcess() +{ + const Configuration& config(GetConfiguration()); + + // These keys may or may not be required, depending on the configured + // store type (e.g. not when using Amazon S3 stores), so they can't be + // verified by BackupDaemonConfigVerify. + std::vector<std::string> requiredKeys; + requiredKeys.push_back("StoreHostname"); + requiredKeys.push_back("AccountNumber"); + requiredKeys.push_back("CertificateFile"); + requiredKeys.push_back("PrivateKeyFile"); + requiredKeys.push_back("TrustedCAsFile"); + bool missingRequiredKeys = false; + + for(std::vector<std::string>::const_iterator i = requiredKeys.begin(); + i != requiredKeys.end(); i++) + { + if(!config.KeyExists(*i)) + { + BOX_ERROR("Missing required configuration key: " << *i); + missingRequiredKeys = true; + } + } + + if(missingRequiredKeys) + { + THROW_EXCEPTION_MESSAGE(CommonException, InvalidConfiguration, + "Some required configuration keys are missing in " << + GetConfigFileName()); + } + +#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET + // Print a warning on this platform if the CommandSocket is used. + if(GetConfiguration().KeyExists("CommandSocket")) + { + BOX_WARNING( + "==============================================================================\n" + "SECURITY WARNING: This platform cannot check the credentials of connections to\n" + "the command socket. This is a potential DoS security problem.\n" + "Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl\n" + "is not used.\n" + "==============================================================================\n" + ); + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteAllLocations() +// Purpose: Deletes all records stored +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteAllLocations() +{ + // Run through, and delete everything + for(Locations::iterator i = mLocations.begin(); + i != mLocations.end(); ++i) + { + delete *i; + } + + // Clear the contents of the map, so it is empty + mLocations.clear(); + + // And delete everything from the associated mount vector + mIDMapMounts.clear(); +} + +#ifdef WIN32 +std::string BackupDaemon::GetOptionString() +{ + std::string oldOpts = this->Daemon::GetOptionString(); + ASSERT(oldOpts.find("s") == std::string::npos); + ASSERT(oldOpts.find("S") == std::string::npos); + ASSERT(oldOpts.find("i") == std::string::npos); + ASSERT(oldOpts.find("r") == std::string::npos); + return oldOpts + "sS:ir"; +} + +int BackupDaemon::ProcessOption(signed int option) +{ + switch(option) + { + case 's': + { + mRunAsService = true; + return 0; + } + + case 'S': + { + mServiceName = optarg; + Logging::SetProgramName(mServiceName); + return 0; + } + + case 'i': + { + mInstallService = true; + return 0; + } + + case 'r': + { + mRemoveService = true; + return 0; + } + + default: + { + return this->Daemon::ProcessOption(option); + } + } +} + +int BackupDaemon::Main(const std::string &rConfigFileName) +{ + if (mInstallService) + { + return InstallService(rConfigFileName.c_str(), mServiceName); + } + + if (mRemoveService) + { + return RemoveService(mServiceName); + } + +#ifdef ENABLE_VSS + HRESULT result = CoInitialize(NULL); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to initialize COM: " << + GetMsgForHresult(result)); + return 1; + } +#endif + + CreateMutexes("__boxbackup_mutex__"); + + int returnCode; + + if (mRunAsService) + { + // We will be called reentrantly by the Service Control + // Manager, and we had better not call OurService again! + mRunAsService = false; + + BOX_INFO("Box Backup service starting"); + returnCode = OurService(rConfigFileName.c_str()); + BOX_INFO("Box Backup service shut down"); + } + else + { + returnCode = this->Daemon::Main(rConfigFileName); + } + + return returnCode; +} +#endif // WIN32 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Run() +// Purpose: Run function for daemon +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Run() +{ + // initialise global timer mechanism + Timers::Init(); + + mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, + "CommandSocketPollTimer")); + + #ifndef WIN32 + // Ignore SIGPIPE so that if a command connection is broken, + // the daemon doesn't terminate. + ::signal(SIGPIPE, SIG_IGN); + #endif + + // Create a command socket? + const Configuration &conf(GetConfiguration()); + if(conf.KeyExists("CommandSocket")) + { + // Yes, create a local UNIX socket + mapCommandSocketInfo.reset(new CommandSocketInfo); + const char *socketName = + conf.GetKeyValue("CommandSocket").c_str(); + #ifdef WIN32 + mapCommandSocketInfo->mListeningSocket.Listen( + socketName); + #else + ::unlink(socketName); + mapCommandSocketInfo->mListeningSocket.Listen( + Socket::TypeUNIX, socketName); + #endif + } + + // Handle things nicely on exceptions + try + { + Run2(); + } + catch(...) + { + try + { + mapCommandSocketInfo.reset(); + } + catch(std::exception &e) + { + BOX_WARNING("Internal error while closing command " + "socket after another exception, ignored: " << + e.what()); + } + catch(...) + { + BOX_WARNING("Error closing command socket after " + "exception, ignored."); + } + + mapCommandSocketPollTimer.reset(); + Timers::Cleanup(); + + throw; + } + + // Clean up + mapCommandSocketInfo.reset(); + mapCommandSocketPollTimer.reset(); + Timers::Cleanup(); +} + +void BackupDaemon::InitCrypto() +{ + // Read in the certificates creating a TLS context + const Configuration &conf(GetConfiguration()); + std::string certFile(conf.GetKeyValue("CertificateFile")); + std::string keyFile(conf.GetKeyValue("PrivateKeyFile")); + std::string caFile(conf.GetKeyValue("TrustedCAsFile")); + mTlsContext.Initialise(false /* as client */, certFile.c_str(), + keyFile.c_str(), caFile.c_str()); + + // Set up the keys for various things + BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile")); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Run2() +// Purpose: Run function for daemon (second stage) +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Run2() +{ + InitCrypto(); + + const Configuration &conf(GetConfiguration()); + + // How often to connect to the store (approximate) + mUpdateStoreInterval = SecondsToBoxTime( + conf.GetKeyValueInt("UpdateStoreInterval")); + mBackupErrorDelay = conf.GetKeyValueInt("BackupErrorDelay"); + + // But are we connecting automatically? + bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup"); + + // When the next sync should take place -- which is ASAP + mNextSyncTime = 0; + + // When the last sync started (only updated if the store was not full when the sync ended) + mLastSyncTime = 0; + + // -------------------------------------------------------------------------------------------- + + mDeleteStoreObjectInfoFile = DeserializeStoreObjectInfo(mLastSyncTime, + mNextSyncTime); + + // -------------------------------------------------------------------------------------------- + + + // Set state + SetState(State_Idle); + + mDoSyncForcedByPreviousSyncError = false; + + // Loop around doing backups + do + { + // Flags used below + bool storageLimitExceeded = false; + bool doSync = false; + bool mDoSyncForcedByCommand = false; + + // Check whether we should be stopping, and if so, + // don't hang around waiting on the command socket. + if(StopRun()) + { + BOX_INFO("Skipping command socket polling " + "due to shutdown request"); + break; + } + + // Is a delay necessary? + box_time_t currentTime = GetCurrentBoxTime(); + box_time_t requiredDelay = (mNextSyncTime < currentTime) + ? (0) : (mNextSyncTime - currentTime); + mNextSyncTime = currentTime + requiredDelay; + + if (mDoSyncForcedByPreviousSyncError) + { + BOX_INFO("Last backup was not successful, " + "next one starting at " << + FormatTime(mNextSyncTime, false, true)); + } + else if (automaticBackup) + { + BOX_INFO("Automatic backups are enabled, " + "next one starting at " << + FormatTime(mNextSyncTime, false, true)); + } + else + { + BOX_INFO("No automatic backups, waiting for " + "bbackupctl snapshot command"); + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } + + if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME)) + { + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } + + // Only delay if necessary + if(requiredDelay == 0) + { + // No sleep necessary, so don't listen on the command + // socket at all right now. + } + else if(mapCommandSocketInfo.get() != 0) + { + // A command socket exists, so sleep by waiting for a + // connection or command on it. + WaitOnCommandSocket(requiredDelay, doSync, + mDoSyncForcedByCommand); + } + else + { + // No command socket or connection, just do a normal + // sleep. + time_t sleepSeconds = + BoxTimeToSeconds(requiredDelay); + ::sleep((sleepSeconds <= 0) + ? 1 : sleepSeconds); + } + + // We have now slept, so if automaticBackup is enabled then + // it's time for a backup now. + + if(StopRun()) + { + BOX_INFO("Stopping idle loop due to shutdown request"); + break; + } + else if(doSync) + { + BOX_INFO("Starting a backup immediately due to " + "bbackupctl sync command"); + } + else if(GetCurrentBoxTime() < mNextSyncTime) + { + BOX_TRACE("Deadline not reached, sleeping again"); + continue; + } + else if(mDoSyncForcedByPreviousSyncError) + { + BOX_INFO("Last backup was not successful, next one " + "starting now"); + } + else if(!automaticBackup) + { + BOX_TRACE("Sleeping again because automatic backups " + "are not enabled"); + continue; + } + else + { + BOX_INFO("Automatic backups are enabled, next one " + "starting now"); + } + + // If we pass this point, or exit the loop, we should have + // logged something at INFO level or higher to explain why. + + // Use a script to see if sync is allowed now? + if(mDoSyncForcedByCommand) + { + BOX_INFO("Skipping SyncAllowScript due to bbackupctl " + "force-sync command"); + } + else + { + int d = UseScriptToSeeIfSyncAllowed(); + if(d > 0) + { + // Script has asked for a delay + mNextSyncTime = GetCurrentBoxTime() + + SecondsToBoxTime(d); + BOX_INFO("Impending backup stopped by " + "SyncAllowScript, next attempt " + "scheduled for " << + FormatTime(mNextSyncTime, false)); + continue; + } + } + + mCurrentSyncStartTime = GetCurrentBoxTime(); + RunSyncNowWithExceptionHandling(); + + // Set state + SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle); + } + while(!StopRun()); + + // Make sure we have a clean start next time round (if restart) + DeleteAllLocations(); + DeleteAllIDMaps(); +} + +std::auto_ptr<BackupClientContext> BackupDaemon::RunSyncNowWithExceptionHandling() +{ + bool errorOccurred = false; + int errorCode = 0, errorSubCode = 0; + std::string errorString = "unknown"; + + try + { + OnBackupStart(); + // Do sync + RunSyncNow(); + } + catch(BoxException &e) + { + errorOccurred = true; + errorString = e.what(); + errorCode = e.GetType(); + errorSubCode = e.GetSubType(); + } + catch(std::exception &e) + { + BOX_ERROR("Internal error during backup run: " << e.what()); + errorOccurred = true; + errorString = e.what(); + } + catch(...) + { + // TODO: better handling of exceptions here... + // need to be very careful + errorOccurred = true; + } + + // do not retry immediately without a good reason + mDoSyncForcedByPreviousSyncError = false; + + // Is it a berkely db failure? + bool isBerkelyDbFailure = false; + + // Notify system administrator about the final state of the backup + if(errorOccurred) + { + if (errorCode == BackupStoreException::ExceptionType + && errorSubCode == BackupStoreException::BerkelyDBFailure) + { + isBerkelyDbFailure = true; + } + + if(isBerkelyDbFailure) + { + // Delete corrupt files + DeleteCorruptBerkelyDbFiles(); + } + + ResetCachedState(); + + // Handle restart? + if(StopRun()) + { + BOX_NOTICE("Exception (" << errorCode << "/" << + errorSubCode << ") due to signal"); + OnBackupFinish(); + return mapClientContext; // releases mapClientContext + } + + NotifySysadmin(SysadminNotifier::BackupError); + + // If the Berkely db files get corrupted, + // delete them and try again immediately. + if(isBerkelyDbFailure) + { + BOX_ERROR("Berkely db inode map files corrupted, " + "deleting and restarting scan. Renamed files " + "and directories will not be tracked until " + "after this scan."); + ::sleep(1); + } + else + { + // Not restart/terminate, pause and retry + // Notify administrator + SetState(State_Error); + BOX_ERROR("Exception caught (" << errorString << + " " << errorCode << "/" << errorSubCode << + "), reset state and waiting to retry..."); + ::sleep(10); + mNextSyncTime = GetCurrentBoxTime() + + SecondsToBoxTime(mBackupErrorDelay) + + Random::RandomInt(mUpdateStoreInterval >> + SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); + } + } + + if(mReadErrorsOnFilesystemObjects) + { + NotifySysadmin(SysadminNotifier::ReadError); + } + + if(mStorageLimitExceeded) + { + NotifySysadmin(SysadminNotifier::StoreFull); + } + + if (!errorOccurred && !mReadErrorsOnFilesystemObjects && + !mStorageLimitExceeded) + { + NotifySysadmin(SysadminNotifier::BackupOK); + } + + // If we were retrying after an error, and this backup succeeded, + // then now would be a good time to stop :-) + mDoSyncForcedByPreviousSyncError = errorOccurred && !isBerkelyDbFailure; + + OnBackupFinish(); + return mapClientContext; // releases mapClientContext +} + +void BackupDaemon::ResetCachedState() +{ + // Clear state data + // Go back to beginning of time + mLastSyncTime = 0; + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything + DeleteAllLocations(); + DeleteAllIDMaps(); +} + +std::auto_ptr<BackupClientContext> BackupDaemon::GetNewContext +( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode +) +{ + std::auto_ptr<BackupClientContext> context(new BackupClientContext( + rResolver, rTLSContext, rHostname, Port, AccountNumber, + ExtendedLogging, ExtendedLogToFile, ExtendedLogFile, + rProgressNotifier, TcpNiceMode)); + return context; +} + +// Returns the BackupClientContext so that tests can use it to hold the +// connection open and prevent housekeeping from running. Otherwise don't use +// it, let it be destroyed and close the connection. +std::auto_ptr<BackupClientContext> BackupDaemon::RunSyncNow() +{ + Timers::AssertInitialised(); + + // Delete the serialised store object file, + // so that we don't try to reload it after a + // partially completed backup + if(mDeleteStoreObjectInfoFile && !DeleteStoreObjectInfo()) + { + BOX_ERROR("Failed to delete the StoreObjectInfoFile, " + "backup cannot continue safely."); + THROW_EXCEPTION(ClientException, + FailedToDeleteStoreObjectInfoFile); + } + + // In case the backup throws an exception, + // we should not try to delete the store info + // object file again. + mDeleteStoreObjectInfoFile = false; + + const Configuration &conf(GetConfiguration()); + + std::auto_ptr<FileLogger> fileLogger; + + if (conf.KeyExists("LogFile")) + { + bool overwrite = false; + if (conf.KeyExists("LogFileOverwrite")) + { + overwrite = conf.GetKeyValueBool("LogFileOverwrite"); + } + + Log::Level level = Log::INFO; + if (conf.KeyExists("LogFileLevel")) + { + level = Logging::GetNamedLevel( + conf.GetKeyValue("LogFileLevel")); + } + + fileLogger.reset(new FileLogger(conf.GetKeyValue("LogFile"), + level, !overwrite)); + } + + std::string extendedLogFile; + if (conf.KeyExists("ExtendedLogFile")) + { + extendedLogFile = conf.GetKeyValue("ExtendedLogFile"); + } + + if (conf.KeyExists("LogAllFileAccess")) + { + mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess"); + } + + // Then create a client context object (don't + // just connect, as this may be unnecessary) + mapClientContext = GetNewContext( + *mpLocationResolver, + mTlsContext, + conf.GetKeyValue("StoreHostname"), + conf.GetKeyValueInt("StorePort"), + conf.GetKeyValueUint32("AccountNumber"), + conf.GetKeyValueBool("ExtendedLogging"), + conf.KeyExists("ExtendedLogFile"), + extendedLogFile, + *mpProgressNotifier, + conf.GetKeyValueBool("TcpNice") + ); + + // The minimum age a file needs to be before it will be + // considered for uploading + box_time_t minimumFileAge = SecondsToBoxTime( + conf.GetKeyValueInt("MinimumFileAge")); + + // The maximum time we'll wait to upload a file, regardless + // of how often it's modified + box_time_t maxUploadWait = SecondsToBoxTime( + conf.GetKeyValueInt("MaxUploadWait")); + // Adjust by subtracting the minimum file age, so is relative + // to sync period end in comparisons + if (maxUploadWait > minimumFileAge) + { + maxUploadWait -= minimumFileAge; + } + else + { + maxUploadWait = 0; + } + + // Calculate the sync period of files to examine + box_time_t syncPeriodStart = mLastSyncTime; + box_time_t syncPeriodEnd = GetCurrentBoxTime() - minimumFileAge; + + if(syncPeriodStart >= syncPeriodEnd && + syncPeriodStart - syncPeriodEnd < minimumFileAge) + { + // This can happen if we receive a force-sync command less + // than minimumFileAge after the last sync. Deal with it by + // moving back syncPeriodStart, which should not do any + // damage. + syncPeriodStart = syncPeriodEnd - SecondsToBoxTime(1); + } + + if(syncPeriodStart >= syncPeriodEnd) + { + BOX_ERROR("Invalid (negative) sync period: perhaps your clock " + "is going backwards? (" << syncPeriodStart << " to " << + syncPeriodEnd << ")"); + THROW_EXCEPTION(ClientException, ClockWentBackwards); + } + + // Check logic + ASSERT(syncPeriodEnd > syncPeriodStart); + // Paranoid check on sync times + if(syncPeriodStart >= syncPeriodEnd) + { + return mapClientContext; // releases mapClientContext + } + + // Adjust syncPeriodEnd to emulate snapshot behaviour properly + box_time_t syncPeriodEndExtended = syncPeriodEnd; + + // Using zero min file age? + if(minimumFileAge == 0) + { + // Add a year on to the end of the end time, + // to make sure we sync files which are + // modified after the scan run started. + // Of course, they may be eligible to be + // synced again the next time round, + // but this should be OK, because the changes + // only upload should upload no data. + syncPeriodEndExtended += SecondsToBoxTime( + (time_t)(356*24*3600)); + } + + // Set up the sync parameters + BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider, + *mpSysadminNotifier, *mpProgressNotifier, *mapClientContext, this); + params.mSyncPeriodStart = syncPeriodStart; + params.mSyncPeriodEnd = syncPeriodEndExtended; + // use potentially extended end time + params.mMaxUploadWait = maxUploadWait; + params.mFileTrackingSizeThreshold = + conf.GetKeyValueInt("FileTrackingSizeThreshold"); + params.mDiffingUploadSizeThreshold = + conf.GetKeyValueInt("DiffingUploadSizeThreshold"); + params.mMaxFileTimeInFuture = + SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture")); + mNumFilesUploaded = 0; + mNumDirsCreated = 0; + + if(conf.KeyExists("MaxUploadRate")) + { + params.mMaxUploadRate = conf.GetKeyValueInt("MaxUploadRate"); + } + + if(mMaxBandwidthFromSyncAllowScript != 0) + { + params.mMaxUploadRate = mMaxBandwidthFromSyncAllowScript; + } + + mDeleteRedundantLocationsAfter = + conf.GetKeyValueInt("DeleteRedundantLocationsAfter"); + mStorageLimitExceeded = false; + mReadErrorsOnFilesystemObjects = false; + + // Setup various timings + int maximumDiffingTime = 600; + int keepAliveTime = 60; + + // max diffing time, keep-alive time + if(conf.KeyExists("MaximumDiffingTime")) + { + maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime"); + } + if(conf.KeyExists("KeepAliveTime")) + { + keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); + } + + mapClientContext->SetMaximumDiffingTime(maximumDiffingTime); + mapClientContext->SetKeepAliveTime(keepAliveTime); + + // Set store marker + mapClientContext->SetClientStoreMarker(mClientStoreMarker); + + // Set up the locations, if necessary -- need to do it here so we have + // a (potential) connection to use. + { + const Configuration &locations( + conf.GetSubConfiguration( + "BackupLocations")); + + // Make sure all the directory records + // are set up + SetupLocations(*mapClientContext, locations); + } + + mpProgressNotifier->NotifyIDMapsSetup(*mapClientContext); + + // Get some ID maps going + SetupIDMapsForSync(); + + // Delete any unused directories? + DeleteUnusedRootDirEntries(*mapClientContext); + +#ifdef ENABLE_VSS + CreateVssBackupComponents(); +#endif + + // Go through the records, syncing them + for(Locations::const_iterator + i(mLocations.begin()); + i != mLocations.end(); ++i) + { + // Set current and new ID map pointers + // in the context + mapClientContext->SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], + mNewIDMaps[(*i)->mIDMapIndex]); + + // Set exclude lists (context doesn't + // take ownership) + mapClientContext->SetExcludeLists( + (*i)->mapExcludeFiles.get(), + (*i)->mapExcludeDirs.get()); + + // Sync the directory + std::string locationPath = (*i)->mPath; +#ifdef ENABLE_VSS + if((*i)->mIsSnapshotCreated) + { + locationPath = (*i)->mSnapshotPath; + } +#endif + + (*i)->mapDirectoryRecord->SyncDirectory(params, + BackupProtocolListDirectory::RootDirectory, + locationPath, std::string("/") + (*i)->mName, **i); + + // Unset exclude lists (just in case) + mapClientContext->SetExcludeLists(0, 0); + } + + // Perform any deletions required -- these are + // delayed until the end to allow renaming to + // happen neatly. + mapClientContext->PerformDeletions(); + +#ifdef ENABLE_VSS + CleanupVssBackupComponents(); +#endif + + // Get the new store marker + mClientStoreMarker = mapClientContext->GetClientStoreMarker(); + mStorageLimitExceeded = mapClientContext->StorageLimitExceeded(); + mReadErrorsOnFilesystemObjects |= + params.mReadErrorsOnFilesystemObjects; + + if(!mStorageLimitExceeded) + { + // The start time of the next run is the end time of this + // run. This is only done if the storage limit wasn't + // exceeded (as things won't have been done properly if + // it was) + mLastSyncTime = syncPeriodEnd; + } + + // Commit the ID Maps + CommitIDMapsAfterSync(); + + // Calculate when the next sync run should be + mNextSyncTime = mCurrentSyncStartTime + + mUpdateStoreInterval + + Random::RandomInt(mUpdateStoreInterval >> + SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); + + // -------------------------------------------------------------------------------------------- + + // We had a successful backup, save the store + // info. If we save successfully, we must + // delete the file next time we start a backup + + mDeleteStoreObjectInfoFile = + SerializeStoreObjectInfo(mLastSyncTime, mNextSyncTime); + + // -------------------------------------------------------------------------------------------- + + return mapClientContext; // releases mapClientContext +} + +#ifdef ENABLE_VSS +bool BackupDaemon::WaitForAsync(IVssAsync *pAsync, + const std::string& description) +{ + BOX_INFO("VSS: waiting for " << description << " to complete"); + HRESULT result; + + do + { + result = pAsync->Wait(1000); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to wait for " << description << + " to complete: " << GetMsgForHresult(result)); + break; + } + + HRESULT result2; + result = pAsync->QueryStatus(&result2, NULL); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query " << description << + " status: " << GetMsgForHresult(result)); + break; + } + + result = result2; + BOX_INFO("VSS: " << description << " status: " << + GetMsgForHresult(result)); + } + while(result == VSS_S_ASYNC_PENDING); + + pAsync->Release(); + + return (result == VSS_S_ASYNC_FINISHED); +} + +#define CALL_MEMBER_FN(object, method) ((object).*(method)) + +bool BackupDaemon::CallAndWaitForAsync(AsyncMethod method, + const std::string& description) +{ + IVssAsync *pAsync; + HRESULT result = CALL_MEMBER_FN(*mpVssBackupComponents, method)(&pAsync); + if(result != S_OK) + { + BOX_ERROR("VSS: " << description << " failed: " << + GetMsgForHresult(result)); + return false; + } + + return WaitForAsync(pAsync, description); +} + +void BackupDaemon::CreateVssBackupComponents() +{ + std::map<char, VSS_ID> volumesIncluded; + + HRESULT result = ::CreateVssBackupComponents(&mpVssBackupComponents); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to create backup components: " << + GetMsgForHresult(result)); + return; + } + + result = mpVssBackupComponents->InitializeForBackup(NULL); + if(result != S_OK) + { + std::string message = GetMsgForHresult(result); + + if (result == VSS_E_UNEXPECTED) + { + message = "Check the Application Log for details, and ensure " + "that the Volume Shadow Copy, COM+ System Application, " + "and Distributed Transaction Coordinator services " + "are running"; + } + + BOX_ERROR("VSS: Failed to initialize for backup: " << message); + return; + } + + result = mpVssBackupComponents->SetContext(VSS_CTX_BACKUP); + if(result == E_NOTIMPL) + { + BOX_INFO("VSS: Failed to set context to VSS_CTX_BACKUP: " + "not implemented, probably Windows XP, ignored."); + } + else if(result != S_OK) + { + BOX_ERROR("VSS: Failed to set context to VSS_CTX_BACKUP: " << + GetMsgForHresult(result)); + return; + } + + result = mpVssBackupComponents->SetBackupState( + false, /* no components for now */ + true, /* might as well ask for a bootable backup */ + VSS_BT_FULL, + false /* what is Partial File Support? */); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to set backup state: " << + GetMsgForHresult(result)); + return; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterMetadata, + "GatherWriterMetadata()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + UINT writerCount; + result = mpVssBackupComponents->GetWriterMetadataCount(&writerCount); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get writer count: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + for(UINT iWriter = 0; iWriter < writerCount; iWriter++) + { + BOX_INFO("VSS: Getting metadata from writer " << iWriter); + VSS_ID writerInstance; + IVssExamineWriterMetadata* pMetadata; + result = mpVssBackupComponents->GetWriterMetadata(iWriter, + &writerInstance, &pMetadata); + if(result != S_OK) + { + BOX_ERROR("Failed to get VSS metadata from writer " << iWriter << + ": " << GetMsgForHresult(result)); + continue; + } + + UINT includeFiles, excludeFiles, numComponents; + result = pMetadata->GetFileCounts(&includeFiles, &excludeFiles, + &numComponents); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata file counts from " + "writer " << iWriter << ": " << + GetMsgForHresult(result)); + pMetadata->Release(); + continue; + } + + for(UINT iComponent = 0; iComponent < numComponents; iComponent++) + { + IVssWMComponent* pComponent; + result = pMetadata->GetComponent(iComponent, &pComponent); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata component " << + iComponent << " from writer " << iWriter << ": " << + GetMsgForHresult(result)); + continue; + } + + PVSSCOMPONENTINFO pComponentInfo; + result = pComponent->GetComponentInfo(&pComponentInfo); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata component " << + iComponent << " info from writer " << iWriter << ": " << + GetMsgForHresult(result)); + pComponent->Release(); + continue; + } + + BOX_TRACE("VSS: writer " << iWriter << " component " << + iComponent << " info:"); + switch(pComponentInfo->type) + { + case VSS_CT_UNDEFINED: BOX_TRACE("VSS: type: undefined"); break; + case VSS_CT_DATABASE: BOX_TRACE("VSS: type: database"); break; + case VSS_CT_FILEGROUP: BOX_TRACE("VSS: type: filegroup"); break; + default: + BOX_WARNING("VSS: type: unknown (" << pComponentInfo->type << ")"); + } + + BOX_TRACE("VSS: logical path: " << + BstrToString(pComponentInfo->bstrLogicalPath)); + BOX_TRACE("VSS: component name: " << + BstrToString(pComponentInfo->bstrComponentName)); + BOX_TRACE("VSS: caption: " << + BstrToString(pComponentInfo->bstrCaption)); + BOX_TRACE("VSS: restore metadata: " << + pComponentInfo->bRestoreMetadata); + BOX_TRACE("VSS: notify on complete: " << + pComponentInfo->bRestoreMetadata); + BOX_TRACE("VSS: selectable: " << + pComponentInfo->bSelectable); + BOX_TRACE("VSS: selectable for restore: " << + pComponentInfo->bSelectableForRestore); + BOX_TRACE("VSS: component flags: " << + BOX_FORMAT_HEX32(pComponentInfo->dwComponentFlags)); + BOX_TRACE("VSS: file count: " << + pComponentInfo->cFileCount); + BOX_TRACE("VSS: databases: " << + pComponentInfo->cDatabases); + BOX_TRACE("VSS: log files: " << + pComponentInfo->cLogFiles); + BOX_TRACE("VSS: dependencies: " << + pComponentInfo->cDependencies); + + pComponent->FreeComponentInfo(pComponentInfo); + pComponent->Release(); + } + + pMetadata->Release(); + } + + VSS_ID snapshotSetId; + result = mpVssBackupComponents->StartSnapshotSet(&snapshotSetId); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to start snapshot set: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + // Add all volumes included as backup locations to the snapshot set + for(Locations::iterator + iLocation = mLocations.begin(); + iLocation != mLocations.end(); + iLocation++) + { + Location& rLocation(**iLocation); + std::string path = rLocation.mPath; + // convert to absolute and remove Unicode prefix + path = ConvertPathToAbsoluteUnicode(path.c_str()).substr(4); + + if(path.length() >= 3 && path[1] == ':' && path[2] == '\\') + { + std::string volumeRoot = path.substr(0, 3); + + std::map<char, VSS_ID>::iterator i = + volumesIncluded.find(path[0]); + + if(i == volumesIncluded.end()) + { + std::wstring volumeRootWide; + volumeRootWide.push_back((WCHAR) path[0]); + volumeRootWide.push_back((WCHAR) ':'); + volumeRootWide.push_back((WCHAR) '\\'); + VSS_ID newVolumeId; + result = mpVssBackupComponents->AddToSnapshotSet( + (VSS_PWSZ)(volumeRootWide.c_str()), GUID_NULL, + &newVolumeId); + if(result == S_OK) + { + BOX_TRACE("VSS: Added volume " << volumeRoot << + " for backup location " << path << + " to snapshot set"); + volumesIncluded[path[0]] = newVolumeId; + rLocation.mSnapshotVolumeId = newVolumeId; + } + else + { + BOX_ERROR("VSS: Failed to add volume " << + volumeRoot << " to snapshot set: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + } + else + { + BOX_TRACE("VSS: Skipping already included volume " << + volumeRoot << " for backup location " << path); + rLocation.mSnapshotVolumeId = i->second; + } + + rLocation.mIsSnapshotCreated = true; + + // If the snapshot path starts with the volume root + // (drive letter), because the path is absolute (as it + // should be), then remove it so that the resulting + // snapshot path can be appended to the snapshot device + // object to make a real path, without a spurious drive + // letter in it. + + if (path.substr(0, volumeRoot.length()) == volumeRoot) + { + path = path.substr(volumeRoot.length()); + } + + rLocation.mSnapshotPath = path; + } + else + { + BOX_WARNING("VSS: Skipping backup location " << path << + " which does not start with a volume specification"); + } + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::PrepareForBackup, + "PrepareForBackup()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::DoSnapshotSet, + "DoSnapshotSet()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterStatus, + "GatherWriterStatus()")) + { + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + result = mpVssBackupComponents->GetWriterStatusCount(&writerCount); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get writer status count: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + for(UINT iWriter = 0; iWriter < writerCount; iWriter++) + { + VSS_ID instance, writer; + BSTR writerNameBstr; + VSS_WRITER_STATE writerState; + HRESULT writerResult; + + result = mpVssBackupComponents->GetWriterStatus(iWriter, + &instance, &writer, &writerNameBstr, &writerState, + &writerResult); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query writer " << iWriter << + " status: " << GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + std::string writerName = BstrToString(writerNameBstr); + ::SysFreeString(writerNameBstr); + + if(writerResult != S_OK) + { + BOX_ERROR("VSS: Writer " << iWriter << " (" << + writerName << ") failed: " << + GetMsgForHresult(writerResult)); + continue; + } + + std::string stateName; + + switch(writerState) + { +#define WRITER_STATE(code) \ + case code: stateName = #code; break; + WRITER_STATE(VSS_WS_UNKNOWN); + WRITER_STATE(VSS_WS_STABLE); + WRITER_STATE(VSS_WS_WAITING_FOR_FREEZE); + WRITER_STATE(VSS_WS_WAITING_FOR_THAW); + WRITER_STATE(VSS_WS_WAITING_FOR_POST_SNAPSHOT); + WRITER_STATE(VSS_WS_WAITING_FOR_BACKUP_COMPLETE); + WRITER_STATE(VSS_WS_FAILED_AT_IDENTIFY); + WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_BACKUP); + WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_SNAPSHOT); + WRITER_STATE(VSS_WS_FAILED_AT_FREEZE); + WRITER_STATE(VSS_WS_FAILED_AT_THAW); + WRITER_STATE(VSS_WS_FAILED_AT_POST_SNAPSHOT); + WRITER_STATE(VSS_WS_FAILED_AT_BACKUP_COMPLETE); + WRITER_STATE(VSS_WS_FAILED_AT_PRE_RESTORE); + WRITER_STATE(VSS_WS_FAILED_AT_POST_RESTORE); + WRITER_STATE(VSS_WS_FAILED_AT_BACKUPSHUTDOWN); +#undef WRITER_STATE + default: + std::ostringstream o; + o << "unknown (" << writerState << ")"; + stateName = o.str(); + } + + BOX_TRACE("VSS: Writer " << iWriter << " (" << + writerName << ") is in state " << stateName); + } + + // lookup new snapshot volume for each location that has a snapshot + for(Locations::iterator + iLocation = mLocations.begin(); + iLocation != mLocations.end(); + iLocation++) + { + Location& rLocation(**iLocation); + if(rLocation.mIsSnapshotCreated) + { + VSS_SNAPSHOT_PROP prop; + result = mpVssBackupComponents->GetSnapshotProperties( + rLocation.mSnapshotVolumeId, &prop); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get snapshot properties " + "for volume " << GuidToString(rLocation.mSnapshotVolumeId) << + " for location " << rLocation.mPath << ": " << + GetMsgForHresult(result)); + rLocation.mIsSnapshotCreated = false; + continue; + } + + rLocation.mSnapshotPath = + WideStringToString(prop.m_pwszSnapshotDeviceObject) + + DIRECTORY_SEPARATOR + rLocation.mSnapshotPath; + VssFreeSnapshotProperties(&prop); + + BOX_INFO("VSS: Location " << rLocation.mPath << " using " + "snapshot path " << rLocation.mSnapshotPath); + } + } + + IVssEnumObject *pEnum; + result = mpVssBackupComponents->Query(GUID_NULL, VSS_OBJECT_NONE, + VSS_OBJECT_SNAPSHOT, &pEnum); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query snapshot list: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + while(result == S_OK) + { + VSS_OBJECT_PROP rgelt; + ULONG count; + result = pEnum->Next(1, &rgelt, &count); + + if(result == S_FALSE) + { + // end of list, break out of the loop + break; + } + else if(result != S_OK) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + GetMsgForHresult(result)); + } + else if(count != 1) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + "Next() returned " << count << " objects instead of 1"); + } + else if(rgelt.Type != VSS_OBJECT_SNAPSHOT) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + "Next() returned a type " << rgelt.Type << " object " + "instead of VSS_OBJECT_SNAPSHOT"); + } + else + { + VSS_SNAPSHOT_PROP *pSnap = &rgelt.Obj.Snap; + BOX_TRACE("VSS: Snapshot ID: " << + GuidToString(pSnap->m_SnapshotId)); + BOX_TRACE("VSS: Snapshot set ID: " << + GuidToString(pSnap->m_SnapshotSetId)); + BOX_TRACE("VSS: Number of volumes: " << + pSnap->m_lSnapshotsCount); + BOX_TRACE("VSS: Snapshot device object: " << + WideStringToString(pSnap->m_pwszSnapshotDeviceObject)); + BOX_TRACE("VSS: Original volume name: " << + WideStringToString(pSnap->m_pwszOriginalVolumeName)); + BOX_TRACE("VSS: Originating machine: " << + WideStringToString(pSnap->m_pwszOriginatingMachine)); + BOX_TRACE("VSS: Service machine: " << + WideStringToString(pSnap->m_pwszServiceMachine)); + BOX_TRACE("VSS: Exposed name: " << + WideStringToString(pSnap->m_pwszExposedName)); + BOX_TRACE("VSS: Exposed path: " << + WideStringToString(pSnap->m_pwszExposedPath)); + BOX_TRACE("VSS: Provider ID: " << + GuidToString(pSnap->m_ProviderId)); + BOX_TRACE("VSS: Snapshot attributes: " << + BOX_FORMAT_HEX32(pSnap->m_lSnapshotAttributes)); + BOX_TRACE("VSS: Snapshot creation time: " << + BOX_FORMAT_HEX32(pSnap->m_tsCreationTimestamp)); + + std::string status; + switch(pSnap->m_eStatus) + { + case VSS_SS_UNKNOWN: status = "Unknown (error)"; break; + case VSS_SS_PREPARING: status = "Preparing"; break; + case VSS_SS_PROCESSING_PREPARE: status = "Preparing (processing)"; break; + case VSS_SS_PREPARED: status = "Prepared"; break; + case VSS_SS_PROCESSING_PRECOMMIT: status = "Precommitting"; break; + case VSS_SS_PRECOMMITTED: status = "Precommitted"; break; + case VSS_SS_PROCESSING_COMMIT: status = "Commiting"; break; + case VSS_SS_COMMITTED: status = "Committed"; break; + case VSS_SS_PROCESSING_POSTCOMMIT: status = "Postcommitting"; break; + case VSS_SS_PROCESSING_PREFINALCOMMIT: status = "Pre final committing"; break; + case VSS_SS_PREFINALCOMMITTED: status = "Pre final committed"; break; + case VSS_SS_PROCESSING_POSTFINALCOMMIT: status = "Post final committing"; break; + case VSS_SS_CREATED: status = "Created"; break; + case VSS_SS_ABORTED: status = "Aborted"; break; + case VSS_SS_DELETED: status = "Deleted"; break; + case VSS_SS_POSTCOMMITTED: status = "Postcommitted"; break; + default: + std::ostringstream buf; + buf << "Unknown code: " << pSnap->m_eStatus; + status = buf.str(); + } + + BOX_TRACE("VSS: Snapshot status: " << status); + VssFreeSnapshotProperties(pSnap); + } + } + + pEnum->Release(); + +CreateVssBackupComponents_cleanup_WriterStatus: + result = mpVssBackupComponents->FreeWriterStatus(); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to free writer status: " << + GetMsgForHresult(result)); + } + +CreateVssBackupComponents_cleanup_WriterMetadata: + result = mpVssBackupComponents->FreeWriterMetadata(); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to free writer metadata: " << + GetMsgForHresult(result)); + } +} + +void BackupDaemon::CleanupVssBackupComponents() +{ + if(mpVssBackupComponents == NULL) + { + return; + } + + CallAndWaitForAsync(&IVssBackupComponents::BackupComplete, + "BackupComplete()"); + + mpVssBackupComponents->Release(); + mpVssBackupComponents = NULL; +} +#endif + +void BackupDaemon::OnBackupStart() +{ + ResetLogFile(); + + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_start"); + + // Reset statistics on uploads + BackupStoreFile::ResetStats(); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(true /* start */); + + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupStart); + + // Setup timer for polling the command socket + mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, + "CommandSocketPollTimer")); + + // Set state and log start + SetState(State_Connected); + BOX_NOTICE("Beginning scan of local files"); +} + +void BackupDaemon::OnBackupFinish() +{ + try + { + // Log + BOX_NOTICE("Finished scan of local files"); + + // Log the stats + BOX_NOTICE("File statistics: total file size uploaded " + << BackupStoreFile::msStats.mBytesInEncodedFiles + << ", bytes already on server " + << BackupStoreFile::msStats.mBytesAlreadyOnServer + << ", encoded size " + << BackupStoreFile::msStats.mTotalFileStreamSize + << ", " << mNumFilesUploaded << " files uploaded, " + << mNumDirsCreated << " dirs created"); + + // Reset statistics again + BackupStoreFile::ResetStats(); + + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupFinish); + + // Stop the timer for polling the command socket, + // to prevent needless alarms while sleeping. + mapCommandSocketPollTimer.reset(); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(false /* finish */); + + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_finish"); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to perform backup finish actions: " << e.what()); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::UseScriptToSeeIfSyncAllowed() +// Purpose: Private. Use a script to see if the sync should be +// allowed now (if configured). Returns -1 if it's +// allowed, time in seconds to wait otherwise. +// Created: 21/6/04 +// +// -------------------------------------------------------------------------- +int BackupDaemon::UseScriptToSeeIfSyncAllowed() +{ + const Configuration &conf(GetConfiguration()); + + // Got a script to run? + if(!conf.KeyExists("SyncAllowScript")) + { + // No. Do sync. + return -1; + } + + // If there's no result, try again in five minutes + int waitInSeconds = (60*5); + + std::string script(conf.GetKeyValue("SyncAllowScript") + + " \"" + GetConfigFileName() + "\""); + + // Run it? + pid_t pid = 0; + try + { + std::auto_ptr<IOStream> pscript(LocalProcessStream(script, + pid)); + + // Read in the result + IOStreamGetLine getLine(*pscript); + std::string line; + if(getLine.GetLine(line, true, 30000)) // 30 seconds should be enough + { + waitInSeconds = BackupDaemon::ParseSyncAllowScriptOutput(script, line); + } + else + { + BOX_ERROR("SyncAllowScript output nothing within " + "30 seconds, waiting 5 minutes to try again" + " (" << script << ")"); + } + } + catch(std::exception &e) + { + BOX_ERROR("Internal error running SyncAllowScript: " + << e.what() << " (" << script << ")"); + } + catch(...) + { + // Ignore any exceptions + // Log that something bad happened + BOX_ERROR("Unknown error running SyncAllowScript (" << + script << ")"); + } + + // Wait and then cleanup child process, if any + if(pid != 0) + { + int status = 0; + ::waitpid(pid, &status, 0); + } + + return waitInSeconds; +} + +int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, + const std::string& output) +{ + int waitInSeconds = (60*5); + std::istringstream iss(output); + + std::string delay; + iss >> delay; + + if(delay == "") + { + BOX_ERROR("SyncAllowScript output an empty line, sleeping for " + << waitInSeconds << " seconds (" << script << ")"); + return waitInSeconds; + } + + // Got a string, interpret + if(delay == "now") + { + // Script says do it now. Obey. + waitInSeconds = -1; + + BOX_NOTICE("SyncAllowScript requested a backup now " + "(" << script << ")"); + } + else + { + try + { + // How many seconds to wait? + waitInSeconds = BoxConvert::Convert<int32_t, const std::string&>(delay); + } + catch(ConversionException &e) + { + BOX_ERROR("SyncAllowScript output an invalid " + "number: '" << output << "' (" << + script << ")"); + throw; + } + + BOX_NOTICE("SyncAllowScript requested a delay of " << + waitInSeconds << " seconds (" << script << ")"); + } + + if(iss.eof()) + { + // No bandwidth limit requested + mMaxBandwidthFromSyncAllowScript = 0; + BOX_NOTICE("SyncAllowScript did not set a maximum bandwidth " + "(" << script << ")"); + } + else + { + std::string maxBandwidth; + iss >> maxBandwidth; + + try + { + // How many seconds to wait? + mMaxBandwidthFromSyncAllowScript = + BoxConvert::Convert<int32_t, const std::string&>(maxBandwidth); + } + catch(ConversionException &e) + { + BOX_ERROR("Invalid maximum bandwidth from " + "SyncAllowScript: '" << + output << "' (" << script << ")"); + throw; + } + + BOX_NOTICE("SyncAllowScript set maximum bandwidth to " << + mMaxBandwidthFromSyncAllowScript << " kB/s (" << + script << ")"); + } + + return waitInSeconds; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::RunBackgroundTask() +// Purpose: Checks for connections or commands on the command +// socket and handles them with minimal delay. Polled +// during lengthy operations such as file uploads. +// Created: 07/04/14 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum) +{ + BOX_TRACE("BackupDaemon::RunBackgroundTask: state = " << state << + ", progress = " << progress << "/" << maximum); + + if(!mapCommandSocketPollTimer.get()) + { + return true; // no background task + } + + if(mapCommandSocketPollTimer->HasExpired()) + { + mapCommandSocketPollTimer->Reset(COMMAND_SOCKET_POLL_INTERVAL); + } + else + { + // Do no more work right now + return true; + } + + if(mapCommandSocketInfo.get()) + { + BOX_TRACE("BackupDaemon::RunBackgroundTask: polling command socket"); + + bool sync_flag_out, sync_is_forced_out; + + WaitOnCommandSocket(0, // RequiredDelay + sync_flag_out, sync_is_forced_out); + + if(sync_flag_out) + { + BOX_WARNING("Ignoring request to sync while " + "already syncing."); + } + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::WaitOnCommandSocket(box_time_t, bool &, bool &) +// Purpose: Waits on a the command socket for a time of UP TO +// the required time but may be much less, and handles +// a command if necessary. +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut) +{ + DoSyncFlagOut = false; + SyncIsForcedOut = false; + + ASSERT(mapCommandSocketInfo.get()); + if(!mapCommandSocketInfo.get()) + { + // failure case isn't too bad + ::sleep(1); + return; + } + + BOX_TRACE("Wait on command socket, delay = " << + BOX_FORMAT_MICROSECONDS(RequiredDelay)); + + try + { + // Timeout value for connections and things + int timeout = ((int)BoxTimeToMilliSeconds(RequiredDelay)) + 1; + // Handle bad boundary cases + if(timeout <= 0) timeout = 1; + if(timeout == INFTIM) timeout = 100000; + + // Wait for socket connection, or handle a command? + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + // No connection, listen for a new one + mapCommandSocketInfo->mpConnectedSocket.reset(mapCommandSocketInfo->mListeningSocket.Accept(timeout).release()); + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + // If a connection didn't arrive, there was a timeout, which means we've + // waited long enough and it's time to go. + return; + } + else + { +#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET + bool uidOK = true; + BOX_WARNING("On this platform, no security check can be made on the credentials of peers connecting to the command socket. (bbackupctl)"); +#else + // Security check -- does the process connecting to this socket have + // the same UID as this process? + bool uidOK = false; + // BLOCK + { + uid_t remoteEUID = 0xffff; + gid_t remoteEGID = 0xffff; + if(mapCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID)) + { + // Credentials are available -- check UID + if(remoteEUID == ::getuid()) + { + // Acceptable + uidOK = true; + } + } + } +#endif // PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET + + // Is this an acceptable connection? + if(!uidOK) + { + // Dump the connection + BOX_ERROR("Incoming command connection from peer had different user ID than this process, or security check could not be completed."); + mapCommandSocketInfo->mpConnectedSocket.reset(); + return; + } + else + { + // Log + BOX_INFO("Connection from command socket"); + + // Send a header line summarising the configuration and current state + const Configuration &conf(GetConfiguration()); + std::ostringstream hello; + hello << "bbackupd: " << + (conf.GetKeyValueBool("AutomaticBackup") ? 1 : 0) + << " " << + conf.GetKeyValueInt("UpdateStoreInterval") + << " " << + conf.GetKeyValueInt("MinimumFileAge") + << " " << + conf.GetKeyValueInt("MaxUploadWait") + << "\nstate " << mState << "\n"; + mapCommandSocketInfo->mpConnectedSocket->Write( + hello.str(), timeout); + + // Set the timeout to something very small, so we don't wait too long on waiting + // for any incoming data + timeout = 10; // milliseconds + } + } + } + + // So there must be a connection now. + ASSERT(mapCommandSocketInfo->mpConnectedSocket.get() != 0); + + // Is there a getline object ready? + if(mapCommandSocketInfo->mpGetLine == 0) + { + // Create a new one + mapCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mapCommandSocketInfo->mpConnectedSocket.get())); + } + + // Ping the remote side, to provide errors which will mean the socket gets closed + mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5, + timeout); + + // Wait for a command or something on the socket + std::string command; + while(mapCommandSocketInfo->mpGetLine != 0 && !mapCommandSocketInfo->mpGetLine->IsEOF() + && mapCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) + { + BOX_TRACE("Receiving command '" << command + << "' over command socket"); + + bool sendOK = false; + bool sendResponse = true; + + // Command to process! + if(command == "quit" || command == "") + { + // Close the socket. + CloseCommandConnection(); + sendResponse = false; + } + else if(command == "sync") + { + // Sync now! + DoSyncFlagOut = true; + SyncIsForcedOut = false; + sendOK = true; + } + else if(command == "force-sync") + { + // Sync now (forced -- overrides any SyncAllowScript) + DoSyncFlagOut = true; + SyncIsForcedOut = true; + sendOK = true; + } + else if(command == "reload") + { + // Reload the configuration + SetReloadConfigWanted(); + sendOK = true; + } + else if(command == "terminate") + { + // Terminate the daemon cleanly + SetTerminateWanted(); + sendOK = true; + } + + // Send a response back? + if(sendResponse) + { + std::string response = sendOK ? "ok\n" : "error\n"; + mapCommandSocketInfo->mpConnectedSocket->Write( + response, timeout); + } + + // Set timeout to something very small, so this just checks for data which is waiting + timeout = 1; + } + + // Close on EOF? + if(mapCommandSocketInfo->mpGetLine != 0 && mapCommandSocketInfo->mpGetLine->IsEOF()) + { + CloseCommandConnection(); + } + } + catch(ConnectionException &ce) + { + BOX_NOTICE("Failed to write to command socket: " << ce.what()); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } + catch(std::exception &e) + { + BOX_ERROR("Failed to write to command socket: " << + e.what()); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } + catch(...) + { + BOX_ERROR("Failed to write to command socket: unknown error"); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CloseCommandConnection() +// Purpose: Close the command connection, ignoring any errors +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::CloseCommandConnection() +{ + try + { + BOX_TRACE("Closing command connection"); + + if(mapCommandSocketInfo->mpGetLine) + { + delete mapCommandSocketInfo->mpGetLine; + mapCommandSocketInfo->mpGetLine = 0; + } + mapCommandSocketInfo->mpConnectedSocket.reset(); + } + catch(std::exception &e) + { + BOX_ERROR("Internal error while closing command " + "socket: " << e.what()); + } + catch(...) + { + // Ignore any errors + } +} + + +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemon.cpp +// Purpose: Send a start or finish sync message to the command socket, if it's connected. +// +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SendSyncStartOrFinish(bool SendStart) +{ + // The bbackupctl program can't rely on a state change, because it + // may never change if the server doesn't need to be contacted. + + if(mapCommandSocketInfo.get() && + mapCommandSocketInfo->mpConnectedSocket.get() != 0) + { + std::string message = SendStart ? "start-sync" : "finish-sync"; + try + { + message += "\n"; + mapCommandSocketInfo->mpConnectedSocket->Write(message, + 1); // short timeout, it's overlapped + } + catch(std::exception &e) + { + BOX_ERROR("Internal error while sending to " + "command socket client: " << e.what()); + CloseCommandConnection(); + } + catch(...) + { + CloseCommandConnection(); + } + } +} + + + + +#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_NMTONNAME) + // string comparison ordering for when mount points are handled + // by code, rather than the OS. + typedef struct + { + bool operator()(const std::string &s1, const std::string &s2) + { + if(s1.size() == s2.size()) + { + // Equal size, sort according to natural sort order + return s1 < s2; + } + else + { + // Make sure longer strings go first + return s1.size() > s2.size(); + } + } + } mntLenCompare; +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetupLocations(BackupClientContext &, const Configuration &) +// Purpose: Makes sure that the list of directories records is correctly set up +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf) +{ + // Going to need a copy of the root directory. Get a connection, + // and fetch it. + BackupProtocolCallable& connection(rClientContext.GetConnection()); + + // Ask server for a list of everything in the root directory, + // which is a directory itself + std::auto_ptr<BackupProtocolSuccess> dirreply( + connection.QueryListDirectory( + BackupProtocolListDirectory::RootDirectory, + // only directories + BackupProtocolListDirectory::Flags_Dir, + // exclude old/deleted stuff + BackupProtocolListDirectory::Flags_Deleted | + BackupProtocolListDirectory::Flags_OldVersion, + false /* no attributes */)); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(connection.ReceiveStream()); + dir.ReadFromStream(*dirstream, connection.GetTimeout()); + + // Map of mount names to ID map index + std::map<std::string, int> mounts; + int numIDMaps = 0; + +#ifdef HAVE_MOUNTS +#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_MNTONNAME) + // Linux and others can't tell you where a directory is mounted. So we + // have to read the mount entries from /etc/mtab! Bizarre that the OS + // itself can't tell you, but there you go. + std::set<std::string, mntLenCompare> mountPoints; + // BLOCK + FILE *mountPointsFile = 0; + +#ifdef HAVE_STRUCT_MNTENT_MNT_DIR + // Open mounts file + mountPointsFile = ::setmntent("/proc/mounts", "r"); + if(mountPointsFile == 0) + { + mountPointsFile = ::setmntent("/etc/mtab", "r"); + } + if(mountPointsFile == 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + + try + { + // Read all the entries, and put them in the set + struct mntent *entry = 0; + while((entry = ::getmntent(mountPointsFile)) != 0) + { + BOX_TRACE("Found mount point at " << entry->mnt_dir); + mountPoints.insert(std::string(entry->mnt_dir)); + } + + // Close mounts file + ::endmntent(mountPointsFile); + } + catch(...) + { + ::endmntent(mountPointsFile); + throw; + } +#else // ! HAVE_STRUCT_MNTENT_MNT_DIR + // Open mounts file + mountPointsFile = ::fopen("/etc/mnttab", "r"); + if(mountPointsFile == 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + + try + { + // Read all the entries, and put them in the set + struct mnttab entry; + while(getmntent(mountPointsFile, &entry) == 0) + { + BOX_TRACE("Found mount point at " << entry.mnt_mountp); + mountPoints.insert(std::string(entry.mnt_mountp)); + } + + // Close mounts file + ::fclose(mountPointsFile); + } + catch(...) + { + ::fclose(mountPointsFile); + throw; + } +#endif // HAVE_STRUCT_MNTENT_MNT_DIR + // Check sorting and that things are as we expect + ASSERT(mountPoints.size() > 0); +#ifndef BOX_RELEASE_BUILD + { + std::set<std::string, mntLenCompare>::reverse_iterator i(mountPoints.rbegin()); + ASSERT(*i == "/"); + } +#endif // n BOX_RELEASE_BUILD +#endif // n HAVE_STRUCT_STATFS_F_MNTONNAME || n HAVE_STRUCT_STATVFS_F_MNTONNAME +#endif // HAVE_MOUNTS + + // Then... go through each of the entries in the configuration, + // making sure there's a directory created for it. + std::vector<std::string> locNames = + rLocationsConf.GetSubConfigurationNames(); + + // We only want completely configured locations to be in the list + // when this function exits, so move them all to a temporary list. + // Entries matching a properly configured location will be moved + // back to mLocations. Anything left in this list after the loop + // finishes will be deleted. + Locations tmpLocations = mLocations; + mLocations.clear(); + + // The ID map list will be repopulated automatically by this loop + mIDMapMounts.clear(); + + for(std::vector<std::string>::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + Location* pLoc = NULL; + + // Try to find and reuse an existing Location object + for(Locations::const_iterator + i = tmpLocations.begin(); + i != tmpLocations.end(); i++) + { + if ((*i)->mName == *pLocName) + { + BOX_TRACE("Location already configured: " << *pLocName); + pLoc = *i; + break; + } + } + + const Configuration& rConfig( + rLocationsConf.GetSubConfiguration(*pLocName)); + std::auto_ptr<Location> apLoc; + + try + { + if(pLoc == NULL) + { + // Create a record for it + BOX_TRACE("New location: " << *pLocName); + pLoc = new Location; + + // ensure deletion if setup fails + apLoc.reset(pLoc); + + // Setup names in the location record + pLoc->mName = *pLocName; + pLoc->mPath = rConfig.GetKeyValue("Path"); + } + + // Read the exclude lists from the Configuration + pLoc->mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rConfig)); + pLoc->mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rConfig)); + + // Does this exist on the server? + // Remove from dir object early, so that if we fail + // to stat the local directory, we still don't + // consider to remote one for deletion. + BackupStoreDirectory::Iterator iter(dir); + BackupStoreFilenameClear dirname(pLoc->mName); // generate the filename + BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname); + int64_t oid = 0; + if(en != 0) + { + oid = en->GetObjectID(); + + // Delete the entry from the directory, so we get a list of + // unused root directories at the end of this. + dir.DeleteEntry(oid); + } + + // Do a fsstat on the pathname to find out which mount it's on + { + +#if defined HAVE_STRUCT_STATFS_F_MNTONNAME || defined HAVE_STRUCT_STATVFS_F_MNTONNAME || defined WIN32 + + // BSD style statfs -- includes mount point, which is nice. +#ifdef HAVE_STRUCT_STATVFS_F_MNTONNAME + struct statvfs s; + if(::statvfs(pLoc->mPath.c_str(), &s) != 0) +#else // HAVE_STRUCT_STATVFS_F_MNTONNAME + struct statfs s; + if(::statfs(pLoc->mPath.c_str(), &s) != 0) +#endif // HAVE_STRUCT_STATVFS_F_MNTONNAME + { + THROW_SYS_ERROR("Failed to stat path " + "'" << pLoc->mPath << "' " + "for location " + "'" << pLoc->mName << "'", + CommonException, OSFileError); + } + + // Where the filesystem is mounted + std::string mountName(s.f_mntonname); + +#else // !HAVE_STRUCT_STATFS_F_MNTONNAME && !WIN32 + + // Warn in logs if the directory isn't absolute + if(pLoc->mPath[0] != '/') + { + BOX_WARNING("Location path '" + << pLoc->mPath + << "' is not absolute"); + } + // Go through the mount points found, and find a suitable one + std::string mountName("/"); + { + std::set<std::string, mntLenCompare>::const_iterator i(mountPoints.begin()); + BOX_TRACE(mountPoints.size() + << " potential mount points"); + for(; i != mountPoints.end(); ++i) + { + // Compare first n characters with the filename + // If it matches, the file belongs in that mount point + // (sorting order ensures this) + BOX_TRACE("checking against mount point " << *i); + if(::strncmp(i->c_str(), pLoc->mPath.c_str(), i->size()) == 0) + { + // Match + mountName = *i; + break; + } + } + BOX_TRACE("mount point chosen for " + << pLoc->mPath << " is " + << mountName); + } + +#endif + + // Got it? + std::map<std::string, int>::iterator f(mounts.find(mountName)); + if(f != mounts.end()) + { + // Yes -- store the index + pLoc->mIDMapIndex = f->second; + } + else + { + // No -- new index + pLoc->mIDMapIndex = numIDMaps; + mounts[mountName] = numIDMaps; + + // Store the mount name + mIDMapMounts.push_back(mountName); + + // Increment number of maps + ++numIDMaps; + } + } + + // Does this exist on the server? + if(en == 0) + { + // Doesn't exist, so it has to be created on the server. Let's go! + // First, get the directory's attributes and modification time + box_time_t attrModTime = 0; + BackupClientFileAttributes attr; + try + { + attr.ReadAttributes(pLoc->mPath.c_str(), + true /* directories have zero mod times */, + 0 /* not interested in mod time */, + &attrModTime /* get the attribute modification time */); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to get attributes " + "for path '" << pLoc->mPath + << "', skipping location '" << + pLoc->mName << "'"); + throw; + } + + // Execute create directory command + try + { + std::auto_ptr<IOStream> attrStream( + new MemBlockStream(attr)); + std::auto_ptr<BackupProtocolSuccess> + dirCreate(connection.QueryCreateDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, // containing directory + attrModTime, dirname, attrStream)); + + // Object ID for later creation + oid = dirCreate->GetObjectID(); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to create remote " + "directory '/" << pLoc->mName << + "', skipping location '" << + pLoc->mName << "'"); + throw; + } + + } + + // Create and store the directory object for the root of this location + ASSERT(oid != 0); + if(pLoc->mapDirectoryRecord.get() == NULL) + { + pLoc->mapDirectoryRecord.reset( + new BackupClientDirectoryRecord(oid, *pLocName)); + } + + // Remove it from the temporary list to avoid deletion + tmpLocations.remove(pLoc); + + // Push it back on the vector of locations + mLocations.push_back(pLoc); + + if(apLoc.get() != NULL) + { + // Don't delete it now! + apLoc.release(); + } + } + catch (std::exception &e) + { + BOX_ERROR("Failed to configure location '" + << pLoc->mName << "' path '" + << pLoc->mPath << "': " << e.what() << + ": please check for previous errors"); + mReadErrorsOnFilesystemObjects = true; + } + catch(...) + { + BOX_ERROR("Failed to configure location '" + << pLoc->mName << "' path '" + << pLoc->mPath << "': please check for " + "previous errors"); + mReadErrorsOnFilesystemObjects = true; + } + } + + // Now remove any leftovers + for(BackupDaemon::Locations::iterator + i = tmpLocations.begin(); + i != tmpLocations.end(); i++) + { + BOX_INFO("Removing obsolete location from memory: " << + (*i)->mName); + delete *i; + } + + tmpLocations.clear(); + + // Any entries in the root directory which need deleting? + if(dir.GetNumberOfEntries() > 0 && + mDeleteRedundantLocationsAfter == 0) + { + BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " + "in root directory found, but will not delete because " + "DeleteRedundantLocationsAfter = 0"); + } + else if(dir.GetNumberOfEntries() > 0) + { + box_time_t now = GetCurrentBoxTime(); + + // This should reset the timer if the list of unused + // locations changes, but it will not if the number of + // unused locations does not change, but the locations + // do change, e.g. one mysteriously appears and another + // mysteriously appears. (FIXME) + if (dir.GetNumberOfEntries() != mUnusedRootDirEntries.size() || + mDeleteUnusedRootDirEntriesAfter == 0) + { + mDeleteUnusedRootDirEntriesAfter = now + + SecondsToBoxTime(mDeleteRedundantLocationsAfter); + } + + int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter + - now); + + BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " + "in root directory found, will delete from store " + "after " << secs << " seconds."); + + // Store directories in list of things to delete + mUnusedRootDirEntries.clear(); + BackupStoreDirectory::Iterator iter(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = iter.Next()) != 0) + { + // Add name to list + BackupStoreFilenameClear clear(en->GetName()); + const std::string &name(clear.GetClearFilename()); + mUnusedRootDirEntries.push_back( + std::pair<int64_t,std::string> + (en->GetObjectID(), name)); + // Log this + BOX_INFO("Unused location in root: " << name); + } + ASSERT(mUnusedRootDirEntries.size() > 0); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetupIDMapsForSync() +// Purpose: Sets up ID maps for the sync process -- make sure they're all there +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetupIDMapsForSync() +{ + // Make sure we have some blank, empty ID maps + DeleteIDMapVector(mNewIDMaps); + FillIDMapVector(mNewIDMaps, true /* new maps */); + DeleteIDMapVector(mCurrentIDMaps); + FillIDMapVector(mCurrentIDMaps, false /* new maps */); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &) +// Purpose: Fills the vector with the right number of empty ID maps +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps) +{ + ASSERT(rVector.size() == 0); + rVector.reserve(mIDMapMounts.size()); + + for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) + { + // Create the object + BackupClientInodeToIDMap *pmap = new BackupClientInodeToIDMap(); + try + { + // Get the base filename of this map + std::string filename; + MakeMapBaseName(l, filename); + + // If it's a new one, add a suffix + if(NewMaps) + { + filename += ".n"; + } + + // The new map file should not exist yet. If there's + // one left over from a previous failed run, it's not + // useful to us because we never read from it and will + // overwrite the entries of all files that still + // exist, so we should just delete it and start afresh. + if(NewMaps && FileExists(filename.c_str())) + { + BOX_NOTICE("Found an incomplete ID map " + "database, deleting it to start " + "afresh: " << filename); + if(unlink(filename.c_str()) != 0) + { + BOX_LOG_NATIVE_ERROR(BOX_FILE_MESSAGE( + filename, "Failed to delete " + "incomplete ID map database")); + } + } + + // If it's not a new map, it may not exist in which case an empty map should be created + if(!NewMaps && !FileExists(filename.c_str())) + { + pmap->OpenEmpty(); + } + else + { + // Open the map + pmap->Open(filename.c_str(), !NewMaps /* read only */, NewMaps /* create new */); + } + + // Store on vector + rVector.push_back(pmap); + } + catch(...) + { + delete pmap; + throw; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteCorruptBerkelyDbFiles() +// Purpose: Delete the Berkely db files from disc after they have been corrupted. +// Created: 14/9/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteCorruptBerkelyDbFiles() +{ + for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) + { + // Get the base filename of this map + std::string filename; + MakeMapBaseName(l, filename); + + // Delete the file + BOX_TRACE("Deleting " << filename); + ::unlink(filename.c_str()); + + // Add a suffix for the new map + filename += ".n"; + + // Delete that too + BOX_TRACE("Deleting " << filename); + ::unlink(filename.c_str()); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: MakeMapBaseName(unsigned int, std::string &) +// Purpose: Makes the base name for a inode map +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const +{ + // Get the directory for the maps + const Configuration &config(GetConfiguration()); + std::string dir(config.GetKeyValue("DataDirectory")); + + // Make a leafname + std::string leaf(mIDMapMounts[MountNumber]); + for(unsigned int z = 0; z < leaf.size(); ++z) + { + if(leaf[z] == DIRECTORY_SEPARATOR_ASCHAR) + { + leaf[z] = '_'; + } + } + + // Build the final filename + rNameOut = dir + DIRECTORY_SEPARATOR "mnt" + leaf; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CommitIDMapsAfterSync() +// Purpose: Commits the new ID maps, so the 'new' maps are now the 'current' maps. +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::CommitIDMapsAfterSync() +{ + // Get rid of the maps in memory (leaving them on disc of course) + DeleteIDMapVector(mCurrentIDMaps); + DeleteIDMapVector(mNewIDMaps); + + // Then move the old maps into the new places + for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) + { + std::string target; + MakeMapBaseName(l, target); + std::string newmap(target + ".n"); + + // Try to rename +#ifdef WIN32 + // win32 rename doesn't overwrite existing files + ::remove(target.c_str()); +#endif + if(::rename(newmap.c_str(), target.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("Failed to rename ID map: " << + newmap << " to " << target); + THROW_EXCEPTION(CommonException, OSFileError) + } + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &) +// Purpose: Deletes the contents of a vector of ID maps +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector) +{ + while(!rVector.empty()) + { + // Pop off list + BackupClientInodeToIDMap *toDel = rVector.back(); + rVector.pop_back(); + + // Close and delete + delete toDel; + } + ASSERT(rVector.size() == 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::FindLocationPathName(const std::string &, std::string &) const +// Purpose: Tries to find the path of the root of a backup location. Returns true (and path in rPathOut) +// if it can be found, false otherwise. +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const +{ + // Search for the location + for(Locations::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i) + { + if((*i)->mName == rLocationName) + { + rPathOut = (*i)->mPath; + return true; + } + } + + // Didn't find it + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetState(int) +// Purpose: Record current action of daemon, and update process title to reflect this +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetState(int State) +{ + // Two little checks + if(State == mState) return; + if(State < 0) return; + + // Update + mState = State; + + // Set process title + const static char *stateText[] = {"idle", "connected", "error -- waiting for retry", "over limit on server -- not backing up"}; + SetProcessTitle(stateText[State]); + + // If there's a command socket connected, then inform it -- disconnecting from the + // command socket if there's an error + + std::ostringstream msg; + msg << "state " << State << "\n"; + + if(!mapCommandSocketInfo.get()) + { + return; + } + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + return; + } + + // Something connected to the command socket, tell it about the new state + try + { + mapCommandSocketInfo->mpConnectedSocket->Write(msg.str(), + 1); // very short timeout, it's overlapped anyway + } + catch(ConnectionException &ce) + { + BOX_NOTICE("Failed to write state to command socket: " << + ce.what()); + CloseCommandConnection(); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to write state to command socket: " << + e.what()); + CloseCommandConnection(); + } + catch(...) + { + BOX_ERROR("Failed to write state to command socket: " + "unknown error"); + CloseCommandConnection(); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::TouchFileInWorkingDir(const char *) +// Purpose: Make sure a zero length file of the name exists in the working directory. +// Use for marking times of events in the filesystem. +// Created: 21/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::TouchFileInWorkingDir(const char *Filename) +{ + // Filename + const Configuration &config(GetConfiguration()); + std::string fn(config.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); + fn += Filename; + + // Open and close it to update the timestamp + try + { + FileStream touch(fn, O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to write to timestamp file: " << fn << ": " << + e.what()); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::NotifySysadmin(int) +// Purpose: Run the script to tell the sysadmin about events +// which need attention. +// Created: 25/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::NotifySysadmin(SysadminNotifier::EventCode Event) +{ + static const char *sEventNames[] = + { + "store-full", + "read-error", + "backup-error", + "backup-start", + "backup-finish", + "backup-ok", + 0 + }; + + // BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames)); + // BOX_TRACE("sizeof(*sEventNames) == " << sizeof(*sEventNames)); + // BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX); + ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == SysadminNotifier::MAX + 1); + + if(Event < 0 || Event >= SysadminNotifier::MAX) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + BadNotifySysadminEventCode, "NotifySysadmin() called " + "for unknown event code " << Event); + } + + BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << + sEventNames[Event]); + + if(!GetConfiguration().KeyExists("NotifyAlways") || + !GetConfiguration().GetKeyValueBool("NotifyAlways")) + { + // Don't send lots of repeated messages + // Note: backup-start and backup-finish will always be + // logged, because mLastNotifiedEvent is never set to + // these values and therefore they are never "duplicates". + if(mLastNotifiedEvent == Event) + { + if(Event == SysadminNotifier::BackupOK) + { + BOX_INFO("Suppressing duplicate notification " + "about " << sEventNames[Event]); + } + else + { + BOX_WARNING("Suppressing duplicate notification " + "about " << sEventNames[Event]); + } + return; + } + } + + // Is there a notification script? + const Configuration &conf(GetConfiguration()); + if(!conf.KeyExists("NotifyScript")) + { + // Log, and then return + if(Event != SysadminNotifier::BackupStart && + Event != SysadminNotifier::BackupFinish) + { + BOX_INFO("Not notifying administrator about event " + << sEventNames[Event] << ", set NotifyScript " + "to do this in future"); + } + return; + } + + // Script to run + std::string script(conf.GetKeyValue("NotifyScript") + " " + + sEventNames[Event] + " \"" + GetConfigFileName() + "\""); + + // Log what we're about to do + BOX_INFO("About to notify administrator about event " + << sEventNames[Event] << ", running script '" << script << "'"); + + // Then do it + int returnCode = ::system(script.c_str()); + if(returnCode != 0) + { + BOX_WARNING("Notify script returned error code: " << + returnCode << " (" << script << ")"); + } + else if(Event != SysadminNotifier::BackupStart && + Event != SysadminNotifier::BackupFinish) + { + mLastNotifiedEvent = Event; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &) +// Purpose: Deletes any unused entries in the root directory, if they're scheduled to be deleted. +// Created: 13/5/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) +{ + if(mUnusedRootDirEntries.empty()) + { + BOX_INFO("Not deleting unused entries - none in list"); + return; + } + + if(mDeleteUnusedRootDirEntriesAfter == 0) + { + BOX_INFO("Not deleting unused entries - " + "zero delete time (bad)"); + return; + } + + // Check time + box_time_t now = GetCurrentBoxTime(); + if(now < mDeleteUnusedRootDirEntriesAfter) + { + int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter + - now); + BOX_INFO("Not deleting unused entries - too early (" + << secs << " seconds remaining)"); + return; + } + + // Entries to delete, and it's the right time to do so... + BOX_NOTICE("Deleting unused locations from store root..."); + BackupProtocolCallable &connection(rContext.GetConnection()); + for(std::vector<std::pair<int64_t,std::string> >::iterator + i(mUnusedRootDirEntries.begin()); + i != mUnusedRootDirEntries.end(); ++i) + { + connection.QueryDeleteDirectory(i->first); + rContext.GetProgressNotifier().NotifyFileDeleted( + i->first, i->second); + } + + // Reset state + mDeleteUnusedRootDirEntriesAfter = 0; + mUnusedRootDirEntries.clear(); +} + +// -------------------------------------------------------------------------- + +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mNumEntries; + int64_t mObjectID; // this object ID + int64_t mContainerID; // ID of container + uint64_t mAttributesModTime; + int32_t mOptionsPresent; // bit mask of optional sections / features present + +} loc_StreamFormat; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CommandSocketInfo::CommandSocketInfo() +// Purpose: Constructor +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +BackupDaemon::CommandSocketInfo::CommandSocketInfo() + : mpGetLine(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CommandSocketInfo::~CommandSocketInfo() +// Purpose: Destructor +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +BackupDaemon::CommandSocketInfo::~CommandSocketInfo() +{ + if(mpGetLine) + { + delete mpGetLine; + mpGetLine = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SerializeStoreObjectInfo( +// box_time_t theLastSyncTime, +// box_time_t theNextSyncTime) +// Purpose: Serializes remote directory and file information +// into a stream of bytes, using an Archive +// abstraction. +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- + +static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; +static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; +static const int STOREOBJECTINFO_VERSION = 2; + +bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime, + box_time_t theNextSyncTime) const +{ + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + return false; + } + + std::string StoreObjectInfoFile = + GetConfiguration().GetKeyValue("StoreObjectInfoFile"); + + if(StoreObjectInfoFile.size() <= 0) + { + return false; + } + + bool created = false; + + try + { + FileStream aFile(StoreObjectInfoFile.c_str(), + O_WRONLY | O_CREAT | O_TRUNC); + created = true; + + Archive anArchive(aFile, 0); + + anArchive.Write(STOREOBJECTINFO_MAGIC_ID_VALUE); + anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); + anArchive.Write(STOREOBJECTINFO_VERSION); + anArchive.Write(GetLoadedConfigModifiedTime()); + anArchive.Write(mClientStoreMarker); + anArchive.Write(theLastSyncTime); + anArchive.Write(theNextSyncTime); + + // + // + // + int64_t iCount = mLocations.size(); + anArchive.Write(iCount); + + for(Locations::const_iterator i = mLocations.begin(); + i != mLocations.end(); i++) + { + ASSERT(*i); + (*i)->Serialize(anArchive); + } + + // + // + // + iCount = mIDMapMounts.size(); + anArchive.Write(iCount); + + for(int v = 0; v < iCount; v++) + anArchive.Write(mIDMapMounts[v]); + + // + // + // + iCount = mUnusedRootDirEntries.size(); + anArchive.Write(iCount); + + for(int v = 0; v < iCount; v++) + { + anArchive.Write(mUnusedRootDirEntries[v].first); + anArchive.Write(mUnusedRootDirEntries[v].second); + } + + if (iCount > 0) + { + anArchive.Write(mDeleteUnusedRootDirEntriesAfter); + } + + // + // + // + aFile.Close(); + BOX_INFO("Saved store object info file version " << + STOREOBJECTINFO_VERSION << " (" << + StoreObjectInfoFile << ")"); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to write StoreObjectInfoFile: " << + StoreObjectInfoFile << ": " << e.what()); + } + catch(...) + { + BOX_ERROR("Failed to write StoreObjectInfoFile: " << + StoreObjectInfoFile << ": unknown error"); + } + + return created; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeserializeStoreObjectInfo( +// box_time_t & theLastSyncTime, +// box_time_t & theNextSyncTime) +// Purpose: Deserializes remote directory and file information +// from a stream of bytes, using an Archive +// abstraction. +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, + box_time_t & theNextSyncTime) +{ + // + // + // + DeleteAllLocations(); + + // + // + // + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + BOX_NOTICE("Store object info file is not enabled. Will " + "download directory listings from store."); + return false; + } + + std::string StoreObjectInfoFile = + GetConfiguration().GetKeyValue("StoreObjectInfoFile"); + + if(StoreObjectInfoFile.size() <= 0) + { + return false; + } + + int64_t fileSize; + if (!FileExists(StoreObjectInfoFile, &fileSize) || fileSize == 0) + { + BOX_NOTICE(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file does not exist or is empty")); + } + else + { + try + { + FileStream aFile(StoreObjectInfoFile, O_RDONLY); + Archive anArchive(aFile, 0); + + // + // see if the content looks like a valid serialised archive + // + int iMagicValue = 0; + anArchive.Read(iMagicValue); + + if(iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is not a valid " + "or compatible serialised archive")); + return false; + } + + // + // get a bit optimistic and read in a string identifier + // + std::string strMagicValue; + anArchive.Read(strMagicValue); + + if(strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is not a valid " + "or compatible serialised archive")); + return false; + } + + // + // check if we are loading some future format + // version by mistake + // + int iVersion = 0; + anArchive.Read(iVersion); + + if(iVersion != STOREOBJECTINFO_VERSION) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file version " << + iVersion << " is not supported")); + return false; + } + + // + // check if this state file is even valid + // for the loaded bbackupd.conf file + // + box_time_t lastKnownConfigModTime; + anArchive.Read(lastKnownConfigModTime); + + if(lastKnownConfigModTime != GetLoadedConfigModifiedTime()) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is older than " + "configuration file")); + return false; + } + + // + // this is it, go at it + // + anArchive.Read(mClientStoreMarker); + anArchive.Read(theLastSyncTime); + anArchive.Read(theNextSyncTime); + + // + // + // + int64_t iCount = 0; + anArchive.Read(iCount); + + for(int v = 0; v < iCount; v++) + { + Location* pLocation = new Location; + if(!pLocation) + { + throw std::bad_alloc(); + } + + pLocation->Deserialize(anArchive); + mLocations.push_back(pLocation); + } + + // + // + // + iCount = 0; + anArchive.Read(iCount); + + for(int v = 0; v < iCount; v++) + { + std::string strItem; + anArchive.Read(strItem); + + mIDMapMounts.push_back(strItem); + } + + // + // + // + iCount = 0; + anArchive.Read(iCount); + + for(int v = 0; v < iCount; v++) + { + int64_t anId; + anArchive.Read(anId); + + std::string aName; + anArchive.Read(aName); + + mUnusedRootDirEntries.push_back(std::pair<int64_t, std::string>(anId, aName)); + } + + if (iCount > 0) + anArchive.Read(mDeleteUnusedRootDirEntriesAfter); + + // + // + // + aFile.Close(); + + BOX_INFO(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Loaded store object info file version " << iVersion)); + return true; + } + catch(std::exception &e) + { + BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Internal error reading store object info " + "file: " << e.what())); + } + catch(...) + { + BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Internal error reading store object info " + "file: unknown error")); + } + } + + BOX_NOTICE("No usable cache, will download directory listings from " + "server."); + + DeleteAllLocations(); + + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; + theLastSyncTime = 0; + theNextSyncTime = 0; + + return false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteStoreObjectInfo() +// Purpose: Deletes the serialised state file, to prevent us +// from using it again if a backup is interrupted. +// +// Created: 2006/02/12 +// +// -------------------------------------------------------------------------- + +bool BackupDaemon::DeleteStoreObjectInfo() const +{ + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + return false; + } + + std::string storeObjectInfoFile(GetConfiguration().GetKeyValue("StoreObjectInfoFile")); + + // Check to see if the file exists + if(!FileExists(storeObjectInfoFile.c_str())) + { + // File doesn't exist -- so can't be deleted. But something + // isn't quite right, so log a message + BOX_WARNING("StoreObjectInfoFile did not exist when it " + "was supposed to: " << storeObjectInfoFile); + + // Return true to stop things going around in a loop + return true; + } + + // Actually delete it + if(::unlink(storeObjectInfoFile.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("Failed to delete the old " + "StoreObjectInfoFile: " << storeObjectInfoFile); + return false; + } + + return true; +} diff --git a/lib/bbackupd/BackupDaemon.h b/lib/bbackupd/BackupDaemon.h new file mode 100644 index 00000000..ffe31247 --- /dev/null +++ b/lib/bbackupd/BackupDaemon.h @@ -0,0 +1,539 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemon.h +// Purpose: Backup daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPDAEMON__H +#define BACKUPDAEMON__H + +#include <vector> +#include <string> +#include <memory> + +#include "BackupClientContext.h" +#include "BackupClientDirectoryRecord.h" +#include "BoxTime.h" +#include "Daemon.h" +#include "Logging.h" +#include "Socket.h" +#include "SocketListen.h" +#include "SocketStream.h" +#include "TLSContext.h" + +#include "autogen_BackupProtocol.h" +#include "autogen_BackupStoreException.h" + +#ifdef WIN32 + #include "WinNamedPipeListener.h" + #include "WinNamedPipeStream.h" +#endif + +#ifdef ENABLE_VSS +# include <comdef.h> +# include <Vss.h> +# include <VsWriter.h> +# include <VsBackup.h> +#endif + +#define COMMAND_SOCKET_POLL_INTERVAL 1000 + +class BackupClientDirectoryRecord; +class BackupClientContext; +class Configuration; +class BackupClientInodeToIDMap; +class ExcludeList; +class IOStreamGetLine; +class Archive; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupDaemon +// Purpose: Backup daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class BackupDaemon : public Daemon, public ProgressNotifier, public LocationResolver, +public RunStatusProvider, public SysadminNotifier, public BackgroundTask +{ +public: + BackupDaemon(); + ~BackupDaemon(); + +private: + // methods below do partial (specialized) serialization of + // client state only + bool SerializeStoreObjectInfo(box_time_t theLastSyncTime, + box_time_t theNextSyncTime) const; + bool DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, + box_time_t & theNextSyncTime); + bool DeleteStoreObjectInfo() const; + BackupDaemon(const BackupDaemon &); + +public: + #ifdef WIN32 + // add command-line options to handle Windows services + std::string GetOptionString(); + int ProcessOption(signed int option); + int Main(const std::string &rConfigFileName); + + // This shouldn't be here, but apparently gcc on + // Windows has no idea about inherited methods... + virtual int Main(const char *DefaultConfigFile, int argc, + const char *argv[]) + { + return Daemon::Main(DefaultConfigFile, argc, argv); + } + #endif + + void Run(); + virtual const char *DaemonName() const; + virtual std::string DaemonBanner() const; + virtual void Usage(); + const ConfigurationVerify *GetConfigVerify() const; + + bool FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const; + + enum + { + // Add stuff to this, make sure the textual equivalents in SetState() are changed too. + State_Initialising = -1, + State_Idle = 0, + State_Connected = 1, + State_Error = 2, + State_StorageLimitExceeded = 3 + }; + + int GetState() {return mState;} + static std::string GetStateName(int state) + { + std::string stateName; + + #define STATE(x) case BackupDaemon::State_ ## x: stateName = #x; break; + switch (state) + { + STATE(Initialising); + STATE(Idle); + STATE(Connected); + STATE(Error); + STATE(StorageLimitExceeded); + default: + stateName = "unknown"; + } + #undef STATE + + return stateName; + } + + // Allow other classes to call this too + void NotifySysadmin(SysadminNotifier::EventCode Event); + +private: + void Run2(); + +public: + void InitCrypto(); + std::auto_ptr<BackupClientContext> RunSyncNowWithExceptionHandling(); + std::auto_ptr<BackupClientContext> RunSyncNow(); + void ResetCachedState(); + void OnBackupStart(); + void OnBackupFinish(); + // TouchFileInWorkingDir is only here for use by Boxi. + // This does NOT constitute an API! + void TouchFileInWorkingDir(const char *Filename); + +protected: + virtual std::auto_ptr<BackupClientContext> GetNewContext + ( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode + ); + +private: + void DeleteAllLocations(); + void SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf); + + void DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector); + void DeleteAllIDMaps() + { + DeleteIDMapVector(mCurrentIDMaps); + DeleteIDMapVector(mNewIDMaps); + } + void FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps); + + void SetupIDMapsForSync(); + void CommitIDMapsAfterSync(); + void DeleteCorruptBerkelyDbFiles(); + + void MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const; + + void SetState(int State); + + void WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut); + void CloseCommandConnection(); + void SendSyncStartOrFinish(bool SendStart); + + void DeleteUnusedRootDirEntries(BackupClientContext &rContext); + + // For warning user about potential security hole + virtual void SetupInInitialProcess(); + + int UseScriptToSeeIfSyncAllowed(); + +public: + int ParseSyncAllowScriptOutput(const std::string& script, + const std::string& output); + typedef std::list<Location *> Locations; + Locations GetLocations() { return mLocations; } + +private: + int mState; // what the daemon is currently doing + + Locations mLocations; + + std::vector<std::string> mIDMapMounts; + std::vector<BackupClientInodeToIDMap *> mCurrentIDMaps; + std::vector<BackupClientInodeToIDMap *> mNewIDMaps; + + int mDeleteRedundantLocationsAfter; + + // For the command socket + class CommandSocketInfo + { + public: + CommandSocketInfo(); + ~CommandSocketInfo(); + private: + CommandSocketInfo(const CommandSocketInfo &); // no copying + CommandSocketInfo &operator=(const CommandSocketInfo &); + public: +#ifdef WIN32 + WinNamedPipeListener<1 /* listen backlog */> mListeningSocket; + std::auto_ptr<WinNamedPipeStream> mpConnectedSocket; +#else + SocketListen<SocketStream, 1 /* listen backlog */> mListeningSocket; + std::auto_ptr<SocketStream> mpConnectedSocket; +#endif + IOStreamGetLine *mpGetLine; + }; + + // Using a socket? + std::auto_ptr<CommandSocketInfo> mapCommandSocketInfo; + + // Stop notifications being repeated. + SysadminNotifier::EventCode mLastNotifiedEvent; + + // Unused entries in the root directory wait a while before being deleted + box_time_t mDeleteUnusedRootDirEntriesAfter; // time to delete them + std::vector<std::pair<int64_t,std::string> > mUnusedRootDirEntries; + + int64_t mClientStoreMarker; + bool mStorageLimitExceeded; + bool mReadErrorsOnFilesystemObjects; + box_time_t mLastSyncTime, mNextSyncTime; + box_time_t mCurrentSyncStartTime, mUpdateStoreInterval, + mBackupErrorDelay; + TLSContext mTlsContext; + bool mDeleteStoreObjectInfoFile; + bool mDoSyncForcedByPreviousSyncError; + int64_t mNumFilesUploaded, mNumDirsCreated; + int mMaxBandwidthFromSyncAllowScript; + +public: + int GetMaxBandwidthFromSyncAllowScript() { return mMaxBandwidthFromSyncAllowScript; } + bool StopRun() { return this->Daemon::StopRun(); } + bool StorageLimitExceeded() { return mStorageLimitExceeded; } + +private: + bool mLogAllFileAccess; + +public: + ProgressNotifier* GetProgressNotifier() { return mpProgressNotifier; } + LocationResolver* GetLocationResolver() { return mpLocationResolver; } + RunStatusProvider* GetRunStatusProvider() { return mpRunStatusProvider; } + SysadminNotifier* GetSysadminNotifier() { return mpSysadminNotifier; } + void SetProgressNotifier (ProgressNotifier* p) { mpProgressNotifier = p; } + void SetLocationResolver (LocationResolver* p) { mpLocationResolver = p; } + void SetRunStatusProvider(RunStatusProvider* p) { mpRunStatusProvider = p; } + void SetSysadminNotifier (SysadminNotifier* p) { mpSysadminNotifier = p; } + virtual bool RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum); + +private: + ProgressNotifier* mpProgressNotifier; + LocationResolver* mpLocationResolver; + RunStatusProvider* mpRunStatusProvider; + SysadminNotifier* mpSysadminNotifier; + std::auto_ptr<Timer> mapCommandSocketPollTimer; + std::auto_ptr<BackupClientContext> mapClientContext; + + /* ProgressNotifier implementation */ +public: + virtual void NotifyIDMapsSetup(BackupClientContext& rContext) { } + + virtual void NotifyScanDirectory( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Scanning directory: " << rLocalPath); + } + + if (!RunBackgroundTask(BackgroundTask::Scanning_Dirs, 0, 0)) + { + THROW_EXCEPTION(BackupStoreException, + CancelledByBackgroundTask); + } + } + virtual void NotifyDirStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to access directory: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyFileStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to access file: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyDirListFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to list directory: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyMountPointSkipped( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + #ifdef WIN32 + BOX_WARNING("Ignored directory: " << rLocalPath << + ": is an NTFS junction/reparse point; create " + "a new location if you want to back it up"); + #else + BOX_WARNING("Ignored directory: " << rLocalPath << + ": is a mount point; create a new location " + "if you want to back it up"); + #endif + } + virtual void NotifyFileExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Skipping excluded file: " << rLocalPath); + } + } + virtual void NotifyDirExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Skipping excluded directory: " << rLocalPath); + } + } + virtual void NotifyUnsupportedFileType( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Ignoring file of unknown type: " << rLocalPath); + } + virtual void NotifyFileReadFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Error reading file: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyFileModifiedInFuture( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Some files have modification times excessively " + "in the future. Check clock synchronisation. " + "Example file (only one shown): " << rLocalPath); + } + virtual void NotifyFileSkippedServerFull( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Skipped file: server is full: " << rLocalPath); + } + virtual void NotifyFileUploadException( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const BoxException& rException) + { + if (rException.GetType() == CommonException::ExceptionType && + rException.GetSubType() == CommonException::AccessDenied) + { + BOX_ERROR("Failed to upload file: " << rLocalPath + << ": Access denied"); + } + else + { + BOX_ERROR("Failed to upload file: " << rLocalPath + << ": caught exception: " << rException.what() + << " (" << rException.GetType() + << "/" << rException.GetSubType() << ")"); + } + } + virtual void NotifyFileUploadServerError( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int type, int subtype) + { + BOX_ERROR("Failed to upload file: " << rLocalPath << + ": server error: " << + BackupProtocolError::GetMessage(subtype)); + } + virtual void NotifyFileUploading( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploading complete file: " << rLocalPath); + } + } + virtual void NotifyFileUploadingPatch( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t EstimatedBytesToUpload) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploading patch to file: " << rLocalPath << + ", estimated upload size = " << + EstimatedBytesToUpload); + } + } + virtual void NotifyFileUploadingAttributes( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploading new file attributes: " << + rLocalPath); + } + } + virtual void NotifyFileUploaded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploaded file: " << rLocalPath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + "): total size = " << FileSize << ", " + "uploaded size = " << UploadedSize); + } + mNumFilesUploaded++; + } + virtual void NotifyFileSynchronised( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) + { + if (mLogAllFileAccess) + { + BOX_INFO("Synchronised file: " << rLocalPath); + } + } + virtual void NotifyDirectoryCreated( + int64_t ObjectID, + const std::string& rLocalPath, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Created directory: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + mNumDirsCreated++; + } + virtual void NotifyDirectoryDeleted( + int64_t ObjectID, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Deleted directory: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + } + virtual void NotifyFileDeleted( + int64_t ObjectID, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Deleted file: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", " << (length - offset) << " remain, eta " << + BoxTimeToSeconds(finish - elapsed) << "s"); + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", " << (length - offset) << " remain"); + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", unknown bytes remaining"); + } + +#ifdef WIN32 + private: + bool mInstallService, mRemoveService, mRunAsService; + std::string mServiceName; +#endif + +#ifdef ENABLE_VSS + IVssBackupComponents* mpVssBackupComponents; + void CreateVssBackupComponents(); + bool WaitForAsync(IVssAsync *pAsync, const std::string& description); + typedef HRESULT (__stdcall IVssBackupComponents::*AsyncMethod)(IVssAsync**); + bool CallAndWaitForAsync(AsyncMethod method, + const std::string& description); + void CleanupVssBackupComponents(); +#endif +}; + +#endif // BACKUPDAEMON__H diff --git a/lib/bbackupd/BackupDaemonInterface.h b/lib/bbackupd/BackupDaemonInterface.h new file mode 100644 index 00000000..d5c47c85 --- /dev/null +++ b/lib/bbackupd/BackupDaemonInterface.h @@ -0,0 +1,166 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemonInterface.h +// Purpose: Interfaces for managing a BackupDaemon +// Created: 2008/12/30 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPDAEMONINTERFACE__H +#define BACKUPDAEMONINTERFACE__H + +#include <string> + +#include "BoxTime.h" + +class Archive; +class BackupClientContext; +class BackupDaemon; + +// -------------------------------------------------------------------------- +// +// Class +// Name: SysadminNotifier +// Purpose: Provides a NotifySysadmin() method to send mail to the sysadmin +// Created: 2005/11/15 +// +// -------------------------------------------------------------------------- +class SysadminNotifier +{ + public: + virtual ~SysadminNotifier() { } + + typedef enum + { + StoreFull = 0, + ReadError, + BackupError, + BackupStart, + BackupFinish, + BackupOK, + MAX + // When adding notifications, remember to add + // strings to NotifySysadmin() + } + EventCode; + + virtual void NotifySysadmin(EventCode Event) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: ProgressNotifier +// Purpose: Provides methods for the backup library to inform the user +// interface about its progress with the backup +// Created: 2005/11/20 +// +// -------------------------------------------------------------------------- + +class BackupClientContext; +class BackupClientDirectoryRecord; + +class ProgressNotifier +{ + public: + virtual ~ProgressNotifier() { } + virtual void NotifyIDMapsSetup(BackupClientContext& rContext) = 0; + virtual void NotifyScanDirectory( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyDirStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyFileStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyDirListFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyMountPointSkipped( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyDirExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyUnsupportedFileType( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileReadFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyFileModifiedInFuture( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileSkippedServerFull( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploadException( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const BoxException& rException) = 0; + virtual void NotifyFileUploadServerError( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int type, int subtype) = 0; + virtual void NotifyFileUploading( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploadingPatch( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t EstimatedBytesToUpload) = 0; + virtual void NotifyFileUploadingAttributes( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploaded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) = 0; + virtual void NotifyFileSynchronised( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) = 0; + virtual void NotifyDirectoryCreated( + int64_t ObjectID, + const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirectoryDeleted( + int64_t ObjectID, + const std::string& rRemotePath) = 0; + virtual void NotifyFileDeleted( + int64_t ObjectID, + const std::string& rRemotePath) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: LocationResolver +// Purpose: Interface for classes that can resolve locations to paths, +// like BackupDaemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class LocationResolver +{ +public: + virtual ~LocationResolver() { } + virtual bool FindLocationPathName(const std::string &rLocationName, + std::string &rPathOut) const = 0; +}; + +#endif // BACKUPDAEMONINTERFACE__H diff --git a/lib/bbackupd/Win32BackupService.cpp b/lib/bbackupd/Win32BackupService.cpp new file mode 100644 index 00000000..6d027abf --- /dev/null +++ b/lib/bbackupd/Win32BackupService.cpp @@ -0,0 +1,48 @@ +// Win32 service functions for Box Backup, by Nick Knight + +#ifdef WIN32 + +#include "Box.h" +#include "BackupDaemon.h" +#include "MainHelper.h" +#include "BoxPortsAndFiles.h" +#include "BackupStoreException.h" + +#include "MemLeakFindOn.h" + +#include "Win32BackupService.h" + +Win32BackupService* gpDaemonService = NULL; +extern HANDLE gStopServiceEvent; +extern DWORD gServiceReturnCode; + +unsigned int WINAPI RunService(LPVOID lpParameter) +{ + DWORD retVal = gpDaemonService->WinService((const char*) lpParameter); + gServiceReturnCode = retVal; + SetEvent(gStopServiceEvent); + return retVal; +} + +void TerminateService(void) +{ + gpDaemonService->SetTerminateWanted(); +} + +DWORD Win32BackupService::WinService(const char* pConfigFileName) +{ + DWORD ret; + + if (pConfigFileName != NULL) + { + ret = this->Main(pConfigFileName); + } + else + { + ret = this->Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE); + } + + return ret; +} + +#endif // WIN32 diff --git a/lib/bbackupd/Win32BackupService.h b/lib/bbackupd/Win32BackupService.h new file mode 100644 index 00000000..e7f077f2 --- /dev/null +++ b/lib/bbackupd/Win32BackupService.h @@ -0,0 +1,21 @@ +// Box Backup service daemon implementation by Nick Knight + +#ifndef WIN32BACKUPSERVICE_H +#define WIN32BACKUPSERVICE_H + +#ifdef WIN32 + +class Configuration; +class ConfigurationVerify; +class BackupDaemon; + +class Win32BackupService : public BackupDaemon +{ +public: + DWORD WinService(const char* pConfigFileName); +}; + +#endif // WIN32 + +#endif // WIN32BACKUPSERVICE_H + diff --git a/lib/bbackupd/Win32ServiceFunctions.cpp b/lib/bbackupd/Win32ServiceFunctions.cpp new file mode 100644 index 00000000..2df914a7 --- /dev/null +++ b/lib/bbackupd/Win32ServiceFunctions.cpp @@ -0,0 +1,384 @@ +//*************************************************************** +// From the book "Win32 System Services: The Heart of Windows 98 +// and Windows 2000" +// by Marshall Brain +// Published by Prentice Hall +// Copyright 1995 Prentice Hall. +// +// This code implements the Windows API Service interface +// for the Box Backup for Windows native port. +// Adapted for Box Backup by Nick Knight. +//*************************************************************** + +#ifdef WIN32 + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif +#ifdef HAVE_PROCESS_H + #include <process.h> +#endif + +extern void TerminateService(void); +extern unsigned int WINAPI RunService(LPVOID lpParameter); + +// Global variables + +TCHAR* gServiceName = TEXT("Box Backup Service"); +SERVICE_STATUS gServiceStatus; +SERVICE_STATUS_HANDLE gServiceStatusHandle = 0; +HANDLE gStopServiceEvent = 0; +DWORD gServiceReturnCode = 0; + +#define SERVICE_NAME "boxbackup" + +void ShowMessage(char *s) +{ + MessageBox(0, s, "Box Backup Message", + MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); +} + +void ErrorHandler(char *s, DWORD err) +{ + char buf[256]; + memset(buf, 0, sizeof(buf)); + _snprintf(buf, sizeof(buf)-1, "%s: %s", s, + GetErrorMessage(err).c_str()); + BOX_ERROR(buf); + MessageBox(0, buf, "Error", + MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); + ExitProcess(err); +} + +void WINAPI ServiceControlHandler( DWORD controlCode ) +{ + switch ( controlCode ) + { + case SERVICE_CONTROL_INTERROGATE: + break; + + case SERVICE_CONTROL_SHUTDOWN: + case SERVICE_CONTROL_STOP: + Beep(1000,100); + TerminateService(); + gServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + SetEvent(gStopServiceEvent); + return; + + case SERVICE_CONTROL_PAUSE: + break; + + case SERVICE_CONTROL_CONTINUE: + break; + + default: + if ( controlCode >= 128 && controlCode <= 255 ) + // user defined control code + break; + else + // unrecognised control code + break; + } + + SetServiceStatus( gServiceStatusHandle, &gServiceStatus ); +} + +// ServiceMain is called when the SCM wants to +// start the service. When it returns, the service +// has stopped. It therefore waits on an event +// just before the end of the function, and +// that event gets set when it is time to stop. +// It also returns on any error because the +// service cannot start if there is an eror. + +static char* spConfigFileName; + +VOID ServiceMain(DWORD argc, LPTSTR *argv) +{ + // initialise service status + gServiceStatus.dwServiceType = SERVICE_WIN32; + gServiceStatus.dwCurrentState = SERVICE_STOPPED; + gServiceStatus.dwControlsAccepted = 0; + gServiceStatus.dwWin32ExitCode = NO_ERROR; + gServiceStatus.dwServiceSpecificExitCode = NO_ERROR; + gServiceStatus.dwCheckPoint = 0; + gServiceStatus.dwWaitHint = 0; + + gServiceStatusHandle = RegisterServiceCtrlHandler(gServiceName, + ServiceControlHandler); + + if (gServiceStatusHandle) + { + // service is starting + gServiceStatus.dwCurrentState = SERVICE_START_PENDING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + // do initialisation here + gStopServiceEvent = CreateEvent(0, TRUE, FALSE, 0); + if (!gStopServiceEvent) + { + gServiceStatus.dwControlsAccepted &= + ~(SERVICE_ACCEPT_STOP | + SERVICE_ACCEPT_SHUTDOWN); + gServiceStatus.dwCurrentState = SERVICE_STOPPED; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + return; + } + + HANDLE ourThread = (HANDLE)_beginthreadex( + NULL, + 0, + RunService, + spConfigFileName, + CREATE_SUSPENDED, + NULL); + + SetThreadPriority(ourThread, THREAD_PRIORITY_LOWEST); + ResumeThread(ourThread); + + // we are now running so tell the SCM + gServiceStatus.dwControlsAccepted |= + (SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); + gServiceStatus.dwCurrentState = SERVICE_RUNNING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + // do cleanup here + WaitForSingleObject(gStopServiceEvent, INFINITE); + CloseHandle(gStopServiceEvent); + gStopServiceEvent = 0; + + // service was stopped + gServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + // service is now stopped + gServiceStatus.dwControlsAccepted &= + ~(SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); + + gServiceStatus.dwCurrentState = SERVICE_STOPPED; + + if (gServiceReturnCode != 0) + { + gServiceStatus.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR; + gServiceStatus.dwServiceSpecificExitCode = gServiceReturnCode; + } + + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + } +} + +int OurService(const char* pConfigFileName) +{ + spConfigFileName = strdup(pConfigFileName); + + SERVICE_TABLE_ENTRY serviceTable[] = + { + { SERVICE_NAME, (LPSERVICE_MAIN_FUNCTION) ServiceMain }, + { NULL, NULL } + }; + BOOL success; + + // Register with the SCM + success = StartServiceCtrlDispatcher(serviceTable); + + free(spConfigFileName); + spConfigFileName = NULL; + + if (!success) + { + ErrorHandler("Failed to start service. Did you start " + "Box Backup from the Service Control Manager? " + "(StartServiceCtrlDispatcher)", GetLastError()); + return 1; + } + + return 0; +} + +int InstallService(const char* pConfigFileName, const std::string& rServiceName) +{ + if (pConfigFileName != NULL) + { + EMU_STRUCT_STAT st; + + if (emu_stat(pConfigFileName, &st) != 0) + { + BOX_LOG_SYS_ERROR("Failed to open configuration file " + "'" << pConfigFileName << "'"); + return 1; + } + + if (!(st.st_mode & S_IFREG)) + { + + BOX_ERROR("Failed to open configuration file '" << + pConfigFileName << "': not a file"); + return 1; + } + } + + SC_HANDLE scm = OpenSCManager(0, 0, SC_MANAGER_CREATE_SERVICE); + + if (!scm) + { + BOX_ERROR("Failed to open service control manager: " << + GetErrorMessage(GetLastError())); + return 1; + } + + char cmd[MAX_PATH]; + GetModuleFileName(NULL, cmd, sizeof(cmd)-1); + cmd[sizeof(cmd)-1] = 0; + + std::string cmdWithArgs(cmd); + cmdWithArgs += " -s -S \"" + rServiceName + "\""; + + if (pConfigFileName != NULL) + { + cmdWithArgs += " \""; + cmdWithArgs += pConfigFileName; + cmdWithArgs += "\""; + } + + std::string serviceDesc = "Box Backup (" + rServiceName + ")"; + + SC_HANDLE newService = CreateService( + scm, + rServiceName.c_str(), + serviceDesc.c_str(), + SERVICE_ALL_ACCESS, + SERVICE_WIN32_OWN_PROCESS, + SERVICE_AUTO_START, + SERVICE_ERROR_NORMAL, + cmdWithArgs.c_str(), + 0,0,0,0,0); + + DWORD err = GetLastError(); + CloseServiceHandle(scm); + + if (!newService) + { + switch (err) + { + case ERROR_SERVICE_EXISTS: + { + BOX_ERROR("Failed to create Box Backup " + "service: it already exists"); + } + break; + + case ERROR_SERVICE_MARKED_FOR_DELETE: + { + BOX_ERROR("Failed to create Box Backup " + "service: it is waiting to be deleted"); + } + break; + + case ERROR_DUPLICATE_SERVICE_NAME: + { + BOX_ERROR("Failed to create Box Backup " + "service: a service with this name " + "already exists"); + } + break; + + default: + { + BOX_ERROR("Failed to create Box Backup " + "service: error " << + GetErrorMessage(GetLastError())); + } + } + + return 1; + } + + BOX_INFO("Created Box Backup service"); + + SERVICE_DESCRIPTION desc; + desc.lpDescription = "Backs up your data files over the Internet"; + + if (!ChangeServiceConfig2(newService, SERVICE_CONFIG_DESCRIPTION, + &desc)) + { + BOX_WARNING("Failed to set description for Box Backup " + "service: " << GetErrorMessage(GetLastError())); + } + + CloseServiceHandle(newService); + + return 0; +} + +int RemoveService(const std::string& rServiceName) +{ + SC_HANDLE scm = OpenSCManager(0,0,SC_MANAGER_CREATE_SERVICE); + + if (!scm) + { + BOX_ERROR("Failed to open service control manager: " << + GetErrorMessage(GetLastError())); + return 1; + } + + SC_HANDLE service = OpenService(scm, rServiceName.c_str(), + SERVICE_ALL_ACCESS|DELETE); + DWORD err = GetLastError(); + CloseServiceHandle(scm); + + if (!service) + { + if (err == ERROR_SERVICE_DOES_NOT_EXIST || + err == ERROR_IO_PENDING) + // hello microsoft? anyone home? + { + BOX_ERROR("Failed to open Box Backup service: " + "not installed or not found"); + } + else + { + BOX_ERROR("Failed to open Box Backup service: " << + GetErrorMessage(err)); + } + return 1; + } + + SERVICE_STATUS status; + if (!ControlService(service, SERVICE_CONTROL_STOP, &status)) + { + err = GetLastError(); + if (err != ERROR_SERVICE_NOT_ACTIVE) + { + BOX_WARNING("Failed to stop Box Backup service: " << + GetErrorMessage(err)); + } + } + + BOOL deleted = DeleteService(service); + err = GetLastError(); + CloseServiceHandle(service); + + if (deleted) + { + BOX_INFO("Box Backup service deleted"); + return 0; + } + else if (err == ERROR_SERVICE_MARKED_FOR_DELETE) + { + BOX_ERROR("Failed to remove Box Backup service: " + "it is already being deleted"); + } + else + { + BOX_ERROR("Failed to remove Box Backup service: " << + GetErrorMessage(err)); + } + + return 1; +} + +#endif // WIN32 diff --git a/lib/bbackupd/Win32ServiceFunctions.h b/lib/bbackupd/Win32ServiceFunctions.h new file mode 100644 index 00000000..e04c368f --- /dev/null +++ b/lib/bbackupd/Win32ServiceFunctions.h @@ -0,0 +1,19 @@ +//*************************************************************** +// From the book "Win32 System Services: The Heart of Windows 98 +// and Windows 2000" +// by Marshall Brain +// Published by Prentice Hall +// Copyright 1995 Prentice Hall. +// +// This code implements the Windows API Service interface +// for the Box Backup for Windows native port. +//*************************************************************** + +#ifndef WIN32SERVICEFUNCTIONS_H +#define WIN32SERVICEFUNCTIONS_H + +int RemoveService (const std::string& rServiceName); +int InstallService (const char* pConfigFilePath, const std::string& rServiceName); +int OurService (const char* pConfigFileName); + +#endif diff --git a/lib/bbackupquery/BackupQueries.cpp b/lib/bbackupquery/BackupQueries.cpp new file mode 100644 index 00000000..bcb1827e --- /dev/null +++ b/lib/bbackupquery/BackupQueries.cpp @@ -0,0 +1,2410 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupQueries.cpp +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include <stdio.h> +#include <errno.h> +#include <stdlib.h> +#include <limits.h> +#include <sys/types.h> +#include <sys/stat.h> + +#ifdef HAVE_DIRENT_H + #include <dirent.h> +#endif + +#include <algorithm> +#include <cstring> +#include <limits> +#include <iostream> +#include <ostream> +#include <set> + +#include "BackupClientFileAttributes.h" +#include "BackupClientMakeExcludeList.h" +#include "BackupClientRestore.h" +#include "BackupQueries.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFilenameClear.h" +#include "BoxTimeToText.h" +#include "CommonException.h" +#include "Configuration.h" +#include "ExcludeList.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "IOStream.h" +#include "Logging.h" +#include "PathUtils.h" +#include "SelfFlushingStream.h" +#include "Utils.h" +#include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" + +#include "MemLeakFindOn.h" + +// min() and max() macros from stdlib.h break numeric_limits<>::min(), etc. +#undef min +#undef max + +#define COMPARE_RETURN_SAME 1 +#define COMPARE_RETURN_DIFFERENT 2 +#define COMPARE_RETURN_ERROR 3 +#define COMMAND_RETURN_ERROR 4 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::BackupQueries() +// Purpose: Constructor +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +BackupQueries::BackupQueries(BackupProtocolCallable &rConnection, + const Configuration &rConfiguration, bool readWrite) + : mReadWrite(readWrite), + mrConnection(rConnection), + mrConfiguration(rConfiguration), + mQuitNow(false), + mRunningAsRoot(false), + mWarnedAboutOwnerAttributes(false), + mReturnCode(0) // default return code +{ + #ifdef WIN32 + mRunningAsRoot = TRUE; + #else + mRunningAsRoot = (::geteuid() == 0); + #endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::~BackupQueries() +// Purpose: Destructor +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +BackupQueries::~BackupQueries() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::DoCommand(const char *, bool) +// Purpose: Perform a command +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::DoCommand(ParsedCommand& rCommand) +{ + // Check... + + if(rCommand.mFailed) + { + BOX_ERROR("Parse failed: unknown command '" << + rCommand.mCmdElements[0] << "' or failed to convert " + "encoding of arguments"); + return; + } + + if(rCommand.mCmdElements.size() < 1) + { + // blank command + return; + } + + if(rCommand.pSpec->type == Command_sh && + rCommand.mCmdElements.size() == 2) + { + // Yes, run shell command + int result = ::system(rCommand.mCmdElements[1].c_str()); + if(result != 0) + { + BOX_WARNING("System command returned error code " << + result); + SetReturnCode(ReturnCode::Command_Error); + } + return; + } + + if(rCommand.pSpec->type == Command_Unknown) + { + // No such command + BOX_ERROR("Unrecognised command: " << rCommand.mCmdElements[0]); + return; + } + + // Arguments + std::vector<std::string> args(rCommand.mCmdElements.begin() + 1, + rCommand.mCmdElements.end()); + + // Set up options + bool opts[256]; + for(int o = 0; o < 256; ++o) opts[o] = false; + // BLOCK + { + // options + const char *c = rCommand.mOptions.c_str(); + while(*c != 0) + { + // Valid option? + if(::strchr(rCommand.pSpec->opts, *c) == NULL) + { + BOX_ERROR("Invalid option '" << *c << "' for " + "command " << rCommand.pSpec->name); + return; + } + opts[(int)*c] = true; + ++c; + } + } + + if(rCommand.pSpec->type != Command_Quit) + { + // If not a quit command, set the return code to zero + SetReturnCode(ReturnCode::Command_OK); + } + + // Handle command + switch(rCommand.pSpec->type) + { + case Command_Quit: + mQuitNow = true; + break; + + case Command_List: + CommandList(args, opts); + break; + + case Command_pwd: + { + // Simple implementation, so do it here + BOX_NOTICE(GetCurrentDirectoryName() << " (" << + BOX_FORMAT_OBJECTID(GetCurrentDirectoryID()) << + ")"); + } + break; + + case Command_cd: + CommandChangeDir(args, opts); + break; + + case Command_lcd: + CommandChangeLocalDir(args); + break; + + case Command_sh: + BOX_ERROR("The command to run must be specified as an argument."); + break; + + case Command_GetObject: + CommandGetObject(args, opts); + break; + + case Command_Get: + CommandGet(args, opts); + break; + + case Command_Compare: + CommandCompare(args, opts); + break; + + case Command_Restore: + CommandRestore(args, opts); + break; + + case Command_Usage: + CommandUsage(opts); + break; + + case Command_Help: + CommandHelp(args); + break; + + case Command_Undelete: + CommandUndelete(args, opts); + break; + + case Command_Delete: + CommandDelete(args, opts); + break; + + default: + BOX_ERROR("Unknown command: " << rCommand.mCmdElements[0]); + break; + } +} + +#define LIST_OPTION_TIMES_ATTRIBS 'a' +#define LIST_OPTION_SORT_NO_DIRS_FIRST 'D' +#define LIST_OPTION_NOFLAGS 'F' +#define LIST_OPTION_DISPLAY_HASH 'h' +#define LIST_OPTION_SORT_ID 'i' +#define LIST_OPTION_NOOBJECTID 'I' +#define LIST_OPTION_SORT_REVERSE 'r' +#define LIST_OPTION_RECURSIVE 'R' +#define LIST_OPTION_SIZEINBLOCKS 's' +#define LIST_OPTION_SORT_SIZE 'S' +#define LIST_OPTION_TIMES_LOCAL 't' +#define LIST_OPTION_TIMES_UTC 'T' +#define LIST_OPTION_SORT_NONE 'U' + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandList(const std::vector<std::string> &, const bool *) +// Purpose: List directories (optionally recursive) +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandList(const std::vector<std::string> &args, const bool *opts) +{ + // default to using the current directory + int64_t rootDir = GetCurrentDirectoryID(); + + // name of base directory + std::string listRoot; // blank + + // Got a directory in the arguments? + if(args.size() > 0) + { +#ifdef WIN32 + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) + return; +#else + const std::string& storeDirEncoded(args[0]); +#endif + + // Attempt to find the directory + rootDir = FindDirectoryObjectID(storeDirEncoded, + opts[LIST_OPTION_ALLOWOLD], + opts[LIST_OPTION_ALLOWDELETED]); + + if(rootDir == 0) + { + BOX_ERROR("Directory '" << args[0] << "' not found " + "on store."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + } + + // List it + List(rootDir, listRoot, opts, true /* first level to list */); +} + +static std::string GetTimeString(BackupStoreDirectory::Entry& en, + bool useLocalTime, bool showAttrModificationTimes) +{ + std::ostringstream out; + box_time_t originalTime, newAttributesTime; + + // there is no attribute modification time in the directory + // entry, unfortunately, so we can't display it. + originalTime = en.GetModificationTime(); + out << BoxTimeToISO8601String(originalTime, useLocalTime); + + if(en.HasAttributes()) + { + const StreamableMemBlock &storeAttr(en.GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + + box_time_t NewModificationTime, NewAttrModificationTime; + attr.GetModificationTimes(&NewModificationTime, + &NewAttrModificationTime); + + if (showAttrModificationTimes) + { + newAttributesTime = NewAttrModificationTime; + } + else + { + newAttributesTime = NewModificationTime; + } + + if (newAttributesTime == originalTime) + { + out << "*"; + } + else + { + out << "~" << BoxTimeToISO8601String(newAttributesTime, + useLocalTime); + } + } + else + { + out << " "; + } + + return out.str(); +} + +/* We need a way to pass options to sort functions for sorting. The algorithm + * doesn't seem to provide a way to do this, so I'm using a global variable. + * Which is not thread safe, but we don't currently use threads so that should + * be OK. Do not use threads without checking! + */ +const bool *gThreadUnsafeOptions; + +int DirsFirst(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + if (a->IsDir() && !b->IsDir()) + { + return -1; // a < b + } + else if (!a->IsDir() && b->IsDir()) + { + return 1; // b > a + } + else + { + return 0; // continue comparison + } +} + +#define MAYBE_DIRS_FIRST(a, b) \ + if (!gThreadUnsafeOptions[LIST_OPTION_SORT_NO_DIRS_FIRST]) \ + { \ + int result = DirsFirst(a, b); \ + if (result < 0) return true; /* a < b */ \ + else if (result > 0) return false; /* a > b */ \ + /* else: fall through */ \ + } + +#define MAYBE_REVERSE(result) \ + (result != gThreadUnsafeOptions[LIST_OPTION_SORT_REVERSE]) +// result is false, opts[reverse] is false => return false +// result is false, opts[reverse] is true => return true +// result is true, opts[reverse] is false => return true +// result is true, opts[reverse] is true => return false +// this is logical XOR, for which the boolean operator is !=. + +bool SortById(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + bool result = (a->GetObjectID() < b->GetObjectID()); + return MAYBE_REVERSE(result); +} + +bool SortBySize(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + bool result = (a->GetSizeInBlocks() < b->GetSizeInBlocks()); + return MAYBE_REVERSE(result); +} + +bool SortByName(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + BackupStoreFilenameClear afc(a->GetName()); + BackupStoreFilenameClear bfc(b->GetName()); + std::string an = afc.GetClearFilename(); + std::string bn = bfc.GetClearFilename(); + bool result = (an < bn); + return MAYBE_REVERSE(result); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::List(int64_t, const std::string &, const bool *, bool) +// Purpose: Do the actual listing of directories and files +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::List(int64_t DirID, const std::string &rListRoot, + const bool *opts, bool FirstLevel, std::ostream* pOut) +{ +#ifdef WIN32 + DWORD n_chars; + HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE); +#endif + + // Generate exclude flags + int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + if(!opts[LIST_OPTION_ALLOWOLD]) excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + if(!opts[LIST_OPTION_ALLOWDELETED]) excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; + + // Do communication + try + { + mrConnection.QueryListDirectory( + DirID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // both files and directories + excludeFlags, + true /* want attributes */); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to list directory: " << e.what()); + SetReturnCode(ReturnCode::Command_Error); + return; + } + catch (...) + { + BOX_ERROR("Failed to list directory: unknown error"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Store entry pointers in a std::vector for sorting + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + std::vector<BackupStoreDirectory::Entry*> sorted_entries; + while((en = i.Next()) != 0) + { + sorted_entries.push_back(en); + } + + // Typedef to avoid mind-bending while dealing with pointers to functions. + typedef bool (EntryComparator_t)(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b); + // Default is no comparator, i.e. no sorting. + EntryComparator_t* pComparator = NULL; + + if (opts[LIST_OPTION_SORT_ID]) + { + pComparator = &SortById; + } + else if (opts[LIST_OPTION_SORT_SIZE]) + { + pComparator = &SortBySize; + } + else if (opts[LIST_OPTION_SORT_NONE]) + { + // do nothing + } + else // sort by name + { + pComparator = &SortByName; + } + + if (pComparator != NULL) + { + gThreadUnsafeOptions = opts; + sort(sorted_entries.begin(), sorted_entries.end(), + pComparator); + gThreadUnsafeOptions = NULL; + } + + for (std::vector<BackupStoreDirectory::Entry*>::const_iterator + i = sorted_entries.begin(); + i != sorted_entries.end(); i++) + { + en = *i; + std::ostringstream buf; + + // Display this entry + BackupStoreFilenameClear clear(en->GetName()); + + // Object ID? + if(!opts[LIST_OPTION_NOOBJECTID]) + { + // add object ID to line + buf << std::hex << std::internal << std::setw(8) << + std::setfill('0') << en->GetObjectID() << + std::dec << " "; + } + + // Flags? + if(!opts[LIST_OPTION_NOFLAGS]) + { + static const char *flags = BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES; + char displayflags[16]; + // make sure f is big enough + ASSERT(sizeof(displayflags) >= sizeof(BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES) + 3); + // Insert flags + char *f = displayflags; + const char *t = flags; + int16_t en_flags = en->GetFlags(); + while(*t != 0) + { + *f = ((en_flags&1) == 0)?'-':*t; + en_flags >>= 1; + f++; + t++; + } + // attributes flags + *(f++) = (en->HasAttributes())?'a':'-'; + + // terminate + *(f++) = ' '; + *(f++) = '\0'; + buf << displayflags; + + if(en_flags != 0) + { + buf << "[ERROR: Entry has additional flags set] "; + } + } + + if(opts[LIST_OPTION_TIMES_UTC]) + { + // Show UTC times... + buf << GetTimeString(*en, false, + opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; + } + + if(opts[LIST_OPTION_TIMES_LOCAL]) + { + // Show local times... + buf << GetTimeString(*en, true, + opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; + } + + if(opts[LIST_OPTION_DISPLAY_HASH]) + { + buf << std::hex << std::internal << std::setw(16) << + std::setfill('0') << en->GetAttributesHash() << + std::dec; + } + + if(opts[LIST_OPTION_SIZEINBLOCKS]) + { + buf << std::internal << std::setw(5) << + std::setfill('0') << en->GetSizeInBlocks() << + " "; + } + + // add name + if(!FirstLevel) + { +#ifdef WIN32 + std::string listRootDecoded; + if(!ConvertUtf8ToConsole(rListRoot.c_str(), + listRootDecoded)) return; + listRootDecoded += "/"; + buf << listRootDecoded; + WriteConsole(hOut, listRootDecoded.c_str(), + strlen(listRootDecoded.c_str()), &n_chars, NULL); +#else + buf << rListRoot << "/"; +#endif + } + + std::string fileName; + try + { + fileName = clear.GetClearFilename(); + } + catch(CipherException &e) + { + fileName = "<decrypt failed>"; + } + +#ifdef WIN32 + std::string fileNameUtf8 = fileName; + if(!ConvertUtf8ToConsole(fileNameUtf8, fileName)) + { + fileName = fileNameUtf8 + " [convert encoding failed]"; + } +#endif + + buf << fileName; + + if(!en->GetName().IsEncrypted()) + { + buf << " [FILENAME NOT ENCRYPTED]"; + } + + buf << std::endl; + + if(pOut) + { + (*pOut) << buf.str(); + } + else + { +#ifdef WIN32 + std::string line = buf.str(); + if (!WriteConsole(hOut, line.c_str(), line.size(), + &n_chars, NULL)) + { + // WriteConsole failed, try standard method + std::cout << buf.str(); + } +#else + std::cout << buf.str(); +#endif + } + + // Directory? + if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) + { + // Recurse? + if(opts[LIST_OPTION_RECURSIVE]) + { + std::string subroot(rListRoot); + if(!FirstLevel) subroot += '/'; + subroot += clear.GetClearFilename(); + List(en->GetObjectID(), subroot, opts, + false /* not the first level to list */, + pOut); + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::FindDirectoryObjectID(const +// std::string &) +// Purpose: Find the object ID of a directory on the store, +// or return 0 for not found. If pStack != 0, the +// object is set to the stack of directories. +// Will start from the current directory stack. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, + bool AllowOldVersion, bool AllowDeletedDirs, + std::vector<std::pair<std::string, int64_t> > *pStack) +{ + // Split up string into elements + std::vector<std::string> dirElements; + SplitString(rDirName, '/', dirElements); + + // Start from current stack, or root, whichever is required + std::vector<std::pair<std::string, int64_t> > stack; + int64_t dirID = BackupProtocolListDirectory::RootDirectory; + if(rDirName.size() > 0 && rDirName[0] == '/') + { + // Root, do nothing + } + else + { + // Copy existing stack + stack = mDirStack; + if(stack.size() > 0) + { + dirID = stack[stack.size() - 1].second; + } + } + + // Generate exclude flags + int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + if(!AllowOldVersion) excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + if(!AllowDeletedDirs) excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; + + // Read directories + for(unsigned int e = 0; e < dirElements.size(); ++e) + { + if(dirElements[e].size() > 0) + { + if(dirElements[e] == ".") + { + // Ignore. + } + else if(dirElements[e] == "..") + { + // Up one! + if(stack.size() > 0) + { + // Remove top element + stack.pop_back(); + + // New dir ID + dirID = (stack.size() > 0)?(stack[stack.size() - 1].second):BackupProtocolListDirectory::RootDirectory; + } + else + { + // At root anyway + dirID = BackupProtocolListDirectory::RootDirectory; + } + } + else + { + // Not blank element. Read current directory. + std::auto_ptr<BackupProtocolSuccess> dirreply(mrConnection.QueryListDirectory( + dirID, + BackupProtocolListDirectory::Flags_Dir, // just directories + excludeFlags, + true /* want attributes */)); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Then... find the directory within it + BackupStoreDirectory::Iterator i(dir); + BackupStoreFilenameClear dirname(dirElements[e]); + BackupStoreDirectory::Entry *en = i.FindMatchingClearName(dirname); + if(en == 0) + { + // Not found + return 0; + } + + // Object ID for next round of searching + dirID = en->GetObjectID(); + + // Push onto stack + stack.push_back(std::pair<std::string, int64_t>(dirElements[e], dirID)); + } + } + } + + // If required, copy the new stack to the caller + if(pStack) + { + *pStack = stack; + } + + return dirID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::GetCurrentDirectoryID() +// Purpose: Returns the ID of the current directory +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::GetCurrentDirectoryID() +{ + // Special case for root + if(mDirStack.size() == 0) + { + return BackupProtocolListDirectory::RootDirectory; + } + + // Otherwise, get from the last entry on the stack + return mDirStack[mDirStack.size() - 1].second; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::GetCurrentDirectoryName() +// Purpose: Gets the name of the current directory +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +std::string BackupQueries::GetCurrentDirectoryName() +{ + // Special case for root + if(mDirStack.size() == 0) + { + return std::string("/"); + } + + // Build path + std::string r; + for(unsigned int l = 0; l < mDirStack.size(); ++l) + { + r += "/"; +#ifdef WIN32 + std::string dirName; + if(!ConvertUtf8ToConsole(mDirStack[l].first.c_str(), dirName)) + return "error"; + r += dirName; +#else + r += mDirStack[l].first; +#endif + } + + return r; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandChangeDir(const std::vector<std::string> &) +// Purpose: Change directory command +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandChangeDir(const std::vector<std::string> &args, const bool *opts) +{ + if(args.size() != 1 || args[0].size() == 0) + { + BOX_ERROR("Incorrect usage. cd [-o] [-d] <directory>"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + +#ifdef WIN32 + std::string dirName; + if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) return; +#else + const std::string& dirName(args[0]); +#endif + + std::vector<std::pair<std::string, int64_t> > newStack; + int64_t id = FindDirectoryObjectID(dirName, opts['o'], opts['d'], + &newStack); + + if(id == 0) + { + BOX_ERROR("Directory '" << args[0] << "' not found."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Store new stack + mDirStack = newStack; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandChangeLocalDir(const std::vector<std::string> &) +// Purpose: Change local directory command +// Created: 2003/10/11 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandChangeLocalDir(const std::vector<std::string> &args) +{ + if(args.size() != 1 || args[0].size() == 0) + { + BOX_ERROR("Incorrect usage. lcd <local-directory>"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Try changing directory +#ifdef WIN32 + std::string dirName; + if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) + { + BOX_ERROR("Failed to convert path from console encoding."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + int result = ::chdir(dirName.c_str()); +#else + int result = ::chdir(args[0].c_str()); +#endif + if(result != 0) + { + if(errno == ENOENT || errno == ENOTDIR) + { + BOX_ERROR("Directory '" << args[0] << "' does not exist."); + } + else + { + BOX_LOG_SYS_ERROR("Failed to change to directory " + "'" << args[0] << "'"); + } + + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Report current dir + char wd[PATH_MAX]; + if(::getcwd(wd, PATH_MAX) == 0) + { + BOX_LOG_SYS_ERROR("Error getting current directory"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + +#ifdef WIN32 + if(!ConvertUtf8ToConsole(wd, dirName)) + { + BOX_ERROR("Failed to convert new path from console encoding."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + BOX_INFO("Local current directory is now '" << dirName << "'."); +#else + BOX_INFO("Local current directory is now '" << wd << "'."); +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandGetObject(const std::vector<std::string> &, const bool *) +// Purpose: Gets an object without any translation. +// Created: 2003/10/11 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandGetObject(const std::vector<std::string> &args, const bool *opts) +{ + // Check args + if(args.size() != 2) + { + BOX_ERROR("Incorrect usage. getobject <object-id> " + "<local-filename>"); + return; + } + + int64_t id = ::strtoll(args[0].c_str(), 0, 16); + if(id == std::numeric_limits<long long>::min() || id == std::numeric_limits<long long>::max() || id == 0) + { + BOX_ERROR("Not a valid object ID (specified in hex): " << + args[0]); + return; + } + + // Does file exist? + EMU_STRUCT_STAT st; + if(EMU_STAT(args[1].c_str(), &st) == 0 || errno != ENOENT) + { + BOX_ERROR("The local file '" << args[1] << " already exists."); + return; + } + + // Open file + FileStream out(args[1].c_str(), O_WRONLY | O_CREAT | O_EXCL); + + // Request that object + try + { + // Request object + std::auto_ptr<BackupProtocolSuccess> getobj(mrConnection.QueryGetObject(id)); + + // Stream that object out to the file + std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); + objectStream->CopyStreamTo(out); + + BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(id) << + " fetched successfully."); + } + catch(ConnectionException &e) + { + if(mrConnection.GetLastErrorType() == BackupProtocolError::Err_DoesNotExist) + { + BOX_ERROR("Object ID " << BOX_FORMAT_OBJECTID(id) << + " does not exist on store."); + ::unlink(args[1].c_str()); + } + else + { + BOX_ERROR("Error occured fetching object."); + ::unlink(args[1].c_str()); + } + } + catch(...) + { + ::unlink(args[1].c_str()); + BOX_ERROR("Error occured fetching object."); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::FindFileID(const std::string& +// rNameOrIdString, const bool *options, +// int64_t *pDirIdOut, std::string* pFileNameOut) +// Purpose: Locate a file on the store (either by name or by +// object ID, depending on opts['i'], where name can +// include a path) and return the file ID, placing the +// directory ID in *pDirIdOut and the filename part +// of the path in *pFileNameOut (if not NULL). +// Created: 2008-09-12 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::FindFileID(const std::string& rNameOrIdString, + const bool *opts, int64_t *pDirIdOut, std::string* pFileNameOut, + int16_t flagsInclude, int16_t flagsExclude, int16_t* pFlagsOut) +{ + // Find object ID somehow + int64_t fileId; + int64_t dirId = GetCurrentDirectoryID(); + std::string fileName = rNameOrIdString; + + if(!opts['i']) + { + // does this remote filename include a path? + std::string::size_type index = fileName.rfind('/'); + if(index != std::string::npos) + { + std::string dirName(fileName.substr(0, index)); + fileName = fileName.substr(index + 1); + + dirId = FindDirectoryObjectID(dirName); + if(dirId == 0) + { + BOX_ERROR("Directory '" << dirName << + "' not found."); + return 0; + } + } + } + + BackupStoreFilenameClear fn(fileName); + + // Need to look it up in the current directory + mrConnection.QueryListDirectory( + dirId, flagsInclude, flagsExclude, + true /* do want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + BackupStoreDirectory::Entry *en; + + if(opts['i']) + { + // Specified as ID. + fileId = ::strtoll(rNameOrIdString.c_str(), 0, 16); + if(fileId == std::numeric_limits<long long>::min() || + fileId == std::numeric_limits<long long>::max() || + fileId == 0) + { + BOX_ERROR("Not a valid object ID (specified in hex): " + << rNameOrIdString); + return 0; + } + + // Check that the item is actually in the directory + en = dir.FindEntryByID(fileId); + if(en == 0) + { + BOX_ERROR("File ID " << + BOX_FORMAT_OBJECTID(fileId) << + " not found in current directory on store.\n" + "(You can only access files by ID from the " + "current directory.)"); + return 0; + } + } + else + { + // Specified by name, find the object in the directory to get the ID + BackupStoreDirectory::Iterator i(dir); + en = i.FindMatchingClearName(fn); + if(en == 0) + { + BOX_ERROR("Filename '" << rNameOrIdString << "' " + "not found in current directory on store.\n" + "(Subdirectories in path not searched.)"); + return 0; + } + + fileId = en->GetObjectID(); + } + + *pDirIdOut = dirId; + + if(pFlagsOut) + { + *pFlagsOut = en->GetFlags(); + } + + if(pFileNameOut) + { + BackupStoreFilenameClear entryName(en->GetName()); + *pFileNameOut = entryName.GetClearFilename(); + } + + return fileId; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandGet(const std::vector<std::string> &, const bool *) +// Purpose: Command to get a file from the store +// Created: 2003/10/12 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) +{ + // At least one argument? + // Check args + if(args.size() < 1 || (opts['i'] && args.size() != 2) || args.size() > 2) + { + BOX_ERROR("Incorrect usage.\n" + "get <remote-filename> [<local-filename>] or\n" + "get -i <object-id> <local-filename>"); + return; + } + + // Find object ID somehow + int64_t fileId, dirId; + std::string localName; + +#ifdef WIN32 + for (std::vector<std::string>::iterator + i = args.begin(); i != args.end(); i++) + { + std::string out; + if(!ConvertConsoleToUtf8(i->c_str(), out)) + { + BOX_ERROR("Failed to convert encoding."); + return; + } + *i = out; + } +#endif + + int16_t flagsExclude; + + if(opts['i']) + { + // can retrieve anything by ID + flagsExclude = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + } + else + { + // only current versions by name + flagsExclude = + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted; + } + + + fileId = FindFileID(args[0], opts, &dirId, &localName, + BackupProtocolListDirectory::Flags_File, // just files + flagsExclude, NULL /* don't care about flags found */); + + if (fileId == 0) + { + // error already reported + return; + } + + if(opts['i']) + { + // Specified as ID. Must have a local name in the arguments + // (check at beginning of function ensures this) + localName = args[1]; + } + else + { + // Specified by name. Local name already set by FindFileID, + // but may be overridden by user supplying a second argument. + if(args.size() == 2) + { + localName = args[1]; + } + } + + // Does local file already exist? (don't want to overwrite) + EMU_STRUCT_STAT st; + if(EMU_STAT(localName.c_str(), &st) == 0 || errno != ENOENT) + { + BOX_ERROR("The local file " << localName << " already exists, " + "will not overwrite it."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Request it from the store + try + { + // Request object + mrConnection.QueryGetFile(dirId, fileId); + + // Stream containing encoded file + std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); + + // Decode it + BackupStoreFile::DecodeFile(*objectStream, localName.c_str(), mrConnection.GetTimeout()); + + // Done. + BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(fileId) << + " fetched successfully."); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to fetch file: " << + e.what()); + ::unlink(localName.c_str()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to fetch file: " << + e.what()); + ::unlink(localName.c_str()); + } + catch(...) + { + BOX_ERROR("Failed to fetch file: unknown error"); + ::unlink(localName.c_str()); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CompareParams::CompareParams() +// Purpose: Constructor +// Created: 29/1/04 +// +// -------------------------------------------------------------------------- +BackupQueries::CompareParams::CompareParams(bool QuickCompare, + bool IgnoreExcludes, bool IgnoreAttributes, + box_time_t LatestFileUploadTime) +: BoxBackupCompareParams(QuickCompare, IgnoreExcludes, IgnoreAttributes, + LatestFileUploadTime), + mDifferences(0), + mDifferencesExplainedByModTime(0), + mUncheckedFiles(0), + mExcludedDirs(0), + mExcludedFiles(0) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandCompare(const std::vector<std::string> &, const bool *) +// Purpose: Command to compare data on the store with local data +// Created: 2003/10/12 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandCompare(const std::vector<std::string> &args, const bool *opts) +{ + box_time_t LatestFileUploadTime = GetCurrentBoxTime(); + + // Try and work out the time before which all files should be on the server + { + std::string syncTimeFilename(mrConfiguration.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); + syncTimeFilename += "last_sync_start"; + // Stat it to get file time + EMU_STRUCT_STAT st; + if(EMU_STAT(syncTimeFilename.c_str(), &st) == 0) + { + // Files modified after this time shouldn't be on the server, so report errors slightly differently + LatestFileUploadTime = FileModificationTime(st) - + SecondsToBoxTime(mrConfiguration.GetKeyValueInt("MinimumFileAge")); + } + else + { + BOX_WARNING("Failed to determine the time of the last " + "synchronisation -- checks not performed."); + } + } + + // Parameters, including count of differences + BackupQueries::CompareParams params(opts['q'], // quick compare? + opts['E'], // ignore excludes + opts['A'], // ignore attributes + LatestFileUploadTime); + + params.mQuietCompare = opts['Q']; + + // Quick compare? + if(params.QuickCompare()) + { + BOX_WARNING("Quick compare used -- file attributes are not " + "checked."); + } + + if(!opts['l'] && opts['a'] && args.size() == 0) + { + // Compare all locations + const Configuration &rLocations( + mrConfiguration.GetSubConfiguration("BackupLocations")); + std::vector<std::string> locNames = + rLocations.GetSubConfigurationNames(); + for(std::vector<std::string>::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + CompareLocation(*pLocName, params); + } + } + else if(opts['l'] && !opts['a'] && args.size() == 1) + { + // Compare one location + CompareLocation(args[0], params); + } + else if(!opts['l'] && !opts['a'] && args.size() == 2) + { + // Compare directory to directory + + // Can't be bothered to do all the hard work to work out which location it's on, and hence which exclude list + if(!params.IgnoreExcludes()) + { + BOX_ERROR("Cannot use excludes on directory to directory comparison -- use -E flag to specify ignored excludes."); + return; + } + else + { + // Do compare + Compare(args[0], args[1], params); + } + } + else + { + BOX_ERROR("Incorrect usage.\ncompare -a\n or compare -l <location-name>\n or compare <store-dir-name> <local-dir-name>"); + return; + } + + if (!params.mQuietCompare) + { + BOX_INFO("[ " << + params.mDifferencesExplainedByModTime << " (of " << + params.mDifferences << ") differences probably " + "due to file modifications after the last upload ]"); + } + + BOX_INFO("Differences: " << params.mDifferences << " (" << + params.mExcludedDirs << " dirs excluded, " << + params.mExcludedFiles << " files excluded, " << + params.mUncheckedFiles << " files not checked)"); + + // Set return code? + if(opts['c']) + { + if (params.mUncheckedFiles != 0) + { + SetReturnCode(ReturnCode::Compare_Error); + } + else if (params.mDifferences != 0) + { + SetReturnCode(ReturnCode::Compare_Different); + } + else + { + SetReturnCode(ReturnCode::Compare_Same); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CompareLocation(const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a location +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::CompareLocation(const std::string &rLocation, + BoxBackupCompareParams &rParams) +{ + // Find the location's sub configuration + const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations")); + if(!locations.SubConfigurationExists(rLocation.c_str())) + { + BOX_ERROR("Location " << rLocation << " does not exist."); + return; + } + const Configuration &loc(locations.GetSubConfiguration(rLocation.c_str())); + + #ifdef WIN32 + { + std::string path = loc.GetKeyValue("Path"); + if (path.size() > 0 && path[path.size()-1] == + DIRECTORY_SEPARATOR_ASCHAR) + { + BOX_WARNING("Location '" << rLocation << "' path ends " + "with '" DIRECTORY_SEPARATOR "', " + "compare may fail!"); + } + } + #endif + + // Generate the exclude lists + if(!rParams.IgnoreExcludes()) + { + rParams.LoadExcludeLists(loc); + } + + // Then get it compared + Compare(std::string("/") + rLocation, loc.GetKeyValue("Path"), rParams); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::Compare(const std::string &, +// const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a store directory against a local directory +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::Compare(const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams) +{ +#ifdef WIN32 + std::string localDirEncoded; + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(rLocalDir.c_str(), localDirEncoded)) return; + if(!ConvertConsoleToUtf8(rStoreDir.c_str(), storeDirEncoded)) return; +#else + const std::string& localDirEncoded(rLocalDir); + const std::string& storeDirEncoded(rStoreDir); +#endif + + // Get the directory ID of the directory -- only use current data + int64_t dirID = FindDirectoryObjectID(storeDirEncoded); + + // Found? + if(dirID == 0) + { + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalDir.c_str(), &st) == 0) + { + if(FileAttrModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyRemoteFileMissing(localDirEncoded, + storeDirEncoded, modifiedAfterLastSync); + return; + } + + // Go! + Compare(dirID, storeDirEncoded, localDirEncoded, rParams); +} + +void BackupQueries::CompareOneFile(int64_t DirID, + BackupStoreDirectory::Entry *pEntry, + const std::string& rLocalPath, + const std::string& rStorePath, + BoxBackupCompareParams &rParams) +{ + int64_t fileId = pEntry->GetObjectID(); + int64_t fileSize = 0; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalPath.c_str(), &st) == 0) + { + fileSize = st.st_size; + } + + try + { + // Files the same flag? + bool equal = true; + + // File modified after last sync flag + bool modifiedAfterLastSync = false; + + bool hasDifferentAttribs = false; + + bool alreadyReported = false; + + if(rParams.QuickCompare()) + { + // Compare file -- fetch it + mrConnection.QueryGetBlockIndexByID(fileId); + + // Stream containing block index + std::auto_ptr<IOStream> blockIndexStream(mrConnection.ReceiveStream()); + + // Compare + equal = BackupStoreFile::CompareFileContentsAgainstBlockIndex( + rLocalPath.c_str(), *blockIndexStream, + mrConnection.GetTimeout()); + } + else + { + // Compare file -- fetch it + mrConnection.QueryGetFile(DirID, pEntry->GetObjectID()); + + // Stream containing encoded file + std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); + + // Decode it + std::auto_ptr<BackupStoreFile::DecodedStream> fileOnServerStream; + + // Got additional attributes? + if(pEntry->HasAttributes()) + { + // Use these attributes + const StreamableMemBlock &storeAttr(pEntry->GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + fileOnServerStream.reset( + BackupStoreFile::DecodeFileStream( + *objectStream, + mrConnection.GetTimeout(), + &attr).release()); + } + else + { + // Use attributes stored in file + fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout()).release()); + } + + // Should always be something in the auto_ptr, it's how the interface is defined. But be paranoid. + if(!fileOnServerStream.get()) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Compare attributes + BackupClientFileAttributes localAttr; + box_time_t fileModTime = 0; + localAttr.ReadAttributes(rLocalPath.c_str(), false /* don't zero mod times */, &fileModTime); + modifiedAfterLastSync = (fileModTime > rParams.LatestFileUploadTime()); + bool ignoreAttrModTime = true; + + #ifdef WIN32 + // attr mod time is really + // creation time, so check it + ignoreAttrModTime = false; + #endif + + if(!rParams.IgnoreAttributes() && + #ifdef PLATFORM_DISABLE_SYMLINK_ATTRIB_COMPARE + !fileOnServerStream->IsSymLink() && + #endif + !localAttr.Compare(fileOnServerStream->GetAttributes(), + ignoreAttrModTime, + fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */)) + { + hasDifferentAttribs = true; + } + + // Compare contents, if it's a regular file not a link + // Remember, we MUST read the entire stream from the server. + SelfFlushingStream flushObject(*objectStream); + + if(!fileOnServerStream->IsSymLink()) + { + SelfFlushingStream flushFile(*fileOnServerStream); + // Open the local file + std::auto_ptr<FileStream> apLocalFile; + + try + { + apLocalFile.reset(new FileStream(rLocalPath.c_str())); + } + catch(std::exception &e) + { + rParams.NotifyLocalFileReadFailed(rLocalPath, + rStorePath, fileSize, e); + alreadyReported = true; + } + catch(...) + { + rParams.NotifyLocalFileReadFailed(rLocalPath, + rStorePath, fileSize); + alreadyReported = true; + } + + if(apLocalFile.get()) + { + equal = apLocalFile->CompareWith(*fileOnServerStream, + mrConnection.GetTimeout()); + } + } + } + + rParams.NotifyFileCompared(rLocalPath, rStorePath, fileSize, + hasDifferentAttribs, !equal, modifiedAfterLastSync, + pEntry->HasAttributes()); + } + catch(BoxException &e) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize, + e); + } + catch(std::exception &e) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize, + e); + } + catch(...) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::Compare(int64_t, const std::string &, +// const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a store directory against a local directory +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams) +{ + rParams.NotifyDirComparing(rLocalDir, rStoreDir); + + // Get info on the local directory + EMU_STRUCT_STAT st; + if(EMU_LSTAT(rLocalDir.c_str(), &st) != 0) + { + // What kind of error? + if(errno == ENOTDIR || errno == ENOENT) + { + rParams.NotifyLocalDirMissing(rLocalDir, rStoreDir); + } + else + { + rParams.NotifyLocalDirAccessFailed(rLocalDir, rStoreDir); + } + return; + } + + // Get the directory listing from the store + mrConnection.QueryListDirectory( + DirID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // get everything + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted, + // except for old versions and deleted files + true /* want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Test out the attributes + if(!dir.HasAttributes()) + { + rParams.NotifyStoreDirMissingAttributes(rLocalDir, rStoreDir); + } + else + { + // Fetch the attributes + const StreamableMemBlock &storeAttr(dir.GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + + // Get attributes of local directory + BackupClientFileAttributes localAttr; + localAttr.ReadAttributes(rLocalDir.c_str(), + true /* directories have zero mod times */); + + if(attr.Compare(localAttr, true, true /* ignore modification times */)) + { + rParams.NotifyDirCompared(rLocalDir, rStoreDir, + false, false /* actually we didn't check :) */); + } + else + { + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalDir.c_str(), &st) == 0) + { + if(FileAttrModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyDirCompared(rLocalDir, rStoreDir, + true, modifiedAfterLastSync); + } + } + + // Open the local directory + DIR *dirhandle = ::opendir(rLocalDir.c_str()); + if(dirhandle == 0) + { + rParams.NotifyLocalDirAccessFailed(rLocalDir, rStoreDir); + return; + } + + try + { + // Read the files and directories into sets + std::set<std::string> localFiles; + std::set<std::string> localDirs; + struct dirent *localDirEn = 0; + while((localDirEn = readdir(dirhandle)) != 0) + { + // Not . and ..! + if(localDirEn->d_name[0] == '.' && + (localDirEn->d_name[1] == '\0' || (localDirEn->d_name[1] == '.' && localDirEn->d_name[2] == '\0'))) + { + // ignore, it's . or .. + +#ifdef HAVE_VALID_DIRENT_D_TYPE + if (localDirEn->d_type != DT_DIR) + { + BOX_ERROR("d_type does not really " + "work on your platform. " + "Reconfigure Box!"); + return; + } +#endif + + continue; + } + + std::string localDirPath(MakeFullPath(rLocalDir, + localDirEn->d_name)); + std::string storeDirPath(rStoreDir + "/" + + localDirEn->d_name); + +#ifndef HAVE_VALID_DIRENT_D_TYPE + EMU_STRUCT_STAT st; + if(EMU_LSTAT(localDirPath.c_str(), &st) != 0) + { + // Check whether dir is excluded before trying + // to stat it, to fix problems with .gvfs + // directories that are not readable by root + // causing compare to crash: + // http://lists.boxbackup.org/pipermail/boxbackup/2010-January/000013.html + if(rParams.IsExcludedDir(localDirPath)) + { + rParams.NotifyExcludedDir(localDirPath, + storeDirPath); + continue; + } + else + { + THROW_EXCEPTION_MESSAGE(CommonException, + OSFileError, localDirPath); + } + } + + // Entry -- file or dir? + if(S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) + { + // File or symbolic link + localFiles.insert(std::string(localDirEn->d_name)); + } + else if(S_ISDIR(st.st_mode)) + { + // Directory + localDirs.insert(std::string(localDirEn->d_name)); + } +#else + // Entry -- file or dir? + if(localDirEn->d_type == DT_REG || localDirEn->d_type == DT_LNK) + { + // File or symbolic link + localFiles.insert(std::string(localDirEn->d_name)); + } + else if(localDirEn->d_type == DT_DIR) + { + // Directory + localDirs.insert(std::string(localDirEn->d_name)); + } +#endif + } + // Close directory + if(::closedir(dirhandle) != 0) + { + BOX_LOG_SYS_ERROR("Failed to close local directory " + "'" << rLocalDir << "'"); + } + dirhandle = 0; + + // Do the same for the store directories + std::set<std::pair<std::string, BackupStoreDirectory::Entry *> > storeFiles; + std::set<std::pair<std::string, BackupStoreDirectory::Entry *> > storeDirs; + + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *storeDirEn = 0; + while((storeDirEn = i.Next()) != 0) + { + // Decrypt filename + BackupStoreFilenameClear name(storeDirEn->GetName()); + + // What is it? + if((storeDirEn->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == BackupStoreDirectory::Entry::Flags_File) + { + // File + storeFiles.insert(std::pair<std::string, BackupStoreDirectory::Entry *>(name.GetClearFilename(), storeDirEn)); + } + else + { + // Dir + storeDirs.insert(std::pair<std::string, BackupStoreDirectory::Entry *>(name.GetClearFilename(), storeDirEn)); + } + } + +#ifdef _MSC_VER + typedef std::set<std::string>::iterator string_set_iter_t; +#else + typedef std::set<std::string>::const_iterator string_set_iter_t; +#endif + + // Now compare files. + for(std::set<std::pair<std::string, BackupStoreDirectory::Entry *> >::const_iterator i = storeFiles.begin(); i != storeFiles.end(); ++i) + { + const std::string& fileName(i->first); + + std::string localPath(MakeFullPath(rLocalDir, fileName)); + std::string storePath(rStoreDir + "/" + fileName); + + rParams.NotifyFileComparing(localPath, storePath); + + // Does the file exist locally? + string_set_iter_t local(localFiles.find(fileName)); + if(local == localFiles.end()) + { + // Not found -- report + rParams.NotifyLocalFileMissing(localPath, + storePath); + } + else + { + CompareOneFile(DirID, i->second, localPath, + storePath, rParams); + + // Remove from set so that we know it's been compared + localFiles.erase(local); + } + } + + // Report any files which exist locally, but not on the store + for(string_set_iter_t i = localFiles.begin(); i != localFiles.end(); ++i) + { + std::string localPath(MakeFullPath(rLocalDir, *i)); + std::string storePath(rStoreDir + "/" + *i); + + // Should this be ignored (ie is excluded)? + if(!rParams.IsExcludedFile(localPath)) + { + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(localPath.c_str(), &st) == 0) + { + if(FileModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyRemoteFileMissing(localPath, + storePath, modifiedAfterLastSync); + } + else + { + rParams.NotifyExcludedFile(localPath, + storePath); + } + } + + // Finished with the files, clear the sets to reduce memory usage slightly + localFiles.clear(); + storeFiles.clear(); + + // Now do the directories, recursively to check subdirectories + for(std::set<std::pair<std::string, BackupStoreDirectory::Entry *> >::const_iterator i = storeDirs.begin(); i != storeDirs.end(); ++i) + { + std::string localPath(MakeFullPath(rLocalDir, i->first)); + std::string storePath(rStoreDir + "/" + i->first); + + // Does the directory exist locally? + string_set_iter_t local(localDirs.find(i->first)); + if(local == localDirs.end() && + rParams.IsExcludedDir(localPath)) + { + rParams.NotifyExcludedFileNotDeleted(localPath, + storePath); + } + else if(local == localDirs.end()) + { + // Not found -- report + rParams.NotifyLocalFileMissing(localPath, + storePath); + } + else if(rParams.IsExcludedDir(localPath)) + { + // don't recurse into excluded directories + } + else + { + // Compare directory + Compare(i->second->GetObjectID(), + storePath, localPath, rParams); + + // Remove from set so that we know it's been compared + localDirs.erase(local); + } + } + + // Report any directories which exist locally, but not on the store + for(std::set<std::string>::const_iterator + i = localDirs.begin(); + i != localDirs.end(); ++i) + { + std::string localPath(MakeFullPath(rLocalDir, *i)); + std::string storePath(rStoreDir + "/" + *i); + + // Should this be ignored (ie is excluded)? + if(!rParams.IsExcludedDir(localPath)) + { + bool modifiedAfterLastSync = false; + + // Check the dir modification time + EMU_STRUCT_STAT st; + if(EMU_STAT(localPath.c_str(), &st) == 0 && + FileModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + + rParams.NotifyRemoteFileMissing(localPath, + storePath, modifiedAfterLastSync); + } + else + { + rParams.NotifyExcludedDir(localPath, storePath); + } + } + } + catch(...) + { + if(dirhandle != 0) + { + ::closedir(dirhandle); + } + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandRestore(const std::vector<std::string> &, const bool *) +// Purpose: Restore a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandRestore(const std::vector<std::string> &args, const bool *opts) +{ + // Check arguments + if(args.size() < 1 || args.size() > 2) + { + BOX_ERROR("Incorrect usage. restore [-drif] <remote-name> " + "[<local-name>]"); + return; + } + + // Restoring deleted things? + bool restoreDeleted = opts['d']; + + std::string storeDirEncoded; + + // Get directory ID + int64_t dirID = 0; + if(opts['i']) + { + // Specified as ID. + dirID = ::strtoll(args[0].c_str(), 0, 16); + if(dirID == std::numeric_limits<long long>::min() || dirID == std::numeric_limits<long long>::max() || dirID == 0) + { + BOX_ERROR("Not a valid object ID (specified in hex): " + << args[0]); + return; + } + std::ostringstream oss; + oss << BOX_FORMAT_OBJECTID(args[0]); + storeDirEncoded = oss.str(); + } + else + { +#ifdef WIN32 + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) + return; +#else + storeDirEncoded = args[0]; +#endif + + // Look up directory ID + dirID = FindDirectoryObjectID(storeDirEncoded, + false /* no old versions */, + restoreDeleted /* find deleted dirs */); + } + + // Allowable? + if(dirID == 0) + { + BOX_ERROR("Directory '" << args[0] << "' not found on server"); + return; + } + + if(dirID == BackupProtocolListDirectory::RootDirectory) + { + BOX_ERROR("Cannot restore the root directory -- restore locations individually."); + return; + } + + std::string localName; + + if(args.size() == 2) + { + #ifdef WIN32 + if(!ConvertConsoleToUtf8(args[1].c_str(), localName)) + { + return; + } + #else + localName = args[1]; + #endif + } + else + { + localName = args[0]; + } + + // Go and restore... + int result; + + try + { + // At TRACE level, we print a line for each file and + // directory, so we don't need dots. + + result = BackupClientRestore(mrConnection, dirID, + storeDirEncoded.c_str(), localName.c_str(), + true /* print progress dots */, restoreDeleted, + false /* don't undelete after restore! */, + opts['r'] /* resume? */, + opts['f'] /* force continue after errors */); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to restore: " << e.what()); + SetReturnCode(ReturnCode::Command_Error); + return; + } + catch(...) + { + BOX_ERROR("Failed to restore: unknown exception"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + switch(result) + { + case Restore_Complete: + BOX_INFO("Restore complete."); + break; + + case Restore_CompleteWithErrors: + BOX_WARNING("Restore complete, but some files could not be " + "restored."); + break; + + case Restore_ResumePossible: + BOX_ERROR("Resume possible -- repeat command with -r flag " + "to resume."); + SetReturnCode(ReturnCode::Command_Error); + break; + + case Restore_TargetExists: + BOX_ERROR("The target directory exists. You cannot restore " + "over an existing directory."); + SetReturnCode(ReturnCode::Command_Error); + break; + + case Restore_TargetPathNotFound: + BOX_ERROR("The target directory path does not exist.\n" + "To restore to a directory whose parent " + "does not exist, create the parent first."); + SetReturnCode(ReturnCode::Command_Error); + break; + + case Restore_UnknownError: + BOX_ERROR("Unknown error during restore."); + SetReturnCode(ReturnCode::Command_Error); + break; + + default: + BOX_ERROR("Unknown restore result " << result << "."); + SetReturnCode(ReturnCode::Command_Error); + break; + } +} + + + +// These are autogenerated by a script. +extern const char *help_commands[]; +extern const char *help_text[]; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandHelp(const std::vector<std::string> &args) +// Purpose: Display help on commands +// Created: 15/2/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandHelp(const std::vector<std::string> &args) +{ + if(args.size() == 0) + { + // Display a list of all commands + printf("Available commands are:\n"); + for(int c = 0; help_commands[c] != 0; ++c) + { + printf(" %s\n", help_commands[c]); + } + printf("Type \"help <command>\" for more information on a command.\n\n"); + } + else + { + // Display help on a particular command + int c; + for(c = 0; help_commands[c] != 0; ++c) + { + if(::strcmp(help_commands[c], args[0].c_str()) == 0) + { + // Found the command, print help + printf("\n%s\n", help_text[c]); + break; + } + } + if(help_commands[c] == 0) + { + printf("No help found for command '%s'\n", args[0].c_str()); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUsage() +// Purpose: Display storage space used on server +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUsage(const bool *opts) +{ + bool MachineReadable = opts['m']; + + // Request full details from the server + std::auto_ptr<BackupProtocolAccountUsage> usage(mrConnection.QueryGetAccountUsage()); + + // Display each entry in turn + int64_t hardLimit = usage->GetBlocksHardLimit(); + int32_t blockSize = usage->GetBlockSize(); + CommandUsageDisplayEntry("Used", usage->GetBlocksUsed(), hardLimit, + blockSize, MachineReadable); + CommandUsageDisplayEntry("Old files", usage->GetBlocksInOldFiles(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Deleted files", usage->GetBlocksInDeletedFiles(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Directories", usage->GetBlocksInDirectories(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Soft limit", usage->GetBlocksSoftLimit(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Hard limit", hardLimit, hardLimit, blockSize, + MachineReadable); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUsageDisplayEntry(const char *, +// int64_t, int64_t, int32_t, bool) +// Purpose: Display an entry in the usage table +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUsageDisplayEntry(const char *Name, int64_t Size, +int64_t HardLimit, int32_t BlockSize, bool MachineReadable) +{ + std::cout << FormatUsageLineStart(Name, MachineReadable) << + FormatUsageBar(Size, Size * BlockSize, HardLimit * BlockSize, + MachineReadable) << std::endl; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUndelete(const std::vector<std::string> &, const bool *) +// Purpose: Undelete a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUndelete(const std::vector<std::string> &args, const bool *opts) +{ + if (!mReadWrite) + { + BOX_ERROR("This command requires a read-write connection. " + "Please reconnect with the -w option."); + return; + } + + // Check arguments + if(args.size() != 1) + { + BOX_ERROR("Incorrect usage. undelete <name> or undelete -i <object-id>"); + return; + } + +#ifdef WIN32 + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) return; +#else + const std::string& storeDirEncoded(args[0]); +#endif + + // Find object ID somehow + int64_t fileId, parentId; + std::string fileName; + int16_t flagsOut; + + fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, + /* include files and directories */ + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + /* include old and deleted files */ + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + &flagsOut); + + if (fileId == 0) + { + // error already reported + return; + } + + // Undelete it on the store + try + { + // Undelete object + if(flagsOut & BackupProtocolListDirectory::Flags_File) + { + mrConnection.QueryUndeleteFile(parentId, fileId); + } + else + { + mrConnection.QueryUndeleteDirectory(fileId); + } + } + catch (BoxException &e) + { + BOX_ERROR("Failed to undelete object: " << + e.what()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to undelete object: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Failed to undelete object: unknown error"); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandDelete(const +// std::vector<std::string> &, const bool *) +// Purpose: Deletes a file +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandDelete(const std::vector<std::string> &args, + const bool *opts) +{ + if (!mReadWrite) + { + BOX_ERROR("This command requires a read-write connection. " + "Please reconnect with the -w option."); + return; + } + + // Check arguments + if(args.size() != 1) + { + BOX_ERROR("Incorrect usage. delete <name>"); + return; + } + +#ifdef WIN32 + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) return; +#else + const std::string& storeDirEncoded(args[0]); +#endif + + // Find object ID somehow + int64_t fileId, parentId; + std::string fileName; + int16_t flagsOut; + + fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, + /* include files and directories */ + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + /* exclude old and deleted files */ + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted, + &flagsOut); + + if (fileId == 0) + { + // error already reported + return; + } + + BackupStoreFilenameClear fn(fileName); + + // Delete it on the store + try + { + // Delete object + if(flagsOut & BackupProtocolListDirectory::Flags_File) + { + mrConnection.QueryDeleteFile(parentId, fn); + } + else + { + mrConnection.QueryDeleteDirectory(fileId); + } + } + catch (BoxException &e) + { + BOX_ERROR("Failed to delete object: " << + e.what()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to delete object: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Failed to delete object: unknown error"); + } +} diff --git a/lib/bbackupquery/BackupQueries.h b/lib/bbackupquery/BackupQueries.h new file mode 100644 index 00000000..96df34f5 --- /dev/null +++ b/lib/bbackupquery/BackupQueries.h @@ -0,0 +1,440 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupQueries.h +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPQUERIES__H +#define BACKUPQUERIES__H + +#include <iostream> +#include <string> +#include <vector> + +#include "BoxTime.h" +#include "BoxBackupCompareParams.h" +#include "BackupStoreDirectory.h" + +class BackupProtocolCallable; +class Configuration; +class ExcludeList; + +typedef enum +{ + Command_Unknown = 0, + Command_Quit, + Command_List, + Command_pwd, + Command_cd, + Command_lcd, + Command_sh, + Command_GetObject, + Command_Get, + Command_Compare, + Command_Restore, + Command_Help, + Command_Usage, + Command_Undelete, + Command_Delete, +} +CommandType; + +struct QueryCommandSpecification; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupQueries +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +class BackupQueries +{ +public: + BackupQueries(BackupProtocolCallable &rConnection, + const Configuration &rConfiguration, + bool readWrite); + ~BackupQueries(); +private: + BackupQueries(const BackupQueries &); +public: + struct ParsedCommand + { + std::vector<std::string> mCmdElements; + std::string mOptions; + std::string mCompleteCommand; + bool mInOptions, mFailed; + QueryCommandSpecification* pSpec; + // mArgCount is the same as mCmdElements.size() for a complete + // command, but if the command line ends in a space, + // e.g. during readline parsing, it can be one greater, + // to indicate that we should complete the next item instead. + size_t mCompleteArgCount; + ParsedCommand(const std::string& Command, + bool isFromCommandLine); + bool IsEmpty() { return mCmdElements.empty(); } + bool IsFailed() { return mFailed; } + }; + + void DoCommand(ParsedCommand& rCommand); + + // Ready to stop? + bool Stop() {return mQuitNow;} + + // Return code? + int GetReturnCode() {return mReturnCode;} + + void List(int64_t DirID, const std::string &rListRoot, const bool *opts, + bool FirstLevel, std::ostream* pOut = NULL); + void CommandList(const std::vector<std::string> &args, const bool *opts); + + // Commands + void CommandChangeDir(const std::vector<std::string> &args, const bool *opts); + void CommandChangeLocalDir(const std::vector<std::string> &args); + void CommandGetObject(const std::vector<std::string> &args, const bool *opts); + void CommandGet(std::vector<std::string> args, const bool *opts); + void CommandCompare(const std::vector<std::string> &args, const bool *opts); + void CommandRestore(const std::vector<std::string> &args, const bool *opts); + void CommandUndelete(const std::vector<std::string> &args, const bool *opts); + void CommandDelete(const std::vector<std::string> &args, + const bool *opts); + void CommandUsage(const bool *opts); + void CommandUsageDisplayEntry(const char *Name, int64_t Size, + int64_t HardLimit, int32_t BlockSize, bool MachineReadable); + void CommandHelp(const std::vector<std::string> &args); + + class CompareParams : public BoxBackupCompareParams + { + public: + CompareParams(bool QuickCompare, bool IgnoreExcludes, + bool IgnoreAttributes, box_time_t LatestFileUploadTime); + + bool mQuietCompare; + int mDifferences; + int mDifferencesExplainedByModTime; + int mUncheckedFiles; + int mExcludedDirs; + int mExcludedFiles; + + std::string ConvertForConsole(const std::string& rUtf8String) + { + #ifdef WIN32 + std::string output; + + if(!ConvertUtf8ToConsole(rUtf8String.c_str(), output)) + { + BOX_WARNING("Character set conversion failed " + "on string: " << rUtf8String); + return rUtf8String; + } + + return output; + #else + return rUtf8String; + #endif + } + + virtual void NotifyLocalDirMissing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Local directory '" << + ConvertForConsole(rLocalPath) << "' " + "does not exist, but remote directory does."); + mDifferences ++; + } + + virtual void NotifyLocalDirAccessFailed( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_LOG_SYS_WARNING("Failed to access local directory " + "'" << ConvertForConsole(rLocalPath) << "'"); + mUncheckedFiles ++; + } + + virtual void NotifyStoreDirMissingAttributes( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Store directory '" << + ConvertForConsole(rRemotePath) << "' " + "doesn't have attributes."); + } + + virtual void NotifyRemoteFileMissing( + const std::string& rLocalPath, + const std::string& rRemotePath, + bool modifiedAfterLastSync) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "exists, but remote file '" << + ConvertForConsole(rRemotePath) << "' " + "does not."); + mDifferences ++; + + if(modifiedAfterLastSync) + { + mDifferencesExplainedByModTime ++; + BOX_INFO("(the file above was modified after " + "the last sync time -- might be " + "reason for difference)"); + } + } + + virtual void NotifyLocalFileMissing( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Remote file '" << + ConvertForConsole(rRemotePath) << "' " + "exists, but local file '" << + ConvertForConsole(rLocalPath) << "' does not."); + mDifferences ++; + } + + virtual void NotifyExcludedFileNotDeleted( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "is excluded, but remote file '" << + ConvertForConsole(rRemotePath) << "' " + "still exists."); + mDifferences ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + BoxException& rException) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath) << "': " << + rException.what() << " (" << + rException.GetType() << "/" << + rException.GetSubType() << ")"); + mUncheckedFiles ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath) << "': " << + rException.what()); + mUncheckedFiles ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath)); + mUncheckedFiles ++; + } + + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) + { + BOX_ERROR("Failed to read local file '" << + ConvertForConsole(rLocalPath) << "': " << + rException.what()); + mUncheckedFiles ++; + } + + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) + { + BOX_ERROR("Failed to read local file '" << + ConvertForConsole(rLocalPath)); + mUncheckedFiles ++; + } + + virtual void NotifyExcludedFile(const std::string& rLocalPath, + const std::string& rRemotePath) + { + mExcludedFiles ++; + } + + virtual void NotifyExcludedDir(const std::string& rLocalPath, + const std::string& rRemotePath) + { + mExcludedDirs ++; + } + + virtual void NotifyDirComparing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_INFO("Comparing directory: " << rLocalPath); + } + + virtual void NotifyDirCompared( + const std::string& rLocalPath, + const std::string& rRemotePath, + bool HasDifferentAttributes, + bool modifiedAfterLastSync) + { + if(HasDifferentAttributes) + { + BOX_WARNING("Local directory '" << + ConvertForConsole(rLocalPath) << "' " + "has different attributes to " + "store directory '" << + ConvertForConsole(rRemotePath) << "'."); + mDifferences ++; + + if(modifiedAfterLastSync) + { + mDifferencesExplainedByModTime ++; + BOX_INFO("(the directory above was " + "modified after the last sync " + "time -- might be reason for " + "difference)"); + } + } + } + + virtual void NotifyFileComparing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_TRACE("Comparing file: " << rLocalPath); + } + + virtual void NotifyFileCompared(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + bool HasDifferentAttributes, bool HasDifferentContents, + bool ModifiedAfterLastSync, bool NewAttributesApplied) + { + int NewDifferences = 0; + + if(HasDifferentAttributes) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "has different attributes to " + "store file '" << + ConvertForConsole(rRemotePath) << "'."); + NewDifferences ++; + } + + if(HasDifferentContents) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "has different contents to " + "store file '" << + ConvertForConsole(rRemotePath) << "'."); + NewDifferences ++; + } + + if(HasDifferentAttributes || HasDifferentContents) + { + if(ModifiedAfterLastSync) + { + mDifferencesExplainedByModTime += + NewDifferences; + BOX_INFO("(the file above was modified " + "after the last sync time -- " + "might be reason for difference)"); + } + else if(NewAttributesApplied) + { + BOX_INFO("(the file above has had new " + "attributes applied)\n"); + } + } + + mDifferences += NewDifferences; + } + }; + void CompareLocation(const std::string &rLocation, + BoxBackupCompareParams &rParams); + void Compare(const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams); + void Compare(int64_t DirID, const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams); + void CompareOneFile(int64_t DirID, BackupStoreDirectory::Entry *pEntry, + const std::string& rLocalPath, const std::string& rStorePath, + BoxBackupCompareParams &rParams); + +public: + + class ReturnCode + { + public: + typedef enum { + Command_OK = 0, + Compare_Same = 1, + Compare_Different, + Compare_Error, + Command_Error, + } Type; + }; + + // Were private, but needed by completion functions: + int64_t GetCurrentDirectoryID(); + int64_t FindDirectoryObjectID(const std::string &rDirName, + bool AllowOldVersion = false, bool AllowDeletedDirs = false, + std::vector<std::pair<std::string, int64_t> > *pStack = 0); + +private: + + // Utility functions + int64_t FindFileID(const std::string& rNameOrIdString, + const bool *opts, int64_t *pDirIdOut, + std::string* pFileNameOut, int16_t flagsInclude, + int16_t flagsExclude, int16_t* pFlagsOut); + std::string GetCurrentDirectoryName(); + void SetReturnCode(int code) {mReturnCode = code;} + +private: + bool mReadWrite; + BackupProtocolCallable &mrConnection; + const Configuration &mrConfiguration; + bool mQuitNow; + std::vector<std::pair<std::string, int64_t> > mDirStack; + bool mRunningAsRoot; + bool mWarnedAboutOwnerAttributes; + int mReturnCode; +}; + +typedef std::vector<std::string> (*CompletionHandler) + (BackupQueries::ParsedCommand& rCommand, const std::string& prefix, + BackupProtocolCallable& rProtocol, const Configuration& rConfig, + BackupQueries& rQueries); + +std::vector<std::string> CompleteCommand(BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolCallable& rProtocol, + const Configuration& rConfig, BackupQueries& rQueries); +std::vector<std::string> CompleteOptions(BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolCallable& rProtocol, + const Configuration& rConfig, BackupQueries& rQueries); + +#define MAX_COMPLETION_HANDLERS 4 + +struct QueryCommandSpecification +{ + const char* name; + const char* opts; + CommandType type; + CompletionHandler complete[MAX_COMPLETION_HANDLERS]; +}; + +// Data about commands +extern QueryCommandSpecification commands[]; + +extern const char *alias[]; +extern const int aliasIs[]; + +#define LIST_OPTION_ALLOWOLD 'o' +#define LIST_OPTION_ALLOWDELETED 'd' + +#endif // BACKUPQUERIES__H + diff --git a/lib/bbackupquery/BoxBackupCompareParams.h b/lib/bbackupquery/BoxBackupCompareParams.h new file mode 100644 index 00000000..655df947 --- /dev/null +++ b/lib/bbackupquery/BoxBackupCompareParams.h @@ -0,0 +1,112 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxBackupCompareParams.h +// Purpose: Parameters and notifiers for a compare operation +// Created: 2008/12/30 +// +// -------------------------------------------------------------------------- + +#ifndef BOXBACKUPCOMPAREPARAMS__H +#define BOXBACKUPCOMPAREPARAMS__H + +#include <memory> +#include <string> + +#include "BoxTime.h" +#include "ExcludeList.h" +#include "BackupClientMakeExcludeList.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: BoxBackupCompareParams +// Purpose: Parameters and notifiers for a compare operation +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +class BoxBackupCompareParams +{ +private: + std::auto_ptr<const ExcludeList> mapExcludeFiles, mapExcludeDirs; + bool mQuickCompare; + bool mIgnoreExcludes; + bool mIgnoreAttributes; + box_time_t mLatestFileUploadTime; + +public: + BoxBackupCompareParams(bool QuickCompare, bool IgnoreExcludes, + bool IgnoreAttributes, box_time_t LatestFileUploadTime) + : mQuickCompare(QuickCompare), + mIgnoreExcludes(IgnoreExcludes), + mIgnoreAttributes(IgnoreAttributes), + mLatestFileUploadTime(LatestFileUploadTime) + { } + + virtual ~BoxBackupCompareParams() { } + + bool QuickCompare() { return mQuickCompare; } + bool IgnoreExcludes() { return mIgnoreExcludes; } + bool IgnoreAttributes() { return mIgnoreAttributes; } + box_time_t LatestFileUploadTime() { return mLatestFileUploadTime; } + + void LoadExcludeLists(const Configuration& rLoc) + { + mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rLoc)); + mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rLoc)); + } + bool IsExcludedFile(const std::string& rLocalPath) + { + if (!mapExcludeFiles.get()) return false; + return mapExcludeFiles->IsExcluded(rLocalPath); + } + bool IsExcludedDir(const std::string& rLocalPath) + { + if (!mapExcludeDirs.get()) return false; + return mapExcludeDirs->IsExcluded(rLocalPath); + } + + virtual void NotifyLocalDirMissing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyLocalDirAccessFailed(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyStoreDirMissingAttributes(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyRemoteFileMissing(const std::string& rLocalPath, + const std::string& rRemotePath, + bool modifiedAfterLastSync) = 0; + virtual void NotifyLocalFileMissing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyExcludedFileNotDeleted(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + BoxException& rException) = 0; + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) = 0; + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) = 0; + virtual void NotifyExcludedFile(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyExcludedDir(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirComparing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirCompared(const std::string& rLocalPath, + const std::string& rRemotePath, bool HasDifferentAttributes, + bool modifiedAfterLastSync) = 0; + virtual void NotifyFileComparing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyFileCompared(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + bool HasDifferentAttributes, bool HasDifferentContents, + bool modifiedAfterLastSync, bool newAttributesApplied) = 0; +}; + +#endif // BOXBACKUPCOMPAREPARAMS__H diff --git a/lib/bbackupquery/CommandCompletion.cpp b/lib/bbackupquery/CommandCompletion.cpp new file mode 100644 index 00000000..761fc97e --- /dev/null +++ b/lib/bbackupquery/CommandCompletion.cpp @@ -0,0 +1,604 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CommandCompletion.cpp +// Purpose: Parts of BackupQueries that depend on readline +// Created: 2011/01/21 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_LIBREADLINE + #ifdef HAVE_READLINE_READLINE_H + #include <readline/readline.h> + #elif defined(HAVE_EDITLINE_READLINE_H) + #include <editline/readline.h> + #elif defined(HAVE_READLINE_H) + #include <readline.h> + #endif +#endif + +#ifdef HAVE_READLINE_HISTORY + #ifdef HAVE_READLINE_HISTORY_H + #include <readline/history.h> + #elif defined(HAVE_HISTORY_H) + #include <history.h> + #endif +#endif + +#include <cstring> + +#include "BackupQueries.h" +#include "Configuration.h" + +#include "autogen_BackupProtocol.h" + +#include "MemLeakFindOn.h" + +#define COMPARE_RETURN_SAME 1 +#define COMPARE_RETURN_DIFFERENT 2 +#define COMPARE_RETURN_ERROR 3 +#define COMMAND_RETURN_ERROR 4 + +#define COMPLETION_FUNCTION(name, code) \ +std::vector<std::string> Complete ## name( \ + BackupQueries::ParsedCommand& rCommand, \ + const std::string& prefix, \ + BackupProtocolCallable& rProtocol, const Configuration& rConfig, \ + BackupQueries& rQueries) \ +{ \ + std::vector<std::string> completions; \ + \ + try \ + { \ + code \ + } \ + catch(std::exception &e) \ + { \ + BOX_TRACE("Failed to complete " << prefix << ": " << e.what()); \ + } \ + catch(...) \ + { \ + BOX_TRACE("Failed to complete " << prefix << ": " \ + "unknown error"); \ + } \ + \ + return completions; \ +} + +#define DELEGATE_COMPLETION(name) \ + completions = Complete ## name(rCommand, prefix, rProtocol, rConfig, \ + rQueries); + +COMPLETION_FUNCTION(None,) + +#ifdef HAVE_RL_FILENAME_COMPLETION_FUNCTION + #define RL_FILENAME_COMPLETION_FUNCTION rl_filename_completion_function + #define HAVE_A_FILENAME_COMPLETION_FUNCTION 1 +#elif defined HAVE_FILENAME_COMPLETION_FUNCTION + #define RL_FILENAME_COMPLETION_FUNCTION filename_completion_function + #define HAVE_A_FILENAME_COMPLETION_FUNCTION 1 +#endif + +#ifdef HAVE_A_FILENAME_COMPLETION_FUNCTION +COMPLETION_FUNCTION(Default, + int i = 0; + + while (const char *match = RL_FILENAME_COMPLETION_FUNCTION(prefix.c_str(), i)) + { + completions.push_back(match); + ++i; + } +) +#else // !HAVE_A_FILENAME_COMPLETION_FUNCTION +COMPLETION_FUNCTION(Default,) +#endif // HAVE_A_FILENAME_COMPLETION_FUNCTION + +COMPLETION_FUNCTION(Command, + int len = prefix.length(); + + for(int i = 0; commands[i].name != NULL; i++) + { + if(::strncmp(commands[i].name, prefix.c_str(), len) == 0) + { + completions.push_back(commands[i].name); + } + } +) + +void CompleteOptionsInternal(const std::string& prefix, + BackupQueries::ParsedCommand& rCommand, + std::vector<std::string>& completions) +{ + std::string availableOptions = rCommand.pSpec->opts; + + for(std::string::iterator + opt = availableOptions.begin(); + opt != availableOptions.end(); opt++) + { + if(rCommand.mOptions.find(*opt) == std::string::npos) + { + if(prefix == "") + { + // complete with possible option strings + completions.push_back(std::string("-") + *opt); + } + else + { + // complete with possible additional options + completions.push_back(prefix + *opt); + } + } + } +} + +COMPLETION_FUNCTION(Options, + CompleteOptionsInternal(prefix, rCommand, completions); +) + +std::string EncodeFileName(const std::string &rUnEncodedName) +{ +#ifdef WIN32 + std::string encodedName; + if(!ConvertConsoleToUtf8(rUnEncodedName, encodedName)) + { + return std::string(); + } + return encodedName; +#else + return rUnEncodedName; +#endif +} + +int16_t GetExcludeFlags(BackupQueries::ParsedCommand& rCommand) +{ + int16_t excludeFlags = 0; + + if (rCommand.mOptions.find(LIST_OPTION_ALLOWOLD) == std::string::npos) + { + excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + } + + if (rCommand.mOptions.find(LIST_OPTION_ALLOWDELETED) == std::string::npos) + { + excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; + } + + return excludeFlags; +} + +std::vector<std::string> CompleteRemoteFileOrDirectory( + BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolCallable& rProtocol, + BackupQueries& rQueries, int16_t includeFlags) +{ + std::vector<std::string> completions; + + // default to using the current directory + int64_t listDirId = rQueries.GetCurrentDirectoryID(); + std::string searchPrefix; + std::string listDir = prefix; + + if(rCommand.mCompleteArgCount == rCommand.mCmdElements.size()) + { + // completing an empty name, from the current directory + // nothing to change + } + else + { + // completing a partially-completed subdirectory name + searchPrefix = prefix; + listDir = ""; + + // do we need to list a subdirectory to complete? + size_t lastSlash = searchPrefix.rfind('/'); + if(lastSlash == std::string::npos) + { + // no slashes, so the whole name is the prefix + // nothing to change + } + else + { + // listing a partially-completed subdirectory name + listDir = searchPrefix.substr(0, lastSlash); + + listDirId = rQueries.FindDirectoryObjectID(listDir, + rCommand.mOptions.find(LIST_OPTION_ALLOWOLD) + != std::string::npos, + rCommand.mOptions.find(LIST_OPTION_ALLOWDELETED) + != std::string::npos); + + if(listDirId == 0) + { + // no matches for subdir to list, + // return empty-handed. + return completions; + } + + // matched, and updated listDir and listDirId already + searchPrefix = searchPrefix.substr(lastSlash + 1); + } + } + + // Always include directories, because they contain files. + // We will append a slash later for each directory if we're + // actually looking for files. + // + // If we're looking for directories, then only list directories. + + bool completeFiles = includeFlags & + BackupProtocolListDirectory::Flags_File; + bool completeDirs = includeFlags & + BackupProtocolListDirectory::Flags_Dir; + int16_t listFlags = 0; + + if(completeFiles) + { + listFlags = BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING; + } + else if(completeDirs) + { + listFlags = BackupProtocolListDirectory::Flags_Dir; + } + + rProtocol.QueryListDirectory(listDirId, + listFlags, GetExcludeFlags(rCommand), + false /* no attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream()); + dir.ReadFromStream(*dirstream, rProtocol.GetTimeout()); + + // Then... display everything + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + BackupStoreFilenameClear clear(en->GetName()); + std::string name = clear.GetClearFilename().c_str(); + if(name.compare(0, searchPrefix.length(), searchPrefix) == 0) + { + bool dir_added = false; + + if(en->IsDir() && + (includeFlags & BackupProtocolListDirectory::Flags_Dir) == 0) + { + // Was looking for a file, but this is a + // directory, so append a slash to the name + name += "/"; + } + + #ifdef HAVE_LIBREADLINE + if(strchr(name.c_str(), ' ')) + { + int n_quote = 0; + + for(int k = strlen(rl_line_buffer); k >= 0; k--) + { + if (rl_line_buffer[k] == '\"') { + ++n_quote; + } + } + + dir_added = false; + + if (!(n_quote % 2)) + { + name = "\"" + (listDir == "" ? name : listDir + "/" + name); + dir_added = true; + } + + name = name + "\""; + } + #endif + + if(listDir == "" || dir_added) + { + completions.push_back(name); + } + else + { + completions.push_back(listDir + "/" + name); + } + } + } + + return completions; +} + +COMPLETION_FUNCTION(RemoteDir, + completions = CompleteRemoteFileOrDirectory(rCommand, prefix, + rProtocol, rQueries, + BackupProtocolListDirectory::Flags_Dir); +) + +COMPLETION_FUNCTION(RemoteFile, + completions = CompleteRemoteFileOrDirectory(rCommand, prefix, + rProtocol, rQueries, + BackupProtocolListDirectory::Flags_File); +) + +COMPLETION_FUNCTION(LocalDir, + DELEGATE_COMPLETION(Default); +) + +COMPLETION_FUNCTION(LocalFile, + DELEGATE_COMPLETION(Default); +) + +COMPLETION_FUNCTION(LocationName, + const Configuration &locations(rConfig.GetSubConfiguration( + "BackupLocations")); + + std::vector<std::string> locNames = + locations.GetSubConfigurationNames(); + + for(std::vector<std::string>::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + if(pLocName->compare(0, pLocName->length(), prefix) == 0) + { + completions.push_back(*pLocName); + } + } +) + +COMPLETION_FUNCTION(RemoteFileIdInCurrentDir, + int64_t listDirId = rQueries.GetCurrentDirectoryID(); + int16_t excludeFlags = GetExcludeFlags(rCommand); + + rProtocol.QueryListDirectory( + listDirId, + BackupProtocolListDirectory::Flags_File, + excludeFlags, false /* no attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream()); + dir.ReadFromStream(*dirstream, rProtocol.GetTimeout()); + + // Then... compare each item + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + std::ostringstream hexId; + hexId << std::hex << en->GetObjectID(); + if(hexId.str().compare(0, prefix.length(), prefix) == 0) + { + completions.push_back(hexId.str()); + } + } +) + +// TODO implement completion of hex IDs up to the maximum according to Usage +COMPLETION_FUNCTION(RemoteId,) + +COMPLETION_FUNCTION(GetFileOrId, + if(rCommand.mOptions.find('i') != std::string::npos) + { + DELEGATE_COMPLETION(RemoteFileIdInCurrentDir); + } + else + { + DELEGATE_COMPLETION(RemoteFile); + } +) + +COMPLETION_FUNCTION(CompareLocationOrRemoteDir, + if(rCommand.mOptions.find('l') != std::string::npos) + { + DELEGATE_COMPLETION(LocationName); + } + else + { + DELEGATE_COMPLETION(RemoteDir); + } +) + +COMPLETION_FUNCTION(CompareNoneOrLocalDir, + if(rCommand.mOptions.find('l') != std::string::npos) + { + // no completions + DELEGATE_COMPLETION(None); + } + else + { + DELEGATE_COMPLETION(LocalDir); + } +) + +COMPLETION_FUNCTION(RestoreRemoteDirOrId, + if(rCommand.mOptions.find('i') != std::string::npos) + { + DELEGATE_COMPLETION(RemoteId); + } + else + { + DELEGATE_COMPLETION(RemoteDir); + } +) + +// Data about commands +QueryCommandSpecification commands[] = +{ + { "quit", "", Command_Quit, {} }, + { "exit", "", Command_Quit, {} }, + { "list", "adDFhiIorRsStTU", Command_List, {CompleteRemoteDir} }, + { "pwd", "", Command_pwd, {} }, + { "cd", "od", Command_cd, {CompleteRemoteDir} }, + { "lcd", "", Command_lcd, {CompleteLocalDir} }, + { "sh", "", Command_sh, {CompleteDefault} }, + { "getobject", "", Command_GetObject, + {CompleteRemoteId, CompleteLocalDir} }, + { "get", "i", Command_Get, + {CompleteGetFileOrId, CompleteLocalDir} }, + { "compare", "alcqAEQ", Command_Compare, + {CompleteCompareLocationOrRemoteDir, CompleteCompareNoneOrLocalDir} }, + { "restore", "drif", Command_Restore, + {CompleteRestoreRemoteDirOrId, CompleteLocalDir} }, + { "help", "", Command_Help, {} }, + { "usage", "m", Command_Usage, {} }, + { "undelete", "i", Command_Undelete, + {CompleteGetFileOrId} }, + { "delete", "i", Command_Delete, {CompleteGetFileOrId} }, + { NULL, NULL, Command_Unknown, {} } +}; + +const char *alias[] = {"ls", 0}; +const int aliasIs[] = {Command_List, 0}; + +BackupQueries::ParsedCommand::ParsedCommand(const std::string& Command, + bool isFromCommandLine) +: mInOptions(false), + mFailed(false), + pSpec(NULL), + mCompleteArgCount(0) +{ + mCompleteCommand = Command; + + // is the command a shell command? + if(Command[0] == 's' && Command[1] == 'h' && Command[2] == ' ' && Command[3] != '\0') + { + // Yes, run shell command + for(int i = 0; commands[i].type != Command_Unknown; i++) + { + if(commands[i].type == Command_sh) + { + pSpec = &(commands[i]); + break; + } + } + + mCmdElements[0] = "sh"; + mCmdElements[1] = Command.c_str() + 3; + return; + } + + // split command into components + bool inQuoted = false; + mInOptions = false; + + std::string currentArg; + for (std::string::const_iterator c = Command.begin(); + c != Command.end(); c++) + { + // Terminating char? + if(*c == ((inQuoted)?'"':' ')) + { + if(!currentArg.empty()) + { + mCmdElements.push_back(currentArg); + + // Because we just found a space, and the last + // word was not options (otherwise currentArg + // would be empty), we've received a complete + // command or non-option argument. + mCompleteArgCount++; + } + + currentArg.resize(0); + inQuoted = false; + mInOptions = false; + } + // Start of quoted parameter? + else if(currentArg.empty() && *c == '"') + { + inQuoted = true; + } + // Start of options? You can't have options if there's no + // command before them, so treat the options as a command (which + // doesn't exist, so it will fail to parse) in that case. + else if(currentArg.empty() && *c == '-' && !mCmdElements.empty()) + { + mInOptions = true; + } + else if(mInOptions) + { + // Option char + mOptions += *c; + } + else + { + // Normal string char, part of current arg + currentArg += *c; + } + } + + if(!currentArg.empty()) + { + mCmdElements.push_back(currentArg); + } + + // If there are no commands then there's nothing to do except return + if(mCmdElements.empty()) + { + return; + } + + // Work out which command it is... + int cmd = 0; + while(commands[cmd].name != 0 && + mCmdElements[0] != commands[cmd].name) + { + cmd++; + } + + if(commands[cmd].name == 0) + { + // Check for aliases + int a; + for(a = 0; alias[a] != 0; ++a) + { + if(mCmdElements[0] == alias[a]) + { + // Found an alias + cmd = aliasIs[a]; + break; + } + } + } + + if(commands[cmd].name == 0) + { + mFailed = true; + return; + } + + pSpec = &(commands[cmd]); + + #ifdef WIN32 + if(isFromCommandLine) + { + std::string converted; + + if(!ConvertEncoding(mCompleteCommand, CP_ACP, converted, + GetConsoleCP())) + { + BOX_ERROR("Failed to convert encoding"); + mFailed = true; + } + + mCompleteCommand = converted; + + for(std::vector<std::string>::iterator + i = mCmdElements.begin(); + i != mCmdElements.end(); i++) + { + if(!ConvertEncoding(*i, CP_ACP, converted, + GetConsoleCP())) + { + BOX_ERROR("Failed to convert encoding"); + mFailed = true; + } + + *i = converted; + } + } + #endif +} + diff --git a/lib/bbackupquery/documentation.txt b/lib/bbackupquery/documentation.txt new file mode 100644 index 00000000..b16a6f7c --- /dev/null +++ b/lib/bbackupquery/documentation.txt @@ -0,0 +1,194 @@ + +bbackupquery utility -- examine store, compare files, restore, etc. + +This file has markers for automatic help generation script -- '>' marks a start of a command/help topic, +and '<' marks the end of a section. + +Command line: +============= + +> bbackupquery [-q] [-c configfile] [commands ...] + + -q -- quiet, no information prompts + -c -- specify another bbackupd configuation file + +The commands following the options are executed, then (if there was no quit +command) an interactive mode is entered. + +If a command contains a space, enclose it in quotes. Example + + bbackupquery "list testdir1" quit + +to list the contents of testdir1, and then exit without interactive mode. +< + +Commands: +========= + +All directory names relative to a "current" directory, or from root if they +start with '/'. The initial directory is always the root directory. + + +> ls [options] [directory-name] + + List contents of current directory, or specified directory. + + -R -- recursively list all files + -d -- list deleted files/directories + -o -- list old versions of files/directories + -I -- don't display object ID + -F -- don't display flags + -t -- show file modification time in local time + (and attr mod time if has the object has attributes, ~ separated) + -T -- show file modification time in GMT + -a -- show updated attribute instead of file modification time + -s -- show file size in blocks used on server + (only very approximate indication of size locally) + -h -- show file attributes hash + -D -- sort directories together with files (not dirs first) + -i -- sort by object ID (the old default) + -S -- sort by object size in blocks + -U -- don't sort the results (new default is to sort by name) + +list can be used as an alias. +< + +> list + + Alias for 'ls'. Type 'help ls' for options. +< + +> cd [options] <directory-name> + + Change directory + + -d -- consider deleted directories for traversal + -o -- consider old versions of directories for traversal + (this option should never be useful in a correctly formed store) +< + +> pwd + + Print current directory, always root relative. +< + +> lcd <local-directory-name> + + Change local directory. + + Type "sh ls" to list the contents. +< + +> sh <shell command> + + All of the parameters after the "sh" are run as a shell command. + + For example, to list the contents of the location directory, type "sh ls" +< + +> get <object-filename> [<local-filename>] +get -i <object-id> <local-filename> + + Gets a file from the store. Object is specified as the filename within + the current directory, and local filename is optional. Ignores old and + deleted files when searching the directory for the file to retrieve. + + To get an old or deleted file, use the -i option and select the object + as a hex object ID (first column in listing). The local filename must + be specified. +< + +> compare -a +compare -l <location-name> +compare <store-dir-name> <local-dir-name> + + Compares the (current) data on the store with the data on the disc. + All the data will be downloaded -- this is potentially a very long + operation. + + -a -- compare all locations + -l -- compare one backup location as specified in the configuration file. + -c -- set return code + -q -- quick compare. Only checks file contents against checksums, + doesn't do a full download + -A -- ignore attribute differences + -E -- ignore exclusion settings + + Comparing with the root directory is an error, use -a option instead. + + If -c is set, then the return code (if quit is the next command) will be + 1 Comparison was exact + 2 Differences were found + 3 An error occured + This can be used for automated tests. +< + +> restore [-drif] <directory-name> [<local-directory-name>] + + Restores a directory to the local disc. The local directory specified + must not exist (unless a previous restore is being restarted). If the + local directory is omitted, the default is to restore to the same + directory name and path, relative to the current local directory, + as set with the "lcd" command. + + The root cannot be restored -- restore locations individually. + + -d -- restore a deleted directory or deleted files inside + -r -- resume an interrupted restoration + -i -- directory name is actually an ID + -f -- force restore to continue if errors are encountered + + If a restore operation is interrupted for any reason, it can be restarted + using the -r switch. Restore progress information is saved in a file at + regular intervals during the restore operation to allow restarts. +< + +> getobject <object-id> <local-filename> + + Gets the object specified by the object id (in hex) and stores the raw + contents in the local file specified. + + This is only useful for debugging as it does not decode files from the + stored format, which is encrypted and compressed. +< + +> usage [-m] + + Show space used on the server for this account. + + -m -- display the output in machine-readable form + + Used: Total amount of space used on the server. + Old files: Space used by old files + Deleted files: Space used by deleted files + Directories: Space used by the directory structure. + + When Used exceeds the soft limit, the server will start to remove old and + deleted files until the usage drops below the soft limit. + + After a while, you would expect to see the usage stay at just below the + soft limit. You only need more space if the space used by old and deleted + files is near zero. +< + +> undelete <directory-name> +undelete -i <object-id> + + Removes the deleted flag from the specified directory name (in the + current directory) or hex object ID. Be careful not to use this + command where a directory already exists with the same name which is + not marked as deleted. +< + +> delete <file-name> + + Sets the deleted flag on the specified file name (in the current + directory, or with a relative path). +< + +> quit + + End session and exit. +< + + diff --git a/lib/bbackupquery/makedocumentation.pl.in b/lib/bbackupquery/makedocumentation.pl.in new file mode 100755 index 00000000..530c4ff6 --- /dev/null +++ b/lib/bbackupquery/makedocumentation.pl.in @@ -0,0 +1,75 @@ +#!@PERL@ +use strict; + +print "Creating built-in documentation for bbackupquery...\n"; + +open DOC,"documentation.txt" or die "Can't open documentation.txt file"; +my $section; +my %help; +my @in_order; + +while(<DOC>) +{ + if(m/\A>\s+(\w+)/) + { + $section = $1; + m/\A>\s+(.+)\Z/; + $help{$section} = $1."\n"; + push @in_order,$section; + } + elsif(m/\A</) + { + $section = ''; + } + elsif($section ne '') + { + $help{$section} .= $_; + } +} + +close DOC; + +open OUT,">autogen_Documentation.cpp" or die "Can't open output file for writing"; + +print OUT <<__E; +// +// Automatically generated file, do not edit. +// + +#include "Box.h" + +#include "MemLeakFindOn.h" + +const char *help_commands[] = +{ +__E + +for(@in_order) +{ + print OUT qq:\t"$_",\n:; +} + +print OUT <<__E; + 0 +}; + +const char *help_text[] = +{ +__E + +for(@in_order) +{ + my $t = $help{$_}; + $t =~ s/\t/ /g; + $t =~ s/\n/\\n/g; + $t =~ s/"/\\"/g; + print OUT qq:\t"$t",\n:; +} + +print OUT <<__E; + 0 +}; + +__E + +close OUT; diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp index 8cacf065..e440408e 100644 --- a/lib/common/NamedLock.cpp +++ b/lib/common/NamedLock.cpp @@ -82,6 +82,9 @@ bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) #if HAVE_DECL_O_EXLOCK flags |= O_NONBLOCK | O_EXLOCK; BOX_TRACE("Trying to create lockfile " << rFilename << " using O_EXLOCK"); +#elif defined BOX_OPEN_LOCK + flags |= BOX_OPEN_LOCK; + BOX_TRACE("Trying to create lockfile " << rFilename << " using BOX_OPEN_LOCK"); #elif !HAVE_DECL_F_SETLK && !defined HAVE_FLOCK // We have no other way to get a lock, so all we can do is fail if // the file already exists, and take the risk of stale locks. diff --git a/lib/common/Test.cpp b/lib/common/Test.cpp index 9385d8c3..df2c3bc6 100644 --- a/lib/common/Test.cpp +++ b/lib/common/Test.cpp @@ -22,7 +22,9 @@ #endif #include "BoxTime.h" +#include "FileStream.h" #include "Test.h" +#include "Utils.h" int num_tests_selected = 0; int num_failures = 0; @@ -81,6 +83,60 @@ bool setUp(const char* function_name) } } +#ifdef _MSC_VER + DIR* pDir = opendir("testfiles"); + if(!pDir) + { + THROW_SYS_FILE_ERROR("Failed to open test temporary directory", + "testfiles", CommonException, Internal); + } + struct dirent* pEntry; + for(pEntry = readdir(pDir); pEntry; pEntry = readdir(pDir)) + { + std::string filename = pEntry->d_name; + if(StartsWith("TestDir", filename) || + StartsWith("0_", filename) || + filename == "accounts.txt" || + StartsWith("file", filename) || + StartsWith("notifyran", filename) || + StartsWith("notifyscript.tag", filename) || + StartsWith("restore", filename) || + filename == "bbackupd-data" || + filename == "syncallowscript.control" || + StartsWith("syncallowscript.notifyran.", filename) || + filename == "test2.downloaded") + { + int filetype = ObjectExists(std::string("testfiles/") + filename); + if(filetype == ObjectExists_File) + { + if(!::unlink(filename.c_str())) + { + TEST_FAIL_WITH_MESSAGE(BOX_SYS_ERROR_MESSAGE("Failed to delete " + "test fixture file: unlink(\"" << filename << "\")")); + } + } + else if(filetype == ObjectExists_Dir) + { + std::string cmd = "rmdir /s /q testfiles\\" + filename; + int status = system(cmd.c_str()); + if(status != 0) + { + TEST_FAIL_WITH_MESSAGE("Failed to delete test fixture " + "file: command '" << cmd << "' exited with " + "status " << status); + } + } + else + { + TEST_FAIL_WITH_MESSAGE("Don't know how to delete file " << filename << + " of type " << filetype); + } + } + } + closedir(pDir); + FileStream touch("testfiles/accounts.txt", O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR); +#else TEST_THAT_THROWONFAIL(system( "rm -rf testfiles/TestDir* testfiles/0_0 testfiles/0_1 " "testfiles/0_2 testfiles/accounts.txt " // testfiles/test* .tgz! @@ -91,11 +147,12 @@ bool setUp(const char* function_name) "testfiles/syncallowscript.notifyran.* " "testfiles/test2.downloaded" ) == 0); + TEST_THAT_THROWONFAIL(system("touch testfiles/accounts.txt") == 0); +#endif TEST_THAT_THROWONFAIL(mkdir("testfiles/0_0", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/0_1", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/0_2", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/bbackupd-data", 0755) == 0); - TEST_THAT_THROWONFAIL(system("touch testfiles/accounts.txt") == 0); return true; } diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp index 658bfbbc..65c12b09 100644 --- a/lib/win32/emu.cpp +++ b/lib/win32/emu.cpp @@ -613,7 +613,7 @@ HANDLE openfile(const char *pFileName, int flags, int mode) createDisposition = CREATE_NEW; } - if (flags & O_LOCK) + if (flags & BOX_OPEN_LOCK) { shareMode = 0; } diff --git a/lib/win32/emu.h b/lib/win32/emu.h index b0fa2832..3cc3979c 100644 --- a/lib/win32/emu.h +++ b/lib/win32/emu.h @@ -228,7 +228,7 @@ struct dirent *readdir(DIR *dp); int closedir(DIR *dp); // local constant to open file exclusively without shared access -#define O_LOCK 0x10000 +#define BOX_OPEN_LOCK 0x10000 extern DWORD winerrno; /* used to report errors from openfile() */ HANDLE openfile(const char *filename, int flags, int mode); |