From 768f2bd2c35d56f0df65c4c9acfe5649e776409e Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sat, 12 Dec 2015 22:49:41 +0000 Subject: Define TEST_EXECUTABLE to allow it to have different names on MSVC/CMake. --- lib/common/Test.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/common/Test.h b/lib/common/Test.h index 15a3db6f..36cd6a59 100644 --- a/lib/common/Test.h +++ b/lib/common/Test.h @@ -243,4 +243,12 @@ void safe_sleep(int seconds); std::auto_ptr load_config_file(const std::string& config_file, const ConfigurationVerify& verify); +#ifdef _MSC_VER + // Our CMakeFiles compile tests to different executable filenames, + // e.g. test_common.exe instead of _test.exe. + #define TEST_EXECUTABLE BOX_MODULE ".exe" +#else + #define TEST_EXECUTABLE "./_test" +#endif + #endif // TEST__H -- cgit v1.2.3 From cdd27f760290c27db7e3e1704e43eeceba07ea77 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 2 Dec 2015 19:53:05 +0000 Subject: Remove the duplicate copy constructor that MSVC warns about. --- lib/common/CollectInBufferStream.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'lib') diff --git a/lib/common/CollectInBufferStream.h b/lib/common/CollectInBufferStream.h index a76daf65..297d2851 100644 --- a/lib/common/CollectInBufferStream.h +++ b/lib/common/CollectInBufferStream.h @@ -38,12 +38,6 @@ public: rOther.Reset(); } -private: - // No copying (only moving, as defined above) - CollectInBufferStream(const CollectInBufferStream &); - CollectInBufferStream(const IOStream &); - -public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, -- cgit v1.2.3 From e77de564aaacaed07075eaac1974d35d09dd2bce Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 26 Nov 2015 23:32:29 +0000 Subject: Add some little string functions to Utils.cpp. These functions check whether the beginning and end of a std::string match a supplied prefix or suffix, and remove that prefix or suffix, returning the remaining part. They are almost but not entirely trivial, and giving them names makes the code that uses them much more readable. --- lib/common/Utils.cpp | 36 ++++++++++++++++++++++++++++++++++++ lib/common/Utils.h | 4 ++++ 2 files changed, 40 insertions(+) (limited to 'lib') diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp index a73a80da..4325dd88 100644 --- a/lib/common/Utils.cpp +++ b/lib/common/Utils.cpp @@ -95,6 +95,42 @@ void SplitString(std::string String, char SplitOn, std::vector &rOu #endif*/ } +bool StartsWith(const std::string& prefix, const std::string& haystack) +{ + return haystack.size() >= prefix.size() && + haystack.substr(0, prefix.size()) == prefix; +} + +bool EndsWith(const std::string& suffix, const std::string& haystack) +{ + return haystack.size() >= suffix.size() && + haystack.substr(haystack.size() - suffix.size()) == suffix; +} + +std::string RemovePrefix(const std::string& prefix, const std::string& haystack) +{ + if(StartsWith(prefix, haystack)) + { + return haystack.substr(prefix.size()); + } + else + { + return ""; + } +} + +std::string RemoveSuffix(const std::string& suffix, const std::string& haystack) +{ + if(EndsWith(suffix, haystack)) + { + return haystack.substr(0, haystack.size() - suffix.size()); + } + else + { + return ""; + } +} + static std::string demangle(const std::string& mangled_name) { std::string demangled_name = mangled_name; diff --git a/lib/common/Utils.h b/lib/common/Utils.h index 3a5f0eb5..d306ce1c 100644 --- a/lib/common/Utils.h +++ b/lib/common/Utils.h @@ -18,6 +18,10 @@ std::string GetBoxBackupVersion(); void SplitString(std::string String, char SplitOn, std::vector &rOutput); +bool StartsWith(const std::string& prefix, const std::string& haystack); +bool EndsWith(const std::string& prefix, const std::string& haystack); +std::string RemovePrefix(const std::string& prefix, const std::string& haystack); +std::string RemoveSuffix(const std::string& suffix, const std::string& haystack); void DumpStackBacktrace(); -- cgit v1.2.3 From 403e7e2051ee3bd3e16a616cfca63b036481282b Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sun, 13 Dec 2015 23:39:18 +0000 Subject: Move reusable code out of bin directories. Allows tests to depend on lib/bbackupd instead of bin/bbackupd, which was always a hack, and really doesn't work with CMake. --- lib/bbackupd/BackupClientContext.cpp | 578 ++++ lib/bbackupd/BackupClientContext.h | 252 ++ lib/bbackupd/BackupClientDeleteList.cpp | 229 ++ lib/bbackupd/BackupClientDeleteList.h | 75 + lib/bbackupd/BackupClientDirectoryRecord.cpp | 2302 ++++++++++++++++ lib/bbackupd/BackupClientDirectoryRecord.h | 244 ++ lib/bbackupd/BackupClientInodeToIDMap.cpp | 320 +++ lib/bbackupd/BackupClientInodeToIDMap.h | 59 + lib/bbackupd/BackupDaemon.cpp | 3654 ++++++++++++++++++++++++++ lib/bbackupd/BackupDaemon.h | 539 ++++ lib/bbackupd/BackupDaemonInterface.h | 166 ++ lib/bbackupd/Win32BackupService.cpp | 48 + lib/bbackupd/Win32BackupService.h | 21 + lib/bbackupd/Win32ServiceFunctions.cpp | 384 +++ lib/bbackupd/Win32ServiceFunctions.h | 19 + lib/bbackupquery/BackupQueries.cpp | 2410 +++++++++++++++++ lib/bbackupquery/BackupQueries.h | 440 ++++ lib/bbackupquery/BoxBackupCompareParams.h | 112 + lib/bbackupquery/CommandCompletion.cpp | 604 +++++ lib/bbackupquery/documentation.txt | 194 ++ lib/bbackupquery/makedocumentation.pl.in | 75 + 21 files changed, 12725 insertions(+) create mode 100644 lib/bbackupd/BackupClientContext.cpp create mode 100644 lib/bbackupd/BackupClientContext.h create mode 100644 lib/bbackupd/BackupClientDeleteList.cpp create mode 100644 lib/bbackupd/BackupClientDeleteList.h create mode 100644 lib/bbackupd/BackupClientDirectoryRecord.cpp create mode 100644 lib/bbackupd/BackupClientDirectoryRecord.h create mode 100644 lib/bbackupd/BackupClientInodeToIDMap.cpp create mode 100644 lib/bbackupd/BackupClientInodeToIDMap.h create mode 100644 lib/bbackupd/BackupDaemon.cpp create mode 100644 lib/bbackupd/BackupDaemon.h create mode 100644 lib/bbackupd/BackupDaemonInterface.h create mode 100644 lib/bbackupd/Win32BackupService.cpp create mode 100644 lib/bbackupd/Win32BackupService.h create mode 100644 lib/bbackupd/Win32ServiceFunctions.cpp create mode 100644 lib/bbackupd/Win32ServiceFunctions.h create mode 100644 lib/bbackupquery/BackupQueries.cpp create mode 100644 lib/bbackupquery/BackupQueries.h create mode 100644 lib/bbackupquery/BoxBackupCompareParams.h create mode 100644 lib/bbackupquery/CommandCompletion.cpp create mode 100644 lib/bbackupquery/documentation.txt create mode 100755 lib/bbackupquery/makedocumentation.pl.in (limited to 'lib') diff --git a/lib/bbackupd/BackupClientContext.cpp b/lib/bbackupd/BackupClientContext.cpp new file mode 100644 index 00000000..4c0b01ce --- /dev/null +++ b/lib/bbackupd/BackupClientContext.cpp @@ -0,0 +1,578 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientContext.cpp +// Purpose: Keep track of context +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_SIGNAL_H + #include +#endif + +#ifdef HAVE_SYS_TIME_H + #include +#endif + +#include "BoxPortsAndFiles.h" +#include "BoxTime.h" +#include "BackupClientContext.h" +#include "SocketStreamTLS.h" +#include "Socket.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" +#include "BackupDaemon.h" +#include "autogen_BackupProtocol.h" +#include "BackupStoreFile.h" +#include "Logging.h" +#include "TcpNice.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::BackupClientContext(BackupDaemon &, TLSContext &, const std::string &, int32_t, bool, bool, std::string) +// Purpose: Constructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientContext::BackupClientContext +( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier& rProgressNotifier, + bool TcpNiceMode +) +: mExperimentalSnapshotMode(false), + mrResolver(rResolver), + mrTLSContext(rTLSContext), + mHostname(rHostname), + mPort(Port), + mAccountNumber(AccountNumber), + mExtendedLogging(ExtendedLogging), + mExtendedLogToFile(ExtendedLogToFile), + mExtendedLogFile(ExtendedLogFile), + mpExtendedLogFileHandle(NULL), + mClientStoreMarker(ClientStoreMarker_NotKnown), + mpDeleteList(0), + mpCurrentIDMap(0), + mpNewIDMap(0), + mStorageLimitExceeded(false), + mpExcludeFiles(0), + mpExcludeDirs(0), + mKeepAliveTimer(0, "KeepAliveTime"), + mbIsManaged(false), + mrProgressNotifier(rProgressNotifier), + mTcpNiceMode(TcpNiceMode), + mpNice(NULL) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::~BackupClientContext() +// Purpose: Destructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientContext::~BackupClientContext() +{ + CloseAnyOpenConnection(); + + // Delete delete list + if(mpDeleteList != 0) + { + delete mpDeleteList; + mpDeleteList = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetConnection() +// Purpose: Returns the connection, making the connection and logging into +// the backup store if necessary. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupProtocolCallable &BackupClientContext::GetConnection() +{ + // Already got it? Just return it. + if(mapConnection.get()) + { + return *mapConnection; + } + + // Defensive. Must close connection before releasing any old socket. + mapConnection.reset(); + + std::auto_ptr apSocket(new SocketStreamTLS); + + try + { + // Defensive. + mapConnection.reset(); + + // Log intention + BOX_INFO("Opening connection to server '" << mHostname << + "'..."); + + // Connect! + ((SocketStreamTLS *)(apSocket.get()))->Open(mrTLSContext, + Socket::TypeINET, mHostname, mPort); + + if(mTcpNiceMode) + { + // Pass control of apSocket to NiceSocketStream, + // which will take care of destroying it for us. + // But we need to hang onto a pointer to the nice + // socket, so we can enable and disable nice mode. + // This is scary, it could be deallocated under us. + mpNice = new NiceSocketStream(apSocket); + apSocket.reset(mpNice); + } + + // We need to call some methods that aren't defined in + // BackupProtocolCallable, so we need to hang onto a more + // strongly typed pointer (to avoid far too many casts). + BackupProtocolClient *pClient = new BackupProtocolClient(apSocket); + mapConnection.reset(pClient); + + // Set logging option + pClient->SetLogToSysLog(mExtendedLogging); + + if (mExtendedLogToFile) + { + ASSERT(mpExtendedLogFileHandle == NULL); + + mpExtendedLogFileHandle = fopen( + mExtendedLogFile.c_str(), "a+"); + + if (!mpExtendedLogFileHandle) + { + BOX_LOG_SYS_ERROR("Failed to open extended " + "log file: " << mExtendedLogFile); + } + else + { + pClient->SetLogToFile(mpExtendedLogFileHandle); + } + } + + // Handshake + pClient->Handshake(); + + // Check the version of the server + { + std::auto_ptr serverVersion( + mapConnection->QueryVersion(BACKUP_STORE_SERVER_VERSION)); + if(serverVersion->GetVersion() != BACKUP_STORE_SERVER_VERSION) + { + THROW_EXCEPTION(BackupStoreException, WrongServerVersion) + } + } + + // Login -- if this fails, the Protocol will exception + std::auto_ptr loginConf( + mapConnection->QueryLogin(mAccountNumber, 0 /* read/write */)); + + // Check that the client store marker is the one we expect + if(mClientStoreMarker != ClientStoreMarker_NotKnown) + { + if(loginConf->GetClientStoreMarker() != mClientStoreMarker) + { + // Not good... finish the connection, abort, etc, ignoring errors + try + { + mapConnection->QueryFinished(); + } + catch(...) + { + // IGNORE + } + + // Then throw an exception about this + THROW_EXCEPTION_MESSAGE(BackupStoreException, + ClientMarkerNotAsExpected, + "Expected " << mClientStoreMarker << + " but found " << loginConf->GetClientStoreMarker() << + ": is someone else writing to the " + "same account?"); + } + } + else // mClientStoreMarker == ClientStoreMarker_NotKnown + { + // Yes, choose one, the current time will do + box_time_t marker = GetCurrentBoxTime(); + + // Set it on the store + mapConnection->QuerySetClientStoreMarker(marker); + + // Record it so that it can be picked up later. + mClientStoreMarker = marker; + } + + // Log success + BOX_INFO("Connection made, login successful"); + + // Check to see if there is any space available on the server + if(loginConf->GetBlocksUsed() >= loginConf->GetBlocksHardLimit()) + { + // no -- flag so only things like deletions happen + mStorageLimitExceeded = true; + // Log + BOX_WARNING("Exceeded storage hard-limit on server, " + "not uploading changes to files"); + } + } + catch(...) + { + // Clean up. + mapConnection.reset(); + throw; + } + + return *mapConnection; +} + +BackupProtocolCallable* BackupClientContext::GetOpenConnection() const +{ + return mapConnection.get(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::CloseAnyOpenConnection() +// Purpose: Closes a connection, if it's open +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupClientContext::CloseAnyOpenConnection() +{ + BackupProtocolCallable* pConnection(GetOpenConnection()); + if(pConnection) + { + try + { + // Quit nicely + pConnection->QueryFinished(); + } + catch(...) + { + // Ignore errors here + } + + // Delete it anyway. + mapConnection.reset(); + } + + // Delete any pending list + if(mpDeleteList != 0) + { + delete mpDeleteList; + mpDeleteList = 0; + } + + if (mpExtendedLogFileHandle != NULL) + { + fclose(mpExtendedLogFileHandle); + mpExtendedLogFileHandle = NULL; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetTimeout() +// Purpose: Gets the current timeout time. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +int BackupClientContext::GetTimeout() const +{ + BackupProtocolCallable* pConnection(GetOpenConnection()); + if(pConnection) + { + return pConnection->GetTimeout(); + } + + return (15*60*1000); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetDeleteList() +// Purpose: Returns the delete list, creating one if necessary +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +BackupClientDeleteList &BackupClientContext::GetDeleteList() +{ + // Already created? + if(mpDeleteList == 0) + { + mpDeleteList = new BackupClientDeleteList; + } + + // Return reference to object + return *mpDeleteList; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::PerformDeletions() +// Purpose: Perform any pending file deletions. +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientContext::PerformDeletions() +{ + // Got a list? + if(mpDeleteList == 0) + { + // Nothing to do + return; + } + + // Delegate to the delete list object + mpDeleteList->PerformDeletions(*this); + + // Delete the object + delete mpDeleteList; + mpDeleteList = 0; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetCurrentIDMap() const +// Purpose: Return a (const) reference to the current ID map +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +const BackupClientInodeToIDMap &BackupClientContext::GetCurrentIDMap() const +{ + ASSERT(mpCurrentIDMap != 0); + if(mpCurrentIDMap == 0) + { + THROW_EXCEPTION(CommonException, Internal) + } + return *mpCurrentIDMap; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetNewIDMap() const +// Purpose: Return a reference to the new ID map +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +BackupClientInodeToIDMap &BackupClientContext::GetNewIDMap() const +{ + ASSERT(mpNewIDMap != 0); + if(mpNewIDMap == 0) + { + THROW_EXCEPTION(CommonException, Internal) + } + return *mpNewIDMap; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::FindFilename(int64_t, int64_t, std::string &, bool &) const +// Purpose: Attempts to find the pathname of an object with a given ID on the server. +// Returns true if it can be found, in which case rPathOut is the local filename, +// and rIsDirectoryOut == true if the local object is a directory. +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut, + bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer, box_time_t *pAttributesHashOnServer, BackupStoreFilenameClear *pLeafname) +{ + // Make a connection to the server + BackupProtocolCallable &connection(GetConnection()); + + // Request filenames from the server, in a "safe" manner to ignore errors properly + { + BackupProtocolGetObjectName send(ObjectID, ContainingDirectory); + connection.Send(send); + } + std::auto_ptr preply(connection.Receive()); + + // Is it of the right type? + if(preply->GetType() != BackupProtocolObjectName::TypeID) + { + // Was an error or something + return false; + } + + // Cast to expected type. + BackupProtocolObjectName *names = (BackupProtocolObjectName *)(preply.get()); + + // Anything found? + int32_t numElements = names->GetNumNameElements(); + if(numElements <= 0) + { + // No. + return false; + } + + // Get the stream containing all the names + std::auto_ptr nameStream(connection.ReceiveStream()); + + // Path + std::string path; + + // Remember this is in reverse order! + for(int l = 0; l < numElements; ++l) + { + BackupStoreFilenameClear elementName; + elementName.ReadFromStream(*nameStream, GetTimeout()); + + // Store leafname for caller? + if(l == 0 && pLeafname) + { + *pLeafname = elementName; + } + + // Is it part of the filename in the location? + if(l < (numElements - 1)) + { + // Part of filename within + path = (path.empty())?(elementName.GetClearFilename()):(elementName.GetClearFilename() + DIRECTORY_SEPARATOR_ASCHAR + path); + } + else + { + // Location name -- look up in daemon's records + std::string locPath; + if(!mrResolver.FindLocationPathName(elementName.GetClearFilename(), locPath)) + { + // Didn't find the location... so can't give the local filename + return false; + } + + // Add in location path + path = (path.empty())?(locPath):(locPath + DIRECTORY_SEPARATOR_ASCHAR + path); + } + } + + // Is it a directory? + rIsDirectoryOut = ((names->GetFlags() & BackupProtocolListDirectory::Flags_Dir) == BackupProtocolListDirectory::Flags_Dir); + + // Is it the current version? + rIsCurrentVersionOut = ((names->GetFlags() & (BackupProtocolListDirectory::Flags_OldVersion | BackupProtocolListDirectory::Flags_Deleted)) == 0); + + // And other information which may be required + if(pModTimeOnServer) *pModTimeOnServer = names->GetModificationTime(); + if(pAttributesHashOnServer) *pAttributesHashOnServer = names->GetAttributesHash(); + + // Tell caller about the pathname + rPathOut = path; + + // Found + return true; +} + +void BackupClientContext::SetMaximumDiffingTime(int iSeconds) +{ + mMaximumDiffingTime = iSeconds < 0 ? 0 : iSeconds; + BOX_TRACE("Set maximum diffing time to " << mMaximumDiffingTime << + " seconds"); +} + +void BackupClientContext::SetKeepAliveTime(int iSeconds) +{ + mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; + BOX_TRACE("Set keep-alive time to " << mKeepAliveTime << " seconds"); + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::ManageDiffProcess() +// Purpose: Initiates a file diff control timer +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::ManageDiffProcess() +{ + ASSERT(!mbIsManaged); + mbIsManaged = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::UnManageDiffProcess() +// Purpose: suspends file diff control timer +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::UnManageDiffProcess() +{ + // ASSERT(mbIsManaged); + mbIsManaged = false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::DoKeepAlive() +// Purpose: Check whether it's time to send a KeepAlive +// message over the SSL link, and if so, send it. +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::DoKeepAlive() +{ + BackupProtocolCallable* pConnection(GetOpenConnection()); + if (!pConnection) + { + return; + } + + if (mKeepAliveTime == 0) + { + return; + } + + if (!mKeepAliveTimer.HasExpired()) + { + return; + } + + BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); + pConnection->QueryGetIsAlive(); + + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); +} + +int BackupClientContext::GetMaximumDiffingTime() +{ + return mMaximumDiffingTime; +} diff --git a/lib/bbackupd/BackupClientContext.h b/lib/bbackupd/BackupClientContext.h new file mode 100644 index 00000000..df43a232 --- /dev/null +++ b/lib/bbackupd/BackupClientContext.h @@ -0,0 +1,252 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientContext.h +// Purpose: Keep track of context +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTCONTEXT__H +#define BACKUPCLIENTCONTEXT__H + +#include "BoxTime.h" +#include "BackupClientDeleteList.h" +#include "BackupClientDirectoryRecord.h" +#include "BackupDaemonInterface.h" +#include "BackupStoreFile.h" +#include "ExcludeList.h" +#include "TcpNice.h" +#include "Timer.h" + +class TLSContext; +class BackupProtocolClient; +class SocketStreamTLS; +class BackupClientInodeToIDMap; +class BackupDaemon; +class BackupStoreFilenameClear; + +#include + + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientContext +// Purpose: +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class BackupClientContext : public DiffTimer +{ +public: + BackupClientContext + ( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode + ); + virtual ~BackupClientContext(); + +private: + BackupClientContext(const BackupClientContext &); + +public: + // GetConnection() will open a connection if none is currently open. + virtual BackupProtocolCallable& GetConnection(); + // GetOpenConnection() will not open a connection, just return NULL if there is + // no connection already open. + virtual BackupProtocolCallable* GetOpenConnection() const; + void CloseAnyOpenConnection(); + int GetTimeout() const; + BackupClientDeleteList &GetDeleteList(); + void PerformDeletions(); + + enum + { + ClientStoreMarker_NotKnown = 0 + }; + + void SetClientStoreMarker(int64_t ClientStoreMarker) {mClientStoreMarker = ClientStoreMarker;} + int64_t GetClientStoreMarker() const {return mClientStoreMarker;} + + bool StorageLimitExceeded() {return mStorageLimitExceeded;} + void SetStorageLimitExceeded() {mStorageLimitExceeded = true;} + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetIDMaps(const BackupClientInodeToIDMap *, BackupClientInodeToIDMap *) + // Purpose: Store pointers to the Current and New ID maps + // Created: 11/11/03 + // + // -------------------------------------------------------------------------- + void SetIDMaps(const BackupClientInodeToIDMap *pCurrent, BackupClientInodeToIDMap *pNew) + { + ASSERT(pCurrent != 0); + ASSERT(pNew != 0); + mpCurrentIDMap = pCurrent; + mpNewIDMap = pNew; + } + const BackupClientInodeToIDMap &GetCurrentIDMap() const; + BackupClientInodeToIDMap &GetNewIDMap() const; + + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetExcludeLists(ExcludeList *, ExcludeList *) + // Purpose: Sets the exclude lists for the operation. Can be 0. + // Created: 28/1/04 + // + // -------------------------------------------------------------------------- + void SetExcludeLists(ExcludeList *pExcludeFiles, ExcludeList *pExcludeDirs) + { + mpExcludeFiles = pExcludeFiles; + mpExcludeDirs = pExcludeDirs; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::ExcludeFile(const std::string &) + // Purpose: Returns true is this file should be excluded from the backup + // Created: 28/1/04 + // + // -------------------------------------------------------------------------- + inline bool ExcludeFile(const std::string &rFullFilename) + { + if(mpExcludeFiles != 0) + { + return mpExcludeFiles->IsExcluded(rFullFilename); + } + // If no list, don't exclude anything + return false; + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::ExcludeDir(const std::string &) + // Purpose: Returns true is this directory should be excluded from the backup + // Created: 28/1/04 + // + // -------------------------------------------------------------------------- + inline bool ExcludeDir(const std::string &rFullDirName) + { + if(mpExcludeDirs != 0) + { + return mpExcludeDirs->IsExcluded(rFullDirName); + } + // If no list, don't exclude anything + return false; + } + + // Utility functions -- may do a lot of work + bool FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut, + bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer = 0, box_time_t *pAttributesHashOnServer = 0, + BackupStoreFilenameClear *pLeafname = 0); // not const as may connect to server + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetMaximumDiffingTime() + // Purpose: Sets the maximum time that will be spent diffing a file + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void SetMaximumDiffingTime(int iSeconds); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetKeepAliveTime() + // Purpose: Sets the time interval for repetitive keep-alive operation + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + virtual void SetKeepAliveTime(int iSeconds); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::ManageDiffProcess() + // Purpose: Initiates an SSL connection/session keep-alive process + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void ManageDiffProcess(); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::UnManageDiffProcess() + // Purpose: Suspends an SSL connection/session keep-alive process + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void UnManageDiffProcess(); + + // ------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::DoKeepAlive() + // Purpose: Check whether it's time to send a KeepAlive + // message over the SSL link, and if so, send it. + // Created: 04/19/2005 + // + // ------------------------------------------------------------------- + virtual void DoKeepAlive(); + virtual int GetMaximumDiffingTime(); + virtual bool IsManaged() { return mbIsManaged; } + + ProgressNotifier& GetProgressNotifier() const + { + return mrProgressNotifier; + } + + void SetNiceMode(bool enabled) + { + if(mTcpNiceMode) + { + mpNice->SetEnabled(enabled); + } + } + + bool mExperimentalSnapshotMode; + +private: + LocationResolver &mrResolver; + TLSContext &mrTLSContext; + std::string mHostname; + int mPort; + uint32_t mAccountNumber; + std::auto_ptr mapConnection; + bool mExtendedLogging; + bool mExtendedLogToFile; + std::string mExtendedLogFile; + FILE* mpExtendedLogFileHandle; + int64_t mClientStoreMarker; + BackupClientDeleteList *mpDeleteList; + const BackupClientInodeToIDMap *mpCurrentIDMap; + BackupClientInodeToIDMap *mpNewIDMap; + bool mStorageLimitExceeded; + ExcludeList *mpExcludeFiles; + ExcludeList *mpExcludeDirs; + Timer mKeepAliveTimer; + bool mbIsManaged; + int mKeepAliveTime; + int mMaximumDiffingTime; + ProgressNotifier &mrProgressNotifier; + bool mTcpNiceMode; + NiceSocketStream *mpNice; +}; + +#endif // BACKUPCLIENTCONTEXT__H diff --git a/lib/bbackupd/BackupClientDeleteList.cpp b/lib/bbackupd/BackupClientDeleteList.cpp new file mode 100644 index 00000000..ce5e6264 --- /dev/null +++ b/lib/bbackupd/BackupClientDeleteList.cpp @@ -0,0 +1,229 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDeleteList.cpp +// Purpose: List of pending deletes for backup +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#include "BackupClientDeleteList.h" +#include "BackupClientContext.h" +#include "autogen_BackupProtocol.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::BackupClientDeleteList() +// Purpose: Constructor +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +BackupClientDeleteList::BackupClientDeleteList() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::~BackupClientDeleteList() +// Purpose: Destructor +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +BackupClientDeleteList::~BackupClientDeleteList() +{ +} + +BackupClientDeleteList::FileToDelete::FileToDelete(int64_t DirectoryID, + const BackupStoreFilename& rFilename, + const std::string& rLocalPath) +: mDirectoryID(DirectoryID), + mFilename(rFilename), + mLocalPath(rLocalPath) +{ } + +BackupClientDeleteList::DirToDelete::DirToDelete(int64_t ObjectID, + const std::string& rLocalPath) +: mObjectID(ObjectID), + mLocalPath(rLocalPath) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::AddDirectoryDelete(int64_t, +// const BackupStoreFilename&) +// Purpose: Add a directory to the list of directories to be deleted. +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID, + const std::string& rLocalPath) +{ + // Only add the delete to the list if it's not in the "no delete" set + if(mDirectoryNoDeleteList.find(ObjectID) == + mDirectoryNoDeleteList.end()) + { + // Not in the list, so should delete it + mDirectoryList.push_back(DirToDelete(ObjectID, rLocalPath)); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::AddFileDelete(int64_t, +// const BackupStoreFilename &) +// Purpose: +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, + const BackupStoreFilename &rFilename, const std::string& rLocalPath) +{ + // Try to find it in the no delete list + std::vector >::iterator + delEntry(mFileNoDeleteList.begin()); + while(delEntry != mFileNoDeleteList.end()) + { + if((delEntry)->first == DirectoryID + && (delEntry)->second == rFilename) + { + // Found! + break; + } + ++delEntry; + } + + // Only add it to the delete list if it wasn't in the no delete list + if(delEntry == mFileNoDeleteList.end()) + { + mFileList.push_back(FileToDelete(DirectoryID, rFilename, + rLocalPath)); + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) +// Purpose: Perform all the pending deletes +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) +{ + // Anything to do? + if(mDirectoryList.empty() && mFileList.empty()) + { + // Nothing! + return; + } + + // Get a connection + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Do the deletes + for(std::vector::iterator i(mDirectoryList.begin()); + i != mDirectoryList.end(); ++i) + { + connection.QueryDeleteDirectory(i->mObjectID); + rContext.GetProgressNotifier().NotifyDirectoryDeleted( + i->mObjectID, i->mLocalPath); + } + + // Clear the directory list + mDirectoryList.clear(); + + // Delete the files + for(std::vector::iterator i(mFileList.begin()); + i != mFileList.end(); ++i) + { + connection.QueryDeleteFile(i->mDirectoryID, i->mFilename); + rContext.GetProgressNotifier().NotifyFileDeleted( + i->mDirectoryID, i->mLocalPath); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::StopDirectoryDeletion(int64_t) +// Purpose: Stop a directory being deleted +// Created: 19/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::StopDirectoryDeletion(int64_t ObjectID) +{ + // First of all, is it in the delete vector? + std::vector::iterator delEntry(mDirectoryList.begin()); + for(; delEntry != mDirectoryList.end(); delEntry++) + { + if(delEntry->mObjectID == ObjectID) + { + // Found! + break; + } + } + if(delEntry != mDirectoryList.end()) + { + // erase this entry + mDirectoryList.erase(delEntry); + } + else + { + // Haven't been asked to delete it yet, put it in the + // no delete list + mDirectoryNoDeleteList.insert(ObjectID); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDeleteList::StopFileDeletion(int64_t, const BackupStoreFilename &) +// Purpose: Stop a file from being deleted +// Created: 19/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, + const BackupStoreFilename &rFilename) +{ + // Find this in the delete list + std::vector::iterator delEntry(mFileList.begin()); + while(delEntry != mFileList.end()) + { + if(delEntry->mDirectoryID == DirectoryID + && delEntry->mFilename == rFilename) + { + // Found! + break; + } + ++delEntry; + } + + if(delEntry != mFileList.end()) + { + // erase this entry + mFileList.erase(delEntry); + } + else + { + // Haven't been asked to delete it yet, put it in the no delete list + mFileNoDeleteList.push_back(std::pair(DirectoryID, rFilename)); + } +} + diff --git a/lib/bbackupd/BackupClientDeleteList.h b/lib/bbackupd/BackupClientDeleteList.h new file mode 100644 index 00000000..b0fbf51a --- /dev/null +++ b/lib/bbackupd/BackupClientDeleteList.h @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDeleteList.h +// Purpose: List of pending deletes for backup +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTDELETELIST__H +#define BACKUPCLIENTDELETELIST__H + +#include "BackupStoreFilename.h" + +class BackupClientContext; + +#include +#include +#include + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientDeleteList +// Purpose: List of pending deletes for backup +// Created: 10/11/03 +// +// -------------------------------------------------------------------------- +class BackupClientDeleteList +{ +private: + class FileToDelete + { + public: + int64_t mDirectoryID; + BackupStoreFilename mFilename; + std::string mLocalPath; + FileToDelete(int64_t DirectoryID, + const BackupStoreFilename& rFilename, + const std::string& rLocalPath); + }; + + class DirToDelete + { + public: + int64_t mObjectID; + std::string mLocalPath; + DirToDelete(int64_t ObjectID, const std::string& rLocalPath); + }; + +public: + BackupClientDeleteList(); + ~BackupClientDeleteList(); + + void AddDirectoryDelete(int64_t ObjectID, + const std::string& rLocalPath); + void AddFileDelete(int64_t DirectoryID, + const BackupStoreFilename &rFilename, + const std::string& rLocalPath); + + void StopDirectoryDeletion(int64_t ObjectID); + void StopFileDeletion(int64_t DirectoryID, + const BackupStoreFilename &rFilename); + + void PerformDeletions(BackupClientContext &rContext); + +private: + std::vector mDirectoryList; + std::set mDirectoryNoDeleteList; // note: things only get in this list if they're not present in mDirectoryList when they are 'added' + std::vector mFileList; + std::vector > mFileNoDeleteList; +}; + +#endif // BACKUPCLIENTDELETELIST__H + diff --git a/lib/bbackupd/BackupClientDirectoryRecord.cpp b/lib/bbackupd/BackupClientDirectoryRecord.cpp new file mode 100644 index 00000000..94cb7965 --- /dev/null +++ b/lib/bbackupd/BackupClientDirectoryRecord.cpp @@ -0,0 +1,2302 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDirectoryRecord.cpp +// Purpose: Implementation of record about directory for +// backup client +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_DIRENT_H + #include +#endif + +#include +#include + +#include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" +#include "autogen_ClientException.h" +#include "Archive.h" +#include "BackupClientContext.h" +#include "BackupClientDirectoryRecord.h" +#include "BackupClientInodeToIDMap.h" +#include "BackupDaemon.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileEncodeStream.h" +#include "BufferedStream.h" +#include "CommonException.h" +#include "CollectInBufferStream.h" +#include "FileModificationTime.h" +#include "IOStream.h" +#include "Logging.h" +#include "MemBlockStream.h" +#include "PathUtils.h" +#include "RateLimitingStream.h" +#include "ReadLoggingStream.h" + +#include "MemLeakFindOn.h" + +typedef std::map DecryptedEntriesMap_t; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::BackupClientDirectoryRecord() +// Purpose: Constructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName) + : mObjectID(ObjectID), + mSubDirName(rSubDirName), + mInitialSyncDone(false), + mSyncDone(false), + mpPendingEntries(0) +{ + ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::~BackupClientDirectoryRecord() +// Purpose: Destructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::~BackupClientDirectoryRecord() +{ + // Make deletion recursive + DeleteSubDirectories(); + + // Delete maps + if(mpPendingEntries != 0) + { + delete mpPendingEntries; + mpPendingEntries = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::DeleteSubDirectories(); +// Purpose: Delete all sub directory entries +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::DeleteSubDirectories() +{ + // Delete all pointers + for(std::map::iterator i = mSubDirectories.begin(); + i != mSubDirectories.end(); ++i) + { + delete i->second; + } + + // Empty list + mSubDirectories.clear(); +} + +std::string BackupClientDirectoryRecord::ConvertVssPathToRealPath( + const std::string &rVssPath, + const Location& rBackupLocation) +{ +#ifdef ENABLE_VSS + BOX_TRACE("VSS: ConvertVssPathToRealPath: mIsSnapshotCreated = " << + rBackupLocation.mIsSnapshotCreated); + BOX_TRACE("VSS: ConvertVssPathToRealPath: File/Directory Path = " << + rVssPath.substr(0, rBackupLocation.mSnapshotPath.length())); + BOX_TRACE("VSS: ConvertVssPathToRealPath: Snapshot Path = " << + rBackupLocation.mSnapshotPath); + if (rBackupLocation.mIsSnapshotCreated && + rVssPath.substr(0, rBackupLocation.mSnapshotPath.length()) == + rBackupLocation.mSnapshotPath) + { + std::string convertedPath = rBackupLocation.mPath + + rVssPath.substr(rBackupLocation.mSnapshotPath.length()); + BOX_TRACE("VSS: ConvertVssPathToRealPath: Converted Path = " << + convertedPath); + return convertedPath; + } +#endif + + return rVssPath; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncDirectory(i +// BackupClientDirectoryRecord::SyncParams &, +// int64_t, const std::string &, +// const std::string &, bool) +// Purpose: Recursively synchronise a local directory +// with the server. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::SyncDirectory( + BackupClientDirectoryRecord::SyncParams &rParams, + int64_t ContainingDirectoryID, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + bool ThisDirHasJustBeenCreated) +{ + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + + // Signal received by daemon? + if(rParams.mrRunStatusProvider.StopRun()) + { + // Yes. Stop now. + THROW_EXCEPTION(BackupStoreException, SignalReceived) + } + + // Start by making some flag changes, marking this sync as not done, + // and on the immediate sub directories. + mSyncDone = false; + for(std::map::iterator + i = mSubDirectories.begin(); + i != mSubDirectories.end(); ++i) + { + i->second->mSyncDone = false; + } + + // Work out the time in the future after which the file should + // be uploaded regardless. This is a simple way to avoid having + // too many problems with file servers when they have clients + // with badly out of sync clocks. + rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + + rParams.mMaxFileTimeInFuture; + + // Build the current state checksum to compare against while + // getting info from dirs. Note checksum is used locally only, + // so byte order isn't considered. + MD5Digest currentStateChecksum; + + EMU_STRUCT_STAT dest_st; + // Stat the directory, to get attribute info + // If it's a symbolic link, we want the link target here + // (as we're about to back up the contents of the directory) + { + if(EMU_STAT(rLocalPath.c_str(), &dest_st) != 0) + { + // The directory has probably been deleted, so + // just ignore this error. In a future scan, this + // deletion will be noticed, deleted from server, + // and this object deleted. + rNotifier.NotifyDirStatFailed(this, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation), + strerror(errno)); + return; + } + + BOX_TRACE("Stat dir '" << rLocalPath << "' " + "found device/inode " << + dest_st.st_dev << "/" << dest_st.st_ino); + + // Store inode number in map so directories are tracked + // in case they're renamed + { + BackupClientInodeToIDMap &idMap( + rParams.mrContext.GetNewIDMap()); + idMap.AddToMap(dest_st.st_ino, mObjectID, ContainingDirectoryID, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation)); + } + // Add attributes to checksum + currentStateChecksum.Add(&dest_st.st_mode, + sizeof(dest_st.st_mode)); + currentStateChecksum.Add(&dest_st.st_uid, + sizeof(dest_st.st_uid)); + currentStateChecksum.Add(&dest_st.st_gid, + sizeof(dest_st.st_gid)); + // Inode to be paranoid about things moving around + currentStateChecksum.Add(&dest_st.st_ino, + sizeof(dest_st.st_ino)); +#ifdef HAVE_STRUCT_STAT_ST_FLAGS + currentStateChecksum.Add(&dest_st.st_flags, + sizeof(dest_st.st_flags)); +#endif + + StreamableMemBlock xattr; + BackupClientFileAttributes::FillExtendedAttr(xattr, + rLocalPath.c_str()); + currentStateChecksum.Add(xattr.GetBuffer(), xattr.GetSize()); + } + + // Read directory entries, building arrays of names + // First, need to read the contents of the directory. + std::vector dirs; + std::vector files; + bool downloadDirectoryRecordBecauseOfFutureFiles = false; + + // BLOCK + { + // read the contents... + DIR *dirHandle = 0; + try + { + std::string nonVssDirPath = ConvertVssPathToRealPath(rLocalPath, + rBackupLocation); + rNotifier.NotifyScanDirectory(this, nonVssDirPath); + + dirHandle = ::opendir(rLocalPath.c_str()); + if(dirHandle == 0) + { + // Report the error (logs and eventual email to administrator) + if (errno == EACCES) + { + rNotifier.NotifyDirListFailed(this, + nonVssDirPath, + "Access denied"); + } + else + { + rNotifier.NotifyDirListFailed(this, + nonVssDirPath, + strerror(errno)); + } + + // Report the error (logs and eventual email + // to administrator) + SetErrorWhenReadingFilesystemObject(rParams, + nonVssDirPath); + // Ignore this directory for now. + return; + } + + struct dirent *en = 0; + int num_entries_found = 0; + + while((en = ::readdir(dirHandle)) != 0) + { + num_entries_found++; + rParams.mrContext.DoKeepAlive(); + if(rParams.mpBackgroundTask) + { + rParams.mpBackgroundTask->RunBackgroundTask( + BackgroundTask::Scanning_Dirs, + num_entries_found, 0); + } + + if (!SyncDirectoryEntry(rParams, rNotifier, + rBackupLocation, rLocalPath, + currentStateChecksum, en, dest_st, dirs, + files, downloadDirectoryRecordBecauseOfFutureFiles)) + { + // This entry is not to be backed up. + continue; + } + } + + if(::closedir(dirHandle) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + dirHandle = 0; + } + catch(...) + { + if(dirHandle != 0) + { + ::closedir(dirHandle); + } + throw; + } + } + + // Finish off the checksum, and compare with the one currently stored + bool checksumDifferent = true; + currentStateChecksum.Finish(); + if(mInitialSyncDone && currentStateChecksum.DigestMatches(mStateChecksum)) + { + // The checksum is the same, and there was one to compare with + checksumDifferent = false; + } + + // Pointer to potentially downloaded store directory info + std::auto_ptr apDirOnStore; + + try + { + // Want to get the directory listing? + if(ThisDirHasJustBeenCreated) + { + // Avoid sending another command to the server when we know it's empty + apDirOnStore.reset(new BackupStoreDirectory(mObjectID, + ContainingDirectoryID)); + } + // Consider asking the store for it + else if(!mInitialSyncDone || checksumDifferent || + downloadDirectoryRecordBecauseOfFutureFiles) + { + apDirOnStore = FetchDirectoryListing(rParams); + } + + // Make sure the attributes are up to date -- if there's space + // on the server and this directory has not just been created + // (because it's attributes will be correct in this case) and + // the checksum is different, implying they *MIGHT* be + // different. + if((!ThisDirHasJustBeenCreated) && checksumDifferent && + !rParams.mrContext.StorageLimitExceeded()) + { + UpdateAttributes(rParams, apDirOnStore.get(), rLocalPath); + } + + // Create the list of pointers to directory entries + std::vector entriesLeftOver; + if(apDirOnStore.get()) + { + entriesLeftOver.resize(apDirOnStore->GetNumberOfEntries(), 0); + BackupStoreDirectory::Iterator i(*apDirOnStore); + // Copy in pointers to all the entries + for(unsigned int l = 0; l < apDirOnStore->GetNumberOfEntries(); ++l) + { + entriesLeftOver[l] = i.Next(); + } + } + + // Do the directory reading + bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, + rRemotePath, rBackupLocation, apDirOnStore.get(), + entriesLeftOver, files, dirs); + + // LAST THING! (think exception safety) + // Store the new checksum -- don't fetch things unnecessarily + // in the future But... only if 1) the storage limit isn't + // exceeded -- make sure things are done again if the directory + // is modified later and 2) All the objects within the + // directory were stored successfully. + if(!rParams.mrContext.StorageLimitExceeded() && + updateCompleteSuccess) + { + currentStateChecksum.CopyDigestTo(mStateChecksum); + } + } + catch(...) + { + // Bad things have happened -- clean up + // Set things so that we get a full go at stuff later + ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); + + throw; + } + + // Flag things as having happened. + mInitialSyncDone = true; + mSyncDone = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncDirectorEntry( +// BackupClientDirectoryRecord::SyncParams &, +// int64_t, const std::string &, +// const std::string &, bool) +// Purpose: Recursively synchronise a local directory +// with the server. +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +bool BackupClientDirectoryRecord::SyncDirectoryEntry( + BackupClientDirectoryRecord::SyncParams &rParams, + ProgressNotifier& rNotifier, + const Location& rBackupLocation, + const std::string &rDirLocalPath, + MD5Digest& currentStateChecksum, + struct dirent *en, + EMU_STRUCT_STAT dir_st, + std::vector& rDirs, + std::vector& rFiles, + bool& rDownloadDirectoryRecordBecauseOfFutureFiles) +{ + std::string entry_name = en->d_name; + if(entry_name == "." || entry_name == "..") + { + // ignore parent directory entries + return false; + } + + // Stat file to get info + std::string filename = MakeFullPath(rDirLocalPath, entry_name); + std::string realFileName = ConvertVssPathToRealPath(filename, + rBackupLocation); + EMU_STRUCT_STAT file_st; + +#ifdef WIN32 + // Don't stat the file just yet, to ensure that users can exclude + // unreadable files to suppress warnings that they are not accessible. + // + // Our emulated readdir() abuses en->d_type, which would normally + // contain DT_REG, DT_DIR, etc, but we only use it here and prefer to + // have the full file attributes. + + int type; + if (en->d_type & FILE_ATTRIBUTE_DIRECTORY) + { + type = S_IFDIR; + } + else + { + type = S_IFREG; + } +#else // !WIN32 + if(EMU_LSTAT(filename.c_str(), &file_st) != 0) + { + // We don't know whether it's a file or a directory, so check + // both. This only affects whether a warning message is + // displayed; the file is not backed up in either case. + if(!(rParams.mrContext.ExcludeFile(filename)) && + !(rParams.mrContext.ExcludeDir(filename))) + { + // Report the error (logs and eventual email to + // administrator) + rNotifier.NotifyFileStatFailed(this, filename, + strerror(errno)); + + // FIXME move to NotifyFileStatFailed() + SetErrorWhenReadingFilesystemObject(rParams, filename); + } + + // Ignore this entry for now. + return false; + } + + BOX_TRACE("Stat entry '" << filename << "' found device/inode " << + file_st.st_dev << "/" << file_st.st_ino); + + // Workaround for apparent btrfs bug, where symlinks appear to be on + // a different filesystem than their containing directory, thanks to + // Toke Hoiland-Jorgensen. + + int type = file_st.st_mode & S_IFMT; + if(type == S_IFDIR && file_st.st_dev != dir_st.st_dev) + { + if(!(rParams.mrContext.ExcludeDir(filename))) + { + rNotifier.NotifyMountPointSkipped(this, filename); + } + return false; + } +#endif + + if(type == S_IFREG || type == S_IFLNK) + { + // File or symbolic link + + // Exclude it? + if(rParams.mrContext.ExcludeFile(realFileName)) + { + rNotifier.NotifyFileExcluded(this, realFileName); + // Next item! + return false; + } + } + else if(type == S_IFDIR) + { + // Directory + + // Exclude it? + if(rParams.mrContext.ExcludeDir(realFileName)) + { + rNotifier.NotifyDirExcluded(this, realFileName); + + // Next item! + return false; + } + + #ifdef WIN32 + // exclude reparse points, as Application Data points to the + // parent directory under Vista and later, and causes an + // infinite loop: + // http://social.msdn.microsoft.com/forums/en-US/windowscompatibility/thread/05d14368-25dd-41c8-bdba-5590bf762a68/ + if (en->d_type & FILE_ATTRIBUTE_REPARSE_POINT) + { + rNotifier.NotifyMountPointSkipped(this, realFileName); + return false; + } + #endif + } + else // not a file or directory, what is it? + { + if (type == S_IFSOCK +#ifndef WIN32 + || type == S_IFIFO +#endif + ) + { + // removed notification for these types + // see Debian bug 479145, no objections + } + else if(rParams.mrContext.ExcludeFile(realFileName)) + { + rNotifier.NotifyFileExcluded(this, realFileName); + } + else + { + rNotifier.NotifyUnsupportedFileType(this, realFileName); + SetErrorWhenReadingFilesystemObject(rParams, + realFileName); + } + + return false; + } + + // The object should be backed up (file, symlink or dir, not excluded). + // So make the information for adding to the checksum. + + #ifdef WIN32 + // We didn't stat the file before, but now we need the information. + if(emu_stat(filename.c_str(), &file_st) != 0) + { + rNotifier.NotifyFileStatFailed(this, + ConvertVssPathToRealPath(filename, rBackupLocation), + strerror(errno)); + + // Report the error (logs and eventual email to administrator) + SetErrorWhenReadingFilesystemObject(rParams, filename); + + // Ignore this entry for now. + return false; + } + + if(file_st.st_dev != dir_st.st_dev) + { + rNotifier.NotifyMountPointSkipped(this, + ConvertVssPathToRealPath(filename, rBackupLocation)); + return false; + } + #endif + + // Basic structure for checksum info + struct { + box_time_t mModificationTime; + box_time_t mAttributeModificationTime; + int64_t mSize; + // And then the name follows + } checksum_info; + + // Be paranoid about structure packing + ::memset(&checksum_info, 0, sizeof(checksum_info)); + + checksum_info.mModificationTime = FileModificationTime(file_st); + checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st); + checksum_info.mSize = file_st.st_size; + currentStateChecksum.Add(&checksum_info, sizeof(checksum_info)); + currentStateChecksum.Add(en->d_name, strlen(en->d_name)); + + // If the file has been modified madly into the future, download the + // directory record anyway to ensure that it doesn't get uploaded + // every single time the disc is scanned. + if(checksum_info.mModificationTime > rParams.mUploadAfterThisTimeInTheFuture) + { + rDownloadDirectoryRecordBecauseOfFutureFiles = true; + // Log that this has happened + if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) + { + rNotifier.NotifyFileModifiedInFuture(this, + ConvertVssPathToRealPath(filename, rBackupLocation)); + rParams.mHaveLoggedWarningAboutFutureFileTimes = true; + } + } + + // We've decided to back it up, so add to file or directory list. + if(type == S_IFREG || type == S_IFLNK) + { + rFiles.push_back(entry_name); + } + else if(type == S_IFDIR) + { + rDirs.push_back(entry_name); + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::FetchDirectoryListing( +// BackupClientDirectoryRecord::SyncParams &) +// Purpose: Fetch the directory listing of this directory from +// the store. +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +std::auto_ptr +BackupClientDirectoryRecord::FetchDirectoryListing( + BackupClientDirectoryRecord::SyncParams &rParams) +{ + std::auto_ptr apDir; + + // Get connection to store + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Query the directory + std::auto_ptr dirreply(connection.QueryListDirectory( + mObjectID, + // both files and directories + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // exclude old/deleted stuff + BackupProtocolListDirectory::Flags_Deleted | + BackupProtocolListDirectory::Flags_OldVersion, + true /* want attributes */)); + + // Retrieve the directory from the stream following + apDir.reset(new BackupStoreDirectory(connection.ReceiveStream(), + connection.GetTimeout())); + return apDir; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::UpdateAttributes( +// BackupClientDirectoryRecord::SyncParams &, +// const std::string &) +// Purpose: Sets the attributes of the directory on the store, +// if necessary. +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::UpdateAttributes( + BackupClientDirectoryRecord::SyncParams &rParams, + BackupStoreDirectory *pDirOnStore, + const std::string &rLocalPath) +{ + // Get attributes for the directory + BackupClientFileAttributes attr; + box_time_t attrModTime = 0; + attr.ReadAttributes(rLocalPath.c_str(), true /* directories have zero mod times */, + 0 /* no modification time */, &attrModTime); + + // Assume attributes need updating, unless proved otherwise + bool updateAttr = true; + + // Got a listing to compare with? + ASSERT(pDirOnStore == 0 || (pDirOnStore != 0 && pDirOnStore->HasAttributes())); + if(pDirOnStore != 0 && pDirOnStore->HasAttributes()) + { + const StreamableMemBlock &storeAttrEnc(pDirOnStore->GetAttributes()); + // Explict decryption + BackupClientFileAttributes storeAttr(storeAttrEnc); + + // Compare the attributes + if(attr.Compare(storeAttr, true, + true /* ignore both modification times */)) + { + // No update necessary + updateAttr = false; + } + } + + // Update them? + if(updateAttr) + { + // Get connection to store + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Exception thrown if this doesn't work + std::auto_ptr attrStream(new MemBlockStream(attr)); + connection.QueryChangeDirAttributes(mObjectID, attrModTime, attrStream); + } +} + +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath) +{ + BackupStoreFilenameClear fn(en->GetName()); + return DecryptFilename(fn, en->GetObjectID(), rRemoteDirectoryPath); +} + +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreFilenameClear fn, int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath) +{ + std::string filenameClear; + try + { + filenameClear = fn.GetClearFilename(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to decrypt filename for object " << + BOX_FORMAT_OBJECTID(filenameObjectID) << " in " + "directory " << BOX_FORMAT_OBJECTID(mObjectID) << + " (" << rRemoteDirectoryPath << ")"); + throw; + } + return filenameClear; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &, const std::string &, BackupStoreDirectory *, std::vector &) +// Purpose: Update the items stored on the server. The rFiles vector will be erased after it's used to save space. +// Returns true if all items were updated successfully. (If not, the failures will have been logged). +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +bool BackupClientDirectoryRecord::UpdateItems( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + BackupStoreDirectory *pDirOnStore, + std::vector &rEntriesLeftOver, + std::vector &rFiles, + const std::vector &rDirs) +{ + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + + bool allUpdatedSuccessfully = true; + + // Decrypt all the directory entries. + // It would be nice to be able to just compare the encrypted versions, however this doesn't work + // in practise because there can be multiple encodings of the same filename using different + // methods (although each method will result in the same string for the same filename.) This + // happens when the server fixes a broken store, and gives plain text generated filenames. + // So if we didn't do things like this, then you wouldn't be able to recover from bad things + // happening with the server. + DecryptedEntriesMap_t decryptedEntries; + if(pDirOnStore != 0) + { + BackupStoreDirectory::Iterator i(*pDirOnStore); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + std::string filenameClear; + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + decryptedEntries[filenameClear] = en; + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "pretending that the file doesn't " + "exist"); + } + } + } + + // Do files + for(std::vector::const_iterator f = rFiles.begin(); + f != rFiles.end(); ++f) + { + // Send keep-alive message if needed + rContext.DoKeepAlive(); + + // Filename of this file + std::string filename(MakeFullPath(rLocalPath, *f)); + std::string nonVssFilePath = ConvertVssPathToRealPath(filename, + rBackupLocation); + + // Get relevant info about file + box_time_t modTime = 0; + uint64_t attributesHash = 0; + int64_t fileSize = 0; + InodeRefType inodeNum = 0; + // BLOCK + { + // Stat the file + EMU_STRUCT_STAT st; + if(EMU_LSTAT(filename.c_str(), &st) != 0) + { + rNotifier.NotifyFileStatFailed(this, nonVssFilePath, + strerror(errno)); + + // Report the error (logs and + // eventual email to administrator) + SetErrorWhenReadingFilesystemObject(rParams, nonVssFilePath); + + // Ignore this entry for now. + continue; + } + + // Extract required data + modTime = FileModificationTime(st); + fileSize = st.st_size; + inodeNum = st.st_ino; + attributesHash = BackupClientFileAttributes::GenerateAttributeHash(st, filename, *f); + } + + // See if it's in the listing (if we have one) + BackupStoreFilenameClear storeFilename(*f); + BackupStoreDirectory::Entry *en = 0; + int64_t latestObjectID = 0; + if(pDirOnStore != 0) + { + DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*f)); + if(i != decryptedEntries.end()) + { + en = i->second; + latestObjectID = en->GetObjectID(); + } + } + + // Check that the entry which might have been found is in fact a file + if((en != 0) && !(en->IsFile())) + { + // Directory exists in the place of this file -- sort it out + RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, + en, *f); + en = 0; + } + + // Check for renaming? + if(pDirOnStore != 0 && en == 0) + { + // We now know... + // 1) File has just been added + // 2) It's not in the store + + // Do we know about the inode number? + const BackupClientInodeToIDMap &idMap(rContext.GetCurrentIDMap()); + int64_t renameObjectID = 0, renameInDirectory = 0; + if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) + { + // Look up on the server to get the name, to build the local filename + std::string localPotentialOldName; + bool isDir = false; + bool isCurrentVersion = false; + box_time_t srvModTime = 0, srvAttributesHash = 0; + BackupStoreFilenameClear oldLeafname; + if(rContext.FindFilename(renameObjectID, renameInDirectory, + localPotentialOldName, isDir, isCurrentVersion, + &srvModTime, &srvAttributesHash, &oldLeafname)) + { + // Only interested if it's a file and the latest version + if(!isDir && isCurrentVersion) + { + // Check that the object we found in the ID map doesn't exist on disc + EMU_STRUCT_STAT st; + if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) + { + // Doesn't exist locally, but does exist on the server. + // Therefore we can safely rename it to this new file. + + // Get the connection to the server + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Only do this step if there is room on the server. + // This step will be repeated later when there is space available + if(!rContext.StorageLimitExceeded()) + { + // Rename the existing files (ie include old versions) on the server + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, + storeFilename); + + // Stop the attempt to delete the file in the original location + BackupClientDeleteList &rdelList(rContext.GetDeleteList()); + rdelList.StopFileDeletion(renameInDirectory, oldLeafname); + + // Create new entry in the directory for it + // -- will be near enough what's actually on the server for the rest to work. + en = pDirOnStore->AddEntry(storeFilename, + srvModTime, renameObjectID, + 0 /* size in blocks unknown, but not needed */, + BackupStoreDirectory::Entry::Flags_File, + srvAttributesHash); + + // Store the object ID for the inode lookup map later + latestObjectID = renameObjectID; + } + } + } + } + } + } + + // Is it in the mPendingEntries list? + box_time_t pendingFirstSeenTime = 0; // ie not seen + if(mpPendingEntries != 0) + { + std::map::const_iterator i(mpPendingEntries->find(*f)); + if(i != mpPendingEntries->end()) + { + // found it -- set flag + pendingFirstSeenTime = i->second; + } + } + + // If pDirOnStore == 0, then this must have been after an initial sync: + ASSERT(pDirOnStore != 0 || mInitialSyncDone); + // So, if pDirOnStore == 0, then we know that everything before syncPeriodStart + // is either on the server, or in the toupload list. If the directory had changed, + // we'd have got a directory listing. + // + // At this point, if (pDirOnStore == 0 && en == 0), we can assume it's on the server with a + // mod time < syncPeriodStart, or didn't exist before that time. + // + // But if en != 0, then we need to compare modification times to avoid uploading it again. + + // Need to update? + // + // Condition for upload: + // modification time within sync period + // if it's been seen before but not uploaded, is the time from this first sight longer than the MaxUploadWait + // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store + + bool doUpload = false; + std::string decisionReason = "unknown"; + + // Only upload a file if the mod time locally is + // different to that on the server. + + if (en == 0 || en->GetModificationTime() != modTime) + { + // Check the file modified within the acceptable time period we're checking + // If the file isn't on the server, the acceptable time starts at zero. + // Check pDirOnStore and en, because if we didn't download a directory listing, + // pDirOnStore will be zero, but we know it's on the server. + if (modTime < rParams.mSyncPeriodEnd) + { + if (pDirOnStore != 0 && en == 0) + { + doUpload = true; + decisionReason = "not on server"; + } + else if (modTime >= rParams.mSyncPeriodStart) + { + doUpload = true; + decisionReason = "modified since last sync"; + } + } + + // However, just in case things are continually + // modified, we check the first seen time. + // The two compares of syncPeriodEnd and + // pendingFirstSeenTime are because the values + // are unsigned. + + if (!doUpload && + pendingFirstSeenTime != 0 && + rParams.mSyncPeriodEnd > pendingFirstSeenTime && + (rParams.mSyncPeriodEnd - pendingFirstSeenTime) + > rParams.mMaxUploadWait) + { + doUpload = true; + decisionReason = "continually modified"; + } + + // Then make sure that if files are added with a + // time less than the sync period start + // (which can easily happen on file server), it + // gets uploaded. The directory contents checksum + // will pick up the fact it has been added, so the + // store listing will be available when this happens. + + if (!doUpload && + modTime <= rParams.mSyncPeriodStart && + en != 0 && + en->GetModificationTime() != modTime) + { + doUpload = true; + decisionReason = "mod time changed"; + } + + // And just to catch really badly off clocks in + // the future for file server clients, + // just upload the file if it's madly in the future. + + if (!doUpload && modTime > + rParams.mUploadAfterThisTimeInTheFuture) + { + doUpload = true; + decisionReason = "mod time in the future"; + } + } + + if (en != 0 && en->GetModificationTime() == modTime) + { + doUpload = false; + decisionReason = "not modified since last upload"; + } + else if (!doUpload) + { + if (modTime > rParams.mSyncPeriodEnd) + { + box_time_t now = GetCurrentBoxTime(); + int age = BoxTimeToSeconds(now - + modTime); + std::ostringstream s; + s << "modified too recently: only " << + age << " seconds ago"; + decisionReason = s.str(); + } + else + { + std::ostringstream s; + s << "mod time is " << modTime << + " which is outside sync window, " + << rParams.mSyncPeriodStart << " to " + << rParams.mSyncPeriodEnd; + decisionReason = s.str(); + } + } + + BOX_TRACE("Upload decision: " << nonVssFilePath << ": " << + (doUpload ? "will upload" : "will not upload") << + " (" << decisionReason << ")"); + + bool fileSynced = true; + + if (doUpload) + { + // Upload needed, don't mark sync success until + // we've actually done it + fileSynced = false; + + // Make sure we're connected -- must connect here so we know whether + // the storage limit has been exceeded, and hence whether or not + // to actually upload the file. + rContext.GetConnection(); + + // Only do this step if there is room on the server. + // This step will be repeated later when there is space available + if(!rContext.StorageLimitExceeded()) + { + // Upload the file to the server, recording the + // object ID it returns + bool noPreviousVersionOnServer = + ((pDirOnStore != 0) && (en == 0)); + + // Surround this in a try/catch block, to + // catch errors, but still continue + bool uploadSuccess = false; + try + { + latestObjectID = UploadFile(rParams, + filename, + nonVssFilePath, + rRemotePath + "/" + *f, + storeFilename, + fileSize, modTime, + attributesHash, + noPreviousVersionOnServer); + + if (latestObjectID == 0) + { + // storage limit exceeded + rParams.mrContext.SetStorageLimitExceeded(); + uploadSuccess = false; + allUpdatedSuccessfully = false; + } + else + { + uploadSuccess = true; + } + } + catch(ConnectionException &e) + { + // Connection errors should just be + // passed on to the main handler, + // retries would probably just cause + // more problems. + rNotifier.NotifyFileUploadException( + this, nonVssFilePath, e); + throw; + } + catch(BoxException &e) + { + if (e.GetType() == BackupStoreException::ExceptionType && + e.GetSubType() == BackupStoreException::SignalReceived) + { + // abort requested, pass the + // exception on up. + throw; + } + + // an error occured -- make return + // code false, to show error in directory + allUpdatedSuccessfully = false; + // Log it. + SetErrorWhenReadingFilesystemObject(rParams, + nonVssFilePath); + rNotifier.NotifyFileUploadException(this, + nonVssFilePath, e); + } + + // Update structures if the file was uploaded + // successfully. + if(uploadSuccess) + { + fileSynced = true; + + // delete from pending entries + if(pendingFirstSeenTime != 0 && mpPendingEntries != 0) + { + mpPendingEntries->erase(*f); + } + } + } + else + { + rNotifier.NotifyFileSkippedServerFull(this, nonVssFilePath); + } + } + else if(en != 0 && en->GetAttributesHash() != attributesHash) + { + // Attributes have probably changed, upload them again. + // If the attributes have changed enough, the directory + // hash will have changed too, and so the dir will have + // been downloaded, and the entry will be available. + + // Get connection + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Only do this step if there is room on the server. + // This step will be repeated later when there is + // space available + if(!rContext.StorageLimitExceeded()) + { + try + { + rNotifier.NotifyFileUploadingAttributes(this, + nonVssFilePath); + + // Update store + BackupClientFileAttributes attr; + attr.ReadAttributes(filename, + false /* put mod times in the attributes, please */); + std::auto_ptr attrStream( + new MemBlockStream(attr)); + connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream); + fileSynced = true; + } + catch (BoxException &e) + { + BOX_ERROR("Failed to read or store file attributes " + "for '" << nonVssFilePath << "', will try again " + "later"); + } + } + } + + if(modTime >= rParams.mSyncPeriodEnd) + { + // Allocate? + if(mpPendingEntries == 0) + { + mpPendingEntries = new std::map; + } + // Adding to mPendingEntries list + if(pendingFirstSeenTime == 0) + { + // Haven't seen this before -- add to list! + (*mpPendingEntries)[*f] = modTime; + } + } + + // Zero pointer in rEntriesLeftOver, if we have a pointer to zero + if(en != 0) + { + for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) + { + if(rEntriesLeftOver[l] == en) + { + rEntriesLeftOver[l] = 0; + break; + } + } + } + + // Does this file need an entry in the ID map? + if(fileSize >= rParams.mFileTrackingSizeThreshold) + { + // Get the map + BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap()); + + // Need to get an ID from somewhere... + if(latestObjectID == 0) + { + // Don't know it -- haven't sent anything to the store, and didn't get a listing. + // Look it up in the current map, and if it's there, use that. + const BackupClientInodeToIDMap ¤tIDMap(rContext.GetCurrentIDMap()); + int64_t objid = 0, dirid = 0; + if(currentIDMap.Lookup(inodeNum, objid, dirid)) + { + // Found + if (dirid != mObjectID) + { + BOX_WARNING("Found conflicting parent ID for " + "file ID " << inodeNum << " (" << + nonVssFilePath << "): expected " << + mObjectID << " but found " << dirid << + " (same directory used in two different " + "locations?)"); + } + + ASSERT(dirid == mObjectID); + + // NOTE: If the above assert fails, an inode number has been reused by the OS, + // or there is a problem somewhere. If this happened on a short test run, look + // into it. However, in a long running process this may happen occasionally and + // not indicate anything wrong. + // Run the release version for real life use, where this check is not made. + + latestObjectID = objid; + } + } + + if(latestObjectID != 0) + { + BOX_TRACE("Storing uploaded file ID " << + inodeNum << " (" << nonVssFilePath << ") " + "in ID map as object " << + latestObjectID << " with parent " << + mObjectID); + idMap.AddToMap(inodeNum, latestObjectID, + mObjectID /* containing directory */, + nonVssFilePath); + } + + } + + if (fileSynced) + { + rNotifier.NotifyFileSynchronised(this, nonVssFilePath, + fileSize); + } + } + + // Erase contents of files to save space when recursing + rFiles.clear(); + + // Delete the pending entries, if the map is empty + if(mpPendingEntries != 0 && mpPendingEntries->size() == 0) + { + BOX_TRACE("Deleting mpPendingEntries from dir ID " << + BOX_FORMAT_OBJECTID(mObjectID)); + delete mpPendingEntries; + mpPendingEntries = 0; + } + + // Do directories + for(std::vector::const_iterator d = rDirs.begin(); + d != rDirs.end(); ++d) + { + // Send keep-alive message if needed + rContext.DoKeepAlive(); + + // Get the local filename + std::string dirname(MakeFullPath(rLocalPath, *d)); + std::string nonVssDirPath = ConvertVssPathToRealPath(dirname, + rBackupLocation); + + // See if it's in the listing (if we have one) + BackupStoreFilenameClear storeFilename(*d); + BackupStoreDirectory::Entry *en = 0; + if(pDirOnStore != 0) + { + DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*d)); + if(i != decryptedEntries.end()) + { + en = i->second; + } + } + + // Check that the entry which might have been found is in fact a directory + if((en != 0) && !(en->IsDir())) + { + // Entry exists, but is not a directory. Bad. + // Get rid of it. + BackupProtocolCallable &connection(rContext.GetConnection()); + connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); + + std::string filenameClear = DecryptFilename(en, + rRemotePath); + rNotifier.NotifyFileDeleted(en->GetObjectID(), + filenameClear); + + // Nothing found + en = 0; + } + + // Zero pointer in rEntriesLeftOver, if we have a pointer to zero + if(en != 0) + { + for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) + { + if(rEntriesLeftOver[l] == en) + { + rEntriesLeftOver[l] = 0; + break; + } + } + } + + // Flag for having created directory, so can optimise the + // recursive call not to read it again, because we know + // it's empty. + bool haveJustCreatedDirOnServer = false; + + // Next, see if it's in the list of sub directories + BackupClientDirectoryRecord *psubDirRecord = 0; + std::map::iterator + e(mSubDirectories.find(*d)); + + if(e != mSubDirectories.end()) + { + // In the list, just use this pointer + psubDirRecord = e->second; + } + else + { + // Note: if we have exceeded our storage limit, then + // we should not upload any more data, nor create any + // DirectoryRecord representing data that would have + // been uploaded. This step will be repeated when + // there is some space available. + bool doCreateDirectoryRecord = true; + + // Need to create the record. But do we need to create the directory on the server? + int64_t subDirObjectID = 0; + if(en != 0) + { + // No. Exists on the server, and we know about it from the listing. + subDirObjectID = en->GetObjectID(); + } + else if(rContext.StorageLimitExceeded()) + // know we've got a connection if we get this far, + // as dir will have been modified. + { + doCreateDirectoryRecord = false; + } + else + { + // Yes, creation required! + // It is known that it doesn't exist: + // + // if en == 0 and pDirOnStore == 0, then the + // directory has had an initial sync, and + // hasn't been modified (Really? then why + // are we here? TODO FIXME) + // so it has definitely been created already + // (so why create it again?) + // + // if en == 0 but pDirOnStore != 0, well... obviously it doesn't exist. + // + subDirObjectID = CreateRemoteDir(dirname, + nonVssDirPath, rRemotePath + "/" + *d, + storeFilename, &haveJustCreatedDirOnServer, + rParams); + doCreateDirectoryRecord = (subDirObjectID != 0); + } + + if (doCreateDirectoryRecord) + { + // New an object for this + psubDirRecord = new BackupClientDirectoryRecord(subDirObjectID, *d); + + // Store in list + try + { + mSubDirectories[*d] = psubDirRecord; + } + catch(...) + { + delete psubDirRecord; + psubDirRecord = 0; + throw; + } + } + } + + // ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded()); + // There's another possible reason now: the directory no longer + // existed when we finally got around to checking its + // attributes. See for example Brendon Baumgartner's reported + // error with Wordpress cache directories. + + if(psubDirRecord) + { + // Sync this sub directory too + psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, + rRemotePath + "/" + *d, rBackupLocation, + haveJustCreatedDirOnServer); + } + } + + // Delete everything which is on the store, but not on disc + for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) + { + if(rEntriesLeftOver[l] != 0) + { + BackupStoreDirectory::Entry *en = rEntriesLeftOver[l]; + + // These entries can't be deleted immediately, as it would prevent + // renaming and moving of objects working properly. So we add them + // to a list, which is actually deleted at the very end of the session. + // If there's an error during the process, it doesn't matter if things + // aren't actually deleted, as the whole state will be reset anyway. + BackupClientDeleteList &rdel(rContext.GetDeleteList()); + std::string filenameClear; + bool isCorruptFilename = false; + + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "scheduling that file for deletion"); + filenameClear = ""; + isCorruptFilename = true; + } + + std::string localName = MakeFullPath(rLocalPath, + filenameClear); + std::string nonVssLocalName = ConvertVssPathToRealPath(localName, + rBackupLocation); + + // Delete this entry -- file or directory? + if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) + { + // Set a pending deletion for the file + rdel.AddFileDelete(mObjectID, en->GetName(), + localName); + } + else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) + { + // Set as a pending deletion for the directory + rdel.AddDirectoryDelete(en->GetObjectID(), + localName); + + // If there's a directory record for it in + // the sub directory map, delete it now + BackupStoreFilenameClear dirname(en->GetName()); + std::map::iterator + e(mSubDirectories.find(filenameClear)); + if(e != mSubDirectories.end() && !isCorruptFilename) + { + // Carefully delete the entry from the map + BackupClientDirectoryRecord *rec = e->second; + mSubDirectories.erase(e); + delete rec; + + BOX_TRACE("Deleted directory record for " << + nonVssLocalName); + } + } + } + } + + // Return success flag (will be false if some files failed) + return allUpdatedSuccessfully; +} + +int64_t BackupClientDirectoryRecord::CreateRemoteDir(const std::string& localDirPath, + const std::string& nonVssDirPath, const std::string& remoteDirPath, + BackupStoreFilenameClear& storeFilename, bool* pHaveJustCreatedDirOnServer, + BackupClientDirectoryRecord::SyncParams &rParams) +{ + // Get attributes + box_time_t attrModTime = 0; + InodeRefType inodeNum = 0; + BackupClientFileAttributes attr; + *pHaveJustCreatedDirOnServer = false; + ProgressNotifier& rNotifier(rParams.mrContext.GetProgressNotifier()); + + try + { + attr.ReadAttributes(localDirPath, + true /* directories have zero mod times */, + 0 /* not interested in mod time */, + &attrModTime, 0 /* not file size */, + &inodeNum); + } + catch (BoxException &e) + { + // We used to try to recover from this, but we'd need an + // attributes block to upload to the server, so we have to + // skip creating the directory instead. + BOX_WARNING("Failed to read attributes of directory, " + "ignoring it for now: " << nonVssDirPath); + return 0; // no object ID + } + + // Check to see if the directory been renamed + // First, do we have a record in the ID map? + int64_t renameObjectID = 0, renameInDirectory = 0; + bool renameDir = false; + const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap()); + + if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) + { + // Look up on the server to get the name, to build the local filename + std::string localPotentialOldName; + bool isDir = false; + bool isCurrentVersion = false; + if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, + localPotentialOldName, isDir, isCurrentVersion)) + { + // Only interested if it's a directory + if(isDir && isCurrentVersion) + { + // Check that the object doesn't exist already + EMU_STRUCT_STAT st; + if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && + errno == ENOENT) + { + // Doesn't exist locally, but does exist + // on the server. Therefore we can + // safely rename it. + renameDir = true; + } + } + } + } + + // Get connection + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Don't do a check for storage limit exceeded here, because if we get to this + // stage, a connection will have been opened, and the status known, so the check + // in the else if(...) above will be correct. + + // Build attribute stream for sending + std::auto_ptr attrStream(new MemBlockStream(attr)); + + if(renameDir) + { + // Rename the existing directory on the server + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, + storeFilename); + + // Put the latest attributes on it + connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream); + + // Stop it being deleted later + BackupClientDeleteList &rdelList( + rParams.mrContext.GetDeleteList()); + rdelList.StopDirectoryDeletion(renameObjectID); + + // This is the ID for the renamed directory + return renameObjectID; + } + else + { + int64_t subDirObjectID = 0; // no object ID + + // Create a new directory + try + { + subDirObjectID = connection.QueryCreateDirectory( + mObjectID, attrModTime, storeFilename, + attrStream)->GetObjectID(); + // Flag as having done this for optimisation later + *pHaveJustCreatedDirOnServer = true; + } + catch(BoxException &e) + { + int type, subtype; + connection.GetLastError(type, subtype); + rNotifier.NotifyFileUploadServerError(this, nonVssDirPath, + type, subtype); + if(e.GetType() == ConnectionException::ExceptionType && + e.GetSubType() == ConnectionException::Protocol_UnexpectedReply && + type == BackupProtocolError::ErrorType && + subtype == BackupProtocolError::Err_StorageLimitExceeded) + { + // The hard limit was exceeded on the server, notify! + rParams.mrContext.SetStorageLimitExceeded(); + rParams.mrSysadminNotifier.NotifySysadmin( + SysadminNotifier::StoreFull); + } + else + { + throw; + } + } + + if(*pHaveJustCreatedDirOnServer) + { + rNotifier.NotifyDirectoryCreated(subDirObjectID, + nonVssDirPath, remoteDirPath); + } + + return subDirObjectID; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &, BackupStoreDirectory *, int64_t, const std::string &) +// Purpose: Called to resolve difficulties when a directory is found on the +// store where a file is to be uploaded. +// Created: 9/7/04 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile( + SyncParams &rParams, + BackupStoreDirectory* pDirOnStore, + BackupStoreDirectory::Entry* pEntry, + const std::string &rFilename) +{ + // First, delete the directory + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + connection.QueryDeleteDirectory(pEntry->GetObjectID()); + + BackupStoreFilenameClear clear(pEntry->GetName()); + rParams.mrContext.GetProgressNotifier().NotifyDirectoryDeleted( + pEntry->GetObjectID(), clear.GetClearFilename()); + + // Then, delete any directory record + std::map::iterator + e(mSubDirectories.find(rFilename)); + + if(e != mSubDirectories.end()) + { + // A record exists for this, remove it + BackupClientDirectoryRecord *psubDirRecord = e->second; + mSubDirectories.erase(e); + + // And delete the object + delete psubDirRecord; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::UploadFile( +// BackupClientDirectoryRecord::SyncParams &, +// const std::string &, +// const BackupStoreFilename &, +// int64_t, box_time_t, box_time_t, bool) +// Purpose: Private. Upload a file to the server. May send +// a patch instead of the whole thing +// Created: 20/1/04 +// +// -------------------------------------------------------------------------- +int64_t BackupClientDirectoryRecord::UploadFile( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string &rLocalPath, + const std::string &rNonVssFilePath, + const std::string &rRemotePath, + const BackupStoreFilenameClear &rStoreFilename, + int64_t FileSize, + box_time_t ModificationTime, + box_time_t AttributesHash, + bool NoPreviousVersionOnServer) +{ + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + + // Get the connection + BackupProtocolCallable &connection(rContext.GetConnection()); + + // Info + int64_t objID = 0; + int64_t uploadedSize = -1; + + // Use a try block to catch store full errors + try + { + std::auto_ptr apStreamToUpload; + int64_t diffFromID = 0; + + // Might an old version be on the server, and is the file + // size over the diffing threshold? + if(!NoPreviousVersionOnServer && + FileSize >= rParams.mDiffingUploadSizeThreshold) + { + // YES -- try to do diff, if possible + // First, query the server to see if there's an old version available + std::auto_ptr getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename)); + diffFromID = getBlockIndex->GetObjectID(); + + if(diffFromID != 0) + { + // Found an old version + + // Get the index + std::auto_ptr blockIndexStream(connection.ReceiveStream()); + + // + // Diff the file + // + + rContext.ManageDiffProcess(); + + bool isCompletelyDifferent = false; + + apStreamToUpload = BackupStoreFile::EncodeFileDiff( + rLocalPath, + mObjectID, /* containing directory */ + rStoreFilename, diffFromID, *blockIndexStream, + connection.GetTimeout(), + &rContext, // DiffTimer implementation + 0 /* not interested in the modification time */, + &isCompletelyDifferent, + rParams.mpBackgroundTask); + + if(isCompletelyDifferent) + { + diffFromID = 0; + } + + rContext.UnManageDiffProcess(); + } + } + + if(apStreamToUpload.get()) + { + rNotifier.NotifyFileUploadingPatch(this, rNonVssFilePath, + apStreamToUpload->GetBytesToUpload()); + } + else // No patch upload, so do a normal upload + { + // below threshold or nothing to diff from, so upload whole + rNotifier.NotifyFileUploading(this, rNonVssFilePath); + + // Prepare to upload, getting a stream which will encode the file as we go along + apStreamToUpload = BackupStoreFile::EncodeFile( + rLocalPath, mObjectID, /* containing directory */ + rStoreFilename, NULL, &rParams, + &(rParams.mrRunStatusProvider), + rParams.mpBackgroundTask); + } + + rContext.SetNiceMode(true); + std::auto_ptr apWrappedStream; + + if(rParams.mMaxUploadRate > 0) + { + apWrappedStream.reset(new RateLimitingStream( + *apStreamToUpload, rParams.mMaxUploadRate)); + } + else + { + // Wrap the stream in *something*, so that + // QueryStoreFile() doesn't delete the original + // stream (upload object) and we can retrieve + // the byte counter. + apWrappedStream.reset(new BufferedStream( + *apStreamToUpload)); + } + + // Send to store + std::auto_ptr stored( + connection.QueryStoreFile(mObjectID, ModificationTime, + AttributesHash, diffFromID, rStoreFilename, + apWrappedStream)); + + rContext.SetNiceMode(false); + + // Get object ID from the result + objID = stored->GetObjectID(); + uploadedSize = apStreamToUpload->GetTotalBytesSent(); + } + catch(BoxException &e) + { + rContext.UnManageDiffProcess(); + + if(e.GetType() == ConnectionException::ExceptionType && + e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) + { + // Check and see what error the protocol has, + // this is more useful to users than the exception. + int type, subtype; + if(connection.GetLastError(type, subtype)) + { + if(type == BackupProtocolError::ErrorType + && subtype == BackupProtocolError::Err_StorageLimitExceeded) + { + // The hard limit was exceeded on the server, notify! + rParams.mrSysadminNotifier.NotifySysadmin( + SysadminNotifier::StoreFull); + // return an error code instead of + // throwing an exception that we + // can't debug. + return 0; + } + rNotifier.NotifyFileUploadServerError(this, + rNonVssFilePath, type, subtype); + } + } + + // Send the error on it's way + throw; + } + + rNotifier.NotifyFileUploaded(this, rNonVssFilePath, FileSize, + uploadedSize, objID); + + // Return the new object ID of this file + return objID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( +// SyncParams &, const char *) +// Purpose: Sets the error state when there were problems +// reading an object from the filesystem. +// Created: 29/3/04 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string& rFilename) +{ + // Zero hash, so it gets synced properly next time round. + ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); + + // More detailed logging was already done by the caller, but if we + // have a read error reported, we need to be able to search the logs + // to find out which file it was, so we need to log a consistent and + // clear error message. + BOX_WARNING("Failed to backup file, see above for details: " << + rFilename); + + // Mark that an error occured in the parameters object + rParams.mReadErrorsOnFilesystemObjects = true; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncParams::SyncParams(BackupClientContext &) +// Purpose: Constructor +// Created: 8/3/04 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::SyncParams::SyncParams( + RunStatusProvider &rRunStatusProvider, + SysadminNotifier &rSysadminNotifier, + ProgressNotifier &rProgressNotifier, + BackupClientContext &rContext, + BackgroundTask *pBackgroundTask) +: mSyncPeriodStart(0), + mSyncPeriodEnd(0), + mMaxUploadWait(0), + mMaxFileTimeInFuture(99999999999999999LL), + mFileTrackingSizeThreshold(16*1024), + mDiffingUploadSizeThreshold(16*1024), + mpBackgroundTask(pBackgroundTask), + mrRunStatusProvider(rRunStatusProvider), + mrSysadminNotifier(rSysadminNotifier), + mrProgressNotifier(rProgressNotifier), + mrContext(rContext), + mReadErrorsOnFilesystemObjects(false), + mMaxUploadRate(0), + mUploadAfterThisTimeInTheFuture(99999999999999999LL), + mHaveLoggedWarningAboutFutureFileTimes(false) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::SyncParams::~SyncParams() +// Purpose: Destructor +// Created: 8/3/04 +// +// -------------------------------------------------------------------------- +BackupClientDirectoryRecord::SyncParams::~SyncParams() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::Deserialize(Archive & rArchive) +{ + // Make deletion recursive + DeleteSubDirectories(); + + // Delete maps + if(mpPendingEntries != 0) + { + delete mpPendingEntries; + mpPendingEntries = 0; + } + + // + // + // + rArchive.Read(mObjectID); + rArchive.Read(mSubDirName); + rArchive.Read(mInitialSyncDone); + rArchive.Read(mSyncDone); + + // + // + // + int64_t iCount = 0; + rArchive.Read(iCount); + + if (iCount != sizeof(mStateChecksum)/sizeof(mStateChecksum[0])) + { + // we have some kind of internal system representation change: throw for now + THROW_EXCEPTION(CommonException, Internal) + } + + for (int v = 0; v < iCount; v++) + { + // Load each checksum entry + rArchive.Read(mStateChecksum[v]); + } + + // + // + // + iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + // load each pending entry + mpPendingEntries = new std::map; + if (!mpPendingEntries) + { + throw std::bad_alloc(); + } + + for (int v = 0; v < iCount; v++) + { + std::string strItem; + box_time_t btItem; + + rArchive.Read(strItem); + rArchive.Read(btItem); + (*mpPendingEntries)[strItem] = btItem; + } + } + + // + // + // + iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + for (int v = 0; v < iCount; v++) + { + std::string strItem; + rArchive.Read(strItem); + + BackupClientDirectoryRecord* pSubDirRecord = + new BackupClientDirectoryRecord(0, ""); + // will be deserialized anyway, give it id 0 for now + + if (!pSubDirRecord) + { + throw std::bad_alloc(); + } + + /***** RECURSE *****/ + pSubDirRecord->Deserialize(rArchive); + mSubDirectories[strItem] = pSubDirRecord; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mObjectID); + rArchive.Write(mSubDirName); + rArchive.Write(mInitialSyncDone); + rArchive.Write(mSyncDone); + + // + // + // + int64_t iCount = 0; + + // when reading back the archive, we will + // need to know how many items there are. + iCount = sizeof(mStateChecksum) / sizeof(mStateChecksum[0]); + rArchive.Write(iCount); + + for (int v = 0; v < iCount; v++) + { + rArchive.Write(mStateChecksum[v]); + } + + // + // + // + if (!mpPendingEntries) + { + iCount = 0; + rArchive.Write(iCount); + } + else + { + iCount = mpPendingEntries->size(); + rArchive.Write(iCount); + + for (std::map::const_iterator + i = mpPendingEntries->begin(); + i != mpPendingEntries->end(); i++) + { + rArchive.Write(i->first); + rArchive.Write(i->second); + } + } + // + // + // + iCount = mSubDirectories.size(); + rArchive.Write(iCount); + + for (std::map::const_iterator + i = mSubDirectories.begin(); + i != mSubDirectories.end(); i++) + { + const BackupClientDirectoryRecord* pSubItem = i->second; + ASSERT(pSubItem); + + rArchive.Write(i->first); + pSubItem->Serialize(rArchive); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Location() +// Purpose: Constructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +Location::Location() +: mIDMapIndex(0) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::~Location() +// Purpose: Destructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +Location::~Location() +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, +// using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void Location::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mName); + rArchive.Write(mPath); + rArchive.Write(mIDMapIndex); + + // + // + // + if(!mapDirectoryRecord.get()) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mapDirectoryRecord->Serialize(rArchive); + } + + // + // + // + if(!mapExcludeFiles.get()) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mapExcludeFiles->Serialize(rArchive); + } + + // + // + // + if(!mapExcludeDirs.get()) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mapExcludeDirs->Serialize(rArchive); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void Location::Deserialize(Archive &rArchive) +{ + // + // + // + mapDirectoryRecord.reset(); + mapExcludeFiles.reset(); + mapExcludeDirs.reset(); + + // + // + // + rArchive.Read(mName); + rArchive.Read(mPath); + rArchive.Read(mIDMapIndex); + + // + // + // + int64_t aMagicMarker = 0; + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, ""); + if(!pSubRecord) + { + throw std::bad_alloc(); + } + + mapDirectoryRecord.reset(pSubRecord); + mapDirectoryRecord->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } + + // + // + // + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mapExcludeFiles.reset(new ExcludeList); + if(!mapExcludeFiles.get()) + { + throw std::bad_alloc(); + } + + mapExcludeFiles->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } + + // + // + // + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mapExcludeDirs.reset(new ExcludeList); + if(!mapExcludeDirs.get()) + { + throw std::bad_alloc(); + } + + mapExcludeDirs->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } +} diff --git a/lib/bbackupd/BackupClientDirectoryRecord.h b/lib/bbackupd/BackupClientDirectoryRecord.h new file mode 100644 index 00000000..865fc747 --- /dev/null +++ b/lib/bbackupd/BackupClientDirectoryRecord.h @@ -0,0 +1,244 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientDirectoryRecord.h +// Purpose: Implementation of record about directory for backup client +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTDIRECTORYRECORD__H +#define BACKUPCLIENTDIRECTORYRECORD__H + +#include +#include +#include + +#include "BackgroundTask.h" +#include "BackupClientFileAttributes.h" +#include "BackupDaemonInterface.h" +#include "BackupStoreDirectory.h" +#include "BoxTime.h" +#include "MD5Digest.h" +#include "ReadLoggingStream.h" +#include "RunStatusProvider.h" + +#ifdef ENABLE_VSS +# include +# include +# include +# include +#endif + +class Archive; +class BackupClientContext; +class BackupDaemon; +class ExcludeList; +class Location; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientDirectoryRecord +// Purpose: Implementation of record about directory for backup client +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class BackupClientDirectoryRecord +{ +public: + BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName); + virtual ~BackupClientDirectoryRecord(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; +private: + BackupClientDirectoryRecord(const BackupClientDirectoryRecord &); +public: + + enum + { + UnknownDirectoryID = 0 + }; + + // -------------------------------------------------------------------------- + // + // Class + // Name: BackupClientDirectoryRecord::SyncParams + // Purpose: Holds parameters etc for directory syncing. Not passed as + // const, some parameters may be modified during sync. + // Created: 8/3/04 + // + // -------------------------------------------------------------------------- + class SyncParams : public ReadLoggingStream::Logger + { + public: + SyncParams( + RunStatusProvider &rRunStatusProvider, + SysadminNotifier &rSysadminNotifier, + ProgressNotifier &rProgressNotifier, + BackupClientContext &rContext, + BackgroundTask *pBackgroundTask); + ~SyncParams(); + private: + // No copying + SyncParams(const SyncParams&); + SyncParams &operator=(const SyncParams&); + + public: + // Data members are public, as accessors are not justified here + box_time_t mSyncPeriodStart; + box_time_t mSyncPeriodEnd; + box_time_t mMaxUploadWait; + box_time_t mMaxFileTimeInFuture; + int32_t mFileTrackingSizeThreshold; + int32_t mDiffingUploadSizeThreshold; + BackgroundTask *mpBackgroundTask; + RunStatusProvider &mrRunStatusProvider; + SysadminNotifier &mrSysadminNotifier; + ProgressNotifier &mrProgressNotifier; + BackupClientContext &mrContext; + bool mReadErrorsOnFilesystemObjects; + int64_t mMaxUploadRate; + + // Member variables modified by syncing process + box_time_t mUploadAfterThisTimeInTheFuture; + bool mHaveLoggedWarningAboutFutureFileTimes; + + bool StopRun() { return mrRunStatusProvider.StopRun(); } + void NotifySysadmin(SysadminNotifier::EventCode Event) + { + mrSysadminNotifier.NotifySysadmin(Event); + } + ProgressNotifier& GetProgressNotifier() const + { + return mrProgressNotifier; + } + + /* ReadLoggingStream::Logger implementation */ + virtual void Log(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset, + length, elapsed, finish); + } + virtual void Log(int64_t readSize, int64_t offset, + int64_t length) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset, + length); + } + virtual void Log(int64_t readSize, int64_t offset) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset); + } + }; + + void SyncDirectory(SyncParams &rParams, + int64_t ContainingDirectoryID, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + bool ThisDirHasJustBeenCreated = false); + + bool SyncDirectoryEntry(SyncParams &rParams, + ProgressNotifier& rNotifier, + const Location& rBackupLocation, + const std::string &rDirLocalPath, + MD5Digest& currentStateChecksum, + struct dirent *en, + EMU_STRUCT_STAT dir_st, + std::vector& rDirs, + std::vector& rFiles, + bool& rDownloadDirectoryRecordBecauseOfFutureFiles); + + std::string ConvertVssPathToRealPath(const std::string &rVssPath, + const Location& rBackupLocation); + + int64_t GetObjectID() const { return mObjectID; } + +private: + void DeleteSubDirectories(); + std::auto_ptr FetchDirectoryListing(SyncParams &rParams); + void UpdateAttributes(SyncParams &rParams, + BackupStoreDirectory *pDirOnStore, + const std::string &rLocalPath); +protected: // to allow tests to hook in before UpdateItems() runs + virtual bool UpdateItems(SyncParams &rParams, + const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + BackupStoreDirectory *pDirOnStore, + std::vector &rEntriesLeftOver, + std::vector &rFiles, + const std::vector &rDirs); +private: + int64_t CreateRemoteDir(const std::string& localDirPath, + const std::string& nonVssDirPath, + const std::string& remoteDirPath, + BackupStoreFilenameClear& storeFilename, + bool* pHaveJustCreatedDirOnServer, + BackupClientDirectoryRecord::SyncParams &rParams); + int64_t UploadFile(SyncParams &rParams, + const std::string &rFilename, + const std::string &rNonVssFilePath, + const std::string &rRemotePath, + const BackupStoreFilenameClear &rStoreFilename, + int64_t FileSize, box_time_t ModificationTime, + box_time_t AttributesHash, bool NoPreviousVersionOnServer); + void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, + const std::string& rFilename); + void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, + BackupStoreDirectory* pDirOnStore, + BackupStoreDirectory::Entry* pEntry, + const std::string &rFilename); + std::string DecryptFilename(BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath); + std::string DecryptFilename(BackupStoreFilenameClear fn, + int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath); + + int64_t mObjectID; + std::string mSubDirName; + bool mInitialSyncDone; + bool mSyncDone; + + // Checksum of directory contents and attributes, used to detect changes + uint8_t mStateChecksum[MD5Digest::DigestLength]; + + std::map *mpPendingEntries; + std::map mSubDirectories; + // mpPendingEntries is a pointer rather than simple a member + // variable, because most of the time it'll be empty. This would + // waste a lot of memory because of STL allocation policies. +}; + +class Location +{ +public: + Location(); + ~Location(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; +private: + Location(const Location &); // copy not allowed + Location &operator=(const Location &); +public: + std::string mName; + std::string mPath; + std::auto_ptr mapDirectoryRecord; + std::auto_ptr mapExcludeFiles; + std::auto_ptr mapExcludeDirs; + int mIDMapIndex; + +#ifdef ENABLE_VSS + bool mIsSnapshotCreated; + VSS_ID mSnapshotVolumeId; + std::string mSnapshotPath; +#endif +}; + +#endif // BACKUPCLIENTDIRECTORYRECORD__H + + diff --git a/lib/bbackupd/BackupClientInodeToIDMap.cpp b/lib/bbackupd/BackupClientInodeToIDMap.cpp new file mode 100644 index 00000000..6eaf7394 --- /dev/null +++ b/lib/bbackupd/BackupClientInodeToIDMap.cpp @@ -0,0 +1,320 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientInodeToIDMap.cpp +// Purpose: Map of inode numbers to file IDs on the store +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include + +#define BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION +#include "BackupClientInodeToIDMap.h" +#undef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION + +#include "Archive.h" +#include "BackupStoreException.h" +#include "CollectInBufferStream.h" +#include "MemBlockStream.h" +#include "autogen_CommonException.h" + +#include "MemLeakFindOn.h" + +#define BOX_DBM_INODE_DB_VERSION_KEY "BackupClientInodeToIDMap.Version" +#define BOX_DBM_INODE_DB_VERSION_CURRENT 2 + +#define BOX_DBM_MESSAGE(stuff) stuff << " (qdbm): " << dperrmsg(dpecode) + +#define BOX_LOG_DBM_ERROR(stuff) \ + BOX_ERROR(BOX_DBM_MESSAGE(stuff)) + +#define THROW_DBM_ERROR(message, filename, exception, subtype) \ + BOX_LOG_DBM_ERROR(message << ": " << filename); \ + THROW_EXCEPTION_MESSAGE(exception, subtype, \ + BOX_DBM_MESSAGE(message << ": " << filename)); + +#define ASSERT_DBM_OK(operation, message, filename, exception, subtype) \ + if(!(operation)) \ + { \ + THROW_DBM_ERROR(message, filename, exception, subtype); \ + } + +#define ASSERT_DBM_OPEN() \ + if(mpDepot == 0) \ + { \ + THROW_EXCEPTION_MESSAGE(BackupStoreException, InodeMapNotOpen, \ + "Inode database not open"); \ + } + +#define ASSERT_DBM_CLOSED() \ + if(mpDepot != 0) \ + { \ + THROW_EXCEPTION_MESSAGE(CommonException, Internal, \ + "Inode database already open: " << mFilename); \ + } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::BackupClientInodeToIDMap() +// Purpose: Constructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +BackupClientInodeToIDMap::BackupClientInodeToIDMap() + : mReadOnly(true), + mEmpty(false), + mpDepot(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::~BackupClientInodeToIDMap() +// Purpose: Destructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +BackupClientInodeToIDMap::~BackupClientInodeToIDMap() +{ + if(mpDepot != 0) + { + Close(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::Open(const char *, bool, bool) +// Purpose: Open the database map, creating a file on disc to store everything +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::Open(const char *Filename, bool ReadOnly, + bool CreateNew) +{ + mFilename = Filename; + + // Correct arguments? + ASSERT(!(CreateNew && ReadOnly)); + + // Correct usage? + ASSERT_DBM_CLOSED(); + ASSERT(!mEmpty); + + // Open the database file + int mode = ReadOnly ? DP_OREADER : DP_OWRITER; + if(CreateNew) + { + mode |= DP_OCREAT; + } + + mpDepot = dpopen(Filename, mode, 0); + + if(!mpDepot) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + BOX_DBM_MESSAGE("Failed to open inode database: " << + mFilename)); + } + + const char* version_key = BOX_DBM_INODE_DB_VERSION_KEY; + int32_t version = 0; + + if(CreateNew) + { + version = BOX_DBM_INODE_DB_VERSION_CURRENT; + + int ret = dpput(mpDepot, version_key, strlen(version_key), + (char *)(&version), sizeof(version), DP_DKEEP); + + if(!ret) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + BOX_DBM_MESSAGE("Failed to write version number to inode " + "database: " << mFilename)); + } + } + else + { + int ret = dpgetwb(mpDepot, version_key, strlen(version_key), 0, + sizeof(version), (char *)(&version)); + + if(ret == -1) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Missing version number in inode database. Perhaps it " + "needs to be recreated: " << mFilename); + } + + if(ret != sizeof(version)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Wrong size version number in inode database: expected " + << sizeof(version) << " bytes but found " << ret); + } + + if(version != BOX_DBM_INODE_DB_VERSION_CURRENT) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Wrong version number in inode database: expected " << + BOX_DBM_INODE_DB_VERSION_CURRENT << " but found " << + version << ". Perhaps it needs to be recreated: " << + mFilename); + } + + // By this point the version number has been checked and is OK. + } + + // Read only flag + mReadOnly = ReadOnly; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::OpenEmpty() +// Purpose: 'Open' this map. Not associated with a disc file. +// Useful for when a map is required, but is against +// an empty file on disc which shouldn't be created. +// Implies read only. +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::OpenEmpty() +{ + ASSERT_DBM_CLOSED(); + ASSERT(mpDepot == 0); + mEmpty = true; + mReadOnly = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::Close() +// Purpose: Close the database file +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::Close() +{ + ASSERT_DBM_OPEN(); + ASSERT_DBM_OK(dpclose(mpDepot), "Failed to close inode database", + mFilename, BackupStoreException, BerkelyDBFailure); + mpDepot = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::AddToMap(InodeRefType, +// int64_t, int64_t) +// Purpose: Adds an entry to the map. Overwrites any existing +// entry. +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, + int64_t InDirectory, const std::string& LocalPath) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, InodeMapIsReadOnly); + } + + if(mpDepot == 0) + { + THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); + } + + ASSERT_DBM_OPEN(); + + // Setup structures + CollectInBufferStream buf; + Archive arc(buf, IOStream::TimeOutInfinite); + arc.WriteExact((uint64_t)ObjectID); + arc.WriteExact((uint64_t)InDirectory); + arc.Write(LocalPath); + buf.SetForReading(); + + ASSERT_DBM_OK(dpput(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), + (const char *)buf.GetBuffer(), buf.GetSize(), DP_DOVER), + "Failed to add record to inode database", mFilename, + BackupStoreException, BerkelyDBFailure); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientInodeToIDMap::Lookup(InodeRefType, +// int64_t &, int64_t &) const +// Purpose: Looks up an inode in the map, returning true if it +// exists, and the object ids of it and the directory +// it's in the reference arguments. +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, + int64_t &rInDirectoryOut, std::string* pLocalPathOut) const +{ + if(mEmpty) + { + // Map is empty + return false; + } + + if(mpDepot == 0) + { + THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); + } + + ASSERT_DBM_OPEN(); + int size; + char* data = dpget(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), + 0, -1, &size); + if(data == NULL) + { + // key not in file + return false; + } + + // Free data automatically when the guard goes out of scope. + MemoryBlockGuard guard(data); + MemBlockStream stream(data, size); + Archive arc(stream, IOStream::TimeOutInfinite); + + // Return data + try + { + arc.Read(rObjectIDOut); + arc.Read(rInDirectoryOut); + if(pLocalPathOut) + { + arc.Read(*pLocalPathOut); + } + } + catch(CommonException &e) + { + if(e.GetSubType() == CommonException::ArchiveBlockIncompleteRead) + { + THROW_FILE_ERROR("Failed to lookup record in inode database: " + << InodeRef << ": not enough data in record", mFilename, + BackupStoreException, BerkelyDBFailure); + // Need to throw precisely that exception to ensure that the + // invalid database is deleted, so that we don't hit the same + // error next time. + } + + throw; + } + + // Found + return true; +} diff --git a/lib/bbackupd/BackupClientInodeToIDMap.h b/lib/bbackupd/BackupClientInodeToIDMap.h new file mode 100644 index 00000000..4bb1e085 --- /dev/null +++ b/lib/bbackupd/BackupClientInodeToIDMap.h @@ -0,0 +1,59 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupClientInodeToIDMap.h +// Purpose: Map of inode numbers to file IDs on the store +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCLIENTINODETOIDMAP_H +#define BACKUPCLIENTINODETOIDMAP_H + +#include + +#include +#include + +// avoid having to include the DB files when not necessary +#ifndef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION + class DEPOT; +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupClientInodeToIDMap +// Purpose: Map of inode numbers to file IDs on the store +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +class BackupClientInodeToIDMap +{ +public: + BackupClientInodeToIDMap(); + ~BackupClientInodeToIDMap(); +private: + BackupClientInodeToIDMap(const BackupClientInodeToIDMap &rToCopy); // not allowed +public: + + void Open(const char *Filename, bool ReadOnly, bool CreateNew); + void OpenEmpty(); + + void AddToMap(InodeRefType InodeRef, int64_t ObjectID, + int64_t InDirectory, const std::string& LocalPath); + bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, + int64_t &rInDirectoryOut, std::string* pLocalPathOut = NULL) const; + + void Close(); + +private: + bool mReadOnly; + bool mEmpty; + std::string mFilename; + DEPOT *mpDepot; +}; + +#endif // BACKUPCLIENTINODETOIDMAP_H + + diff --git a/lib/bbackupd/BackupDaemon.cpp b/lib/bbackupd/BackupDaemon.cpp new file mode 100644 index 00000000..03ce7454 --- /dev/null +++ b/lib/bbackupd/BackupDaemon.cpp @@ -0,0 +1,3654 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemon.cpp +// Purpose: Backup daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include +#include + +#ifdef HAVE_UNISTD_H + #include +#endif +#ifdef HAVE_SIGNAL_H + #include +#endif +#ifdef HAVE_SYS_PARAM_H + #include +#endif +#ifdef HAVE_SYS_WAIT_H + #include +#endif +#ifdef HAVE_SYS_MOUNT_H + #include +#endif +#ifdef HAVE_MNTENT_H + #include +#endif +#ifdef HAVE_SYS_MNTTAB_H + #include + #include +#endif +#ifdef HAVE_PROCESS_H + #include +#endif + +#include +#include +#include + +#include "Configuration.h" +#include "IOStream.h" +#include "MemBlockStream.h" +#include "CommonException.h" +#include "BoxPortsAndFiles.h" + +#include "SSLLib.h" + +#include "autogen_BackupProtocol.h" +#include "autogen_ClientException.h" +#include "autogen_CommonException.h" +#include "autogen_ConversionException.h" +#include "Archive.h" +#include "BackupClientContext.h" +#include "BackupClientCryptoKeys.h" +#include "BackupClientDirectoryRecord.h" +#include "BackupClientFileAttributes.h" +#include "BackupClientInodeToIDMap.h" +#include "BackupClientMakeExcludeList.h" +#include "BackupConstants.h" +#include "BackupDaemon.h" +#include "BackupDaemonConfigVerify.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFilenameClear.h" +#include "BannerText.h" +#include "Conversion.h" +#include "ExcludeList.h" +#include "FileStream.h" +#include "IOStreamGetLine.h" +#include "LocalProcessStream.h" +#include "Logging.h" +#include "Random.h" +#include "Timer.h" +#include "Utils.h" + +#ifdef WIN32 + #include "Win32ServiceFunctions.h" + #include "Win32BackupService.h" + + extern Win32BackupService* gpDaemonService; + +# ifdef ENABLE_VSS +# include +# include +# include +# include + + // http://www.flounder.com/cstring.htm + std::string GetMsgForHresult(HRESULT hr) + { + std::ostringstream buf; + + if(hr == VSS_S_ASYNC_CANCELLED) + { + buf << "VSS async operation cancelled"; + } + else if(hr == VSS_S_ASYNC_FINISHED) + { + buf << "VSS async operation finished"; + } + else if(hr == VSS_S_ASYNC_PENDING) + { + buf << "VSS async operation pending"; + } + else + { + buf << _com_error(hr).ErrorMessage(); + } + + buf << " (" << BOX_FORMAT_HEX32(hr) << ")"; + return buf.str(); + } + + std::string WideStringToString(WCHAR *buf) + { + if (buf == NULL) + { + return "(null)"; + } + + char* pStr = ConvertFromWideString(buf, CP_UTF8); + + if(pStr == NULL) + { + return "(conversion failed)"; + } + + std::string result(pStr); + free(pStr); + return result; + } + + std::string GuidToString(GUID guid) + { + wchar_t buf[64]; + StringFromGUID2(guid, buf, sizeof(buf)); + return WideStringToString(buf); + } + + std::string BstrToString(const BSTR arg) + { + if(arg == NULL) + { + return std::string("(null)"); + } + else + { + // Extract the *long* before where the arg points to + long len = ((long *)arg)[-1] / 2; + std::wstring wstr((WCHAR *)arg, len); + std::string str; + if(!ConvertFromWideString(wstr, &str, CP_UTF8)) + { + throw std::exception("string conversion failed"); + } + return str; + } + } +# endif + + // Mutex support by Achim: see https://www.boxbackup.org/ticket/67 + + // Creates the two mutexes checked for by the installer/uninstaller to + // see if the program is still running. One of the mutexes is created + // in the global name space (which makes it possible to access the + // mutex across user sessions in Windows XP); the other is created in + // the session name space (because versions of Windows NT prior to + // 4.0 TSE don't have a global name space and don't support the + // 'Global\' prefix). + + void CreateMutexes(const std::string& rName) + { + SECURITY_DESCRIPTOR SecurityDesc; + SECURITY_ATTRIBUTES SecurityAttr; + + /* By default on Windows NT, created mutexes are accessible only by the user + running the process. We need our mutexes to be accessible to all users, so + that the mutex detection can work across user sessions in Windows XP. To + do this we use a security descriptor with a null DACL. + */ + + InitializeSecurityDescriptor(&SecurityDesc, SECURITY_DESCRIPTOR_REVISION); + SetSecurityDescriptorDacl(&SecurityDesc, TRUE, NULL, FALSE); + SecurityAttr.nLength = sizeof(SecurityAttr); + SecurityAttr.lpSecurityDescriptor = &SecurityDesc; + SecurityAttr.bInheritHandle = FALSE; + // We don't care if this succeeds or fails. It's only used to + // ensure that an installer can detect if Box Backup is running. + CreateMutexA(&SecurityAttr, FALSE, rName.c_str()); + std::string global_name = "Global\\" + rName; + CreateMutexA(&SecurityAttr, FALSE, global_name.c_str()); + } +#endif + +#include "MemLeakFindOn.h" + +static const time_t MAX_SLEEP_TIME = 1024; + +// Make the actual sync period have a little bit of extra time, up to a 64th of the main sync period. +// This prevents repetative cycles of load on the server +#define SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY 6 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::BackupDaemon() +// Purpose: constructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupDaemon::BackupDaemon() + : mState(BackupDaemon::State_Initialising), + mDeleteRedundantLocationsAfter(0), + mLastNotifiedEvent(SysadminNotifier::MAX), + mDeleteUnusedRootDirEntriesAfter(0), + mClientStoreMarker(BackupClientContext::ClientStoreMarker_NotKnown), + mStorageLimitExceeded(false), + mReadErrorsOnFilesystemObjects(false), + mLastSyncTime(0), + mNextSyncTime(0), + mCurrentSyncStartTime(0), + mUpdateStoreInterval(0), + mDeleteStoreObjectInfoFile(false), + mDoSyncForcedByPreviousSyncError(false), + mNumFilesUploaded(-1), + mNumDirsCreated(-1), + mMaxBandwidthFromSyncAllowScript(0), + mLogAllFileAccess(false), + mpProgressNotifier(this), + mpLocationResolver(this), + mpRunStatusProvider(this), + mpSysadminNotifier(this), + mapCommandSocketPollTimer(NULL) + #ifdef WIN32 + , mInstallService(false), + mRemoveService(false), + mRunAsService(false), + mServiceName("bbackupd") + #endif +#ifdef ENABLE_VSS + , mpVssBackupComponents(NULL) +#endif +{ + // Only ever one instance of a daemon + SSLLib::Initialise(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::~BackupDaemon() +// Purpose: Destructor +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +BackupDaemon::~BackupDaemon() +{ + DeleteAllLocations(); + DeleteAllIDMaps(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DaemonName() +// Purpose: Get name of daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +const char *BackupDaemon::DaemonName() const +{ + return "bbackupd"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DaemonBanner() +// Purpose: Daemon banner +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +std::string BackupDaemon::DaemonBanner() const +{ + return BANNER_TEXT("Backup Client"); +} + +void BackupDaemon::Usage() +{ + this->Daemon::Usage(); + +#ifdef WIN32 + std::cout << + " -s Run as a Windows Service, for internal use only\n" + " -i Install Windows Service (you may want to specify a config file)\n" + " -r Remove Windows Service\n" + " -S Service name for -i and -r options\n"; +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::GetConfigVerify() +// Purpose: Get configuration specification +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify *BackupDaemon::GetConfigVerify() const +{ + // Defined elsewhere + return &BackupDaemonConfigVerify; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetupInInitialProcess() +// Purpose: Platforms with non-checkable credentials on +// local sockets only. +// Prints a warning if the command socket is used. +// Created: 25/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetupInInitialProcess() +{ + const Configuration& config(GetConfiguration()); + + // These keys may or may not be required, depending on the configured + // store type (e.g. not when using Amazon S3 stores), so they can't be + // verified by BackupDaemonConfigVerify. + std::vector requiredKeys; + requiredKeys.push_back("StoreHostname"); + requiredKeys.push_back("AccountNumber"); + requiredKeys.push_back("CertificateFile"); + requiredKeys.push_back("PrivateKeyFile"); + requiredKeys.push_back("TrustedCAsFile"); + bool missingRequiredKeys = false; + + for(std::vector::const_iterator i = requiredKeys.begin(); + i != requiredKeys.end(); i++) + { + if(!config.KeyExists(*i)) + { + BOX_ERROR("Missing required configuration key: " << *i); + missingRequiredKeys = true; + } + } + + if(missingRequiredKeys) + { + THROW_EXCEPTION_MESSAGE(CommonException, InvalidConfiguration, + "Some required configuration keys are missing in " << + GetConfigFileName()); + } + +#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET + // Print a warning on this platform if the CommandSocket is used. + if(GetConfiguration().KeyExists("CommandSocket")) + { + BOX_WARNING( + "==============================================================================\n" + "SECURITY WARNING: This platform cannot check the credentials of connections to\n" + "the command socket. This is a potential DoS security problem.\n" + "Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl\n" + "is not used.\n" + "==============================================================================\n" + ); + } +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteAllLocations() +// Purpose: Deletes all records stored +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteAllLocations() +{ + // Run through, and delete everything + for(Locations::iterator i = mLocations.begin(); + i != mLocations.end(); ++i) + { + delete *i; + } + + // Clear the contents of the map, so it is empty + mLocations.clear(); + + // And delete everything from the associated mount vector + mIDMapMounts.clear(); +} + +#ifdef WIN32 +std::string BackupDaemon::GetOptionString() +{ + std::string oldOpts = this->Daemon::GetOptionString(); + ASSERT(oldOpts.find("s") == std::string::npos); + ASSERT(oldOpts.find("S") == std::string::npos); + ASSERT(oldOpts.find("i") == std::string::npos); + ASSERT(oldOpts.find("r") == std::string::npos); + return oldOpts + "sS:ir"; +} + +int BackupDaemon::ProcessOption(signed int option) +{ + switch(option) + { + case 's': + { + mRunAsService = true; + return 0; + } + + case 'S': + { + mServiceName = optarg; + Logging::SetProgramName(mServiceName); + return 0; + } + + case 'i': + { + mInstallService = true; + return 0; + } + + case 'r': + { + mRemoveService = true; + return 0; + } + + default: + { + return this->Daemon::ProcessOption(option); + } + } +} + +int BackupDaemon::Main(const std::string &rConfigFileName) +{ + if (mInstallService) + { + return InstallService(rConfigFileName.c_str(), mServiceName); + } + + if (mRemoveService) + { + return RemoveService(mServiceName); + } + +#ifdef ENABLE_VSS + HRESULT result = CoInitialize(NULL); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to initialize COM: " << + GetMsgForHresult(result)); + return 1; + } +#endif + + CreateMutexes("__boxbackup_mutex__"); + + int returnCode; + + if (mRunAsService) + { + // We will be called reentrantly by the Service Control + // Manager, and we had better not call OurService again! + mRunAsService = false; + + BOX_INFO("Box Backup service starting"); + returnCode = OurService(rConfigFileName.c_str()); + BOX_INFO("Box Backup service shut down"); + } + else + { + returnCode = this->Daemon::Main(rConfigFileName); + } + + return returnCode; +} +#endif // WIN32 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Run() +// Purpose: Run function for daemon +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Run() +{ + // initialise global timer mechanism + Timers::Init(); + + mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, + "CommandSocketPollTimer")); + + #ifndef WIN32 + // Ignore SIGPIPE so that if a command connection is broken, + // the daemon doesn't terminate. + ::signal(SIGPIPE, SIG_IGN); + #endif + + // Create a command socket? + const Configuration &conf(GetConfiguration()); + if(conf.KeyExists("CommandSocket")) + { + // Yes, create a local UNIX socket + mapCommandSocketInfo.reset(new CommandSocketInfo); + const char *socketName = + conf.GetKeyValue("CommandSocket").c_str(); + #ifdef WIN32 + mapCommandSocketInfo->mListeningSocket.Listen( + socketName); + #else + ::unlink(socketName); + mapCommandSocketInfo->mListeningSocket.Listen( + Socket::TypeUNIX, socketName); + #endif + } + + // Handle things nicely on exceptions + try + { + Run2(); + } + catch(...) + { + try + { + mapCommandSocketInfo.reset(); + } + catch(std::exception &e) + { + BOX_WARNING("Internal error while closing command " + "socket after another exception, ignored: " << + e.what()); + } + catch(...) + { + BOX_WARNING("Error closing command socket after " + "exception, ignored."); + } + + mapCommandSocketPollTimer.reset(); + Timers::Cleanup(); + + throw; + } + + // Clean up + mapCommandSocketInfo.reset(); + mapCommandSocketPollTimer.reset(); + Timers::Cleanup(); +} + +void BackupDaemon::InitCrypto() +{ + // Read in the certificates creating a TLS context + const Configuration &conf(GetConfiguration()); + std::string certFile(conf.GetKeyValue("CertificateFile")); + std::string keyFile(conf.GetKeyValue("PrivateKeyFile")); + std::string caFile(conf.GetKeyValue("TrustedCAsFile")); + mTlsContext.Initialise(false /* as client */, certFile.c_str(), + keyFile.c_str(), caFile.c_str()); + + // Set up the keys for various things + BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile")); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Run2() +// Purpose: Run function for daemon (second stage) +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Run2() +{ + InitCrypto(); + + const Configuration &conf(GetConfiguration()); + + // How often to connect to the store (approximate) + mUpdateStoreInterval = SecondsToBoxTime( + conf.GetKeyValueInt("UpdateStoreInterval")); + mBackupErrorDelay = conf.GetKeyValueInt("BackupErrorDelay"); + + // But are we connecting automatically? + bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup"); + + // When the next sync should take place -- which is ASAP + mNextSyncTime = 0; + + // When the last sync started (only updated if the store was not full when the sync ended) + mLastSyncTime = 0; + + // -------------------------------------------------------------------------------------------- + + mDeleteStoreObjectInfoFile = DeserializeStoreObjectInfo(mLastSyncTime, + mNextSyncTime); + + // -------------------------------------------------------------------------------------------- + + + // Set state + SetState(State_Idle); + + mDoSyncForcedByPreviousSyncError = false; + + // Loop around doing backups + do + { + // Flags used below + bool storageLimitExceeded = false; + bool doSync = false; + bool mDoSyncForcedByCommand = false; + + // Check whether we should be stopping, and if so, + // don't hang around waiting on the command socket. + if(StopRun()) + { + BOX_INFO("Skipping command socket polling " + "due to shutdown request"); + break; + } + + // Is a delay necessary? + box_time_t currentTime = GetCurrentBoxTime(); + box_time_t requiredDelay = (mNextSyncTime < currentTime) + ? (0) : (mNextSyncTime - currentTime); + mNextSyncTime = currentTime + requiredDelay; + + if (mDoSyncForcedByPreviousSyncError) + { + BOX_INFO("Last backup was not successful, " + "next one starting at " << + FormatTime(mNextSyncTime, false, true)); + } + else if (automaticBackup) + { + BOX_INFO("Automatic backups are enabled, " + "next one starting at " << + FormatTime(mNextSyncTime, false, true)); + } + else + { + BOX_INFO("No automatic backups, waiting for " + "bbackupctl snapshot command"); + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } + + if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME)) + { + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } + + // Only delay if necessary + if(requiredDelay == 0) + { + // No sleep necessary, so don't listen on the command + // socket at all right now. + } + else if(mapCommandSocketInfo.get() != 0) + { + // A command socket exists, so sleep by waiting for a + // connection or command on it. + WaitOnCommandSocket(requiredDelay, doSync, + mDoSyncForcedByCommand); + } + else + { + // No command socket or connection, just do a normal + // sleep. + time_t sleepSeconds = + BoxTimeToSeconds(requiredDelay); + ::sleep((sleepSeconds <= 0) + ? 1 : sleepSeconds); + } + + // We have now slept, so if automaticBackup is enabled then + // it's time for a backup now. + + if(StopRun()) + { + BOX_INFO("Stopping idle loop due to shutdown request"); + break; + } + else if(doSync) + { + BOX_INFO("Starting a backup immediately due to " + "bbackupctl sync command"); + } + else if(GetCurrentBoxTime() < mNextSyncTime) + { + BOX_TRACE("Deadline not reached, sleeping again"); + continue; + } + else if(mDoSyncForcedByPreviousSyncError) + { + BOX_INFO("Last backup was not successful, next one " + "starting now"); + } + else if(!automaticBackup) + { + BOX_TRACE("Sleeping again because automatic backups " + "are not enabled"); + continue; + } + else + { + BOX_INFO("Automatic backups are enabled, next one " + "starting now"); + } + + // If we pass this point, or exit the loop, we should have + // logged something at INFO level or higher to explain why. + + // Use a script to see if sync is allowed now? + if(mDoSyncForcedByCommand) + { + BOX_INFO("Skipping SyncAllowScript due to bbackupctl " + "force-sync command"); + } + else + { + int d = UseScriptToSeeIfSyncAllowed(); + if(d > 0) + { + // Script has asked for a delay + mNextSyncTime = GetCurrentBoxTime() + + SecondsToBoxTime(d); + BOX_INFO("Impending backup stopped by " + "SyncAllowScript, next attempt " + "scheduled for " << + FormatTime(mNextSyncTime, false)); + continue; + } + } + + mCurrentSyncStartTime = GetCurrentBoxTime(); + RunSyncNowWithExceptionHandling(); + + // Set state + SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle); + } + while(!StopRun()); + + // Make sure we have a clean start next time round (if restart) + DeleteAllLocations(); + DeleteAllIDMaps(); +} + +std::auto_ptr BackupDaemon::RunSyncNowWithExceptionHandling() +{ + bool errorOccurred = false; + int errorCode = 0, errorSubCode = 0; + std::string errorString = "unknown"; + + try + { + OnBackupStart(); + // Do sync + RunSyncNow(); + } + catch(BoxException &e) + { + errorOccurred = true; + errorString = e.what(); + errorCode = e.GetType(); + errorSubCode = e.GetSubType(); + } + catch(std::exception &e) + { + BOX_ERROR("Internal error during backup run: " << e.what()); + errorOccurred = true; + errorString = e.what(); + } + catch(...) + { + // TODO: better handling of exceptions here... + // need to be very careful + errorOccurred = true; + } + + // do not retry immediately without a good reason + mDoSyncForcedByPreviousSyncError = false; + + // Is it a berkely db failure? + bool isBerkelyDbFailure = false; + + // Notify system administrator about the final state of the backup + if(errorOccurred) + { + if (errorCode == BackupStoreException::ExceptionType + && errorSubCode == BackupStoreException::BerkelyDBFailure) + { + isBerkelyDbFailure = true; + } + + if(isBerkelyDbFailure) + { + // Delete corrupt files + DeleteCorruptBerkelyDbFiles(); + } + + ResetCachedState(); + + // Handle restart? + if(StopRun()) + { + BOX_NOTICE("Exception (" << errorCode << "/" << + errorSubCode << ") due to signal"); + OnBackupFinish(); + return mapClientContext; // releases mapClientContext + } + + NotifySysadmin(SysadminNotifier::BackupError); + + // If the Berkely db files get corrupted, + // delete them and try again immediately. + if(isBerkelyDbFailure) + { + BOX_ERROR("Berkely db inode map files corrupted, " + "deleting and restarting scan. Renamed files " + "and directories will not be tracked until " + "after this scan."); + ::sleep(1); + } + else + { + // Not restart/terminate, pause and retry + // Notify administrator + SetState(State_Error); + BOX_ERROR("Exception caught (" << errorString << + " " << errorCode << "/" << errorSubCode << + "), reset state and waiting to retry..."); + ::sleep(10); + mNextSyncTime = GetCurrentBoxTime() + + SecondsToBoxTime(mBackupErrorDelay) + + Random::RandomInt(mUpdateStoreInterval >> + SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); + } + } + + if(mReadErrorsOnFilesystemObjects) + { + NotifySysadmin(SysadminNotifier::ReadError); + } + + if(mStorageLimitExceeded) + { + NotifySysadmin(SysadminNotifier::StoreFull); + } + + if (!errorOccurred && !mReadErrorsOnFilesystemObjects && + !mStorageLimitExceeded) + { + NotifySysadmin(SysadminNotifier::BackupOK); + } + + // If we were retrying after an error, and this backup succeeded, + // then now would be a good time to stop :-) + mDoSyncForcedByPreviousSyncError = errorOccurred && !isBerkelyDbFailure; + + OnBackupFinish(); + return mapClientContext; // releases mapClientContext +} + +void BackupDaemon::ResetCachedState() +{ + // Clear state data + // Go back to beginning of time + mLastSyncTime = 0; + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything + DeleteAllLocations(); + DeleteAllIDMaps(); +} + +std::auto_ptr BackupDaemon::GetNewContext +( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode +) +{ + std::auto_ptr context(new BackupClientContext( + rResolver, rTLSContext, rHostname, Port, AccountNumber, + ExtendedLogging, ExtendedLogToFile, ExtendedLogFile, + rProgressNotifier, TcpNiceMode)); + return context; +} + +// Returns the BackupClientContext so that tests can use it to hold the +// connection open and prevent housekeeping from running. Otherwise don't use +// it, let it be destroyed and close the connection. +std::auto_ptr BackupDaemon::RunSyncNow() +{ + Timers::AssertInitialised(); + + // Delete the serialised store object file, + // so that we don't try to reload it after a + // partially completed backup + if(mDeleteStoreObjectInfoFile && !DeleteStoreObjectInfo()) + { + BOX_ERROR("Failed to delete the StoreObjectInfoFile, " + "backup cannot continue safely."); + THROW_EXCEPTION(ClientException, + FailedToDeleteStoreObjectInfoFile); + } + + // In case the backup throws an exception, + // we should not try to delete the store info + // object file again. + mDeleteStoreObjectInfoFile = false; + + const Configuration &conf(GetConfiguration()); + + std::auto_ptr fileLogger; + + if (conf.KeyExists("LogFile")) + { + bool overwrite = false; + if (conf.KeyExists("LogFileOverwrite")) + { + overwrite = conf.GetKeyValueBool("LogFileOverwrite"); + } + + Log::Level level = Log::INFO; + if (conf.KeyExists("LogFileLevel")) + { + level = Logging::GetNamedLevel( + conf.GetKeyValue("LogFileLevel")); + } + + fileLogger.reset(new FileLogger(conf.GetKeyValue("LogFile"), + level, !overwrite)); + } + + std::string extendedLogFile; + if (conf.KeyExists("ExtendedLogFile")) + { + extendedLogFile = conf.GetKeyValue("ExtendedLogFile"); + } + + if (conf.KeyExists("LogAllFileAccess")) + { + mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess"); + } + + // Then create a client context object (don't + // just connect, as this may be unnecessary) + mapClientContext = GetNewContext( + *mpLocationResolver, + mTlsContext, + conf.GetKeyValue("StoreHostname"), + conf.GetKeyValueInt("StorePort"), + conf.GetKeyValueUint32("AccountNumber"), + conf.GetKeyValueBool("ExtendedLogging"), + conf.KeyExists("ExtendedLogFile"), + extendedLogFile, + *mpProgressNotifier, + conf.GetKeyValueBool("TcpNice") + ); + + // The minimum age a file needs to be before it will be + // considered for uploading + box_time_t minimumFileAge = SecondsToBoxTime( + conf.GetKeyValueInt("MinimumFileAge")); + + // The maximum time we'll wait to upload a file, regardless + // of how often it's modified + box_time_t maxUploadWait = SecondsToBoxTime( + conf.GetKeyValueInt("MaxUploadWait")); + // Adjust by subtracting the minimum file age, so is relative + // to sync period end in comparisons + if (maxUploadWait > minimumFileAge) + { + maxUploadWait -= minimumFileAge; + } + else + { + maxUploadWait = 0; + } + + // Calculate the sync period of files to examine + box_time_t syncPeriodStart = mLastSyncTime; + box_time_t syncPeriodEnd = GetCurrentBoxTime() - minimumFileAge; + + if(syncPeriodStart >= syncPeriodEnd && + syncPeriodStart - syncPeriodEnd < minimumFileAge) + { + // This can happen if we receive a force-sync command less + // than minimumFileAge after the last sync. Deal with it by + // moving back syncPeriodStart, which should not do any + // damage. + syncPeriodStart = syncPeriodEnd - SecondsToBoxTime(1); + } + + if(syncPeriodStart >= syncPeriodEnd) + { + BOX_ERROR("Invalid (negative) sync period: perhaps your clock " + "is going backwards? (" << syncPeriodStart << " to " << + syncPeriodEnd << ")"); + THROW_EXCEPTION(ClientException, ClockWentBackwards); + } + + // Check logic + ASSERT(syncPeriodEnd > syncPeriodStart); + // Paranoid check on sync times + if(syncPeriodStart >= syncPeriodEnd) + { + return mapClientContext; // releases mapClientContext + } + + // Adjust syncPeriodEnd to emulate snapshot behaviour properly + box_time_t syncPeriodEndExtended = syncPeriodEnd; + + // Using zero min file age? + if(minimumFileAge == 0) + { + // Add a year on to the end of the end time, + // to make sure we sync files which are + // modified after the scan run started. + // Of course, they may be eligible to be + // synced again the next time round, + // but this should be OK, because the changes + // only upload should upload no data. + syncPeriodEndExtended += SecondsToBoxTime( + (time_t)(356*24*3600)); + } + + // Set up the sync parameters + BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider, + *mpSysadminNotifier, *mpProgressNotifier, *mapClientContext, this); + params.mSyncPeriodStart = syncPeriodStart; + params.mSyncPeriodEnd = syncPeriodEndExtended; + // use potentially extended end time + params.mMaxUploadWait = maxUploadWait; + params.mFileTrackingSizeThreshold = + conf.GetKeyValueInt("FileTrackingSizeThreshold"); + params.mDiffingUploadSizeThreshold = + conf.GetKeyValueInt("DiffingUploadSizeThreshold"); + params.mMaxFileTimeInFuture = + SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture")); + mNumFilesUploaded = 0; + mNumDirsCreated = 0; + + if(conf.KeyExists("MaxUploadRate")) + { + params.mMaxUploadRate = conf.GetKeyValueInt("MaxUploadRate"); + } + + if(mMaxBandwidthFromSyncAllowScript != 0) + { + params.mMaxUploadRate = mMaxBandwidthFromSyncAllowScript; + } + + mDeleteRedundantLocationsAfter = + conf.GetKeyValueInt("DeleteRedundantLocationsAfter"); + mStorageLimitExceeded = false; + mReadErrorsOnFilesystemObjects = false; + + // Setup various timings + int maximumDiffingTime = 600; + int keepAliveTime = 60; + + // max diffing time, keep-alive time + if(conf.KeyExists("MaximumDiffingTime")) + { + maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime"); + } + if(conf.KeyExists("KeepAliveTime")) + { + keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); + } + + mapClientContext->SetMaximumDiffingTime(maximumDiffingTime); + mapClientContext->SetKeepAliveTime(keepAliveTime); + + // Set store marker + mapClientContext->SetClientStoreMarker(mClientStoreMarker); + + // Set up the locations, if necessary -- need to do it here so we have + // a (potential) connection to use. + { + const Configuration &locations( + conf.GetSubConfiguration( + "BackupLocations")); + + // Make sure all the directory records + // are set up + SetupLocations(*mapClientContext, locations); + } + + mpProgressNotifier->NotifyIDMapsSetup(*mapClientContext); + + // Get some ID maps going + SetupIDMapsForSync(); + + // Delete any unused directories? + DeleteUnusedRootDirEntries(*mapClientContext); + +#ifdef ENABLE_VSS + CreateVssBackupComponents(); +#endif + + // Go through the records, syncing them + for(Locations::const_iterator + i(mLocations.begin()); + i != mLocations.end(); ++i) + { + // Set current and new ID map pointers + // in the context + mapClientContext->SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], + mNewIDMaps[(*i)->mIDMapIndex]); + + // Set exclude lists (context doesn't + // take ownership) + mapClientContext->SetExcludeLists( + (*i)->mapExcludeFiles.get(), + (*i)->mapExcludeDirs.get()); + + // Sync the directory + std::string locationPath = (*i)->mPath; +#ifdef ENABLE_VSS + if((*i)->mIsSnapshotCreated) + { + locationPath = (*i)->mSnapshotPath; + } +#endif + + (*i)->mapDirectoryRecord->SyncDirectory(params, + BackupProtocolListDirectory::RootDirectory, + locationPath, std::string("/") + (*i)->mName, **i); + + // Unset exclude lists (just in case) + mapClientContext->SetExcludeLists(0, 0); + } + + // Perform any deletions required -- these are + // delayed until the end to allow renaming to + // happen neatly. + mapClientContext->PerformDeletions(); + +#ifdef ENABLE_VSS + CleanupVssBackupComponents(); +#endif + + // Get the new store marker + mClientStoreMarker = mapClientContext->GetClientStoreMarker(); + mStorageLimitExceeded = mapClientContext->StorageLimitExceeded(); + mReadErrorsOnFilesystemObjects |= + params.mReadErrorsOnFilesystemObjects; + + if(!mStorageLimitExceeded) + { + // The start time of the next run is the end time of this + // run. This is only done if the storage limit wasn't + // exceeded (as things won't have been done properly if + // it was) + mLastSyncTime = syncPeriodEnd; + } + + // Commit the ID Maps + CommitIDMapsAfterSync(); + + // Calculate when the next sync run should be + mNextSyncTime = mCurrentSyncStartTime + + mUpdateStoreInterval + + Random::RandomInt(mUpdateStoreInterval >> + SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); + + // -------------------------------------------------------------------------------------------- + + // We had a successful backup, save the store + // info. If we save successfully, we must + // delete the file next time we start a backup + + mDeleteStoreObjectInfoFile = + SerializeStoreObjectInfo(mLastSyncTime, mNextSyncTime); + + // -------------------------------------------------------------------------------------------- + + return mapClientContext; // releases mapClientContext +} + +#ifdef ENABLE_VSS +bool BackupDaemon::WaitForAsync(IVssAsync *pAsync, + const std::string& description) +{ + BOX_INFO("VSS: waiting for " << description << " to complete"); + HRESULT result; + + do + { + result = pAsync->Wait(1000); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to wait for " << description << + " to complete: " << GetMsgForHresult(result)); + break; + } + + HRESULT result2; + result = pAsync->QueryStatus(&result2, NULL); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query " << description << + " status: " << GetMsgForHresult(result)); + break; + } + + result = result2; + BOX_INFO("VSS: " << description << " status: " << + GetMsgForHresult(result)); + } + while(result == VSS_S_ASYNC_PENDING); + + pAsync->Release(); + + return (result == VSS_S_ASYNC_FINISHED); +} + +#define CALL_MEMBER_FN(object, method) ((object).*(method)) + +bool BackupDaemon::CallAndWaitForAsync(AsyncMethod method, + const std::string& description) +{ + IVssAsync *pAsync; + HRESULT result = CALL_MEMBER_FN(*mpVssBackupComponents, method)(&pAsync); + if(result != S_OK) + { + BOX_ERROR("VSS: " << description << " failed: " << + GetMsgForHresult(result)); + return false; + } + + return WaitForAsync(pAsync, description); +} + +void BackupDaemon::CreateVssBackupComponents() +{ + std::map volumesIncluded; + + HRESULT result = ::CreateVssBackupComponents(&mpVssBackupComponents); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to create backup components: " << + GetMsgForHresult(result)); + return; + } + + result = mpVssBackupComponents->InitializeForBackup(NULL); + if(result != S_OK) + { + std::string message = GetMsgForHresult(result); + + if (result == VSS_E_UNEXPECTED) + { + message = "Check the Application Log for details, and ensure " + "that the Volume Shadow Copy, COM+ System Application, " + "and Distributed Transaction Coordinator services " + "are running"; + } + + BOX_ERROR("VSS: Failed to initialize for backup: " << message); + return; + } + + result = mpVssBackupComponents->SetContext(VSS_CTX_BACKUP); + if(result == E_NOTIMPL) + { + BOX_INFO("VSS: Failed to set context to VSS_CTX_BACKUP: " + "not implemented, probably Windows XP, ignored."); + } + else if(result != S_OK) + { + BOX_ERROR("VSS: Failed to set context to VSS_CTX_BACKUP: " << + GetMsgForHresult(result)); + return; + } + + result = mpVssBackupComponents->SetBackupState( + false, /* no components for now */ + true, /* might as well ask for a bootable backup */ + VSS_BT_FULL, + false /* what is Partial File Support? */); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to set backup state: " << + GetMsgForHresult(result)); + return; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterMetadata, + "GatherWriterMetadata()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + UINT writerCount; + result = mpVssBackupComponents->GetWriterMetadataCount(&writerCount); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get writer count: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + for(UINT iWriter = 0; iWriter < writerCount; iWriter++) + { + BOX_INFO("VSS: Getting metadata from writer " << iWriter); + VSS_ID writerInstance; + IVssExamineWriterMetadata* pMetadata; + result = mpVssBackupComponents->GetWriterMetadata(iWriter, + &writerInstance, &pMetadata); + if(result != S_OK) + { + BOX_ERROR("Failed to get VSS metadata from writer " << iWriter << + ": " << GetMsgForHresult(result)); + continue; + } + + UINT includeFiles, excludeFiles, numComponents; + result = pMetadata->GetFileCounts(&includeFiles, &excludeFiles, + &numComponents); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata file counts from " + "writer " << iWriter << ": " << + GetMsgForHresult(result)); + pMetadata->Release(); + continue; + } + + for(UINT iComponent = 0; iComponent < numComponents; iComponent++) + { + IVssWMComponent* pComponent; + result = pMetadata->GetComponent(iComponent, &pComponent); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata component " << + iComponent << " from writer " << iWriter << ": " << + GetMsgForHresult(result)); + continue; + } + + PVSSCOMPONENTINFO pComponentInfo; + result = pComponent->GetComponentInfo(&pComponentInfo); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata component " << + iComponent << " info from writer " << iWriter << ": " << + GetMsgForHresult(result)); + pComponent->Release(); + continue; + } + + BOX_TRACE("VSS: writer " << iWriter << " component " << + iComponent << " info:"); + switch(pComponentInfo->type) + { + case VSS_CT_UNDEFINED: BOX_TRACE("VSS: type: undefined"); break; + case VSS_CT_DATABASE: BOX_TRACE("VSS: type: database"); break; + case VSS_CT_FILEGROUP: BOX_TRACE("VSS: type: filegroup"); break; + default: + BOX_WARNING("VSS: type: unknown (" << pComponentInfo->type << ")"); + } + + BOX_TRACE("VSS: logical path: " << + BstrToString(pComponentInfo->bstrLogicalPath)); + BOX_TRACE("VSS: component name: " << + BstrToString(pComponentInfo->bstrComponentName)); + BOX_TRACE("VSS: caption: " << + BstrToString(pComponentInfo->bstrCaption)); + BOX_TRACE("VSS: restore metadata: " << + pComponentInfo->bRestoreMetadata); + BOX_TRACE("VSS: notify on complete: " << + pComponentInfo->bRestoreMetadata); + BOX_TRACE("VSS: selectable: " << + pComponentInfo->bSelectable); + BOX_TRACE("VSS: selectable for restore: " << + pComponentInfo->bSelectableForRestore); + BOX_TRACE("VSS: component flags: " << + BOX_FORMAT_HEX32(pComponentInfo->dwComponentFlags)); + BOX_TRACE("VSS: file count: " << + pComponentInfo->cFileCount); + BOX_TRACE("VSS: databases: " << + pComponentInfo->cDatabases); + BOX_TRACE("VSS: log files: " << + pComponentInfo->cLogFiles); + BOX_TRACE("VSS: dependencies: " << + pComponentInfo->cDependencies); + + pComponent->FreeComponentInfo(pComponentInfo); + pComponent->Release(); + } + + pMetadata->Release(); + } + + VSS_ID snapshotSetId; + result = mpVssBackupComponents->StartSnapshotSet(&snapshotSetId); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to start snapshot set: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + // Add all volumes included as backup locations to the snapshot set + for(Locations::iterator + iLocation = mLocations.begin(); + iLocation != mLocations.end(); + iLocation++) + { + Location& rLocation(**iLocation); + std::string path = rLocation.mPath; + // convert to absolute and remove Unicode prefix + path = ConvertPathToAbsoluteUnicode(path.c_str()).substr(4); + + if(path.length() >= 3 && path[1] == ':' && path[2] == '\\') + { + std::string volumeRoot = path.substr(0, 3); + + std::map::iterator i = + volumesIncluded.find(path[0]); + + if(i == volumesIncluded.end()) + { + std::wstring volumeRootWide; + volumeRootWide.push_back((WCHAR) path[0]); + volumeRootWide.push_back((WCHAR) ':'); + volumeRootWide.push_back((WCHAR) '\\'); + VSS_ID newVolumeId; + result = mpVssBackupComponents->AddToSnapshotSet( + (VSS_PWSZ)(volumeRootWide.c_str()), GUID_NULL, + &newVolumeId); + if(result == S_OK) + { + BOX_TRACE("VSS: Added volume " << volumeRoot << + " for backup location " << path << + " to snapshot set"); + volumesIncluded[path[0]] = newVolumeId; + rLocation.mSnapshotVolumeId = newVolumeId; + } + else + { + BOX_ERROR("VSS: Failed to add volume " << + volumeRoot << " to snapshot set: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + } + else + { + BOX_TRACE("VSS: Skipping already included volume " << + volumeRoot << " for backup location " << path); + rLocation.mSnapshotVolumeId = i->second; + } + + rLocation.mIsSnapshotCreated = true; + + // If the snapshot path starts with the volume root + // (drive letter), because the path is absolute (as it + // should be), then remove it so that the resulting + // snapshot path can be appended to the snapshot device + // object to make a real path, without a spurious drive + // letter in it. + + if (path.substr(0, volumeRoot.length()) == volumeRoot) + { + path = path.substr(volumeRoot.length()); + } + + rLocation.mSnapshotPath = path; + } + else + { + BOX_WARNING("VSS: Skipping backup location " << path << + " which does not start with a volume specification"); + } + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::PrepareForBackup, + "PrepareForBackup()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::DoSnapshotSet, + "DoSnapshotSet()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterStatus, + "GatherWriterStatus()")) + { + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + result = mpVssBackupComponents->GetWriterStatusCount(&writerCount); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get writer status count: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + for(UINT iWriter = 0; iWriter < writerCount; iWriter++) + { + VSS_ID instance, writer; + BSTR writerNameBstr; + VSS_WRITER_STATE writerState; + HRESULT writerResult; + + result = mpVssBackupComponents->GetWriterStatus(iWriter, + &instance, &writer, &writerNameBstr, &writerState, + &writerResult); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query writer " << iWriter << + " status: " << GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + std::string writerName = BstrToString(writerNameBstr); + ::SysFreeString(writerNameBstr); + + if(writerResult != S_OK) + { + BOX_ERROR("VSS: Writer " << iWriter << " (" << + writerName << ") failed: " << + GetMsgForHresult(writerResult)); + continue; + } + + std::string stateName; + + switch(writerState) + { +#define WRITER_STATE(code) \ + case code: stateName = #code; break; + WRITER_STATE(VSS_WS_UNKNOWN); + WRITER_STATE(VSS_WS_STABLE); + WRITER_STATE(VSS_WS_WAITING_FOR_FREEZE); + WRITER_STATE(VSS_WS_WAITING_FOR_THAW); + WRITER_STATE(VSS_WS_WAITING_FOR_POST_SNAPSHOT); + WRITER_STATE(VSS_WS_WAITING_FOR_BACKUP_COMPLETE); + WRITER_STATE(VSS_WS_FAILED_AT_IDENTIFY); + WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_BACKUP); + WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_SNAPSHOT); + WRITER_STATE(VSS_WS_FAILED_AT_FREEZE); + WRITER_STATE(VSS_WS_FAILED_AT_THAW); + WRITER_STATE(VSS_WS_FAILED_AT_POST_SNAPSHOT); + WRITER_STATE(VSS_WS_FAILED_AT_BACKUP_COMPLETE); + WRITER_STATE(VSS_WS_FAILED_AT_PRE_RESTORE); + WRITER_STATE(VSS_WS_FAILED_AT_POST_RESTORE); + WRITER_STATE(VSS_WS_FAILED_AT_BACKUPSHUTDOWN); +#undef WRITER_STATE + default: + std::ostringstream o; + o << "unknown (" << writerState << ")"; + stateName = o.str(); + } + + BOX_TRACE("VSS: Writer " << iWriter << " (" << + writerName << ") is in state " << stateName); + } + + // lookup new snapshot volume for each location that has a snapshot + for(Locations::iterator + iLocation = mLocations.begin(); + iLocation != mLocations.end(); + iLocation++) + { + Location& rLocation(**iLocation); + if(rLocation.mIsSnapshotCreated) + { + VSS_SNAPSHOT_PROP prop; + result = mpVssBackupComponents->GetSnapshotProperties( + rLocation.mSnapshotVolumeId, &prop); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get snapshot properties " + "for volume " << GuidToString(rLocation.mSnapshotVolumeId) << + " for location " << rLocation.mPath << ": " << + GetMsgForHresult(result)); + rLocation.mIsSnapshotCreated = false; + continue; + } + + rLocation.mSnapshotPath = + WideStringToString(prop.m_pwszSnapshotDeviceObject) + + DIRECTORY_SEPARATOR + rLocation.mSnapshotPath; + VssFreeSnapshotProperties(&prop); + + BOX_INFO("VSS: Location " << rLocation.mPath << " using " + "snapshot path " << rLocation.mSnapshotPath); + } + } + + IVssEnumObject *pEnum; + result = mpVssBackupComponents->Query(GUID_NULL, VSS_OBJECT_NONE, + VSS_OBJECT_SNAPSHOT, &pEnum); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query snapshot list: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + while(result == S_OK) + { + VSS_OBJECT_PROP rgelt; + ULONG count; + result = pEnum->Next(1, &rgelt, &count); + + if(result == S_FALSE) + { + // end of list, break out of the loop + break; + } + else if(result != S_OK) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + GetMsgForHresult(result)); + } + else if(count != 1) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + "Next() returned " << count << " objects instead of 1"); + } + else if(rgelt.Type != VSS_OBJECT_SNAPSHOT) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + "Next() returned a type " << rgelt.Type << " object " + "instead of VSS_OBJECT_SNAPSHOT"); + } + else + { + VSS_SNAPSHOT_PROP *pSnap = &rgelt.Obj.Snap; + BOX_TRACE("VSS: Snapshot ID: " << + GuidToString(pSnap->m_SnapshotId)); + BOX_TRACE("VSS: Snapshot set ID: " << + GuidToString(pSnap->m_SnapshotSetId)); + BOX_TRACE("VSS: Number of volumes: " << + pSnap->m_lSnapshotsCount); + BOX_TRACE("VSS: Snapshot device object: " << + WideStringToString(pSnap->m_pwszSnapshotDeviceObject)); + BOX_TRACE("VSS: Original volume name: " << + WideStringToString(pSnap->m_pwszOriginalVolumeName)); + BOX_TRACE("VSS: Originating machine: " << + WideStringToString(pSnap->m_pwszOriginatingMachine)); + BOX_TRACE("VSS: Service machine: " << + WideStringToString(pSnap->m_pwszServiceMachine)); + BOX_TRACE("VSS: Exposed name: " << + WideStringToString(pSnap->m_pwszExposedName)); + BOX_TRACE("VSS: Exposed path: " << + WideStringToString(pSnap->m_pwszExposedPath)); + BOX_TRACE("VSS: Provider ID: " << + GuidToString(pSnap->m_ProviderId)); + BOX_TRACE("VSS: Snapshot attributes: " << + BOX_FORMAT_HEX32(pSnap->m_lSnapshotAttributes)); + BOX_TRACE("VSS: Snapshot creation time: " << + BOX_FORMAT_HEX32(pSnap->m_tsCreationTimestamp)); + + std::string status; + switch(pSnap->m_eStatus) + { + case VSS_SS_UNKNOWN: status = "Unknown (error)"; break; + case VSS_SS_PREPARING: status = "Preparing"; break; + case VSS_SS_PROCESSING_PREPARE: status = "Preparing (processing)"; break; + case VSS_SS_PREPARED: status = "Prepared"; break; + case VSS_SS_PROCESSING_PRECOMMIT: status = "Precommitting"; break; + case VSS_SS_PRECOMMITTED: status = "Precommitted"; break; + case VSS_SS_PROCESSING_COMMIT: status = "Commiting"; break; + case VSS_SS_COMMITTED: status = "Committed"; break; + case VSS_SS_PROCESSING_POSTCOMMIT: status = "Postcommitting"; break; + case VSS_SS_PROCESSING_PREFINALCOMMIT: status = "Pre final committing"; break; + case VSS_SS_PREFINALCOMMITTED: status = "Pre final committed"; break; + case VSS_SS_PROCESSING_POSTFINALCOMMIT: status = "Post final committing"; break; + case VSS_SS_CREATED: status = "Created"; break; + case VSS_SS_ABORTED: status = "Aborted"; break; + case VSS_SS_DELETED: status = "Deleted"; break; + case VSS_SS_POSTCOMMITTED: status = "Postcommitted"; break; + default: + std::ostringstream buf; + buf << "Unknown code: " << pSnap->m_eStatus; + status = buf.str(); + } + + BOX_TRACE("VSS: Snapshot status: " << status); + VssFreeSnapshotProperties(pSnap); + } + } + + pEnum->Release(); + +CreateVssBackupComponents_cleanup_WriterStatus: + result = mpVssBackupComponents->FreeWriterStatus(); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to free writer status: " << + GetMsgForHresult(result)); + } + +CreateVssBackupComponents_cleanup_WriterMetadata: + result = mpVssBackupComponents->FreeWriterMetadata(); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to free writer metadata: " << + GetMsgForHresult(result)); + } +} + +void BackupDaemon::CleanupVssBackupComponents() +{ + if(mpVssBackupComponents == NULL) + { + return; + } + + CallAndWaitForAsync(&IVssBackupComponents::BackupComplete, + "BackupComplete()"); + + mpVssBackupComponents->Release(); + mpVssBackupComponents = NULL; +} +#endif + +void BackupDaemon::OnBackupStart() +{ + ResetLogFile(); + + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_start"); + + // Reset statistics on uploads + BackupStoreFile::ResetStats(); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(true /* start */); + + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupStart); + + // Setup timer for polling the command socket + mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, + "CommandSocketPollTimer")); + + // Set state and log start + SetState(State_Connected); + BOX_NOTICE("Beginning scan of local files"); +} + +void BackupDaemon::OnBackupFinish() +{ + try + { + // Log + BOX_NOTICE("Finished scan of local files"); + + // Log the stats + BOX_NOTICE("File statistics: total file size uploaded " + << BackupStoreFile::msStats.mBytesInEncodedFiles + << ", bytes already on server " + << BackupStoreFile::msStats.mBytesAlreadyOnServer + << ", encoded size " + << BackupStoreFile::msStats.mTotalFileStreamSize + << ", " << mNumFilesUploaded << " files uploaded, " + << mNumDirsCreated << " dirs created"); + + // Reset statistics again + BackupStoreFile::ResetStats(); + + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupFinish); + + // Stop the timer for polling the command socket, + // to prevent needless alarms while sleeping. + mapCommandSocketPollTimer.reset(); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(false /* finish */); + + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_finish"); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to perform backup finish actions: " << e.what()); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::UseScriptToSeeIfSyncAllowed() +// Purpose: Private. Use a script to see if the sync should be +// allowed now (if configured). Returns -1 if it's +// allowed, time in seconds to wait otherwise. +// Created: 21/6/04 +// +// -------------------------------------------------------------------------- +int BackupDaemon::UseScriptToSeeIfSyncAllowed() +{ + const Configuration &conf(GetConfiguration()); + + // Got a script to run? + if(!conf.KeyExists("SyncAllowScript")) + { + // No. Do sync. + return -1; + } + + // If there's no result, try again in five minutes + int waitInSeconds = (60*5); + + std::string script(conf.GetKeyValue("SyncAllowScript") + + " \"" + GetConfigFileName() + "\""); + + // Run it? + pid_t pid = 0; + try + { + std::auto_ptr pscript(LocalProcessStream(script, + pid)); + + // Read in the result + IOStreamGetLine getLine(*pscript); + std::string line; + if(getLine.GetLine(line, true, 30000)) // 30 seconds should be enough + { + waitInSeconds = BackupDaemon::ParseSyncAllowScriptOutput(script, line); + } + else + { + BOX_ERROR("SyncAllowScript output nothing within " + "30 seconds, waiting 5 minutes to try again" + " (" << script << ")"); + } + } + catch(std::exception &e) + { + BOX_ERROR("Internal error running SyncAllowScript: " + << e.what() << " (" << script << ")"); + } + catch(...) + { + // Ignore any exceptions + // Log that something bad happened + BOX_ERROR("Unknown error running SyncAllowScript (" << + script << ")"); + } + + // Wait and then cleanup child process, if any + if(pid != 0) + { + int status = 0; + ::waitpid(pid, &status, 0); + } + + return waitInSeconds; +} + +int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, + const std::string& output) +{ + int waitInSeconds = (60*5); + std::istringstream iss(output); + + std::string delay; + iss >> delay; + + if(delay == "") + { + BOX_ERROR("SyncAllowScript output an empty line, sleeping for " + << waitInSeconds << " seconds (" << script << ")"); + return waitInSeconds; + } + + // Got a string, interpret + if(delay == "now") + { + // Script says do it now. Obey. + waitInSeconds = -1; + + BOX_NOTICE("SyncAllowScript requested a backup now " + "(" << script << ")"); + } + else + { + try + { + // How many seconds to wait? + waitInSeconds = BoxConvert::Convert(delay); + } + catch(ConversionException &e) + { + BOX_ERROR("SyncAllowScript output an invalid " + "number: '" << output << "' (" << + script << ")"); + throw; + } + + BOX_NOTICE("SyncAllowScript requested a delay of " << + waitInSeconds << " seconds (" << script << ")"); + } + + if(iss.eof()) + { + // No bandwidth limit requested + mMaxBandwidthFromSyncAllowScript = 0; + BOX_NOTICE("SyncAllowScript did not set a maximum bandwidth " + "(" << script << ")"); + } + else + { + std::string maxBandwidth; + iss >> maxBandwidth; + + try + { + // How many seconds to wait? + mMaxBandwidthFromSyncAllowScript = + BoxConvert::Convert(maxBandwidth); + } + catch(ConversionException &e) + { + BOX_ERROR("Invalid maximum bandwidth from " + "SyncAllowScript: '" << + output << "' (" << script << ")"); + throw; + } + + BOX_NOTICE("SyncAllowScript set maximum bandwidth to " << + mMaxBandwidthFromSyncAllowScript << " kB/s (" << + script << ")"); + } + + return waitInSeconds; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::RunBackgroundTask() +// Purpose: Checks for connections or commands on the command +// socket and handles them with minimal delay. Polled +// during lengthy operations such as file uploads. +// Created: 07/04/14 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum) +{ + BOX_TRACE("BackupDaemon::RunBackgroundTask: state = " << state << + ", progress = " << progress << "/" << maximum); + + if(!mapCommandSocketPollTimer.get()) + { + return true; // no background task + } + + if(mapCommandSocketPollTimer->HasExpired()) + { + mapCommandSocketPollTimer->Reset(COMMAND_SOCKET_POLL_INTERVAL); + } + else + { + // Do no more work right now + return true; + } + + if(mapCommandSocketInfo.get()) + { + BOX_TRACE("BackupDaemon::RunBackgroundTask: polling command socket"); + + bool sync_flag_out, sync_is_forced_out; + + WaitOnCommandSocket(0, // RequiredDelay + sync_flag_out, sync_is_forced_out); + + if(sync_flag_out) + { + BOX_WARNING("Ignoring request to sync while " + "already syncing."); + } + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::WaitOnCommandSocket(box_time_t, bool &, bool &) +// Purpose: Waits on a the command socket for a time of UP TO +// the required time but may be much less, and handles +// a command if necessary. +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut) +{ + DoSyncFlagOut = false; + SyncIsForcedOut = false; + + ASSERT(mapCommandSocketInfo.get()); + if(!mapCommandSocketInfo.get()) + { + // failure case isn't too bad + ::sleep(1); + return; + } + + BOX_TRACE("Wait on command socket, delay = " << + BOX_FORMAT_MICROSECONDS(RequiredDelay)); + + try + { + // Timeout value for connections and things + int timeout = ((int)BoxTimeToMilliSeconds(RequiredDelay)) + 1; + // Handle bad boundary cases + if(timeout <= 0) timeout = 1; + if(timeout == INFTIM) timeout = 100000; + + // Wait for socket connection, or handle a command? + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + // No connection, listen for a new one + mapCommandSocketInfo->mpConnectedSocket.reset(mapCommandSocketInfo->mListeningSocket.Accept(timeout).release()); + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + // If a connection didn't arrive, there was a timeout, which means we've + // waited long enough and it's time to go. + return; + } + else + { +#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET + bool uidOK = true; + BOX_WARNING("On this platform, no security check can be made on the credentials of peers connecting to the command socket. (bbackupctl)"); +#else + // Security check -- does the process connecting to this socket have + // the same UID as this process? + bool uidOK = false; + // BLOCK + { + uid_t remoteEUID = 0xffff; + gid_t remoteEGID = 0xffff; + if(mapCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID)) + { + // Credentials are available -- check UID + if(remoteEUID == ::getuid()) + { + // Acceptable + uidOK = true; + } + } + } +#endif // PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET + + // Is this an acceptable connection? + if(!uidOK) + { + // Dump the connection + BOX_ERROR("Incoming command connection from peer had different user ID than this process, or security check could not be completed."); + mapCommandSocketInfo->mpConnectedSocket.reset(); + return; + } + else + { + // Log + BOX_INFO("Connection from command socket"); + + // Send a header line summarising the configuration and current state + const Configuration &conf(GetConfiguration()); + std::ostringstream hello; + hello << "bbackupd: " << + (conf.GetKeyValueBool("AutomaticBackup") ? 1 : 0) + << " " << + conf.GetKeyValueInt("UpdateStoreInterval") + << " " << + conf.GetKeyValueInt("MinimumFileAge") + << " " << + conf.GetKeyValueInt("MaxUploadWait") + << "\nstate " << mState << "\n"; + mapCommandSocketInfo->mpConnectedSocket->Write( + hello.str(), timeout); + + // Set the timeout to something very small, so we don't wait too long on waiting + // for any incoming data + timeout = 10; // milliseconds + } + } + } + + // So there must be a connection now. + ASSERT(mapCommandSocketInfo->mpConnectedSocket.get() != 0); + + // Is there a getline object ready? + if(mapCommandSocketInfo->mpGetLine == 0) + { + // Create a new one + mapCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mapCommandSocketInfo->mpConnectedSocket.get())); + } + + // Ping the remote side, to provide errors which will mean the socket gets closed + mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5, + timeout); + + // Wait for a command or something on the socket + std::string command; + while(mapCommandSocketInfo->mpGetLine != 0 && !mapCommandSocketInfo->mpGetLine->IsEOF() + && mapCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) + { + BOX_TRACE("Receiving command '" << command + << "' over command socket"); + + bool sendOK = false; + bool sendResponse = true; + + // Command to process! + if(command == "quit" || command == "") + { + // Close the socket. + CloseCommandConnection(); + sendResponse = false; + } + else if(command == "sync") + { + // Sync now! + DoSyncFlagOut = true; + SyncIsForcedOut = false; + sendOK = true; + } + else if(command == "force-sync") + { + // Sync now (forced -- overrides any SyncAllowScript) + DoSyncFlagOut = true; + SyncIsForcedOut = true; + sendOK = true; + } + else if(command == "reload") + { + // Reload the configuration + SetReloadConfigWanted(); + sendOK = true; + } + else if(command == "terminate") + { + // Terminate the daemon cleanly + SetTerminateWanted(); + sendOK = true; + } + + // Send a response back? + if(sendResponse) + { + std::string response = sendOK ? "ok\n" : "error\n"; + mapCommandSocketInfo->mpConnectedSocket->Write( + response, timeout); + } + + // Set timeout to something very small, so this just checks for data which is waiting + timeout = 1; + } + + // Close on EOF? + if(mapCommandSocketInfo->mpGetLine != 0 && mapCommandSocketInfo->mpGetLine->IsEOF()) + { + CloseCommandConnection(); + } + } + catch(ConnectionException &ce) + { + BOX_NOTICE("Failed to write to command socket: " << ce.what()); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } + catch(std::exception &e) + { + BOX_ERROR("Failed to write to command socket: " << + e.what()); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } + catch(...) + { + BOX_ERROR("Failed to write to command socket: unknown error"); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CloseCommandConnection() +// Purpose: Close the command connection, ignoring any errors +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::CloseCommandConnection() +{ + try + { + BOX_TRACE("Closing command connection"); + + if(mapCommandSocketInfo->mpGetLine) + { + delete mapCommandSocketInfo->mpGetLine; + mapCommandSocketInfo->mpGetLine = 0; + } + mapCommandSocketInfo->mpConnectedSocket.reset(); + } + catch(std::exception &e) + { + BOX_ERROR("Internal error while closing command " + "socket: " << e.what()); + } + catch(...) + { + // Ignore any errors + } +} + + +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemon.cpp +// Purpose: Send a start or finish sync message to the command socket, if it's connected. +// +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SendSyncStartOrFinish(bool SendStart) +{ + // The bbackupctl program can't rely on a state change, because it + // may never change if the server doesn't need to be contacted. + + if(mapCommandSocketInfo.get() && + mapCommandSocketInfo->mpConnectedSocket.get() != 0) + { + std::string message = SendStart ? "start-sync" : "finish-sync"; + try + { + message += "\n"; + mapCommandSocketInfo->mpConnectedSocket->Write(message, + 1); // short timeout, it's overlapped + } + catch(std::exception &e) + { + BOX_ERROR("Internal error while sending to " + "command socket client: " << e.what()); + CloseCommandConnection(); + } + catch(...) + { + CloseCommandConnection(); + } + } +} + + + + +#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_NMTONNAME) + // string comparison ordering for when mount points are handled + // by code, rather than the OS. + typedef struct + { + bool operator()(const std::string &s1, const std::string &s2) + { + if(s1.size() == s2.size()) + { + // Equal size, sort according to natural sort order + return s1 < s2; + } + else + { + // Make sure longer strings go first + return s1.size() > s2.size(); + } + } + } mntLenCompare; +#endif + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetupLocations(BackupClientContext &, const Configuration &) +// Purpose: Makes sure that the list of directories records is correctly set up +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf) +{ + // Going to need a copy of the root directory. Get a connection, + // and fetch it. + BackupProtocolCallable& connection(rClientContext.GetConnection()); + + // Ask server for a list of everything in the root directory, + // which is a directory itself + std::auto_ptr dirreply( + connection.QueryListDirectory( + BackupProtocolListDirectory::RootDirectory, + // only directories + BackupProtocolListDirectory::Flags_Dir, + // exclude old/deleted stuff + BackupProtocolListDirectory::Flags_Deleted | + BackupProtocolListDirectory::Flags_OldVersion, + false /* no attributes */)); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(connection.ReceiveStream()); + dir.ReadFromStream(*dirstream, connection.GetTimeout()); + + // Map of mount names to ID map index + std::map mounts; + int numIDMaps = 0; + +#ifdef HAVE_MOUNTS +#if !defined(HAVE_STRUCT_STATFS_F_MNTONNAME) && !defined(HAVE_STRUCT_STATVFS_F_MNTONNAME) + // Linux and others can't tell you where a directory is mounted. So we + // have to read the mount entries from /etc/mtab! Bizarre that the OS + // itself can't tell you, but there you go. + std::set mountPoints; + // BLOCK + FILE *mountPointsFile = 0; + +#ifdef HAVE_STRUCT_MNTENT_MNT_DIR + // Open mounts file + mountPointsFile = ::setmntent("/proc/mounts", "r"); + if(mountPointsFile == 0) + { + mountPointsFile = ::setmntent("/etc/mtab", "r"); + } + if(mountPointsFile == 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + + try + { + // Read all the entries, and put them in the set + struct mntent *entry = 0; + while((entry = ::getmntent(mountPointsFile)) != 0) + { + BOX_TRACE("Found mount point at " << entry->mnt_dir); + mountPoints.insert(std::string(entry->mnt_dir)); + } + + // Close mounts file + ::endmntent(mountPointsFile); + } + catch(...) + { + ::endmntent(mountPointsFile); + throw; + } +#else // ! HAVE_STRUCT_MNTENT_MNT_DIR + // Open mounts file + mountPointsFile = ::fopen("/etc/mnttab", "r"); + if(mountPointsFile == 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + + try + { + // Read all the entries, and put them in the set + struct mnttab entry; + while(getmntent(mountPointsFile, &entry) == 0) + { + BOX_TRACE("Found mount point at " << entry.mnt_mountp); + mountPoints.insert(std::string(entry.mnt_mountp)); + } + + // Close mounts file + ::fclose(mountPointsFile); + } + catch(...) + { + ::fclose(mountPointsFile); + throw; + } +#endif // HAVE_STRUCT_MNTENT_MNT_DIR + // Check sorting and that things are as we expect + ASSERT(mountPoints.size() > 0); +#ifndef BOX_RELEASE_BUILD + { + std::set::reverse_iterator i(mountPoints.rbegin()); + ASSERT(*i == "/"); + } +#endif // n BOX_RELEASE_BUILD +#endif // n HAVE_STRUCT_STATFS_F_MNTONNAME || n HAVE_STRUCT_STATVFS_F_MNTONNAME +#endif // HAVE_MOUNTS + + // Then... go through each of the entries in the configuration, + // making sure there's a directory created for it. + std::vector locNames = + rLocationsConf.GetSubConfigurationNames(); + + // We only want completely configured locations to be in the list + // when this function exits, so move them all to a temporary list. + // Entries matching a properly configured location will be moved + // back to mLocations. Anything left in this list after the loop + // finishes will be deleted. + Locations tmpLocations = mLocations; + mLocations.clear(); + + // The ID map list will be repopulated automatically by this loop + mIDMapMounts.clear(); + + for(std::vector::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + Location* pLoc = NULL; + + // Try to find and reuse an existing Location object + for(Locations::const_iterator + i = tmpLocations.begin(); + i != tmpLocations.end(); i++) + { + if ((*i)->mName == *pLocName) + { + BOX_TRACE("Location already configured: " << *pLocName); + pLoc = *i; + break; + } + } + + const Configuration& rConfig( + rLocationsConf.GetSubConfiguration(*pLocName)); + std::auto_ptr apLoc; + + try + { + if(pLoc == NULL) + { + // Create a record for it + BOX_TRACE("New location: " << *pLocName); + pLoc = new Location; + + // ensure deletion if setup fails + apLoc.reset(pLoc); + + // Setup names in the location record + pLoc->mName = *pLocName; + pLoc->mPath = rConfig.GetKeyValue("Path"); + } + + // Read the exclude lists from the Configuration + pLoc->mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rConfig)); + pLoc->mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rConfig)); + + // Does this exist on the server? + // Remove from dir object early, so that if we fail + // to stat the local directory, we still don't + // consider to remote one for deletion. + BackupStoreDirectory::Iterator iter(dir); + BackupStoreFilenameClear dirname(pLoc->mName); // generate the filename + BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname); + int64_t oid = 0; + if(en != 0) + { + oid = en->GetObjectID(); + + // Delete the entry from the directory, so we get a list of + // unused root directories at the end of this. + dir.DeleteEntry(oid); + } + + // Do a fsstat on the pathname to find out which mount it's on + { + +#if defined HAVE_STRUCT_STATFS_F_MNTONNAME || defined HAVE_STRUCT_STATVFS_F_MNTONNAME || defined WIN32 + + // BSD style statfs -- includes mount point, which is nice. +#ifdef HAVE_STRUCT_STATVFS_F_MNTONNAME + struct statvfs s; + if(::statvfs(pLoc->mPath.c_str(), &s) != 0) +#else // HAVE_STRUCT_STATVFS_F_MNTONNAME + struct statfs s; + if(::statfs(pLoc->mPath.c_str(), &s) != 0) +#endif // HAVE_STRUCT_STATVFS_F_MNTONNAME + { + THROW_SYS_ERROR("Failed to stat path " + "'" << pLoc->mPath << "' " + "for location " + "'" << pLoc->mName << "'", + CommonException, OSFileError); + } + + // Where the filesystem is mounted + std::string mountName(s.f_mntonname); + +#else // !HAVE_STRUCT_STATFS_F_MNTONNAME && !WIN32 + + // Warn in logs if the directory isn't absolute + if(pLoc->mPath[0] != '/') + { + BOX_WARNING("Location path '" + << pLoc->mPath + << "' is not absolute"); + } + // Go through the mount points found, and find a suitable one + std::string mountName("/"); + { + std::set::const_iterator i(mountPoints.begin()); + BOX_TRACE(mountPoints.size() + << " potential mount points"); + for(; i != mountPoints.end(); ++i) + { + // Compare first n characters with the filename + // If it matches, the file belongs in that mount point + // (sorting order ensures this) + BOX_TRACE("checking against mount point " << *i); + if(::strncmp(i->c_str(), pLoc->mPath.c_str(), i->size()) == 0) + { + // Match + mountName = *i; + break; + } + } + BOX_TRACE("mount point chosen for " + << pLoc->mPath << " is " + << mountName); + } + +#endif + + // Got it? + std::map::iterator f(mounts.find(mountName)); + if(f != mounts.end()) + { + // Yes -- store the index + pLoc->mIDMapIndex = f->second; + } + else + { + // No -- new index + pLoc->mIDMapIndex = numIDMaps; + mounts[mountName] = numIDMaps; + + // Store the mount name + mIDMapMounts.push_back(mountName); + + // Increment number of maps + ++numIDMaps; + } + } + + // Does this exist on the server? + if(en == 0) + { + // Doesn't exist, so it has to be created on the server. Let's go! + // First, get the directory's attributes and modification time + box_time_t attrModTime = 0; + BackupClientFileAttributes attr; + try + { + attr.ReadAttributes(pLoc->mPath.c_str(), + true /* directories have zero mod times */, + 0 /* not interested in mod time */, + &attrModTime /* get the attribute modification time */); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to get attributes " + "for path '" << pLoc->mPath + << "', skipping location '" << + pLoc->mName << "'"); + throw; + } + + // Execute create directory command + try + { + std::auto_ptr attrStream( + new MemBlockStream(attr)); + std::auto_ptr + dirCreate(connection.QueryCreateDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, // containing directory + attrModTime, dirname, attrStream)); + + // Object ID for later creation + oid = dirCreate->GetObjectID(); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to create remote " + "directory '/" << pLoc->mName << + "', skipping location '" << + pLoc->mName << "'"); + throw; + } + + } + + // Create and store the directory object for the root of this location + ASSERT(oid != 0); + if(pLoc->mapDirectoryRecord.get() == NULL) + { + pLoc->mapDirectoryRecord.reset( + new BackupClientDirectoryRecord(oid, *pLocName)); + } + + // Remove it from the temporary list to avoid deletion + tmpLocations.remove(pLoc); + + // Push it back on the vector of locations + mLocations.push_back(pLoc); + + if(apLoc.get() != NULL) + { + // Don't delete it now! + apLoc.release(); + } + } + catch (std::exception &e) + { + BOX_ERROR("Failed to configure location '" + << pLoc->mName << "' path '" + << pLoc->mPath << "': " << e.what() << + ": please check for previous errors"); + mReadErrorsOnFilesystemObjects = true; + } + catch(...) + { + BOX_ERROR("Failed to configure location '" + << pLoc->mName << "' path '" + << pLoc->mPath << "': please check for " + "previous errors"); + mReadErrorsOnFilesystemObjects = true; + } + } + + // Now remove any leftovers + for(BackupDaemon::Locations::iterator + i = tmpLocations.begin(); + i != tmpLocations.end(); i++) + { + BOX_INFO("Removing obsolete location from memory: " << + (*i)->mName); + delete *i; + } + + tmpLocations.clear(); + + // Any entries in the root directory which need deleting? + if(dir.GetNumberOfEntries() > 0 && + mDeleteRedundantLocationsAfter == 0) + { + BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " + "in root directory found, but will not delete because " + "DeleteRedundantLocationsAfter = 0"); + } + else if(dir.GetNumberOfEntries() > 0) + { + box_time_t now = GetCurrentBoxTime(); + + // This should reset the timer if the list of unused + // locations changes, but it will not if the number of + // unused locations does not change, but the locations + // do change, e.g. one mysteriously appears and another + // mysteriously appears. (FIXME) + if (dir.GetNumberOfEntries() != mUnusedRootDirEntries.size() || + mDeleteUnusedRootDirEntriesAfter == 0) + { + mDeleteUnusedRootDirEntriesAfter = now + + SecondsToBoxTime(mDeleteRedundantLocationsAfter); + } + + int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter + - now); + + BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " + "in root directory found, will delete from store " + "after " << secs << " seconds."); + + // Store directories in list of things to delete + mUnusedRootDirEntries.clear(); + BackupStoreDirectory::Iterator iter(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = iter.Next()) != 0) + { + // Add name to list + BackupStoreFilenameClear clear(en->GetName()); + const std::string &name(clear.GetClearFilename()); + mUnusedRootDirEntries.push_back( + std::pair + (en->GetObjectID(), name)); + // Log this + BOX_INFO("Unused location in root: " << name); + } + ASSERT(mUnusedRootDirEntries.size() > 0); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetupIDMapsForSync() +// Purpose: Sets up ID maps for the sync process -- make sure they're all there +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetupIDMapsForSync() +{ + // Make sure we have some blank, empty ID maps + DeleteIDMapVector(mNewIDMaps); + FillIDMapVector(mNewIDMaps, true /* new maps */); + DeleteIDMapVector(mCurrentIDMaps); + FillIDMapVector(mCurrentIDMaps, false /* new maps */); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::FillIDMapVector(std::vector &) +// Purpose: Fills the vector with the right number of empty ID maps +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::FillIDMapVector(std::vector &rVector, bool NewMaps) +{ + ASSERT(rVector.size() == 0); + rVector.reserve(mIDMapMounts.size()); + + for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) + { + // Create the object + BackupClientInodeToIDMap *pmap = new BackupClientInodeToIDMap(); + try + { + // Get the base filename of this map + std::string filename; + MakeMapBaseName(l, filename); + + // If it's a new one, add a suffix + if(NewMaps) + { + filename += ".n"; + } + + // The new map file should not exist yet. If there's + // one left over from a previous failed run, it's not + // useful to us because we never read from it and will + // overwrite the entries of all files that still + // exist, so we should just delete it and start afresh. + if(NewMaps && FileExists(filename.c_str())) + { + BOX_NOTICE("Found an incomplete ID map " + "database, deleting it to start " + "afresh: " << filename); + if(unlink(filename.c_str()) != 0) + { + BOX_LOG_NATIVE_ERROR(BOX_FILE_MESSAGE( + filename, "Failed to delete " + "incomplete ID map database")); + } + } + + // If it's not a new map, it may not exist in which case an empty map should be created + if(!NewMaps && !FileExists(filename.c_str())) + { + pmap->OpenEmpty(); + } + else + { + // Open the map + pmap->Open(filename.c_str(), !NewMaps /* read only */, NewMaps /* create new */); + } + + // Store on vector + rVector.push_back(pmap); + } + catch(...) + { + delete pmap; + throw; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteCorruptBerkelyDbFiles() +// Purpose: Delete the Berkely db files from disc after they have been corrupted. +// Created: 14/9/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteCorruptBerkelyDbFiles() +{ + for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) + { + // Get the base filename of this map + std::string filename; + MakeMapBaseName(l, filename); + + // Delete the file + BOX_TRACE("Deleting " << filename); + ::unlink(filename.c_str()); + + // Add a suffix for the new map + filename += ".n"; + + // Delete that too + BOX_TRACE("Deleting " << filename); + ::unlink(filename.c_str()); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: MakeMapBaseName(unsigned int, std::string &) +// Purpose: Makes the base name for a inode map +// Created: 20/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const +{ + // Get the directory for the maps + const Configuration &config(GetConfiguration()); + std::string dir(config.GetKeyValue("DataDirectory")); + + // Make a leafname + std::string leaf(mIDMapMounts[MountNumber]); + for(unsigned int z = 0; z < leaf.size(); ++z) + { + if(leaf[z] == DIRECTORY_SEPARATOR_ASCHAR) + { + leaf[z] = '_'; + } + } + + // Build the final filename + rNameOut = dir + DIRECTORY_SEPARATOR "mnt" + leaf; +} + + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CommitIDMapsAfterSync() +// Purpose: Commits the new ID maps, so the 'new' maps are now the 'current' maps. +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::CommitIDMapsAfterSync() +{ + // Get rid of the maps in memory (leaving them on disc of course) + DeleteIDMapVector(mCurrentIDMaps); + DeleteIDMapVector(mNewIDMaps); + + // Then move the old maps into the new places + for(unsigned int l = 0; l < mIDMapMounts.size(); ++l) + { + std::string target; + MakeMapBaseName(l, target); + std::string newmap(target + ".n"); + + // Try to rename +#ifdef WIN32 + // win32 rename doesn't overwrite existing files + ::remove(target.c_str()); +#endif + if(::rename(newmap.c_str(), target.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("Failed to rename ID map: " << + newmap << " to " << target); + THROW_EXCEPTION(CommonException, OSFileError) + } + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteIDMapVector(std::vector &) +// Purpose: Deletes the contents of a vector of ID maps +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteIDMapVector(std::vector &rVector) +{ + while(!rVector.empty()) + { + // Pop off list + BackupClientInodeToIDMap *toDel = rVector.back(); + rVector.pop_back(); + + // Close and delete + delete toDel; + } + ASSERT(rVector.size() == 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::FindLocationPathName(const std::string &, std::string &) const +// Purpose: Tries to find the path of the root of a backup location. Returns true (and path in rPathOut) +// if it can be found, false otherwise. +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const +{ + // Search for the location + for(Locations::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i) + { + if((*i)->mName == rLocationName) + { + rPathOut = (*i)->mPath; + return true; + } + } + + // Didn't find it + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SetState(int) +// Purpose: Record current action of daemon, and update process title to reflect this +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void BackupDaemon::SetState(int State) +{ + // Two little checks + if(State == mState) return; + if(State < 0) return; + + // Update + mState = State; + + // Set process title + const static char *stateText[] = {"idle", "connected", "error -- waiting for retry", "over limit on server -- not backing up"}; + SetProcessTitle(stateText[State]); + + // If there's a command socket connected, then inform it -- disconnecting from the + // command socket if there's an error + + std::ostringstream msg; + msg << "state " << State << "\n"; + + if(!mapCommandSocketInfo.get()) + { + return; + } + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + return; + } + + // Something connected to the command socket, tell it about the new state + try + { + mapCommandSocketInfo->mpConnectedSocket->Write(msg.str(), + 1); // very short timeout, it's overlapped anyway + } + catch(ConnectionException &ce) + { + BOX_NOTICE("Failed to write state to command socket: " << + ce.what()); + CloseCommandConnection(); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to write state to command socket: " << + e.what()); + CloseCommandConnection(); + } + catch(...) + { + BOX_ERROR("Failed to write state to command socket: " + "unknown error"); + CloseCommandConnection(); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::TouchFileInWorkingDir(const char *) +// Purpose: Make sure a zero length file of the name exists in the working directory. +// Use for marking times of events in the filesystem. +// Created: 21/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::TouchFileInWorkingDir(const char *Filename) +{ + // Filename + const Configuration &config(GetConfiguration()); + std::string fn(config.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); + fn += Filename; + + // Open and close it to update the timestamp + try + { + FileStream touch(fn, O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to write to timestamp file: " << fn << ": " << + e.what()); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::NotifySysadmin(int) +// Purpose: Run the script to tell the sysadmin about events +// which need attention. +// Created: 25/2/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::NotifySysadmin(SysadminNotifier::EventCode Event) +{ + static const char *sEventNames[] = + { + "store-full", + "read-error", + "backup-error", + "backup-start", + "backup-finish", + "backup-ok", + 0 + }; + + // BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames)); + // BOX_TRACE("sizeof(*sEventNames) == " << sizeof(*sEventNames)); + // BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX); + ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == SysadminNotifier::MAX + 1); + + if(Event < 0 || Event >= SysadminNotifier::MAX) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + BadNotifySysadminEventCode, "NotifySysadmin() called " + "for unknown event code " << Event); + } + + BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << + sEventNames[Event]); + + if(!GetConfiguration().KeyExists("NotifyAlways") || + !GetConfiguration().GetKeyValueBool("NotifyAlways")) + { + // Don't send lots of repeated messages + // Note: backup-start and backup-finish will always be + // logged, because mLastNotifiedEvent is never set to + // these values and therefore they are never "duplicates". + if(mLastNotifiedEvent == Event) + { + if(Event == SysadminNotifier::BackupOK) + { + BOX_INFO("Suppressing duplicate notification " + "about " << sEventNames[Event]); + } + else + { + BOX_WARNING("Suppressing duplicate notification " + "about " << sEventNames[Event]); + } + return; + } + } + + // Is there a notification script? + const Configuration &conf(GetConfiguration()); + if(!conf.KeyExists("NotifyScript")) + { + // Log, and then return + if(Event != SysadminNotifier::BackupStart && + Event != SysadminNotifier::BackupFinish) + { + BOX_INFO("Not notifying administrator about event " + << sEventNames[Event] << ", set NotifyScript " + "to do this in future"); + } + return; + } + + // Script to run + std::string script(conf.GetKeyValue("NotifyScript") + " " + + sEventNames[Event] + " \"" + GetConfigFileName() + "\""); + + // Log what we're about to do + BOX_INFO("About to notify administrator about event " + << sEventNames[Event] << ", running script '" << script << "'"); + + // Then do it + int returnCode = ::system(script.c_str()); + if(returnCode != 0) + { + BOX_WARNING("Notify script returned error code: " << + returnCode << " (" << script << ")"); + } + else if(Event != SysadminNotifier::BackupStart && + Event != SysadminNotifier::BackupFinish) + { + mLastNotifiedEvent = Event; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &) +// Purpose: Deletes any unused entries in the root directory, if they're scheduled to be deleted. +// Created: 13/5/04 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) +{ + if(mUnusedRootDirEntries.empty()) + { + BOX_INFO("Not deleting unused entries - none in list"); + return; + } + + if(mDeleteUnusedRootDirEntriesAfter == 0) + { + BOX_INFO("Not deleting unused entries - " + "zero delete time (bad)"); + return; + } + + // Check time + box_time_t now = GetCurrentBoxTime(); + if(now < mDeleteUnusedRootDirEntriesAfter) + { + int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter + - now); + BOX_INFO("Not deleting unused entries - too early (" + << secs << " seconds remaining)"); + return; + } + + // Entries to delete, and it's the right time to do so... + BOX_NOTICE("Deleting unused locations from store root..."); + BackupProtocolCallable &connection(rContext.GetConnection()); + for(std::vector >::iterator + i(mUnusedRootDirEntries.begin()); + i != mUnusedRootDirEntries.end(); ++i) + { + connection.QueryDeleteDirectory(i->first); + rContext.GetProgressNotifier().NotifyFileDeleted( + i->first, i->second); + } + + // Reset state + mDeleteUnusedRootDirEntriesAfter = 0; + mUnusedRootDirEntries.clear(); +} + +// -------------------------------------------------------------------------- + +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mNumEntries; + int64_t mObjectID; // this object ID + int64_t mContainerID; // ID of container + uint64_t mAttributesModTime; + int32_t mOptionsPresent; // bit mask of optional sections / features present + +} loc_StreamFormat; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CommandSocketInfo::CommandSocketInfo() +// Purpose: Constructor +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +BackupDaemon::CommandSocketInfo::CommandSocketInfo() + : mpGetLine(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::CommandSocketInfo::~CommandSocketInfo() +// Purpose: Destructor +// Created: 18/2/04 +// +// -------------------------------------------------------------------------- +BackupDaemon::CommandSocketInfo::~CommandSocketInfo() +{ + if(mpGetLine) + { + delete mpGetLine; + mpGetLine = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SerializeStoreObjectInfo( +// box_time_t theLastSyncTime, +// box_time_t theNextSyncTime) +// Purpose: Serializes remote directory and file information +// into a stream of bytes, using an Archive +// abstraction. +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- + +static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; +static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; +static const int STOREOBJECTINFO_VERSION = 2; + +bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime, + box_time_t theNextSyncTime) const +{ + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + return false; + } + + std::string StoreObjectInfoFile = + GetConfiguration().GetKeyValue("StoreObjectInfoFile"); + + if(StoreObjectInfoFile.size() <= 0) + { + return false; + } + + bool created = false; + + try + { + FileStream aFile(StoreObjectInfoFile.c_str(), + O_WRONLY | O_CREAT | O_TRUNC); + created = true; + + Archive anArchive(aFile, 0); + + anArchive.Write(STOREOBJECTINFO_MAGIC_ID_VALUE); + anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); + anArchive.Write(STOREOBJECTINFO_VERSION); + anArchive.Write(GetLoadedConfigModifiedTime()); + anArchive.Write(mClientStoreMarker); + anArchive.Write(theLastSyncTime); + anArchive.Write(theNextSyncTime); + + // + // + // + int64_t iCount = mLocations.size(); + anArchive.Write(iCount); + + for(Locations::const_iterator i = mLocations.begin(); + i != mLocations.end(); i++) + { + ASSERT(*i); + (*i)->Serialize(anArchive); + } + + // + // + // + iCount = mIDMapMounts.size(); + anArchive.Write(iCount); + + for(int v = 0; v < iCount; v++) + anArchive.Write(mIDMapMounts[v]); + + // + // + // + iCount = mUnusedRootDirEntries.size(); + anArchive.Write(iCount); + + for(int v = 0; v < iCount; v++) + { + anArchive.Write(mUnusedRootDirEntries[v].first); + anArchive.Write(mUnusedRootDirEntries[v].second); + } + + if (iCount > 0) + { + anArchive.Write(mDeleteUnusedRootDirEntriesAfter); + } + + // + // + // + aFile.Close(); + BOX_INFO("Saved store object info file version " << + STOREOBJECTINFO_VERSION << " (" << + StoreObjectInfoFile << ")"); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to write StoreObjectInfoFile: " << + StoreObjectInfoFile << ": " << e.what()); + } + catch(...) + { + BOX_ERROR("Failed to write StoreObjectInfoFile: " << + StoreObjectInfoFile << ": unknown error"); + } + + return created; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeserializeStoreObjectInfo( +// box_time_t & theLastSyncTime, +// box_time_t & theNextSyncTime) +// Purpose: Deserializes remote directory and file information +// from a stream of bytes, using an Archive +// abstraction. +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, + box_time_t & theNextSyncTime) +{ + // + // + // + DeleteAllLocations(); + + // + // + // + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + BOX_NOTICE("Store object info file is not enabled. Will " + "download directory listings from store."); + return false; + } + + std::string StoreObjectInfoFile = + GetConfiguration().GetKeyValue("StoreObjectInfoFile"); + + if(StoreObjectInfoFile.size() <= 0) + { + return false; + } + + int64_t fileSize; + if (!FileExists(StoreObjectInfoFile, &fileSize) || fileSize == 0) + { + BOX_NOTICE(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file does not exist or is empty")); + } + else + { + try + { + FileStream aFile(StoreObjectInfoFile, O_RDONLY); + Archive anArchive(aFile, 0); + + // + // see if the content looks like a valid serialised archive + // + int iMagicValue = 0; + anArchive.Read(iMagicValue); + + if(iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is not a valid " + "or compatible serialised archive")); + return false; + } + + // + // get a bit optimistic and read in a string identifier + // + std::string strMagicValue; + anArchive.Read(strMagicValue); + + if(strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is not a valid " + "or compatible serialised archive")); + return false; + } + + // + // check if we are loading some future format + // version by mistake + // + int iVersion = 0; + anArchive.Read(iVersion); + + if(iVersion != STOREOBJECTINFO_VERSION) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file version " << + iVersion << " is not supported")); + return false; + } + + // + // check if this state file is even valid + // for the loaded bbackupd.conf file + // + box_time_t lastKnownConfigModTime; + anArchive.Read(lastKnownConfigModTime); + + if(lastKnownConfigModTime != GetLoadedConfigModifiedTime()) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is older than " + "configuration file")); + return false; + } + + // + // this is it, go at it + // + anArchive.Read(mClientStoreMarker); + anArchive.Read(theLastSyncTime); + anArchive.Read(theNextSyncTime); + + // + // + // + int64_t iCount = 0; + anArchive.Read(iCount); + + for(int v = 0; v < iCount; v++) + { + Location* pLocation = new Location; + if(!pLocation) + { + throw std::bad_alloc(); + } + + pLocation->Deserialize(anArchive); + mLocations.push_back(pLocation); + } + + // + // + // + iCount = 0; + anArchive.Read(iCount); + + for(int v = 0; v < iCount; v++) + { + std::string strItem; + anArchive.Read(strItem); + + mIDMapMounts.push_back(strItem); + } + + // + // + // + iCount = 0; + anArchive.Read(iCount); + + for(int v = 0; v < iCount; v++) + { + int64_t anId; + anArchive.Read(anId); + + std::string aName; + anArchive.Read(aName); + + mUnusedRootDirEntries.push_back(std::pair(anId, aName)); + } + + if (iCount > 0) + anArchive.Read(mDeleteUnusedRootDirEntriesAfter); + + // + // + // + aFile.Close(); + + BOX_INFO(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Loaded store object info file version " << iVersion)); + return true; + } + catch(std::exception &e) + { + BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Internal error reading store object info " + "file: " << e.what())); + } + catch(...) + { + BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Internal error reading store object info " + "file: unknown error")); + } + } + + BOX_NOTICE("No usable cache, will download directory listings from " + "server."); + + DeleteAllLocations(); + + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; + theLastSyncTime = 0; + theNextSyncTime = 0; + + return false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeleteStoreObjectInfo() +// Purpose: Deletes the serialised state file, to prevent us +// from using it again if a backup is interrupted. +// +// Created: 2006/02/12 +// +// -------------------------------------------------------------------------- + +bool BackupDaemon::DeleteStoreObjectInfo() const +{ + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + return false; + } + + std::string storeObjectInfoFile(GetConfiguration().GetKeyValue("StoreObjectInfoFile")); + + // Check to see if the file exists + if(!FileExists(storeObjectInfoFile.c_str())) + { + // File doesn't exist -- so can't be deleted. But something + // isn't quite right, so log a message + BOX_WARNING("StoreObjectInfoFile did not exist when it " + "was supposed to: " << storeObjectInfoFile); + + // Return true to stop things going around in a loop + return true; + } + + // Actually delete it + if(::unlink(storeObjectInfoFile.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("Failed to delete the old " + "StoreObjectInfoFile: " << storeObjectInfoFile); + return false; + } + + return true; +} diff --git a/lib/bbackupd/BackupDaemon.h b/lib/bbackupd/BackupDaemon.h new file mode 100644 index 00000000..ffe31247 --- /dev/null +++ b/lib/bbackupd/BackupDaemon.h @@ -0,0 +1,539 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemon.h +// Purpose: Backup daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPDAEMON__H +#define BACKUPDAEMON__H + +#include +#include +#include + +#include "BackupClientContext.h" +#include "BackupClientDirectoryRecord.h" +#include "BoxTime.h" +#include "Daemon.h" +#include "Logging.h" +#include "Socket.h" +#include "SocketListen.h" +#include "SocketStream.h" +#include "TLSContext.h" + +#include "autogen_BackupProtocol.h" +#include "autogen_BackupStoreException.h" + +#ifdef WIN32 + #include "WinNamedPipeListener.h" + #include "WinNamedPipeStream.h" +#endif + +#ifdef ENABLE_VSS +# include +# include +# include +# include +#endif + +#define COMMAND_SOCKET_POLL_INTERVAL 1000 + +class BackupClientDirectoryRecord; +class BackupClientContext; +class Configuration; +class BackupClientInodeToIDMap; +class ExcludeList; +class IOStreamGetLine; +class Archive; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupDaemon +// Purpose: Backup daemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class BackupDaemon : public Daemon, public ProgressNotifier, public LocationResolver, +public RunStatusProvider, public SysadminNotifier, public BackgroundTask +{ +public: + BackupDaemon(); + ~BackupDaemon(); + +private: + // methods below do partial (specialized) serialization of + // client state only + bool SerializeStoreObjectInfo(box_time_t theLastSyncTime, + box_time_t theNextSyncTime) const; + bool DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, + box_time_t & theNextSyncTime); + bool DeleteStoreObjectInfo() const; + BackupDaemon(const BackupDaemon &); + +public: + #ifdef WIN32 + // add command-line options to handle Windows services + std::string GetOptionString(); + int ProcessOption(signed int option); + int Main(const std::string &rConfigFileName); + + // This shouldn't be here, but apparently gcc on + // Windows has no idea about inherited methods... + virtual int Main(const char *DefaultConfigFile, int argc, + const char *argv[]) + { + return Daemon::Main(DefaultConfigFile, argc, argv); + } + #endif + + void Run(); + virtual const char *DaemonName() const; + virtual std::string DaemonBanner() const; + virtual void Usage(); + const ConfigurationVerify *GetConfigVerify() const; + + bool FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const; + + enum + { + // Add stuff to this, make sure the textual equivalents in SetState() are changed too. + State_Initialising = -1, + State_Idle = 0, + State_Connected = 1, + State_Error = 2, + State_StorageLimitExceeded = 3 + }; + + int GetState() {return mState;} + static std::string GetStateName(int state) + { + std::string stateName; + + #define STATE(x) case BackupDaemon::State_ ## x: stateName = #x; break; + switch (state) + { + STATE(Initialising); + STATE(Idle); + STATE(Connected); + STATE(Error); + STATE(StorageLimitExceeded); + default: + stateName = "unknown"; + } + #undef STATE + + return stateName; + } + + // Allow other classes to call this too + void NotifySysadmin(SysadminNotifier::EventCode Event); + +private: + void Run2(); + +public: + void InitCrypto(); + std::auto_ptr RunSyncNowWithExceptionHandling(); + std::auto_ptr RunSyncNow(); + void ResetCachedState(); + void OnBackupStart(); + void OnBackupFinish(); + // TouchFileInWorkingDir is only here for use by Boxi. + // This does NOT constitute an API! + void TouchFileInWorkingDir(const char *Filename); + +protected: + virtual std::auto_ptr GetNewContext + ( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode + ); + +private: + void DeleteAllLocations(); + void SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf); + + void DeleteIDMapVector(std::vector &rVector); + void DeleteAllIDMaps() + { + DeleteIDMapVector(mCurrentIDMaps); + DeleteIDMapVector(mNewIDMaps); + } + void FillIDMapVector(std::vector &rVector, bool NewMaps); + + void SetupIDMapsForSync(); + void CommitIDMapsAfterSync(); + void DeleteCorruptBerkelyDbFiles(); + + void MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const; + + void SetState(int State); + + void WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut); + void CloseCommandConnection(); + void SendSyncStartOrFinish(bool SendStart); + + void DeleteUnusedRootDirEntries(BackupClientContext &rContext); + + // For warning user about potential security hole + virtual void SetupInInitialProcess(); + + int UseScriptToSeeIfSyncAllowed(); + +public: + int ParseSyncAllowScriptOutput(const std::string& script, + const std::string& output); + typedef std::list Locations; + Locations GetLocations() { return mLocations; } + +private: + int mState; // what the daemon is currently doing + + Locations mLocations; + + std::vector mIDMapMounts; + std::vector mCurrentIDMaps; + std::vector mNewIDMaps; + + int mDeleteRedundantLocationsAfter; + + // For the command socket + class CommandSocketInfo + { + public: + CommandSocketInfo(); + ~CommandSocketInfo(); + private: + CommandSocketInfo(const CommandSocketInfo &); // no copying + CommandSocketInfo &operator=(const CommandSocketInfo &); + public: +#ifdef WIN32 + WinNamedPipeListener<1 /* listen backlog */> mListeningSocket; + std::auto_ptr mpConnectedSocket; +#else + SocketListen mListeningSocket; + std::auto_ptr mpConnectedSocket; +#endif + IOStreamGetLine *mpGetLine; + }; + + // Using a socket? + std::auto_ptr mapCommandSocketInfo; + + // Stop notifications being repeated. + SysadminNotifier::EventCode mLastNotifiedEvent; + + // Unused entries in the root directory wait a while before being deleted + box_time_t mDeleteUnusedRootDirEntriesAfter; // time to delete them + std::vector > mUnusedRootDirEntries; + + int64_t mClientStoreMarker; + bool mStorageLimitExceeded; + bool mReadErrorsOnFilesystemObjects; + box_time_t mLastSyncTime, mNextSyncTime; + box_time_t mCurrentSyncStartTime, mUpdateStoreInterval, + mBackupErrorDelay; + TLSContext mTlsContext; + bool mDeleteStoreObjectInfoFile; + bool mDoSyncForcedByPreviousSyncError; + int64_t mNumFilesUploaded, mNumDirsCreated; + int mMaxBandwidthFromSyncAllowScript; + +public: + int GetMaxBandwidthFromSyncAllowScript() { return mMaxBandwidthFromSyncAllowScript; } + bool StopRun() { return this->Daemon::StopRun(); } + bool StorageLimitExceeded() { return mStorageLimitExceeded; } + +private: + bool mLogAllFileAccess; + +public: + ProgressNotifier* GetProgressNotifier() { return mpProgressNotifier; } + LocationResolver* GetLocationResolver() { return mpLocationResolver; } + RunStatusProvider* GetRunStatusProvider() { return mpRunStatusProvider; } + SysadminNotifier* GetSysadminNotifier() { return mpSysadminNotifier; } + void SetProgressNotifier (ProgressNotifier* p) { mpProgressNotifier = p; } + void SetLocationResolver (LocationResolver* p) { mpLocationResolver = p; } + void SetRunStatusProvider(RunStatusProvider* p) { mpRunStatusProvider = p; } + void SetSysadminNotifier (SysadminNotifier* p) { mpSysadminNotifier = p; } + virtual bool RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum); + +private: + ProgressNotifier* mpProgressNotifier; + LocationResolver* mpLocationResolver; + RunStatusProvider* mpRunStatusProvider; + SysadminNotifier* mpSysadminNotifier; + std::auto_ptr mapCommandSocketPollTimer; + std::auto_ptr mapClientContext; + + /* ProgressNotifier implementation */ +public: + virtual void NotifyIDMapsSetup(BackupClientContext& rContext) { } + + virtual void NotifyScanDirectory( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Scanning directory: " << rLocalPath); + } + + if (!RunBackgroundTask(BackgroundTask::Scanning_Dirs, 0, 0)) + { + THROW_EXCEPTION(BackupStoreException, + CancelledByBackgroundTask); + } + } + virtual void NotifyDirStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to access directory: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyFileStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to access file: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyDirListFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to list directory: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyMountPointSkipped( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + #ifdef WIN32 + BOX_WARNING("Ignored directory: " << rLocalPath << + ": is an NTFS junction/reparse point; create " + "a new location if you want to back it up"); + #else + BOX_WARNING("Ignored directory: " << rLocalPath << + ": is a mount point; create a new location " + "if you want to back it up"); + #endif + } + virtual void NotifyFileExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Skipping excluded file: " << rLocalPath); + } + } + virtual void NotifyDirExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Skipping excluded directory: " << rLocalPath); + } + } + virtual void NotifyUnsupportedFileType( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Ignoring file of unknown type: " << rLocalPath); + } + virtual void NotifyFileReadFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Error reading file: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyFileModifiedInFuture( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Some files have modification times excessively " + "in the future. Check clock synchronisation. " + "Example file (only one shown): " << rLocalPath); + } + virtual void NotifyFileSkippedServerFull( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Skipped file: server is full: " << rLocalPath); + } + virtual void NotifyFileUploadException( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const BoxException& rException) + { + if (rException.GetType() == CommonException::ExceptionType && + rException.GetSubType() == CommonException::AccessDenied) + { + BOX_ERROR("Failed to upload file: " << rLocalPath + << ": Access denied"); + } + else + { + BOX_ERROR("Failed to upload file: " << rLocalPath + << ": caught exception: " << rException.what() + << " (" << rException.GetType() + << "/" << rException.GetSubType() << ")"); + } + } + virtual void NotifyFileUploadServerError( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int type, int subtype) + { + BOX_ERROR("Failed to upload file: " << rLocalPath << + ": server error: " << + BackupProtocolError::GetMessage(subtype)); + } + virtual void NotifyFileUploading( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploading complete file: " << rLocalPath); + } + } + virtual void NotifyFileUploadingPatch( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t EstimatedBytesToUpload) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploading patch to file: " << rLocalPath << + ", estimated upload size = " << + EstimatedBytesToUpload); + } + } + virtual void NotifyFileUploadingAttributes( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploading new file attributes: " << + rLocalPath); + } + } + virtual void NotifyFileUploaded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Uploaded file: " << rLocalPath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + "): total size = " << FileSize << ", " + "uploaded size = " << UploadedSize); + } + mNumFilesUploaded++; + } + virtual void NotifyFileSynchronised( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) + { + if (mLogAllFileAccess) + { + BOX_INFO("Synchronised file: " << rLocalPath); + } + } + virtual void NotifyDirectoryCreated( + int64_t ObjectID, + const std::string& rLocalPath, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Created directory: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + mNumDirsCreated++; + } + virtual void NotifyDirectoryDeleted( + int64_t ObjectID, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Deleted directory: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + } + virtual void NotifyFileDeleted( + int64_t ObjectID, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Deleted file: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", " << (length - offset) << " remain, eta " << + BoxTimeToSeconds(finish - elapsed) << "s"); + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", " << (length - offset) << " remain"); + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", unknown bytes remaining"); + } + +#ifdef WIN32 + private: + bool mInstallService, mRemoveService, mRunAsService; + std::string mServiceName; +#endif + +#ifdef ENABLE_VSS + IVssBackupComponents* mpVssBackupComponents; + void CreateVssBackupComponents(); + bool WaitForAsync(IVssAsync *pAsync, const std::string& description); + typedef HRESULT (__stdcall IVssBackupComponents::*AsyncMethod)(IVssAsync**); + bool CallAndWaitForAsync(AsyncMethod method, + const std::string& description); + void CleanupVssBackupComponents(); +#endif +}; + +#endif // BACKUPDAEMON__H diff --git a/lib/bbackupd/BackupDaemonInterface.h b/lib/bbackupd/BackupDaemonInterface.h new file mode 100644 index 00000000..d5c47c85 --- /dev/null +++ b/lib/bbackupd/BackupDaemonInterface.h @@ -0,0 +1,166 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemonInterface.h +// Purpose: Interfaces for managing a BackupDaemon +// Created: 2008/12/30 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPDAEMONINTERFACE__H +#define BACKUPDAEMONINTERFACE__H + +#include + +#include "BoxTime.h" + +class Archive; +class BackupClientContext; +class BackupDaemon; + +// -------------------------------------------------------------------------- +// +// Class +// Name: SysadminNotifier +// Purpose: Provides a NotifySysadmin() method to send mail to the sysadmin +// Created: 2005/11/15 +// +// -------------------------------------------------------------------------- +class SysadminNotifier +{ + public: + virtual ~SysadminNotifier() { } + + typedef enum + { + StoreFull = 0, + ReadError, + BackupError, + BackupStart, + BackupFinish, + BackupOK, + MAX + // When adding notifications, remember to add + // strings to NotifySysadmin() + } + EventCode; + + virtual void NotifySysadmin(EventCode Event) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: ProgressNotifier +// Purpose: Provides methods for the backup library to inform the user +// interface about its progress with the backup +// Created: 2005/11/20 +// +// -------------------------------------------------------------------------- + +class BackupClientContext; +class BackupClientDirectoryRecord; + +class ProgressNotifier +{ + public: + virtual ~ProgressNotifier() { } + virtual void NotifyIDMapsSetup(BackupClientContext& rContext) = 0; + virtual void NotifyScanDirectory( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyDirStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyFileStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyDirListFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyMountPointSkipped( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyDirExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyUnsupportedFileType( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileReadFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyFileModifiedInFuture( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileSkippedServerFull( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploadException( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const BoxException& rException) = 0; + virtual void NotifyFileUploadServerError( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int type, int subtype) = 0; + virtual void NotifyFileUploading( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploadingPatch( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t EstimatedBytesToUpload) = 0; + virtual void NotifyFileUploadingAttributes( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploaded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) = 0; + virtual void NotifyFileSynchronised( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) = 0; + virtual void NotifyDirectoryCreated( + int64_t ObjectID, + const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirectoryDeleted( + int64_t ObjectID, + const std::string& rRemotePath) = 0; + virtual void NotifyFileDeleted( + int64_t ObjectID, + const std::string& rRemotePath) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: LocationResolver +// Purpose: Interface for classes that can resolve locations to paths, +// like BackupDaemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class LocationResolver +{ +public: + virtual ~LocationResolver() { } + virtual bool FindLocationPathName(const std::string &rLocationName, + std::string &rPathOut) const = 0; +}; + +#endif // BACKUPDAEMONINTERFACE__H diff --git a/lib/bbackupd/Win32BackupService.cpp b/lib/bbackupd/Win32BackupService.cpp new file mode 100644 index 00000000..6d027abf --- /dev/null +++ b/lib/bbackupd/Win32BackupService.cpp @@ -0,0 +1,48 @@ +// Win32 service functions for Box Backup, by Nick Knight + +#ifdef WIN32 + +#include "Box.h" +#include "BackupDaemon.h" +#include "MainHelper.h" +#include "BoxPortsAndFiles.h" +#include "BackupStoreException.h" + +#include "MemLeakFindOn.h" + +#include "Win32BackupService.h" + +Win32BackupService* gpDaemonService = NULL; +extern HANDLE gStopServiceEvent; +extern DWORD gServiceReturnCode; + +unsigned int WINAPI RunService(LPVOID lpParameter) +{ + DWORD retVal = gpDaemonService->WinService((const char*) lpParameter); + gServiceReturnCode = retVal; + SetEvent(gStopServiceEvent); + return retVal; +} + +void TerminateService(void) +{ + gpDaemonService->SetTerminateWanted(); +} + +DWORD Win32BackupService::WinService(const char* pConfigFileName) +{ + DWORD ret; + + if (pConfigFileName != NULL) + { + ret = this->Main(pConfigFileName); + } + else + { + ret = this->Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE); + } + + return ret; +} + +#endif // WIN32 diff --git a/lib/bbackupd/Win32BackupService.h b/lib/bbackupd/Win32BackupService.h new file mode 100644 index 00000000..e7f077f2 --- /dev/null +++ b/lib/bbackupd/Win32BackupService.h @@ -0,0 +1,21 @@ +// Box Backup service daemon implementation by Nick Knight + +#ifndef WIN32BACKUPSERVICE_H +#define WIN32BACKUPSERVICE_H + +#ifdef WIN32 + +class Configuration; +class ConfigurationVerify; +class BackupDaemon; + +class Win32BackupService : public BackupDaemon +{ +public: + DWORD WinService(const char* pConfigFileName); +}; + +#endif // WIN32 + +#endif // WIN32BACKUPSERVICE_H + diff --git a/lib/bbackupd/Win32ServiceFunctions.cpp b/lib/bbackupd/Win32ServiceFunctions.cpp new file mode 100644 index 00000000..2df914a7 --- /dev/null +++ b/lib/bbackupd/Win32ServiceFunctions.cpp @@ -0,0 +1,384 @@ +//*************************************************************** +// From the book "Win32 System Services: The Heart of Windows 98 +// and Windows 2000" +// by Marshall Brain +// Published by Prentice Hall +// Copyright 1995 Prentice Hall. +// +// This code implements the Windows API Service interface +// for the Box Backup for Windows native port. +// Adapted for Box Backup by Nick Knight. +//*************************************************************** + +#ifdef WIN32 + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include +#endif +#ifdef HAVE_PROCESS_H + #include +#endif + +extern void TerminateService(void); +extern unsigned int WINAPI RunService(LPVOID lpParameter); + +// Global variables + +TCHAR* gServiceName = TEXT("Box Backup Service"); +SERVICE_STATUS gServiceStatus; +SERVICE_STATUS_HANDLE gServiceStatusHandle = 0; +HANDLE gStopServiceEvent = 0; +DWORD gServiceReturnCode = 0; + +#define SERVICE_NAME "boxbackup" + +void ShowMessage(char *s) +{ + MessageBox(0, s, "Box Backup Message", + MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); +} + +void ErrorHandler(char *s, DWORD err) +{ + char buf[256]; + memset(buf, 0, sizeof(buf)); + _snprintf(buf, sizeof(buf)-1, "%s: %s", s, + GetErrorMessage(err).c_str()); + BOX_ERROR(buf); + MessageBox(0, buf, "Error", + MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); + ExitProcess(err); +} + +void WINAPI ServiceControlHandler( DWORD controlCode ) +{ + switch ( controlCode ) + { + case SERVICE_CONTROL_INTERROGATE: + break; + + case SERVICE_CONTROL_SHUTDOWN: + case SERVICE_CONTROL_STOP: + Beep(1000,100); + TerminateService(); + gServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + SetEvent(gStopServiceEvent); + return; + + case SERVICE_CONTROL_PAUSE: + break; + + case SERVICE_CONTROL_CONTINUE: + break; + + default: + if ( controlCode >= 128 && controlCode <= 255 ) + // user defined control code + break; + else + // unrecognised control code + break; + } + + SetServiceStatus( gServiceStatusHandle, &gServiceStatus ); +} + +// ServiceMain is called when the SCM wants to +// start the service. When it returns, the service +// has stopped. It therefore waits on an event +// just before the end of the function, and +// that event gets set when it is time to stop. +// It also returns on any error because the +// service cannot start if there is an eror. + +static char* spConfigFileName; + +VOID ServiceMain(DWORD argc, LPTSTR *argv) +{ + // initialise service status + gServiceStatus.dwServiceType = SERVICE_WIN32; + gServiceStatus.dwCurrentState = SERVICE_STOPPED; + gServiceStatus.dwControlsAccepted = 0; + gServiceStatus.dwWin32ExitCode = NO_ERROR; + gServiceStatus.dwServiceSpecificExitCode = NO_ERROR; + gServiceStatus.dwCheckPoint = 0; + gServiceStatus.dwWaitHint = 0; + + gServiceStatusHandle = RegisterServiceCtrlHandler(gServiceName, + ServiceControlHandler); + + if (gServiceStatusHandle) + { + // service is starting + gServiceStatus.dwCurrentState = SERVICE_START_PENDING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + // do initialisation here + gStopServiceEvent = CreateEvent(0, TRUE, FALSE, 0); + if (!gStopServiceEvent) + { + gServiceStatus.dwControlsAccepted &= + ~(SERVICE_ACCEPT_STOP | + SERVICE_ACCEPT_SHUTDOWN); + gServiceStatus.dwCurrentState = SERVICE_STOPPED; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + return; + } + + HANDLE ourThread = (HANDLE)_beginthreadex( + NULL, + 0, + RunService, + spConfigFileName, + CREATE_SUSPENDED, + NULL); + + SetThreadPriority(ourThread, THREAD_PRIORITY_LOWEST); + ResumeThread(ourThread); + + // we are now running so tell the SCM + gServiceStatus.dwControlsAccepted |= + (SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); + gServiceStatus.dwCurrentState = SERVICE_RUNNING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + // do cleanup here + WaitForSingleObject(gStopServiceEvent, INFINITE); + CloseHandle(gStopServiceEvent); + gStopServiceEvent = 0; + + // service was stopped + gServiceStatus.dwCurrentState = SERVICE_STOP_PENDING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + // service is now stopped + gServiceStatus.dwControlsAccepted &= + ~(SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); + + gServiceStatus.dwCurrentState = SERVICE_STOPPED; + + if (gServiceReturnCode != 0) + { + gServiceStatus.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR; + gServiceStatus.dwServiceSpecificExitCode = gServiceReturnCode; + } + + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + } +} + +int OurService(const char* pConfigFileName) +{ + spConfigFileName = strdup(pConfigFileName); + + SERVICE_TABLE_ENTRY serviceTable[] = + { + { SERVICE_NAME, (LPSERVICE_MAIN_FUNCTION) ServiceMain }, + { NULL, NULL } + }; + BOOL success; + + // Register with the SCM + success = StartServiceCtrlDispatcher(serviceTable); + + free(spConfigFileName); + spConfigFileName = NULL; + + if (!success) + { + ErrorHandler("Failed to start service. Did you start " + "Box Backup from the Service Control Manager? " + "(StartServiceCtrlDispatcher)", GetLastError()); + return 1; + } + + return 0; +} + +int InstallService(const char* pConfigFileName, const std::string& rServiceName) +{ + if (pConfigFileName != NULL) + { + EMU_STRUCT_STAT st; + + if (emu_stat(pConfigFileName, &st) != 0) + { + BOX_LOG_SYS_ERROR("Failed to open configuration file " + "'" << pConfigFileName << "'"); + return 1; + } + + if (!(st.st_mode & S_IFREG)) + { + + BOX_ERROR("Failed to open configuration file '" << + pConfigFileName << "': not a file"); + return 1; + } + } + + SC_HANDLE scm = OpenSCManager(0, 0, SC_MANAGER_CREATE_SERVICE); + + if (!scm) + { + BOX_ERROR("Failed to open service control manager: " << + GetErrorMessage(GetLastError())); + return 1; + } + + char cmd[MAX_PATH]; + GetModuleFileName(NULL, cmd, sizeof(cmd)-1); + cmd[sizeof(cmd)-1] = 0; + + std::string cmdWithArgs(cmd); + cmdWithArgs += " -s -S \"" + rServiceName + "\""; + + if (pConfigFileName != NULL) + { + cmdWithArgs += " \""; + cmdWithArgs += pConfigFileName; + cmdWithArgs += "\""; + } + + std::string serviceDesc = "Box Backup (" + rServiceName + ")"; + + SC_HANDLE newService = CreateService( + scm, + rServiceName.c_str(), + serviceDesc.c_str(), + SERVICE_ALL_ACCESS, + SERVICE_WIN32_OWN_PROCESS, + SERVICE_AUTO_START, + SERVICE_ERROR_NORMAL, + cmdWithArgs.c_str(), + 0,0,0,0,0); + + DWORD err = GetLastError(); + CloseServiceHandle(scm); + + if (!newService) + { + switch (err) + { + case ERROR_SERVICE_EXISTS: + { + BOX_ERROR("Failed to create Box Backup " + "service: it already exists"); + } + break; + + case ERROR_SERVICE_MARKED_FOR_DELETE: + { + BOX_ERROR("Failed to create Box Backup " + "service: it is waiting to be deleted"); + } + break; + + case ERROR_DUPLICATE_SERVICE_NAME: + { + BOX_ERROR("Failed to create Box Backup " + "service: a service with this name " + "already exists"); + } + break; + + default: + { + BOX_ERROR("Failed to create Box Backup " + "service: error " << + GetErrorMessage(GetLastError())); + } + } + + return 1; + } + + BOX_INFO("Created Box Backup service"); + + SERVICE_DESCRIPTION desc; + desc.lpDescription = "Backs up your data files over the Internet"; + + if (!ChangeServiceConfig2(newService, SERVICE_CONFIG_DESCRIPTION, + &desc)) + { + BOX_WARNING("Failed to set description for Box Backup " + "service: " << GetErrorMessage(GetLastError())); + } + + CloseServiceHandle(newService); + + return 0; +} + +int RemoveService(const std::string& rServiceName) +{ + SC_HANDLE scm = OpenSCManager(0,0,SC_MANAGER_CREATE_SERVICE); + + if (!scm) + { + BOX_ERROR("Failed to open service control manager: " << + GetErrorMessage(GetLastError())); + return 1; + } + + SC_HANDLE service = OpenService(scm, rServiceName.c_str(), + SERVICE_ALL_ACCESS|DELETE); + DWORD err = GetLastError(); + CloseServiceHandle(scm); + + if (!service) + { + if (err == ERROR_SERVICE_DOES_NOT_EXIST || + err == ERROR_IO_PENDING) + // hello microsoft? anyone home? + { + BOX_ERROR("Failed to open Box Backup service: " + "not installed or not found"); + } + else + { + BOX_ERROR("Failed to open Box Backup service: " << + GetErrorMessage(err)); + } + return 1; + } + + SERVICE_STATUS status; + if (!ControlService(service, SERVICE_CONTROL_STOP, &status)) + { + err = GetLastError(); + if (err != ERROR_SERVICE_NOT_ACTIVE) + { + BOX_WARNING("Failed to stop Box Backup service: " << + GetErrorMessage(err)); + } + } + + BOOL deleted = DeleteService(service); + err = GetLastError(); + CloseServiceHandle(service); + + if (deleted) + { + BOX_INFO("Box Backup service deleted"); + return 0; + } + else if (err == ERROR_SERVICE_MARKED_FOR_DELETE) + { + BOX_ERROR("Failed to remove Box Backup service: " + "it is already being deleted"); + } + else + { + BOX_ERROR("Failed to remove Box Backup service: " << + GetErrorMessage(err)); + } + + return 1; +} + +#endif // WIN32 diff --git a/lib/bbackupd/Win32ServiceFunctions.h b/lib/bbackupd/Win32ServiceFunctions.h new file mode 100644 index 00000000..e04c368f --- /dev/null +++ b/lib/bbackupd/Win32ServiceFunctions.h @@ -0,0 +1,19 @@ +//*************************************************************** +// From the book "Win32 System Services: The Heart of Windows 98 +// and Windows 2000" +// by Marshall Brain +// Published by Prentice Hall +// Copyright 1995 Prentice Hall. +// +// This code implements the Windows API Service interface +// for the Box Backup for Windows native port. +//*************************************************************** + +#ifndef WIN32SERVICEFUNCTIONS_H +#define WIN32SERVICEFUNCTIONS_H + +int RemoveService (const std::string& rServiceName); +int InstallService (const char* pConfigFilePath, const std::string& rServiceName); +int OurService (const char* pConfigFileName); + +#endif diff --git a/lib/bbackupquery/BackupQueries.cpp b/lib/bbackupquery/BackupQueries.cpp new file mode 100644 index 00000000..bcb1827e --- /dev/null +++ b/lib/bbackupquery/BackupQueries.cpp @@ -0,0 +1,2410 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupQueries.cpp +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_UNISTD_H + #include +#endif + +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_DIRENT_H + #include +#endif + +#include +#include +#include +#include +#include +#include + +#include "BackupClientFileAttributes.h" +#include "BackupClientMakeExcludeList.h" +#include "BackupClientRestore.h" +#include "BackupQueries.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreFile.h" +#include "BackupStoreFilenameClear.h" +#include "BoxTimeToText.h" +#include "CommonException.h" +#include "Configuration.h" +#include "ExcludeList.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "IOStream.h" +#include "Logging.h" +#include "PathUtils.h" +#include "SelfFlushingStream.h" +#include "Utils.h" +#include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" + +#include "MemLeakFindOn.h" + +// min() and max() macros from stdlib.h break numeric_limits<>::min(), etc. +#undef min +#undef max + +#define COMPARE_RETURN_SAME 1 +#define COMPARE_RETURN_DIFFERENT 2 +#define COMPARE_RETURN_ERROR 3 +#define COMMAND_RETURN_ERROR 4 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::BackupQueries() +// Purpose: Constructor +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +BackupQueries::BackupQueries(BackupProtocolCallable &rConnection, + const Configuration &rConfiguration, bool readWrite) + : mReadWrite(readWrite), + mrConnection(rConnection), + mrConfiguration(rConfiguration), + mQuitNow(false), + mRunningAsRoot(false), + mWarnedAboutOwnerAttributes(false), + mReturnCode(0) // default return code +{ + #ifdef WIN32 + mRunningAsRoot = TRUE; + #else + mRunningAsRoot = (::geteuid() == 0); + #endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::~BackupQueries() +// Purpose: Destructor +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +BackupQueries::~BackupQueries() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::DoCommand(const char *, bool) +// Purpose: Perform a command +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::DoCommand(ParsedCommand& rCommand) +{ + // Check... + + if(rCommand.mFailed) + { + BOX_ERROR("Parse failed: unknown command '" << + rCommand.mCmdElements[0] << "' or failed to convert " + "encoding of arguments"); + return; + } + + if(rCommand.mCmdElements.size() < 1) + { + // blank command + return; + } + + if(rCommand.pSpec->type == Command_sh && + rCommand.mCmdElements.size() == 2) + { + // Yes, run shell command + int result = ::system(rCommand.mCmdElements[1].c_str()); + if(result != 0) + { + BOX_WARNING("System command returned error code " << + result); + SetReturnCode(ReturnCode::Command_Error); + } + return; + } + + if(rCommand.pSpec->type == Command_Unknown) + { + // No such command + BOX_ERROR("Unrecognised command: " << rCommand.mCmdElements[0]); + return; + } + + // Arguments + std::vector args(rCommand.mCmdElements.begin() + 1, + rCommand.mCmdElements.end()); + + // Set up options + bool opts[256]; + for(int o = 0; o < 256; ++o) opts[o] = false; + // BLOCK + { + // options + const char *c = rCommand.mOptions.c_str(); + while(*c != 0) + { + // Valid option? + if(::strchr(rCommand.pSpec->opts, *c) == NULL) + { + BOX_ERROR("Invalid option '" << *c << "' for " + "command " << rCommand.pSpec->name); + return; + } + opts[(int)*c] = true; + ++c; + } + } + + if(rCommand.pSpec->type != Command_Quit) + { + // If not a quit command, set the return code to zero + SetReturnCode(ReturnCode::Command_OK); + } + + // Handle command + switch(rCommand.pSpec->type) + { + case Command_Quit: + mQuitNow = true; + break; + + case Command_List: + CommandList(args, opts); + break; + + case Command_pwd: + { + // Simple implementation, so do it here + BOX_NOTICE(GetCurrentDirectoryName() << " (" << + BOX_FORMAT_OBJECTID(GetCurrentDirectoryID()) << + ")"); + } + break; + + case Command_cd: + CommandChangeDir(args, opts); + break; + + case Command_lcd: + CommandChangeLocalDir(args); + break; + + case Command_sh: + BOX_ERROR("The command to run must be specified as an argument."); + break; + + case Command_GetObject: + CommandGetObject(args, opts); + break; + + case Command_Get: + CommandGet(args, opts); + break; + + case Command_Compare: + CommandCompare(args, opts); + break; + + case Command_Restore: + CommandRestore(args, opts); + break; + + case Command_Usage: + CommandUsage(opts); + break; + + case Command_Help: + CommandHelp(args); + break; + + case Command_Undelete: + CommandUndelete(args, opts); + break; + + case Command_Delete: + CommandDelete(args, opts); + break; + + default: + BOX_ERROR("Unknown command: " << rCommand.mCmdElements[0]); + break; + } +} + +#define LIST_OPTION_TIMES_ATTRIBS 'a' +#define LIST_OPTION_SORT_NO_DIRS_FIRST 'D' +#define LIST_OPTION_NOFLAGS 'F' +#define LIST_OPTION_DISPLAY_HASH 'h' +#define LIST_OPTION_SORT_ID 'i' +#define LIST_OPTION_NOOBJECTID 'I' +#define LIST_OPTION_SORT_REVERSE 'r' +#define LIST_OPTION_RECURSIVE 'R' +#define LIST_OPTION_SIZEINBLOCKS 's' +#define LIST_OPTION_SORT_SIZE 'S' +#define LIST_OPTION_TIMES_LOCAL 't' +#define LIST_OPTION_TIMES_UTC 'T' +#define LIST_OPTION_SORT_NONE 'U' + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandList(const std::vector &, const bool *) +// Purpose: List directories (optionally recursive) +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandList(const std::vector &args, const bool *opts) +{ + // default to using the current directory + int64_t rootDir = GetCurrentDirectoryID(); + + // name of base directory + std::string listRoot; // blank + + // Got a directory in the arguments? + if(args.size() > 0) + { +#ifdef WIN32 + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) + return; +#else + const std::string& storeDirEncoded(args[0]); +#endif + + // Attempt to find the directory + rootDir = FindDirectoryObjectID(storeDirEncoded, + opts[LIST_OPTION_ALLOWOLD], + opts[LIST_OPTION_ALLOWDELETED]); + + if(rootDir == 0) + { + BOX_ERROR("Directory '" << args[0] << "' not found " + "on store."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + } + + // List it + List(rootDir, listRoot, opts, true /* first level to list */); +} + +static std::string GetTimeString(BackupStoreDirectory::Entry& en, + bool useLocalTime, bool showAttrModificationTimes) +{ + std::ostringstream out; + box_time_t originalTime, newAttributesTime; + + // there is no attribute modification time in the directory + // entry, unfortunately, so we can't display it. + originalTime = en.GetModificationTime(); + out << BoxTimeToISO8601String(originalTime, useLocalTime); + + if(en.HasAttributes()) + { + const StreamableMemBlock &storeAttr(en.GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + + box_time_t NewModificationTime, NewAttrModificationTime; + attr.GetModificationTimes(&NewModificationTime, + &NewAttrModificationTime); + + if (showAttrModificationTimes) + { + newAttributesTime = NewAttrModificationTime; + } + else + { + newAttributesTime = NewModificationTime; + } + + if (newAttributesTime == originalTime) + { + out << "*"; + } + else + { + out << "~" << BoxTimeToISO8601String(newAttributesTime, + useLocalTime); + } + } + else + { + out << " "; + } + + return out.str(); +} + +/* We need a way to pass options to sort functions for sorting. The algorithm + * doesn't seem to provide a way to do this, so I'm using a global variable. + * Which is not thread safe, but we don't currently use threads so that should + * be OK. Do not use threads without checking! + */ +const bool *gThreadUnsafeOptions; + +int DirsFirst(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + if (a->IsDir() && !b->IsDir()) + { + return -1; // a < b + } + else if (!a->IsDir() && b->IsDir()) + { + return 1; // b > a + } + else + { + return 0; // continue comparison + } +} + +#define MAYBE_DIRS_FIRST(a, b) \ + if (!gThreadUnsafeOptions[LIST_OPTION_SORT_NO_DIRS_FIRST]) \ + { \ + int result = DirsFirst(a, b); \ + if (result < 0) return true; /* a < b */ \ + else if (result > 0) return false; /* a > b */ \ + /* else: fall through */ \ + } + +#define MAYBE_REVERSE(result) \ + (result != gThreadUnsafeOptions[LIST_OPTION_SORT_REVERSE]) +// result is false, opts[reverse] is false => return false +// result is false, opts[reverse] is true => return true +// result is true, opts[reverse] is false => return true +// result is true, opts[reverse] is true => return false +// this is logical XOR, for which the boolean operator is !=. + +bool SortById(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + bool result = (a->GetObjectID() < b->GetObjectID()); + return MAYBE_REVERSE(result); +} + +bool SortBySize(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + bool result = (a->GetSizeInBlocks() < b->GetSizeInBlocks()); + return MAYBE_REVERSE(result); +} + +bool SortByName(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + BackupStoreFilenameClear afc(a->GetName()); + BackupStoreFilenameClear bfc(b->GetName()); + std::string an = afc.GetClearFilename(); + std::string bn = bfc.GetClearFilename(); + bool result = (an < bn); + return MAYBE_REVERSE(result); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::List(int64_t, const std::string &, const bool *, bool) +// Purpose: Do the actual listing of directories and files +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::List(int64_t DirID, const std::string &rListRoot, + const bool *opts, bool FirstLevel, std::ostream* pOut) +{ +#ifdef WIN32 + DWORD n_chars; + HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE); +#endif + + // Generate exclude flags + int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + if(!opts[LIST_OPTION_ALLOWOLD]) excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + if(!opts[LIST_OPTION_ALLOWDELETED]) excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; + + // Do communication + try + { + mrConnection.QueryListDirectory( + DirID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // both files and directories + excludeFlags, + true /* want attributes */); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to list directory: " << e.what()); + SetReturnCode(ReturnCode::Command_Error); + return; + } + catch (...) + { + BOX_ERROR("Failed to list directory: unknown error"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Store entry pointers in a std::vector for sorting + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + std::vector sorted_entries; + while((en = i.Next()) != 0) + { + sorted_entries.push_back(en); + } + + // Typedef to avoid mind-bending while dealing with pointers to functions. + typedef bool (EntryComparator_t)(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b); + // Default is no comparator, i.e. no sorting. + EntryComparator_t* pComparator = NULL; + + if (opts[LIST_OPTION_SORT_ID]) + { + pComparator = &SortById; + } + else if (opts[LIST_OPTION_SORT_SIZE]) + { + pComparator = &SortBySize; + } + else if (opts[LIST_OPTION_SORT_NONE]) + { + // do nothing + } + else // sort by name + { + pComparator = &SortByName; + } + + if (pComparator != NULL) + { + gThreadUnsafeOptions = opts; + sort(sorted_entries.begin(), sorted_entries.end(), + pComparator); + gThreadUnsafeOptions = NULL; + } + + for (std::vector::const_iterator + i = sorted_entries.begin(); + i != sorted_entries.end(); i++) + { + en = *i; + std::ostringstream buf; + + // Display this entry + BackupStoreFilenameClear clear(en->GetName()); + + // Object ID? + if(!opts[LIST_OPTION_NOOBJECTID]) + { + // add object ID to line + buf << std::hex << std::internal << std::setw(8) << + std::setfill('0') << en->GetObjectID() << + std::dec << " "; + } + + // Flags? + if(!opts[LIST_OPTION_NOFLAGS]) + { + static const char *flags = BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES; + char displayflags[16]; + // make sure f is big enough + ASSERT(sizeof(displayflags) >= sizeof(BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES) + 3); + // Insert flags + char *f = displayflags; + const char *t = flags; + int16_t en_flags = en->GetFlags(); + while(*t != 0) + { + *f = ((en_flags&1) == 0)?'-':*t; + en_flags >>= 1; + f++; + t++; + } + // attributes flags + *(f++) = (en->HasAttributes())?'a':'-'; + + // terminate + *(f++) = ' '; + *(f++) = '\0'; + buf << displayflags; + + if(en_flags != 0) + { + buf << "[ERROR: Entry has additional flags set] "; + } + } + + if(opts[LIST_OPTION_TIMES_UTC]) + { + // Show UTC times... + buf << GetTimeString(*en, false, + opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; + } + + if(opts[LIST_OPTION_TIMES_LOCAL]) + { + // Show local times... + buf << GetTimeString(*en, true, + opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; + } + + if(opts[LIST_OPTION_DISPLAY_HASH]) + { + buf << std::hex << std::internal << std::setw(16) << + std::setfill('0') << en->GetAttributesHash() << + std::dec; + } + + if(opts[LIST_OPTION_SIZEINBLOCKS]) + { + buf << std::internal << std::setw(5) << + std::setfill('0') << en->GetSizeInBlocks() << + " "; + } + + // add name + if(!FirstLevel) + { +#ifdef WIN32 + std::string listRootDecoded; + if(!ConvertUtf8ToConsole(rListRoot.c_str(), + listRootDecoded)) return; + listRootDecoded += "/"; + buf << listRootDecoded; + WriteConsole(hOut, listRootDecoded.c_str(), + strlen(listRootDecoded.c_str()), &n_chars, NULL); +#else + buf << rListRoot << "/"; +#endif + } + + std::string fileName; + try + { + fileName = clear.GetClearFilename(); + } + catch(CipherException &e) + { + fileName = ""; + } + +#ifdef WIN32 + std::string fileNameUtf8 = fileName; + if(!ConvertUtf8ToConsole(fileNameUtf8, fileName)) + { + fileName = fileNameUtf8 + " [convert encoding failed]"; + } +#endif + + buf << fileName; + + if(!en->GetName().IsEncrypted()) + { + buf << " [FILENAME NOT ENCRYPTED]"; + } + + buf << std::endl; + + if(pOut) + { + (*pOut) << buf.str(); + } + else + { +#ifdef WIN32 + std::string line = buf.str(); + if (!WriteConsole(hOut, line.c_str(), line.size(), + &n_chars, NULL)) + { + // WriteConsole failed, try standard method + std::cout << buf.str(); + } +#else + std::cout << buf.str(); +#endif + } + + // Directory? + if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) + { + // Recurse? + if(opts[LIST_OPTION_RECURSIVE]) + { + std::string subroot(rListRoot); + if(!FirstLevel) subroot += '/'; + subroot += clear.GetClearFilename(); + List(en->GetObjectID(), subroot, opts, + false /* not the first level to list */, + pOut); + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::FindDirectoryObjectID(const +// std::string &) +// Purpose: Find the object ID of a directory on the store, +// or return 0 for not found. If pStack != 0, the +// object is set to the stack of directories. +// Will start from the current directory stack. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, + bool AllowOldVersion, bool AllowDeletedDirs, + std::vector > *pStack) +{ + // Split up string into elements + std::vector dirElements; + SplitString(rDirName, '/', dirElements); + + // Start from current stack, or root, whichever is required + std::vector > stack; + int64_t dirID = BackupProtocolListDirectory::RootDirectory; + if(rDirName.size() > 0 && rDirName[0] == '/') + { + // Root, do nothing + } + else + { + // Copy existing stack + stack = mDirStack; + if(stack.size() > 0) + { + dirID = stack[stack.size() - 1].second; + } + } + + // Generate exclude flags + int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + if(!AllowOldVersion) excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + if(!AllowDeletedDirs) excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; + + // Read directories + for(unsigned int e = 0; e < dirElements.size(); ++e) + { + if(dirElements[e].size() > 0) + { + if(dirElements[e] == ".") + { + // Ignore. + } + else if(dirElements[e] == "..") + { + // Up one! + if(stack.size() > 0) + { + // Remove top element + stack.pop_back(); + + // New dir ID + dirID = (stack.size() > 0)?(stack[stack.size() - 1].second):BackupProtocolListDirectory::RootDirectory; + } + else + { + // At root anyway + dirID = BackupProtocolListDirectory::RootDirectory; + } + } + else + { + // Not blank element. Read current directory. + std::auto_ptr dirreply(mrConnection.QueryListDirectory( + dirID, + BackupProtocolListDirectory::Flags_Dir, // just directories + excludeFlags, + true /* want attributes */)); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Then... find the directory within it + BackupStoreDirectory::Iterator i(dir); + BackupStoreFilenameClear dirname(dirElements[e]); + BackupStoreDirectory::Entry *en = i.FindMatchingClearName(dirname); + if(en == 0) + { + // Not found + return 0; + } + + // Object ID for next round of searching + dirID = en->GetObjectID(); + + // Push onto stack + stack.push_back(std::pair(dirElements[e], dirID)); + } + } + } + + // If required, copy the new stack to the caller + if(pStack) + { + *pStack = stack; + } + + return dirID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::GetCurrentDirectoryID() +// Purpose: Returns the ID of the current directory +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::GetCurrentDirectoryID() +{ + // Special case for root + if(mDirStack.size() == 0) + { + return BackupProtocolListDirectory::RootDirectory; + } + + // Otherwise, get from the last entry on the stack + return mDirStack[mDirStack.size() - 1].second; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::GetCurrentDirectoryName() +// Purpose: Gets the name of the current directory +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +std::string BackupQueries::GetCurrentDirectoryName() +{ + // Special case for root + if(mDirStack.size() == 0) + { + return std::string("/"); + } + + // Build path + std::string r; + for(unsigned int l = 0; l < mDirStack.size(); ++l) + { + r += "/"; +#ifdef WIN32 + std::string dirName; + if(!ConvertUtf8ToConsole(mDirStack[l].first.c_str(), dirName)) + return "error"; + r += dirName; +#else + r += mDirStack[l].first; +#endif + } + + return r; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandChangeDir(const std::vector &) +// Purpose: Change directory command +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandChangeDir(const std::vector &args, const bool *opts) +{ + if(args.size() != 1 || args[0].size() == 0) + { + BOX_ERROR("Incorrect usage. cd [-o] [-d] "); + SetReturnCode(ReturnCode::Command_Error); + return; + } + +#ifdef WIN32 + std::string dirName; + if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) return; +#else + const std::string& dirName(args[0]); +#endif + + std::vector > newStack; + int64_t id = FindDirectoryObjectID(dirName, opts['o'], opts['d'], + &newStack); + + if(id == 0) + { + BOX_ERROR("Directory '" << args[0] << "' not found."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Store new stack + mDirStack = newStack; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandChangeLocalDir(const std::vector &) +// Purpose: Change local directory command +// Created: 2003/10/11 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandChangeLocalDir(const std::vector &args) +{ + if(args.size() != 1 || args[0].size() == 0) + { + BOX_ERROR("Incorrect usage. lcd "); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Try changing directory +#ifdef WIN32 + std::string dirName; + if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) + { + BOX_ERROR("Failed to convert path from console encoding."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + int result = ::chdir(dirName.c_str()); +#else + int result = ::chdir(args[0].c_str()); +#endif + if(result != 0) + { + if(errno == ENOENT || errno == ENOTDIR) + { + BOX_ERROR("Directory '" << args[0] << "' does not exist."); + } + else + { + BOX_LOG_SYS_ERROR("Failed to change to directory " + "'" << args[0] << "'"); + } + + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Report current dir + char wd[PATH_MAX]; + if(::getcwd(wd, PATH_MAX) == 0) + { + BOX_LOG_SYS_ERROR("Error getting current directory"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + +#ifdef WIN32 + if(!ConvertUtf8ToConsole(wd, dirName)) + { + BOX_ERROR("Failed to convert new path from console encoding."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + BOX_INFO("Local current directory is now '" << dirName << "'."); +#else + BOX_INFO("Local current directory is now '" << wd << "'."); +#endif +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandGetObject(const std::vector &, const bool *) +// Purpose: Gets an object without any translation. +// Created: 2003/10/11 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandGetObject(const std::vector &args, const bool *opts) +{ + // Check args + if(args.size() != 2) + { + BOX_ERROR("Incorrect usage. getobject " + ""); + return; + } + + int64_t id = ::strtoll(args[0].c_str(), 0, 16); + if(id == std::numeric_limits::min() || id == std::numeric_limits::max() || id == 0) + { + BOX_ERROR("Not a valid object ID (specified in hex): " << + args[0]); + return; + } + + // Does file exist? + EMU_STRUCT_STAT st; + if(EMU_STAT(args[1].c_str(), &st) == 0 || errno != ENOENT) + { + BOX_ERROR("The local file '" << args[1] << " already exists."); + return; + } + + // Open file + FileStream out(args[1].c_str(), O_WRONLY | O_CREAT | O_EXCL); + + // Request that object + try + { + // Request object + std::auto_ptr getobj(mrConnection.QueryGetObject(id)); + + // Stream that object out to the file + std::auto_ptr objectStream(mrConnection.ReceiveStream()); + objectStream->CopyStreamTo(out); + + BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(id) << + " fetched successfully."); + } + catch(ConnectionException &e) + { + if(mrConnection.GetLastErrorType() == BackupProtocolError::Err_DoesNotExist) + { + BOX_ERROR("Object ID " << BOX_FORMAT_OBJECTID(id) << + " does not exist on store."); + ::unlink(args[1].c_str()); + } + else + { + BOX_ERROR("Error occured fetching object."); + ::unlink(args[1].c_str()); + } + } + catch(...) + { + ::unlink(args[1].c_str()); + BOX_ERROR("Error occured fetching object."); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::FindFileID(const std::string& +// rNameOrIdString, const bool *options, +// int64_t *pDirIdOut, std::string* pFileNameOut) +// Purpose: Locate a file on the store (either by name or by +// object ID, depending on opts['i'], where name can +// include a path) and return the file ID, placing the +// directory ID in *pDirIdOut and the filename part +// of the path in *pFileNameOut (if not NULL). +// Created: 2008-09-12 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::FindFileID(const std::string& rNameOrIdString, + const bool *opts, int64_t *pDirIdOut, std::string* pFileNameOut, + int16_t flagsInclude, int16_t flagsExclude, int16_t* pFlagsOut) +{ + // Find object ID somehow + int64_t fileId; + int64_t dirId = GetCurrentDirectoryID(); + std::string fileName = rNameOrIdString; + + if(!opts['i']) + { + // does this remote filename include a path? + std::string::size_type index = fileName.rfind('/'); + if(index != std::string::npos) + { + std::string dirName(fileName.substr(0, index)); + fileName = fileName.substr(index + 1); + + dirId = FindDirectoryObjectID(dirName); + if(dirId == 0) + { + BOX_ERROR("Directory '" << dirName << + "' not found."); + return 0; + } + } + } + + BackupStoreFilenameClear fn(fileName); + + // Need to look it up in the current directory + mrConnection.QueryListDirectory( + dirId, flagsInclude, flagsExclude, + true /* do want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + BackupStoreDirectory::Entry *en; + + if(opts['i']) + { + // Specified as ID. + fileId = ::strtoll(rNameOrIdString.c_str(), 0, 16); + if(fileId == std::numeric_limits::min() || + fileId == std::numeric_limits::max() || + fileId == 0) + { + BOX_ERROR("Not a valid object ID (specified in hex): " + << rNameOrIdString); + return 0; + } + + // Check that the item is actually in the directory + en = dir.FindEntryByID(fileId); + if(en == 0) + { + BOX_ERROR("File ID " << + BOX_FORMAT_OBJECTID(fileId) << + " not found in current directory on store.\n" + "(You can only access files by ID from the " + "current directory.)"); + return 0; + } + } + else + { + // Specified by name, find the object in the directory to get the ID + BackupStoreDirectory::Iterator i(dir); + en = i.FindMatchingClearName(fn); + if(en == 0) + { + BOX_ERROR("Filename '" << rNameOrIdString << "' " + "not found in current directory on store.\n" + "(Subdirectories in path not searched.)"); + return 0; + } + + fileId = en->GetObjectID(); + } + + *pDirIdOut = dirId; + + if(pFlagsOut) + { + *pFlagsOut = en->GetFlags(); + } + + if(pFileNameOut) + { + BackupStoreFilenameClear entryName(en->GetName()); + *pFileNameOut = entryName.GetClearFilename(); + } + + return fileId; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandGet(const std::vector &, const bool *) +// Purpose: Command to get a file from the store +// Created: 2003/10/12 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandGet(std::vector args, const bool *opts) +{ + // At least one argument? + // Check args + if(args.size() < 1 || (opts['i'] && args.size() != 2) || args.size() > 2) + { + BOX_ERROR("Incorrect usage.\n" + "get [] or\n" + "get -i "); + return; + } + + // Find object ID somehow + int64_t fileId, dirId; + std::string localName; + +#ifdef WIN32 + for (std::vector::iterator + i = args.begin(); i != args.end(); i++) + { + std::string out; + if(!ConvertConsoleToUtf8(i->c_str(), out)) + { + BOX_ERROR("Failed to convert encoding."); + return; + } + *i = out; + } +#endif + + int16_t flagsExclude; + + if(opts['i']) + { + // can retrieve anything by ID + flagsExclude = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + } + else + { + // only current versions by name + flagsExclude = + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted; + } + + + fileId = FindFileID(args[0], opts, &dirId, &localName, + BackupProtocolListDirectory::Flags_File, // just files + flagsExclude, NULL /* don't care about flags found */); + + if (fileId == 0) + { + // error already reported + return; + } + + if(opts['i']) + { + // Specified as ID. Must have a local name in the arguments + // (check at beginning of function ensures this) + localName = args[1]; + } + else + { + // Specified by name. Local name already set by FindFileID, + // but may be overridden by user supplying a second argument. + if(args.size() == 2) + { + localName = args[1]; + } + } + + // Does local file already exist? (don't want to overwrite) + EMU_STRUCT_STAT st; + if(EMU_STAT(localName.c_str(), &st) == 0 || errno != ENOENT) + { + BOX_ERROR("The local file " << localName << " already exists, " + "will not overwrite it."); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + // Request it from the store + try + { + // Request object + mrConnection.QueryGetFile(dirId, fileId); + + // Stream containing encoded file + std::auto_ptr objectStream(mrConnection.ReceiveStream()); + + // Decode it + BackupStoreFile::DecodeFile(*objectStream, localName.c_str(), mrConnection.GetTimeout()); + + // Done. + BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(fileId) << + " fetched successfully."); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to fetch file: " << + e.what()); + ::unlink(localName.c_str()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to fetch file: " << + e.what()); + ::unlink(localName.c_str()); + } + catch(...) + { + BOX_ERROR("Failed to fetch file: unknown error"); + ::unlink(localName.c_str()); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CompareParams::CompareParams() +// Purpose: Constructor +// Created: 29/1/04 +// +// -------------------------------------------------------------------------- +BackupQueries::CompareParams::CompareParams(bool QuickCompare, + bool IgnoreExcludes, bool IgnoreAttributes, + box_time_t LatestFileUploadTime) +: BoxBackupCompareParams(QuickCompare, IgnoreExcludes, IgnoreAttributes, + LatestFileUploadTime), + mDifferences(0), + mDifferencesExplainedByModTime(0), + mUncheckedFiles(0), + mExcludedDirs(0), + mExcludedFiles(0) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandCompare(const std::vector &, const bool *) +// Purpose: Command to compare data on the store with local data +// Created: 2003/10/12 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandCompare(const std::vector &args, const bool *opts) +{ + box_time_t LatestFileUploadTime = GetCurrentBoxTime(); + + // Try and work out the time before which all files should be on the server + { + std::string syncTimeFilename(mrConfiguration.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); + syncTimeFilename += "last_sync_start"; + // Stat it to get file time + EMU_STRUCT_STAT st; + if(EMU_STAT(syncTimeFilename.c_str(), &st) == 0) + { + // Files modified after this time shouldn't be on the server, so report errors slightly differently + LatestFileUploadTime = FileModificationTime(st) - + SecondsToBoxTime(mrConfiguration.GetKeyValueInt("MinimumFileAge")); + } + else + { + BOX_WARNING("Failed to determine the time of the last " + "synchronisation -- checks not performed."); + } + } + + // Parameters, including count of differences + BackupQueries::CompareParams params(opts['q'], // quick compare? + opts['E'], // ignore excludes + opts['A'], // ignore attributes + LatestFileUploadTime); + + params.mQuietCompare = opts['Q']; + + // Quick compare? + if(params.QuickCompare()) + { + BOX_WARNING("Quick compare used -- file attributes are not " + "checked."); + } + + if(!opts['l'] && opts['a'] && args.size() == 0) + { + // Compare all locations + const Configuration &rLocations( + mrConfiguration.GetSubConfiguration("BackupLocations")); + std::vector locNames = + rLocations.GetSubConfigurationNames(); + for(std::vector::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + CompareLocation(*pLocName, params); + } + } + else if(opts['l'] && !opts['a'] && args.size() == 1) + { + // Compare one location + CompareLocation(args[0], params); + } + else if(!opts['l'] && !opts['a'] && args.size() == 2) + { + // Compare directory to directory + + // Can't be bothered to do all the hard work to work out which location it's on, and hence which exclude list + if(!params.IgnoreExcludes()) + { + BOX_ERROR("Cannot use excludes on directory to directory comparison -- use -E flag to specify ignored excludes."); + return; + } + else + { + // Do compare + Compare(args[0], args[1], params); + } + } + else + { + BOX_ERROR("Incorrect usage.\ncompare -a\n or compare -l \n or compare "); + return; + } + + if (!params.mQuietCompare) + { + BOX_INFO("[ " << + params.mDifferencesExplainedByModTime << " (of " << + params.mDifferences << ") differences probably " + "due to file modifications after the last upload ]"); + } + + BOX_INFO("Differences: " << params.mDifferences << " (" << + params.mExcludedDirs << " dirs excluded, " << + params.mExcludedFiles << " files excluded, " << + params.mUncheckedFiles << " files not checked)"); + + // Set return code? + if(opts['c']) + { + if (params.mUncheckedFiles != 0) + { + SetReturnCode(ReturnCode::Compare_Error); + } + else if (params.mDifferences != 0) + { + SetReturnCode(ReturnCode::Compare_Different); + } + else + { + SetReturnCode(ReturnCode::Compare_Same); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CompareLocation(const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a location +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::CompareLocation(const std::string &rLocation, + BoxBackupCompareParams &rParams) +{ + // Find the location's sub configuration + const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations")); + if(!locations.SubConfigurationExists(rLocation.c_str())) + { + BOX_ERROR("Location " << rLocation << " does not exist."); + return; + } + const Configuration &loc(locations.GetSubConfiguration(rLocation.c_str())); + + #ifdef WIN32 + { + std::string path = loc.GetKeyValue("Path"); + if (path.size() > 0 && path[path.size()-1] == + DIRECTORY_SEPARATOR_ASCHAR) + { + BOX_WARNING("Location '" << rLocation << "' path ends " + "with '" DIRECTORY_SEPARATOR "', " + "compare may fail!"); + } + } + #endif + + // Generate the exclude lists + if(!rParams.IgnoreExcludes()) + { + rParams.LoadExcludeLists(loc); + } + + // Then get it compared + Compare(std::string("/") + rLocation, loc.GetKeyValue("Path"), rParams); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::Compare(const std::string &, +// const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a store directory against a local directory +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::Compare(const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams) +{ +#ifdef WIN32 + std::string localDirEncoded; + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(rLocalDir.c_str(), localDirEncoded)) return; + if(!ConvertConsoleToUtf8(rStoreDir.c_str(), storeDirEncoded)) return; +#else + const std::string& localDirEncoded(rLocalDir); + const std::string& storeDirEncoded(rStoreDir); +#endif + + // Get the directory ID of the directory -- only use current data + int64_t dirID = FindDirectoryObjectID(storeDirEncoded); + + // Found? + if(dirID == 0) + { + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalDir.c_str(), &st) == 0) + { + if(FileAttrModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyRemoteFileMissing(localDirEncoded, + storeDirEncoded, modifiedAfterLastSync); + return; + } + + // Go! + Compare(dirID, storeDirEncoded, localDirEncoded, rParams); +} + +void BackupQueries::CompareOneFile(int64_t DirID, + BackupStoreDirectory::Entry *pEntry, + const std::string& rLocalPath, + const std::string& rStorePath, + BoxBackupCompareParams &rParams) +{ + int64_t fileId = pEntry->GetObjectID(); + int64_t fileSize = 0; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalPath.c_str(), &st) == 0) + { + fileSize = st.st_size; + } + + try + { + // Files the same flag? + bool equal = true; + + // File modified after last sync flag + bool modifiedAfterLastSync = false; + + bool hasDifferentAttribs = false; + + bool alreadyReported = false; + + if(rParams.QuickCompare()) + { + // Compare file -- fetch it + mrConnection.QueryGetBlockIndexByID(fileId); + + // Stream containing block index + std::auto_ptr blockIndexStream(mrConnection.ReceiveStream()); + + // Compare + equal = BackupStoreFile::CompareFileContentsAgainstBlockIndex( + rLocalPath.c_str(), *blockIndexStream, + mrConnection.GetTimeout()); + } + else + { + // Compare file -- fetch it + mrConnection.QueryGetFile(DirID, pEntry->GetObjectID()); + + // Stream containing encoded file + std::auto_ptr objectStream(mrConnection.ReceiveStream()); + + // Decode it + std::auto_ptr fileOnServerStream; + + // Got additional attributes? + if(pEntry->HasAttributes()) + { + // Use these attributes + const StreamableMemBlock &storeAttr(pEntry->GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + fileOnServerStream.reset( + BackupStoreFile::DecodeFileStream( + *objectStream, + mrConnection.GetTimeout(), + &attr).release()); + } + else + { + // Use attributes stored in file + fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout()).release()); + } + + // Should always be something in the auto_ptr, it's how the interface is defined. But be paranoid. + if(!fileOnServerStream.get()) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Compare attributes + BackupClientFileAttributes localAttr; + box_time_t fileModTime = 0; + localAttr.ReadAttributes(rLocalPath.c_str(), false /* don't zero mod times */, &fileModTime); + modifiedAfterLastSync = (fileModTime > rParams.LatestFileUploadTime()); + bool ignoreAttrModTime = true; + + #ifdef WIN32 + // attr mod time is really + // creation time, so check it + ignoreAttrModTime = false; + #endif + + if(!rParams.IgnoreAttributes() && + #ifdef PLATFORM_DISABLE_SYMLINK_ATTRIB_COMPARE + !fileOnServerStream->IsSymLink() && + #endif + !localAttr.Compare(fileOnServerStream->GetAttributes(), + ignoreAttrModTime, + fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */)) + { + hasDifferentAttribs = true; + } + + // Compare contents, if it's a regular file not a link + // Remember, we MUST read the entire stream from the server. + SelfFlushingStream flushObject(*objectStream); + + if(!fileOnServerStream->IsSymLink()) + { + SelfFlushingStream flushFile(*fileOnServerStream); + // Open the local file + std::auto_ptr apLocalFile; + + try + { + apLocalFile.reset(new FileStream(rLocalPath.c_str())); + } + catch(std::exception &e) + { + rParams.NotifyLocalFileReadFailed(rLocalPath, + rStorePath, fileSize, e); + alreadyReported = true; + } + catch(...) + { + rParams.NotifyLocalFileReadFailed(rLocalPath, + rStorePath, fileSize); + alreadyReported = true; + } + + if(apLocalFile.get()) + { + equal = apLocalFile->CompareWith(*fileOnServerStream, + mrConnection.GetTimeout()); + } + } + } + + rParams.NotifyFileCompared(rLocalPath, rStorePath, fileSize, + hasDifferentAttribs, !equal, modifiedAfterLastSync, + pEntry->HasAttributes()); + } + catch(BoxException &e) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize, + e); + } + catch(std::exception &e) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize, + e); + } + catch(...) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::Compare(int64_t, const std::string &, +// const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a store directory against a local directory +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams) +{ + rParams.NotifyDirComparing(rLocalDir, rStoreDir); + + // Get info on the local directory + EMU_STRUCT_STAT st; + if(EMU_LSTAT(rLocalDir.c_str(), &st) != 0) + { + // What kind of error? + if(errno == ENOTDIR || errno == ENOENT) + { + rParams.NotifyLocalDirMissing(rLocalDir, rStoreDir); + } + else + { + rParams.NotifyLocalDirAccessFailed(rLocalDir, rStoreDir); + } + return; + } + + // Get the directory listing from the store + mrConnection.QueryListDirectory( + DirID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // get everything + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted, + // except for old versions and deleted files + true /* want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Test out the attributes + if(!dir.HasAttributes()) + { + rParams.NotifyStoreDirMissingAttributes(rLocalDir, rStoreDir); + } + else + { + // Fetch the attributes + const StreamableMemBlock &storeAttr(dir.GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + + // Get attributes of local directory + BackupClientFileAttributes localAttr; + localAttr.ReadAttributes(rLocalDir.c_str(), + true /* directories have zero mod times */); + + if(attr.Compare(localAttr, true, true /* ignore modification times */)) + { + rParams.NotifyDirCompared(rLocalDir, rStoreDir, + false, false /* actually we didn't check :) */); + } + else + { + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalDir.c_str(), &st) == 0) + { + if(FileAttrModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyDirCompared(rLocalDir, rStoreDir, + true, modifiedAfterLastSync); + } + } + + // Open the local directory + DIR *dirhandle = ::opendir(rLocalDir.c_str()); + if(dirhandle == 0) + { + rParams.NotifyLocalDirAccessFailed(rLocalDir, rStoreDir); + return; + } + + try + { + // Read the files and directories into sets + std::set localFiles; + std::set localDirs; + struct dirent *localDirEn = 0; + while((localDirEn = readdir(dirhandle)) != 0) + { + // Not . and ..! + if(localDirEn->d_name[0] == '.' && + (localDirEn->d_name[1] == '\0' || (localDirEn->d_name[1] == '.' && localDirEn->d_name[2] == '\0'))) + { + // ignore, it's . or .. + +#ifdef HAVE_VALID_DIRENT_D_TYPE + if (localDirEn->d_type != DT_DIR) + { + BOX_ERROR("d_type does not really " + "work on your platform. " + "Reconfigure Box!"); + return; + } +#endif + + continue; + } + + std::string localDirPath(MakeFullPath(rLocalDir, + localDirEn->d_name)); + std::string storeDirPath(rStoreDir + "/" + + localDirEn->d_name); + +#ifndef HAVE_VALID_DIRENT_D_TYPE + EMU_STRUCT_STAT st; + if(EMU_LSTAT(localDirPath.c_str(), &st) != 0) + { + // Check whether dir is excluded before trying + // to stat it, to fix problems with .gvfs + // directories that are not readable by root + // causing compare to crash: + // http://lists.boxbackup.org/pipermail/boxbackup/2010-January/000013.html + if(rParams.IsExcludedDir(localDirPath)) + { + rParams.NotifyExcludedDir(localDirPath, + storeDirPath); + continue; + } + else + { + THROW_EXCEPTION_MESSAGE(CommonException, + OSFileError, localDirPath); + } + } + + // Entry -- file or dir? + if(S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) + { + // File or symbolic link + localFiles.insert(std::string(localDirEn->d_name)); + } + else if(S_ISDIR(st.st_mode)) + { + // Directory + localDirs.insert(std::string(localDirEn->d_name)); + } +#else + // Entry -- file or dir? + if(localDirEn->d_type == DT_REG || localDirEn->d_type == DT_LNK) + { + // File or symbolic link + localFiles.insert(std::string(localDirEn->d_name)); + } + else if(localDirEn->d_type == DT_DIR) + { + // Directory + localDirs.insert(std::string(localDirEn->d_name)); + } +#endif + } + // Close directory + if(::closedir(dirhandle) != 0) + { + BOX_LOG_SYS_ERROR("Failed to close local directory " + "'" << rLocalDir << "'"); + } + dirhandle = 0; + + // Do the same for the store directories + std::set > storeFiles; + std::set > storeDirs; + + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *storeDirEn = 0; + while((storeDirEn = i.Next()) != 0) + { + // Decrypt filename + BackupStoreFilenameClear name(storeDirEn->GetName()); + + // What is it? + if((storeDirEn->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == BackupStoreDirectory::Entry::Flags_File) + { + // File + storeFiles.insert(std::pair(name.GetClearFilename(), storeDirEn)); + } + else + { + // Dir + storeDirs.insert(std::pair(name.GetClearFilename(), storeDirEn)); + } + } + +#ifdef _MSC_VER + typedef std::set::iterator string_set_iter_t; +#else + typedef std::set::const_iterator string_set_iter_t; +#endif + + // Now compare files. + for(std::set >::const_iterator i = storeFiles.begin(); i != storeFiles.end(); ++i) + { + const std::string& fileName(i->first); + + std::string localPath(MakeFullPath(rLocalDir, fileName)); + std::string storePath(rStoreDir + "/" + fileName); + + rParams.NotifyFileComparing(localPath, storePath); + + // Does the file exist locally? + string_set_iter_t local(localFiles.find(fileName)); + if(local == localFiles.end()) + { + // Not found -- report + rParams.NotifyLocalFileMissing(localPath, + storePath); + } + else + { + CompareOneFile(DirID, i->second, localPath, + storePath, rParams); + + // Remove from set so that we know it's been compared + localFiles.erase(local); + } + } + + // Report any files which exist locally, but not on the store + for(string_set_iter_t i = localFiles.begin(); i != localFiles.end(); ++i) + { + std::string localPath(MakeFullPath(rLocalDir, *i)); + std::string storePath(rStoreDir + "/" + *i); + + // Should this be ignored (ie is excluded)? + if(!rParams.IsExcludedFile(localPath)) + { + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(localPath.c_str(), &st) == 0) + { + if(FileModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyRemoteFileMissing(localPath, + storePath, modifiedAfterLastSync); + } + else + { + rParams.NotifyExcludedFile(localPath, + storePath); + } + } + + // Finished with the files, clear the sets to reduce memory usage slightly + localFiles.clear(); + storeFiles.clear(); + + // Now do the directories, recursively to check subdirectories + for(std::set >::const_iterator i = storeDirs.begin(); i != storeDirs.end(); ++i) + { + std::string localPath(MakeFullPath(rLocalDir, i->first)); + std::string storePath(rStoreDir + "/" + i->first); + + // Does the directory exist locally? + string_set_iter_t local(localDirs.find(i->first)); + if(local == localDirs.end() && + rParams.IsExcludedDir(localPath)) + { + rParams.NotifyExcludedFileNotDeleted(localPath, + storePath); + } + else if(local == localDirs.end()) + { + // Not found -- report + rParams.NotifyLocalFileMissing(localPath, + storePath); + } + else if(rParams.IsExcludedDir(localPath)) + { + // don't recurse into excluded directories + } + else + { + // Compare directory + Compare(i->second->GetObjectID(), + storePath, localPath, rParams); + + // Remove from set so that we know it's been compared + localDirs.erase(local); + } + } + + // Report any directories which exist locally, but not on the store + for(std::set::const_iterator + i = localDirs.begin(); + i != localDirs.end(); ++i) + { + std::string localPath(MakeFullPath(rLocalDir, *i)); + std::string storePath(rStoreDir + "/" + *i); + + // Should this be ignored (ie is excluded)? + if(!rParams.IsExcludedDir(localPath)) + { + bool modifiedAfterLastSync = false; + + // Check the dir modification time + EMU_STRUCT_STAT st; + if(EMU_STAT(localPath.c_str(), &st) == 0 && + FileModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + + rParams.NotifyRemoteFileMissing(localPath, + storePath, modifiedAfterLastSync); + } + else + { + rParams.NotifyExcludedDir(localPath, storePath); + } + } + } + catch(...) + { + if(dirhandle != 0) + { + ::closedir(dirhandle); + } + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandRestore(const std::vector &, const bool *) +// Purpose: Restore a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandRestore(const std::vector &args, const bool *opts) +{ + // Check arguments + if(args.size() < 1 || args.size() > 2) + { + BOX_ERROR("Incorrect usage. restore [-drif] " + "[]"); + return; + } + + // Restoring deleted things? + bool restoreDeleted = opts['d']; + + std::string storeDirEncoded; + + // Get directory ID + int64_t dirID = 0; + if(opts['i']) + { + // Specified as ID. + dirID = ::strtoll(args[0].c_str(), 0, 16); + if(dirID == std::numeric_limits::min() || dirID == std::numeric_limits::max() || dirID == 0) + { + BOX_ERROR("Not a valid object ID (specified in hex): " + << args[0]); + return; + } + std::ostringstream oss; + oss << BOX_FORMAT_OBJECTID(args[0]); + storeDirEncoded = oss.str(); + } + else + { +#ifdef WIN32 + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) + return; +#else + storeDirEncoded = args[0]; +#endif + + // Look up directory ID + dirID = FindDirectoryObjectID(storeDirEncoded, + false /* no old versions */, + restoreDeleted /* find deleted dirs */); + } + + // Allowable? + if(dirID == 0) + { + BOX_ERROR("Directory '" << args[0] << "' not found on server"); + return; + } + + if(dirID == BackupProtocolListDirectory::RootDirectory) + { + BOX_ERROR("Cannot restore the root directory -- restore locations individually."); + return; + } + + std::string localName; + + if(args.size() == 2) + { + #ifdef WIN32 + if(!ConvertConsoleToUtf8(args[1].c_str(), localName)) + { + return; + } + #else + localName = args[1]; + #endif + } + else + { + localName = args[0]; + } + + // Go and restore... + int result; + + try + { + // At TRACE level, we print a line for each file and + // directory, so we don't need dots. + + result = BackupClientRestore(mrConnection, dirID, + storeDirEncoded.c_str(), localName.c_str(), + true /* print progress dots */, restoreDeleted, + false /* don't undelete after restore! */, + opts['r'] /* resume? */, + opts['f'] /* force continue after errors */); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to restore: " << e.what()); + SetReturnCode(ReturnCode::Command_Error); + return; + } + catch(...) + { + BOX_ERROR("Failed to restore: unknown exception"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + + switch(result) + { + case Restore_Complete: + BOX_INFO("Restore complete."); + break; + + case Restore_CompleteWithErrors: + BOX_WARNING("Restore complete, but some files could not be " + "restored."); + break; + + case Restore_ResumePossible: + BOX_ERROR("Resume possible -- repeat command with -r flag " + "to resume."); + SetReturnCode(ReturnCode::Command_Error); + break; + + case Restore_TargetExists: + BOX_ERROR("The target directory exists. You cannot restore " + "over an existing directory."); + SetReturnCode(ReturnCode::Command_Error); + break; + + case Restore_TargetPathNotFound: + BOX_ERROR("The target directory path does not exist.\n" + "To restore to a directory whose parent " + "does not exist, create the parent first."); + SetReturnCode(ReturnCode::Command_Error); + break; + + case Restore_UnknownError: + BOX_ERROR("Unknown error during restore."); + SetReturnCode(ReturnCode::Command_Error); + break; + + default: + BOX_ERROR("Unknown restore result " << result << "."); + SetReturnCode(ReturnCode::Command_Error); + break; + } +} + + + +// These are autogenerated by a script. +extern const char *help_commands[]; +extern const char *help_text[]; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandHelp(const std::vector &args) +// Purpose: Display help on commands +// Created: 15/2/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandHelp(const std::vector &args) +{ + if(args.size() == 0) + { + // Display a list of all commands + printf("Available commands are:\n"); + for(int c = 0; help_commands[c] != 0; ++c) + { + printf(" %s\n", help_commands[c]); + } + printf("Type \"help \" for more information on a command.\n\n"); + } + else + { + // Display help on a particular command + int c; + for(c = 0; help_commands[c] != 0; ++c) + { + if(::strcmp(help_commands[c], args[0].c_str()) == 0) + { + // Found the command, print help + printf("\n%s\n", help_text[c]); + break; + } + } + if(help_commands[c] == 0) + { + printf("No help found for command '%s'\n", args[0].c_str()); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUsage() +// Purpose: Display storage space used on server +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUsage(const bool *opts) +{ + bool MachineReadable = opts['m']; + + // Request full details from the server + std::auto_ptr usage(mrConnection.QueryGetAccountUsage()); + + // Display each entry in turn + int64_t hardLimit = usage->GetBlocksHardLimit(); + int32_t blockSize = usage->GetBlockSize(); + CommandUsageDisplayEntry("Used", usage->GetBlocksUsed(), hardLimit, + blockSize, MachineReadable); + CommandUsageDisplayEntry("Old files", usage->GetBlocksInOldFiles(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Deleted files", usage->GetBlocksInDeletedFiles(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Directories", usage->GetBlocksInDirectories(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Soft limit", usage->GetBlocksSoftLimit(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Hard limit", hardLimit, hardLimit, blockSize, + MachineReadable); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUsageDisplayEntry(const char *, +// int64_t, int64_t, int32_t, bool) +// Purpose: Display an entry in the usage table +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUsageDisplayEntry(const char *Name, int64_t Size, +int64_t HardLimit, int32_t BlockSize, bool MachineReadable) +{ + std::cout << FormatUsageLineStart(Name, MachineReadable) << + FormatUsageBar(Size, Size * BlockSize, HardLimit * BlockSize, + MachineReadable) << std::endl; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUndelete(const std::vector &, const bool *) +// Purpose: Undelete a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUndelete(const std::vector &args, const bool *opts) +{ + if (!mReadWrite) + { + BOX_ERROR("This command requires a read-write connection. " + "Please reconnect with the -w option."); + return; + } + + // Check arguments + if(args.size() != 1) + { + BOX_ERROR("Incorrect usage. undelete or undelete -i "); + return; + } + +#ifdef WIN32 + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) return; +#else + const std::string& storeDirEncoded(args[0]); +#endif + + // Find object ID somehow + int64_t fileId, parentId; + std::string fileName; + int16_t flagsOut; + + fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, + /* include files and directories */ + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + /* include old and deleted files */ + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + &flagsOut); + + if (fileId == 0) + { + // error already reported + return; + } + + // Undelete it on the store + try + { + // Undelete object + if(flagsOut & BackupProtocolListDirectory::Flags_File) + { + mrConnection.QueryUndeleteFile(parentId, fileId); + } + else + { + mrConnection.QueryUndeleteDirectory(fileId); + } + } + catch (BoxException &e) + { + BOX_ERROR("Failed to undelete object: " << + e.what()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to undelete object: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Failed to undelete object: unknown error"); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandDelete(const +// std::vector &, const bool *) +// Purpose: Deletes a file +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandDelete(const std::vector &args, + const bool *opts) +{ + if (!mReadWrite) + { + BOX_ERROR("This command requires a read-write connection. " + "Please reconnect with the -w option."); + return; + } + + // Check arguments + if(args.size() != 1) + { + BOX_ERROR("Incorrect usage. delete "); + return; + } + +#ifdef WIN32 + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) return; +#else + const std::string& storeDirEncoded(args[0]); +#endif + + // Find object ID somehow + int64_t fileId, parentId; + std::string fileName; + int16_t flagsOut; + + fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, + /* include files and directories */ + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + /* exclude old and deleted files */ + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted, + &flagsOut); + + if (fileId == 0) + { + // error already reported + return; + } + + BackupStoreFilenameClear fn(fileName); + + // Delete it on the store + try + { + // Delete object + if(flagsOut & BackupProtocolListDirectory::Flags_File) + { + mrConnection.QueryDeleteFile(parentId, fn); + } + else + { + mrConnection.QueryDeleteDirectory(fileId); + } + } + catch (BoxException &e) + { + BOX_ERROR("Failed to delete object: " << + e.what()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to delete object: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Failed to delete object: unknown error"); + } +} diff --git a/lib/bbackupquery/BackupQueries.h b/lib/bbackupquery/BackupQueries.h new file mode 100644 index 00000000..96df34f5 --- /dev/null +++ b/lib/bbackupquery/BackupQueries.h @@ -0,0 +1,440 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupQueries.h +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPQUERIES__H +#define BACKUPQUERIES__H + +#include +#include +#include + +#include "BoxTime.h" +#include "BoxBackupCompareParams.h" +#include "BackupStoreDirectory.h" + +class BackupProtocolCallable; +class Configuration; +class ExcludeList; + +typedef enum +{ + Command_Unknown = 0, + Command_Quit, + Command_List, + Command_pwd, + Command_cd, + Command_lcd, + Command_sh, + Command_GetObject, + Command_Get, + Command_Compare, + Command_Restore, + Command_Help, + Command_Usage, + Command_Undelete, + Command_Delete, +} +CommandType; + +struct QueryCommandSpecification; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupQueries +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +class BackupQueries +{ +public: + BackupQueries(BackupProtocolCallable &rConnection, + const Configuration &rConfiguration, + bool readWrite); + ~BackupQueries(); +private: + BackupQueries(const BackupQueries &); +public: + struct ParsedCommand + { + std::vector mCmdElements; + std::string mOptions; + std::string mCompleteCommand; + bool mInOptions, mFailed; + QueryCommandSpecification* pSpec; + // mArgCount is the same as mCmdElements.size() for a complete + // command, but if the command line ends in a space, + // e.g. during readline parsing, it can be one greater, + // to indicate that we should complete the next item instead. + size_t mCompleteArgCount; + ParsedCommand(const std::string& Command, + bool isFromCommandLine); + bool IsEmpty() { return mCmdElements.empty(); } + bool IsFailed() { return mFailed; } + }; + + void DoCommand(ParsedCommand& rCommand); + + // Ready to stop? + bool Stop() {return mQuitNow;} + + // Return code? + int GetReturnCode() {return mReturnCode;} + + void List(int64_t DirID, const std::string &rListRoot, const bool *opts, + bool FirstLevel, std::ostream* pOut = NULL); + void CommandList(const std::vector &args, const bool *opts); + + // Commands + void CommandChangeDir(const std::vector &args, const bool *opts); + void CommandChangeLocalDir(const std::vector &args); + void CommandGetObject(const std::vector &args, const bool *opts); + void CommandGet(std::vector args, const bool *opts); + void CommandCompare(const std::vector &args, const bool *opts); + void CommandRestore(const std::vector &args, const bool *opts); + void CommandUndelete(const std::vector &args, const bool *opts); + void CommandDelete(const std::vector &args, + const bool *opts); + void CommandUsage(const bool *opts); + void CommandUsageDisplayEntry(const char *Name, int64_t Size, + int64_t HardLimit, int32_t BlockSize, bool MachineReadable); + void CommandHelp(const std::vector &args); + + class CompareParams : public BoxBackupCompareParams + { + public: + CompareParams(bool QuickCompare, bool IgnoreExcludes, + bool IgnoreAttributes, box_time_t LatestFileUploadTime); + + bool mQuietCompare; + int mDifferences; + int mDifferencesExplainedByModTime; + int mUncheckedFiles; + int mExcludedDirs; + int mExcludedFiles; + + std::string ConvertForConsole(const std::string& rUtf8String) + { + #ifdef WIN32 + std::string output; + + if(!ConvertUtf8ToConsole(rUtf8String.c_str(), output)) + { + BOX_WARNING("Character set conversion failed " + "on string: " << rUtf8String); + return rUtf8String; + } + + return output; + #else + return rUtf8String; + #endif + } + + virtual void NotifyLocalDirMissing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Local directory '" << + ConvertForConsole(rLocalPath) << "' " + "does not exist, but remote directory does."); + mDifferences ++; + } + + virtual void NotifyLocalDirAccessFailed( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_LOG_SYS_WARNING("Failed to access local directory " + "'" << ConvertForConsole(rLocalPath) << "'"); + mUncheckedFiles ++; + } + + virtual void NotifyStoreDirMissingAttributes( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Store directory '" << + ConvertForConsole(rRemotePath) << "' " + "doesn't have attributes."); + } + + virtual void NotifyRemoteFileMissing( + const std::string& rLocalPath, + const std::string& rRemotePath, + bool modifiedAfterLastSync) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "exists, but remote file '" << + ConvertForConsole(rRemotePath) << "' " + "does not."); + mDifferences ++; + + if(modifiedAfterLastSync) + { + mDifferencesExplainedByModTime ++; + BOX_INFO("(the file above was modified after " + "the last sync time -- might be " + "reason for difference)"); + } + } + + virtual void NotifyLocalFileMissing( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Remote file '" << + ConvertForConsole(rRemotePath) << "' " + "exists, but local file '" << + ConvertForConsole(rLocalPath) << "' does not."); + mDifferences ++; + } + + virtual void NotifyExcludedFileNotDeleted( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "is excluded, but remote file '" << + ConvertForConsole(rRemotePath) << "' " + "still exists."); + mDifferences ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + BoxException& rException) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath) << "': " << + rException.what() << " (" << + rException.GetType() << "/" << + rException.GetSubType() << ")"); + mUncheckedFiles ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath) << "': " << + rException.what()); + mUncheckedFiles ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath)); + mUncheckedFiles ++; + } + + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) + { + BOX_ERROR("Failed to read local file '" << + ConvertForConsole(rLocalPath) << "': " << + rException.what()); + mUncheckedFiles ++; + } + + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) + { + BOX_ERROR("Failed to read local file '" << + ConvertForConsole(rLocalPath)); + mUncheckedFiles ++; + } + + virtual void NotifyExcludedFile(const std::string& rLocalPath, + const std::string& rRemotePath) + { + mExcludedFiles ++; + } + + virtual void NotifyExcludedDir(const std::string& rLocalPath, + const std::string& rRemotePath) + { + mExcludedDirs ++; + } + + virtual void NotifyDirComparing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_INFO("Comparing directory: " << rLocalPath); + } + + virtual void NotifyDirCompared( + const std::string& rLocalPath, + const std::string& rRemotePath, + bool HasDifferentAttributes, + bool modifiedAfterLastSync) + { + if(HasDifferentAttributes) + { + BOX_WARNING("Local directory '" << + ConvertForConsole(rLocalPath) << "' " + "has different attributes to " + "store directory '" << + ConvertForConsole(rRemotePath) << "'."); + mDifferences ++; + + if(modifiedAfterLastSync) + { + mDifferencesExplainedByModTime ++; + BOX_INFO("(the directory above was " + "modified after the last sync " + "time -- might be reason for " + "difference)"); + } + } + } + + virtual void NotifyFileComparing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_TRACE("Comparing file: " << rLocalPath); + } + + virtual void NotifyFileCompared(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + bool HasDifferentAttributes, bool HasDifferentContents, + bool ModifiedAfterLastSync, bool NewAttributesApplied) + { + int NewDifferences = 0; + + if(HasDifferentAttributes) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "has different attributes to " + "store file '" << + ConvertForConsole(rRemotePath) << "'."); + NewDifferences ++; + } + + if(HasDifferentContents) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "has different contents to " + "store file '" << + ConvertForConsole(rRemotePath) << "'."); + NewDifferences ++; + } + + if(HasDifferentAttributes || HasDifferentContents) + { + if(ModifiedAfterLastSync) + { + mDifferencesExplainedByModTime += + NewDifferences; + BOX_INFO("(the file above was modified " + "after the last sync time -- " + "might be reason for difference)"); + } + else if(NewAttributesApplied) + { + BOX_INFO("(the file above has had new " + "attributes applied)\n"); + } + } + + mDifferences += NewDifferences; + } + }; + void CompareLocation(const std::string &rLocation, + BoxBackupCompareParams &rParams); + void Compare(const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams); + void Compare(int64_t DirID, const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams); + void CompareOneFile(int64_t DirID, BackupStoreDirectory::Entry *pEntry, + const std::string& rLocalPath, const std::string& rStorePath, + BoxBackupCompareParams &rParams); + +public: + + class ReturnCode + { + public: + typedef enum { + Command_OK = 0, + Compare_Same = 1, + Compare_Different, + Compare_Error, + Command_Error, + } Type; + }; + + // Were private, but needed by completion functions: + int64_t GetCurrentDirectoryID(); + int64_t FindDirectoryObjectID(const std::string &rDirName, + bool AllowOldVersion = false, bool AllowDeletedDirs = false, + std::vector > *pStack = 0); + +private: + + // Utility functions + int64_t FindFileID(const std::string& rNameOrIdString, + const bool *opts, int64_t *pDirIdOut, + std::string* pFileNameOut, int16_t flagsInclude, + int16_t flagsExclude, int16_t* pFlagsOut); + std::string GetCurrentDirectoryName(); + void SetReturnCode(int code) {mReturnCode = code;} + +private: + bool mReadWrite; + BackupProtocolCallable &mrConnection; + const Configuration &mrConfiguration; + bool mQuitNow; + std::vector > mDirStack; + bool mRunningAsRoot; + bool mWarnedAboutOwnerAttributes; + int mReturnCode; +}; + +typedef std::vector (*CompletionHandler) + (BackupQueries::ParsedCommand& rCommand, const std::string& prefix, + BackupProtocolCallable& rProtocol, const Configuration& rConfig, + BackupQueries& rQueries); + +std::vector CompleteCommand(BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolCallable& rProtocol, + const Configuration& rConfig, BackupQueries& rQueries); +std::vector CompleteOptions(BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolCallable& rProtocol, + const Configuration& rConfig, BackupQueries& rQueries); + +#define MAX_COMPLETION_HANDLERS 4 + +struct QueryCommandSpecification +{ + const char* name; + const char* opts; + CommandType type; + CompletionHandler complete[MAX_COMPLETION_HANDLERS]; +}; + +// Data about commands +extern QueryCommandSpecification commands[]; + +extern const char *alias[]; +extern const int aliasIs[]; + +#define LIST_OPTION_ALLOWOLD 'o' +#define LIST_OPTION_ALLOWDELETED 'd' + +#endif // BACKUPQUERIES__H + diff --git a/lib/bbackupquery/BoxBackupCompareParams.h b/lib/bbackupquery/BoxBackupCompareParams.h new file mode 100644 index 00000000..655df947 --- /dev/null +++ b/lib/bbackupquery/BoxBackupCompareParams.h @@ -0,0 +1,112 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxBackupCompareParams.h +// Purpose: Parameters and notifiers for a compare operation +// Created: 2008/12/30 +// +// -------------------------------------------------------------------------- + +#ifndef BOXBACKUPCOMPAREPARAMS__H +#define BOXBACKUPCOMPAREPARAMS__H + +#include +#include + +#include "BoxTime.h" +#include "ExcludeList.h" +#include "BackupClientMakeExcludeList.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: BoxBackupCompareParams +// Purpose: Parameters and notifiers for a compare operation +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +class BoxBackupCompareParams +{ +private: + std::auto_ptr mapExcludeFiles, mapExcludeDirs; + bool mQuickCompare; + bool mIgnoreExcludes; + bool mIgnoreAttributes; + box_time_t mLatestFileUploadTime; + +public: + BoxBackupCompareParams(bool QuickCompare, bool IgnoreExcludes, + bool IgnoreAttributes, box_time_t LatestFileUploadTime) + : mQuickCompare(QuickCompare), + mIgnoreExcludes(IgnoreExcludes), + mIgnoreAttributes(IgnoreAttributes), + mLatestFileUploadTime(LatestFileUploadTime) + { } + + virtual ~BoxBackupCompareParams() { } + + bool QuickCompare() { return mQuickCompare; } + bool IgnoreExcludes() { return mIgnoreExcludes; } + bool IgnoreAttributes() { return mIgnoreAttributes; } + box_time_t LatestFileUploadTime() { return mLatestFileUploadTime; } + + void LoadExcludeLists(const Configuration& rLoc) + { + mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rLoc)); + mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rLoc)); + } + bool IsExcludedFile(const std::string& rLocalPath) + { + if (!mapExcludeFiles.get()) return false; + return mapExcludeFiles->IsExcluded(rLocalPath); + } + bool IsExcludedDir(const std::string& rLocalPath) + { + if (!mapExcludeDirs.get()) return false; + return mapExcludeDirs->IsExcluded(rLocalPath); + } + + virtual void NotifyLocalDirMissing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyLocalDirAccessFailed(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyStoreDirMissingAttributes(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyRemoteFileMissing(const std::string& rLocalPath, + const std::string& rRemotePath, + bool modifiedAfterLastSync) = 0; + virtual void NotifyLocalFileMissing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyExcludedFileNotDeleted(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + BoxException& rException) = 0; + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) = 0; + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) = 0; + virtual void NotifyExcludedFile(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyExcludedDir(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirComparing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirCompared(const std::string& rLocalPath, + const std::string& rRemotePath, bool HasDifferentAttributes, + bool modifiedAfterLastSync) = 0; + virtual void NotifyFileComparing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyFileCompared(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + bool HasDifferentAttributes, bool HasDifferentContents, + bool modifiedAfterLastSync, bool newAttributesApplied) = 0; +}; + +#endif // BOXBACKUPCOMPAREPARAMS__H diff --git a/lib/bbackupquery/CommandCompletion.cpp b/lib/bbackupquery/CommandCompletion.cpp new file mode 100644 index 00000000..761fc97e --- /dev/null +++ b/lib/bbackupquery/CommandCompletion.cpp @@ -0,0 +1,604 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CommandCompletion.cpp +// Purpose: Parts of BackupQueries that depend on readline +// Created: 2011/01/21 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_LIBREADLINE + #ifdef HAVE_READLINE_READLINE_H + #include + #elif defined(HAVE_EDITLINE_READLINE_H) + #include + #elif defined(HAVE_READLINE_H) + #include + #endif +#endif + +#ifdef HAVE_READLINE_HISTORY + #ifdef HAVE_READLINE_HISTORY_H + #include + #elif defined(HAVE_HISTORY_H) + #include + #endif +#endif + +#include + +#include "BackupQueries.h" +#include "Configuration.h" + +#include "autogen_BackupProtocol.h" + +#include "MemLeakFindOn.h" + +#define COMPARE_RETURN_SAME 1 +#define COMPARE_RETURN_DIFFERENT 2 +#define COMPARE_RETURN_ERROR 3 +#define COMMAND_RETURN_ERROR 4 + +#define COMPLETION_FUNCTION(name, code) \ +std::vector Complete ## name( \ + BackupQueries::ParsedCommand& rCommand, \ + const std::string& prefix, \ + BackupProtocolCallable& rProtocol, const Configuration& rConfig, \ + BackupQueries& rQueries) \ +{ \ + std::vector completions; \ + \ + try \ + { \ + code \ + } \ + catch(std::exception &e) \ + { \ + BOX_TRACE("Failed to complete " << prefix << ": " << e.what()); \ + } \ + catch(...) \ + { \ + BOX_TRACE("Failed to complete " << prefix << ": " \ + "unknown error"); \ + } \ + \ + return completions; \ +} + +#define DELEGATE_COMPLETION(name) \ + completions = Complete ## name(rCommand, prefix, rProtocol, rConfig, \ + rQueries); + +COMPLETION_FUNCTION(None,) + +#ifdef HAVE_RL_FILENAME_COMPLETION_FUNCTION + #define RL_FILENAME_COMPLETION_FUNCTION rl_filename_completion_function + #define HAVE_A_FILENAME_COMPLETION_FUNCTION 1 +#elif defined HAVE_FILENAME_COMPLETION_FUNCTION + #define RL_FILENAME_COMPLETION_FUNCTION filename_completion_function + #define HAVE_A_FILENAME_COMPLETION_FUNCTION 1 +#endif + +#ifdef HAVE_A_FILENAME_COMPLETION_FUNCTION +COMPLETION_FUNCTION(Default, + int i = 0; + + while (const char *match = RL_FILENAME_COMPLETION_FUNCTION(prefix.c_str(), i)) + { + completions.push_back(match); + ++i; + } +) +#else // !HAVE_A_FILENAME_COMPLETION_FUNCTION +COMPLETION_FUNCTION(Default,) +#endif // HAVE_A_FILENAME_COMPLETION_FUNCTION + +COMPLETION_FUNCTION(Command, + int len = prefix.length(); + + for(int i = 0; commands[i].name != NULL; i++) + { + if(::strncmp(commands[i].name, prefix.c_str(), len) == 0) + { + completions.push_back(commands[i].name); + } + } +) + +void CompleteOptionsInternal(const std::string& prefix, + BackupQueries::ParsedCommand& rCommand, + std::vector& completions) +{ + std::string availableOptions = rCommand.pSpec->opts; + + for(std::string::iterator + opt = availableOptions.begin(); + opt != availableOptions.end(); opt++) + { + if(rCommand.mOptions.find(*opt) == std::string::npos) + { + if(prefix == "") + { + // complete with possible option strings + completions.push_back(std::string("-") + *opt); + } + else + { + // complete with possible additional options + completions.push_back(prefix + *opt); + } + } + } +} + +COMPLETION_FUNCTION(Options, + CompleteOptionsInternal(prefix, rCommand, completions); +) + +std::string EncodeFileName(const std::string &rUnEncodedName) +{ +#ifdef WIN32 + std::string encodedName; + if(!ConvertConsoleToUtf8(rUnEncodedName, encodedName)) + { + return std::string(); + } + return encodedName; +#else + return rUnEncodedName; +#endif +} + +int16_t GetExcludeFlags(BackupQueries::ParsedCommand& rCommand) +{ + int16_t excludeFlags = 0; + + if (rCommand.mOptions.find(LIST_OPTION_ALLOWOLD) == std::string::npos) + { + excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + } + + if (rCommand.mOptions.find(LIST_OPTION_ALLOWDELETED) == std::string::npos) + { + excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; + } + + return excludeFlags; +} + +std::vector CompleteRemoteFileOrDirectory( + BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolCallable& rProtocol, + BackupQueries& rQueries, int16_t includeFlags) +{ + std::vector completions; + + // default to using the current directory + int64_t listDirId = rQueries.GetCurrentDirectoryID(); + std::string searchPrefix; + std::string listDir = prefix; + + if(rCommand.mCompleteArgCount == rCommand.mCmdElements.size()) + { + // completing an empty name, from the current directory + // nothing to change + } + else + { + // completing a partially-completed subdirectory name + searchPrefix = prefix; + listDir = ""; + + // do we need to list a subdirectory to complete? + size_t lastSlash = searchPrefix.rfind('/'); + if(lastSlash == std::string::npos) + { + // no slashes, so the whole name is the prefix + // nothing to change + } + else + { + // listing a partially-completed subdirectory name + listDir = searchPrefix.substr(0, lastSlash); + + listDirId = rQueries.FindDirectoryObjectID(listDir, + rCommand.mOptions.find(LIST_OPTION_ALLOWOLD) + != std::string::npos, + rCommand.mOptions.find(LIST_OPTION_ALLOWDELETED) + != std::string::npos); + + if(listDirId == 0) + { + // no matches for subdir to list, + // return empty-handed. + return completions; + } + + // matched, and updated listDir and listDirId already + searchPrefix = searchPrefix.substr(lastSlash + 1); + } + } + + // Always include directories, because they contain files. + // We will append a slash later for each directory if we're + // actually looking for files. + // + // If we're looking for directories, then only list directories. + + bool completeFiles = includeFlags & + BackupProtocolListDirectory::Flags_File; + bool completeDirs = includeFlags & + BackupProtocolListDirectory::Flags_Dir; + int16_t listFlags = 0; + + if(completeFiles) + { + listFlags = BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING; + } + else if(completeDirs) + { + listFlags = BackupProtocolListDirectory::Flags_Dir; + } + + rProtocol.QueryListDirectory(listDirId, + listFlags, GetExcludeFlags(rCommand), + false /* no attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(rProtocol.ReceiveStream()); + dir.ReadFromStream(*dirstream, rProtocol.GetTimeout()); + + // Then... display everything + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + BackupStoreFilenameClear clear(en->GetName()); + std::string name = clear.GetClearFilename().c_str(); + if(name.compare(0, searchPrefix.length(), searchPrefix) == 0) + { + bool dir_added = false; + + if(en->IsDir() && + (includeFlags & BackupProtocolListDirectory::Flags_Dir) == 0) + { + // Was looking for a file, but this is a + // directory, so append a slash to the name + name += "/"; + } + + #ifdef HAVE_LIBREADLINE + if(strchr(name.c_str(), ' ')) + { + int n_quote = 0; + + for(int k = strlen(rl_line_buffer); k >= 0; k--) + { + if (rl_line_buffer[k] == '\"') { + ++n_quote; + } + } + + dir_added = false; + + if (!(n_quote % 2)) + { + name = "\"" + (listDir == "" ? name : listDir + "/" + name); + dir_added = true; + } + + name = name + "\""; + } + #endif + + if(listDir == "" || dir_added) + { + completions.push_back(name); + } + else + { + completions.push_back(listDir + "/" + name); + } + } + } + + return completions; +} + +COMPLETION_FUNCTION(RemoteDir, + completions = CompleteRemoteFileOrDirectory(rCommand, prefix, + rProtocol, rQueries, + BackupProtocolListDirectory::Flags_Dir); +) + +COMPLETION_FUNCTION(RemoteFile, + completions = CompleteRemoteFileOrDirectory(rCommand, prefix, + rProtocol, rQueries, + BackupProtocolListDirectory::Flags_File); +) + +COMPLETION_FUNCTION(LocalDir, + DELEGATE_COMPLETION(Default); +) + +COMPLETION_FUNCTION(LocalFile, + DELEGATE_COMPLETION(Default); +) + +COMPLETION_FUNCTION(LocationName, + const Configuration &locations(rConfig.GetSubConfiguration( + "BackupLocations")); + + std::vector locNames = + locations.GetSubConfigurationNames(); + + for(std::vector::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + if(pLocName->compare(0, pLocName->length(), prefix) == 0) + { + completions.push_back(*pLocName); + } + } +) + +COMPLETION_FUNCTION(RemoteFileIdInCurrentDir, + int64_t listDirId = rQueries.GetCurrentDirectoryID(); + int16_t excludeFlags = GetExcludeFlags(rCommand); + + rProtocol.QueryListDirectory( + listDirId, + BackupProtocolListDirectory::Flags_File, + excludeFlags, false /* no attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(rProtocol.ReceiveStream()); + dir.ReadFromStream(*dirstream, rProtocol.GetTimeout()); + + // Then... compare each item + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + std::ostringstream hexId; + hexId << std::hex << en->GetObjectID(); + if(hexId.str().compare(0, prefix.length(), prefix) == 0) + { + completions.push_back(hexId.str()); + } + } +) + +// TODO implement completion of hex IDs up to the maximum according to Usage +COMPLETION_FUNCTION(RemoteId,) + +COMPLETION_FUNCTION(GetFileOrId, + if(rCommand.mOptions.find('i') != std::string::npos) + { + DELEGATE_COMPLETION(RemoteFileIdInCurrentDir); + } + else + { + DELEGATE_COMPLETION(RemoteFile); + } +) + +COMPLETION_FUNCTION(CompareLocationOrRemoteDir, + if(rCommand.mOptions.find('l') != std::string::npos) + { + DELEGATE_COMPLETION(LocationName); + } + else + { + DELEGATE_COMPLETION(RemoteDir); + } +) + +COMPLETION_FUNCTION(CompareNoneOrLocalDir, + if(rCommand.mOptions.find('l') != std::string::npos) + { + // no completions + DELEGATE_COMPLETION(None); + } + else + { + DELEGATE_COMPLETION(LocalDir); + } +) + +COMPLETION_FUNCTION(RestoreRemoteDirOrId, + if(rCommand.mOptions.find('i') != std::string::npos) + { + DELEGATE_COMPLETION(RemoteId); + } + else + { + DELEGATE_COMPLETION(RemoteDir); + } +) + +// Data about commands +QueryCommandSpecification commands[] = +{ + { "quit", "", Command_Quit, {} }, + { "exit", "", Command_Quit, {} }, + { "list", "adDFhiIorRsStTU", Command_List, {CompleteRemoteDir} }, + { "pwd", "", Command_pwd, {} }, + { "cd", "od", Command_cd, {CompleteRemoteDir} }, + { "lcd", "", Command_lcd, {CompleteLocalDir} }, + { "sh", "", Command_sh, {CompleteDefault} }, + { "getobject", "", Command_GetObject, + {CompleteRemoteId, CompleteLocalDir} }, + { "get", "i", Command_Get, + {CompleteGetFileOrId, CompleteLocalDir} }, + { "compare", "alcqAEQ", Command_Compare, + {CompleteCompareLocationOrRemoteDir, CompleteCompareNoneOrLocalDir} }, + { "restore", "drif", Command_Restore, + {CompleteRestoreRemoteDirOrId, CompleteLocalDir} }, + { "help", "", Command_Help, {} }, + { "usage", "m", Command_Usage, {} }, + { "undelete", "i", Command_Undelete, + {CompleteGetFileOrId} }, + { "delete", "i", Command_Delete, {CompleteGetFileOrId} }, + { NULL, NULL, Command_Unknown, {} } +}; + +const char *alias[] = {"ls", 0}; +const int aliasIs[] = {Command_List, 0}; + +BackupQueries::ParsedCommand::ParsedCommand(const std::string& Command, + bool isFromCommandLine) +: mInOptions(false), + mFailed(false), + pSpec(NULL), + mCompleteArgCount(0) +{ + mCompleteCommand = Command; + + // is the command a shell command? + if(Command[0] == 's' && Command[1] == 'h' && Command[2] == ' ' && Command[3] != '\0') + { + // Yes, run shell command + for(int i = 0; commands[i].type != Command_Unknown; i++) + { + if(commands[i].type == Command_sh) + { + pSpec = &(commands[i]); + break; + } + } + + mCmdElements[0] = "sh"; + mCmdElements[1] = Command.c_str() + 3; + return; + } + + // split command into components + bool inQuoted = false; + mInOptions = false; + + std::string currentArg; + for (std::string::const_iterator c = Command.begin(); + c != Command.end(); c++) + { + // Terminating char? + if(*c == ((inQuoted)?'"':' ')) + { + if(!currentArg.empty()) + { + mCmdElements.push_back(currentArg); + + // Because we just found a space, and the last + // word was not options (otherwise currentArg + // would be empty), we've received a complete + // command or non-option argument. + mCompleteArgCount++; + } + + currentArg.resize(0); + inQuoted = false; + mInOptions = false; + } + // Start of quoted parameter? + else if(currentArg.empty() && *c == '"') + { + inQuoted = true; + } + // Start of options? You can't have options if there's no + // command before them, so treat the options as a command (which + // doesn't exist, so it will fail to parse) in that case. + else if(currentArg.empty() && *c == '-' && !mCmdElements.empty()) + { + mInOptions = true; + } + else if(mInOptions) + { + // Option char + mOptions += *c; + } + else + { + // Normal string char, part of current arg + currentArg += *c; + } + } + + if(!currentArg.empty()) + { + mCmdElements.push_back(currentArg); + } + + // If there are no commands then there's nothing to do except return + if(mCmdElements.empty()) + { + return; + } + + // Work out which command it is... + int cmd = 0; + while(commands[cmd].name != 0 && + mCmdElements[0] != commands[cmd].name) + { + cmd++; + } + + if(commands[cmd].name == 0) + { + // Check for aliases + int a; + for(a = 0; alias[a] != 0; ++a) + { + if(mCmdElements[0] == alias[a]) + { + // Found an alias + cmd = aliasIs[a]; + break; + } + } + } + + if(commands[cmd].name == 0) + { + mFailed = true; + return; + } + + pSpec = &(commands[cmd]); + + #ifdef WIN32 + if(isFromCommandLine) + { + std::string converted; + + if(!ConvertEncoding(mCompleteCommand, CP_ACP, converted, + GetConsoleCP())) + { + BOX_ERROR("Failed to convert encoding"); + mFailed = true; + } + + mCompleteCommand = converted; + + for(std::vector::iterator + i = mCmdElements.begin(); + i != mCmdElements.end(); i++) + { + if(!ConvertEncoding(*i, CP_ACP, converted, + GetConsoleCP())) + { + BOX_ERROR("Failed to convert encoding"); + mFailed = true; + } + + *i = converted; + } + } + #endif +} + diff --git a/lib/bbackupquery/documentation.txt b/lib/bbackupquery/documentation.txt new file mode 100644 index 00000000..b16a6f7c --- /dev/null +++ b/lib/bbackupquery/documentation.txt @@ -0,0 +1,194 @@ + +bbackupquery utility -- examine store, compare files, restore, etc. + +This file has markers for automatic help generation script -- '>' marks a start of a command/help topic, +and '<' marks the end of a section. + +Command line: +============= + +> bbackupquery [-q] [-c configfile] [commands ...] + + -q -- quiet, no information prompts + -c -- specify another bbackupd configuation file + +The commands following the options are executed, then (if there was no quit +command) an interactive mode is entered. + +If a command contains a space, enclose it in quotes. Example + + bbackupquery "list testdir1" quit + +to list the contents of testdir1, and then exit without interactive mode. +< + +Commands: +========= + +All directory names relative to a "current" directory, or from root if they +start with '/'. The initial directory is always the root directory. + + +> ls [options] [directory-name] + + List contents of current directory, or specified directory. + + -R -- recursively list all files + -d -- list deleted files/directories + -o -- list old versions of files/directories + -I -- don't display object ID + -F -- don't display flags + -t -- show file modification time in local time + (and attr mod time if has the object has attributes, ~ separated) + -T -- show file modification time in GMT + -a -- show updated attribute instead of file modification time + -s -- show file size in blocks used on server + (only very approximate indication of size locally) + -h -- show file attributes hash + -D -- sort directories together with files (not dirs first) + -i -- sort by object ID (the old default) + -S -- sort by object size in blocks + -U -- don't sort the results (new default is to sort by name) + +list can be used as an alias. +< + +> list + + Alias for 'ls'. Type 'help ls' for options. +< + +> cd [options] + + Change directory + + -d -- consider deleted directories for traversal + -o -- consider old versions of directories for traversal + (this option should never be useful in a correctly formed store) +< + +> pwd + + Print current directory, always root relative. +< + +> lcd + + Change local directory. + + Type "sh ls" to list the contents. +< + +> sh + + All of the parameters after the "sh" are run as a shell command. + + For example, to list the contents of the location directory, type "sh ls" +< + +> get [] +get -i + + Gets a file from the store. Object is specified as the filename within + the current directory, and local filename is optional. Ignores old and + deleted files when searching the directory for the file to retrieve. + + To get an old or deleted file, use the -i option and select the object + as a hex object ID (first column in listing). The local filename must + be specified. +< + +> compare -a +compare -l +compare + + Compares the (current) data on the store with the data on the disc. + All the data will be downloaded -- this is potentially a very long + operation. + + -a -- compare all locations + -l -- compare one backup location as specified in the configuration file. + -c -- set return code + -q -- quick compare. Only checks file contents against checksums, + doesn't do a full download + -A -- ignore attribute differences + -E -- ignore exclusion settings + + Comparing with the root directory is an error, use -a option instead. + + If -c is set, then the return code (if quit is the next command) will be + 1 Comparison was exact + 2 Differences were found + 3 An error occured + This can be used for automated tests. +< + +> restore [-drif] [] + + Restores a directory to the local disc. The local directory specified + must not exist (unless a previous restore is being restarted). If the + local directory is omitted, the default is to restore to the same + directory name and path, relative to the current local directory, + as set with the "lcd" command. + + The root cannot be restored -- restore locations individually. + + -d -- restore a deleted directory or deleted files inside + -r -- resume an interrupted restoration + -i -- directory name is actually an ID + -f -- force restore to continue if errors are encountered + + If a restore operation is interrupted for any reason, it can be restarted + using the -r switch. Restore progress information is saved in a file at + regular intervals during the restore operation to allow restarts. +< + +> getobject + + Gets the object specified by the object id (in hex) and stores the raw + contents in the local file specified. + + This is only useful for debugging as it does not decode files from the + stored format, which is encrypted and compressed. +< + +> usage [-m] + + Show space used on the server for this account. + + -m -- display the output in machine-readable form + + Used: Total amount of space used on the server. + Old files: Space used by old files + Deleted files: Space used by deleted files + Directories: Space used by the directory structure. + + When Used exceeds the soft limit, the server will start to remove old and + deleted files until the usage drops below the soft limit. + + After a while, you would expect to see the usage stay at just below the + soft limit. You only need more space if the space used by old and deleted + files is near zero. +< + +> undelete +undelete -i + + Removes the deleted flag from the specified directory name (in the + current directory) or hex object ID. Be careful not to use this + command where a directory already exists with the same name which is + not marked as deleted. +< + +> delete + + Sets the deleted flag on the specified file name (in the current + directory, or with a relative path). +< + +> quit + + End session and exit. +< + + diff --git a/lib/bbackupquery/makedocumentation.pl.in b/lib/bbackupquery/makedocumentation.pl.in new file mode 100755 index 00000000..530c4ff6 --- /dev/null +++ b/lib/bbackupquery/makedocumentation.pl.in @@ -0,0 +1,75 @@ +#!@PERL@ +use strict; + +print "Creating built-in documentation for bbackupquery...\n"; + +open DOC,"documentation.txt" or die "Can't open documentation.txt file"; +my $section; +my %help; +my @in_order; + +while() +{ + if(m/\A>\s+(\w+)/) + { + $section = $1; + m/\A>\s+(.+)\Z/; + $help{$section} = $1."\n"; + push @in_order,$section; + } + elsif(m/\Aautogen_Documentation.cpp" or die "Can't open output file for writing"; + +print OUT <<__E; +// +// Automatically generated file, do not edit. +// + +#include "Box.h" + +#include "MemLeakFindOn.h" + +const char *help_commands[] = +{ +__E + +for(@in_order) +{ + print OUT qq:\t"$_",\n:; +} + +print OUT <<__E; + 0 +}; + +const char *help_text[] = +{ +__E + +for(@in_order) +{ + my $t = $help{$_}; + $t =~ s/\t/ /g; + $t =~ s/\n/\\n/g; + $t =~ s/"/\\"/g; + print OUT qq:\t"$t",\n:; +} + +print OUT <<__E; + 0 +}; + +__E + +close OUT; -- cgit v1.2.3 From 004c1d3b39f045f0f3935f6ca7cb9cb1960573ea Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sun, 13 Dec 2015 23:44:05 +0000 Subject: Add CTest configurations to CMake. Make some tests work on Windows/MSVC. --- lib/common/Test.cpp | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/common/Test.cpp b/lib/common/Test.cpp index 9385d8c3..df2c3bc6 100644 --- a/lib/common/Test.cpp +++ b/lib/common/Test.cpp @@ -22,7 +22,9 @@ #endif #include "BoxTime.h" +#include "FileStream.h" #include "Test.h" +#include "Utils.h" int num_tests_selected = 0; int num_failures = 0; @@ -81,6 +83,60 @@ bool setUp(const char* function_name) } } +#ifdef _MSC_VER + DIR* pDir = opendir("testfiles"); + if(!pDir) + { + THROW_SYS_FILE_ERROR("Failed to open test temporary directory", + "testfiles", CommonException, Internal); + } + struct dirent* pEntry; + for(pEntry = readdir(pDir); pEntry; pEntry = readdir(pDir)) + { + std::string filename = pEntry->d_name; + if(StartsWith("TestDir", filename) || + StartsWith("0_", filename) || + filename == "accounts.txt" || + StartsWith("file", filename) || + StartsWith("notifyran", filename) || + StartsWith("notifyscript.tag", filename) || + StartsWith("restore", filename) || + filename == "bbackupd-data" || + filename == "syncallowscript.control" || + StartsWith("syncallowscript.notifyran.", filename) || + filename == "test2.downloaded") + { + int filetype = ObjectExists(std::string("testfiles/") + filename); + if(filetype == ObjectExists_File) + { + if(!::unlink(filename.c_str())) + { + TEST_FAIL_WITH_MESSAGE(BOX_SYS_ERROR_MESSAGE("Failed to delete " + "test fixture file: unlink(\"" << filename << "\")")); + } + } + else if(filetype == ObjectExists_Dir) + { + std::string cmd = "rmdir /s /q testfiles\\" + filename; + int status = system(cmd.c_str()); + if(status != 0) + { + TEST_FAIL_WITH_MESSAGE("Failed to delete test fixture " + "file: command '" << cmd << "' exited with " + "status " << status); + } + } + else + { + TEST_FAIL_WITH_MESSAGE("Don't know how to delete file " << filename << + " of type " << filetype); + } + } + } + closedir(pDir); + FileStream touch("testfiles/accounts.txt", O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR); +#else TEST_THAT_THROWONFAIL(system( "rm -rf testfiles/TestDir* testfiles/0_0 testfiles/0_1 " "testfiles/0_2 testfiles/accounts.txt " // testfiles/test* .tgz! @@ -91,11 +147,12 @@ bool setUp(const char* function_name) "testfiles/syncallowscript.notifyran.* " "testfiles/test2.downloaded" ) == 0); + TEST_THAT_THROWONFAIL(system("touch testfiles/accounts.txt") == 0); +#endif TEST_THAT_THROWONFAIL(mkdir("testfiles/0_0", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/0_1", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/0_2", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/bbackupd-data", 0755) == 0); - TEST_THAT_THROWONFAIL(system("touch testfiles/accounts.txt") == 0); return true; } -- cgit v1.2.3 From e9ea5f21a90cca3623115a3c901d033b81f97eb6 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sun, 13 Dec 2015 23:46:22 +0000 Subject: Fix Windows file locking to work with MSVC --- lib/common/NamedLock.cpp | 3 +++ lib/win32/emu.cpp | 2 +- lib/win32/emu.h | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp index 8cacf065..e440408e 100644 --- a/lib/common/NamedLock.cpp +++ b/lib/common/NamedLock.cpp @@ -82,6 +82,9 @@ bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) #if HAVE_DECL_O_EXLOCK flags |= O_NONBLOCK | O_EXLOCK; BOX_TRACE("Trying to create lockfile " << rFilename << " using O_EXLOCK"); +#elif defined BOX_OPEN_LOCK + flags |= BOX_OPEN_LOCK; + BOX_TRACE("Trying to create lockfile " << rFilename << " using BOX_OPEN_LOCK"); #elif !HAVE_DECL_F_SETLK && !defined HAVE_FLOCK // We have no other way to get a lock, so all we can do is fail if // the file already exists, and take the risk of stale locks. diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp index 658bfbbc..65c12b09 100644 --- a/lib/win32/emu.cpp +++ b/lib/win32/emu.cpp @@ -613,7 +613,7 @@ HANDLE openfile(const char *pFileName, int flags, int mode) createDisposition = CREATE_NEW; } - if (flags & O_LOCK) + if (flags & BOX_OPEN_LOCK) { shareMode = 0; } diff --git a/lib/win32/emu.h b/lib/win32/emu.h index 8014316a..80c1d5d8 100644 --- a/lib/win32/emu.h +++ b/lib/win32/emu.h @@ -224,7 +224,7 @@ struct dirent *readdir(DIR *dp); int closedir(DIR *dp); // local constant to open file exclusively without shared access -#define O_LOCK 0x10000 +#define BOX_OPEN_LOCK 0x10000 extern DWORD winerrno; /* used to report errors from openfile() */ HANDLE openfile(const char *filename, int flags, int mode); -- cgit v1.2.3