diff options
Diffstat (limited to 'bin')
25 files changed, 3521 insertions, 1633 deletions
diff --git a/bin/bbackupctl/bbackupctl.cpp b/bin/bbackupctl/bbackupctl.cpp index 8dc8f30e..39f15baf 100644 --- a/bin/bbackupctl/bbackupctl.cpp +++ b/bin/bbackupctl/bbackupctl.cpp @@ -67,13 +67,7 @@ int main(int argc, const char *argv[]) Logging::SetProgramName("bbackupctl"); // Filename for configuration file? - std::string configFilename; - - #ifdef WIN32 - configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; - #else - configFilename = BOX_FILE_BBACKUPD_DEFAULT_CONFIG; - #endif + std::string configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; // Quiet? bool quiet = false; diff --git a/bin/bbackupd/BackupClientContext.cpp b/bin/bbackupd/BackupClientContext.cpp index 6b51b9e8..26df04be 100644 --- a/bin/bbackupd/BackupClientContext.cpp +++ b/bin/bbackupd/BackupClientContext.cpp @@ -25,9 +25,10 @@ #include "BackupStoreConstants.h" #include "BackupStoreException.h" #include "BackupDaemon.h" -#include "autogen_BackupProtocolClient.h" +#include "autogen_BackupProtocol.h" #include "BackupStoreFile.h" #include "Logging.h" +#include "TcpNice.h" #include "MemLeakFindOn.h" @@ -49,29 +50,30 @@ BackupClientContext::BackupClientContext bool ExtendedLogging, bool ExtendedLogToFile, std::string ExtendedLogFile, - ProgressNotifier& rProgressNotifier + ProgressNotifier& rProgressNotifier, + bool TcpNiceMode ) - : mrResolver(rResolver), - mrTLSContext(rTLSContext), - mHostname(rHostname), - mPort(Port), - mAccountNumber(AccountNumber), - mpSocket(0), - mpConnection(0), - mExtendedLogging(ExtendedLogging), - mExtendedLogToFile(ExtendedLogToFile), - mExtendedLogFile(ExtendedLogFile), - mpExtendedLogFileHandle(NULL), - mClientStoreMarker(ClientStoreMarker_NotKnown), - mpDeleteList(0), - mpCurrentIDMap(0), - mpNewIDMap(0), - mStorageLimitExceeded(false), - mpExcludeFiles(0), - mpExcludeDirs(0), - mKeepAliveTimer(0, "KeepAliveTime"), - mbIsManaged(false), - mrProgressNotifier(rProgressNotifier) +: mExperimentalSnapshotMode(false), + mrResolver(rResolver), + mrTLSContext(rTLSContext), + mHostname(rHostname), + mPort(Port), + mAccountNumber(AccountNumber), + mExtendedLogging(ExtendedLogging), + mExtendedLogToFile(ExtendedLogToFile), + mExtendedLogFile(ExtendedLogFile), + mpExtendedLogFileHandle(NULL), + mClientStoreMarker(ClientStoreMarker_NotKnown), + mpDeleteList(0), + mpCurrentIDMap(0), + mpNewIDMap(0), + mStorageLimitExceeded(false), + mpExcludeFiles(0), + mpExcludeDirs(0), + mKeepAliveTimer(0, "KeepAliveTime"), + mbIsManaged(false), + mrProgressNotifier(rProgressNotifier), + mTcpNiceMode(TcpNiceMode) { } @@ -107,40 +109,44 @@ BackupClientContext::~BackupClientContext() BackupProtocolClient &BackupClientContext::GetConnection() { // Already got it? Just return it. - if(mpConnection != 0) + if(mapConnection.get()) { - return *mpConnection; + return *mapConnection; } - // Get a socket connection - if(mpSocket == 0) - { - mpSocket = new SocketStreamTLS; - ASSERT(mpSocket != 0); // will have exceptioned if this was a problem - } + // there shouldn't be a connection open + ASSERT(mapSocket.get() == 0); + // Defensive. Must close connection before releasing any old socket. + mapConnection.reset(); + mapSocket.reset(new SocketStreamTLS); try { // Defensive. - if(mpConnection != 0) - { - delete mpConnection; - mpConnection = 0; - } + mapConnection.reset(); // Log intention BOX_INFO("Opening connection to server '" << mHostname << "'..."); // Connect! - mpSocket->Open(mrTLSContext, Socket::TypeINET, - mHostname.c_str(), mPort); + ((SocketStreamTLS *)(mapSocket.get()))->Open(mrTLSContext, + Socket::TypeINET, mHostname, mPort); - // And create a procotol object - mpConnection = new BackupProtocolClient(*mpSocket); + if(mTcpNiceMode) + { + // Pass control of mapSocket to NiceSocketStream, + // which will take care of destroying it for us. + mapNice.reset(new NiceSocketStream(mapSocket)); + mapConnection.reset(new BackupProtocolClient(*mapNice)); + } + else + { + mapConnection.reset(new BackupProtocolClient(*mapSocket)); + } // Set logging option - mpConnection->SetLogToSysLog(mExtendedLogging); + mapConnection->SetLogToSysLog(mExtendedLogging); if (mExtendedLogToFile) { @@ -156,16 +162,17 @@ BackupProtocolClient &BackupClientContext::GetConnection() } else { - mpConnection->SetLogToFile(mpExtendedLogFileHandle); + mapConnection->SetLogToFile(mpExtendedLogFileHandle); } } // Handshake - mpConnection->Handshake(); + mapConnection->Handshake(); // Check the version of the server { - std::auto_ptr<BackupProtocolClientVersion> serverVersion(mpConnection->QueryVersion(BACKUP_STORE_SERVER_VERSION)); + std::auto_ptr<BackupProtocolVersion> serverVersion( + mapConnection->QueryVersion(BACKUP_STORE_SERVER_VERSION)); if(serverVersion->GetVersion() != BACKUP_STORE_SERVER_VERSION) { THROW_EXCEPTION(BackupStoreException, WrongServerVersion) @@ -173,7 +180,8 @@ BackupProtocolClient &BackupClientContext::GetConnection() } // Login -- if this fails, the Protocol will exception - std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(mpConnection->QueryLogin(mAccountNumber, 0 /* read/write */)); + std::auto_ptr<BackupProtocolLoginConfirmed> loginConf( + mapConnection->QueryLogin(mAccountNumber, 0 /* read/write */)); // Check that the client store marker is the one we expect if(mClientStoreMarker != ClientStoreMarker_NotKnown) @@ -183,9 +191,9 @@ BackupProtocolClient &BackupClientContext::GetConnection() // Not good... finish the connection, abort, etc, ignoring errors try { - mpConnection->QueryFinished(); - mpSocket->Shutdown(); - mpSocket->Close(); + mapConnection->QueryFinished(); + mapNice.reset(); + mapSocket.reset(); } catch(...) { @@ -213,14 +221,13 @@ BackupProtocolClient &BackupClientContext::GetConnection() catch(...) { // Clean up. - delete mpConnection; - mpConnection = 0; - delete mpSocket; - mpSocket = 0; + mapConnection.reset(); + mapNice.reset(); + mapSocket.reset(); throw; } - return *mpConnection; + return *mapConnection; } // -------------------------------------------------------------------------- @@ -233,7 +240,7 @@ BackupProtocolClient &BackupClientContext::GetConnection() // -------------------------------------------------------------------------- void BackupClientContext::CloseAnyOpenConnection() { - if(mpConnection) + if(mapConnection.get()) { try { @@ -244,14 +251,14 @@ void BackupClientContext::CloseAnyOpenConnection() box_time_t marker = GetCurrentBoxTime(); // Set it on the store - mpConnection->QuerySetClientStoreMarker(marker); + mapConnection->QuerySetClientStoreMarker(marker); // Record it so that it can be picked up later. mClientStoreMarker = marker; } // Quit nicely - mpConnection->QueryFinished(); + mapConnection->QueryFinished(); } catch(...) { @@ -259,26 +266,18 @@ void BackupClientContext::CloseAnyOpenConnection() } // Delete it anyway. - delete mpConnection; - mpConnection = 0; + mapConnection.reset(); } - - if(mpSocket) + + try { - try - { - // Be nice about closing the socket - mpSocket->Shutdown(); - mpSocket->Close(); - } - catch(...) - { - // Ignore errors - } - - // Delete object - delete mpSocket; - mpSocket = 0; + // Be nice about closing the socket + mapNice.reset(); + mapSocket.reset(); + } + catch(...) + { + // Ignore errors } // Delete any pending list @@ -307,9 +306,9 @@ void BackupClientContext::CloseAnyOpenConnection() // -------------------------------------------------------------------------- int BackupClientContext::GetTimeout() const { - if(mpConnection) + if(mapConnection.get()) { - return mpConnection->GetTimeout(); + return mapConnection->GetTimeout(); } return (15*60*1000); @@ -419,20 +418,20 @@ bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirec // Request filenames from the server, in a "safe" manner to ignore errors properly { - BackupProtocolClientGetObjectName send(ObjectID, ContainingDirectory); + BackupProtocolGetObjectName send(ObjectID, ContainingDirectory); connection.Send(send); } - std::auto_ptr<BackupProtocolObjectCl> preply(connection.Receive()); + std::auto_ptr<BackupProtocolMessage> preply(connection.Receive()); // Is it of the right type? - if(preply->GetType() != BackupProtocolClientObjectName::TypeID) + if(preply->GetType() != BackupProtocolObjectName::TypeID) { // Was an error or something return false; } // Cast to expected type. - BackupProtocolClientObjectName *names = (BackupProtocolClientObjectName *)(preply.get()); + BackupProtocolObjectName *names = (BackupProtocolObjectName *)(preply.get()); // Anything found? int32_t numElements = names->GetNumNameElements(); @@ -482,10 +481,10 @@ bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirec } // Is it a directory? - rIsDirectoryOut = ((names->GetFlags() & BackupProtocolClientListDirectory::Flags_Dir) == BackupProtocolClientListDirectory::Flags_Dir); + rIsDirectoryOut = ((names->GetFlags() & BackupProtocolListDirectory::Flags_Dir) == BackupProtocolListDirectory::Flags_Dir); // Is it the current version? - rIsCurrentVersionOut = ((names->GetFlags() & (BackupProtocolClientListDirectory::Flags_OldVersion | BackupProtocolClientListDirectory::Flags_Deleted)) == 0); + rIsCurrentVersionOut = ((names->GetFlags() & (BackupProtocolListDirectory::Flags_OldVersion | BackupProtocolListDirectory::Flags_Deleted)) == 0); // And other information which may be required if(pModTimeOnServer) *pModTimeOnServer = names->GetModificationTime(); @@ -509,7 +508,7 @@ void BackupClientContext::SetKeepAliveTime(int iSeconds) { mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; BOX_TRACE("Set keep-alive time to " << mKeepAliveTime << " seconds"); - mKeepAliveTimer = Timer(mKeepAliveTime, "KeepAliveTime"); + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); } // -------------------------------------------------------------------------- @@ -551,7 +550,7 @@ void BackupClientContext::UnManageDiffProcess() // -------------------------------------------------------------------------- void BackupClientContext::DoKeepAlive() { - if (!mpConnection) + if (!mapConnection.get()) { return; } @@ -567,9 +566,9 @@ void BackupClientContext::DoKeepAlive() } BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); - mpConnection->QueryGetIsAlive(); + mapConnection->QueryGetIsAlive(); - mKeepAliveTimer = Timer(mKeepAliveTime, "KeepAliveTime"); + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); } int BackupClientContext::GetMaximumDiffingTime() diff --git a/bin/bbackupd/BackupClientContext.h b/bin/bbackupd/BackupClientContext.h index 404d2d77..1fcc6ede 100644 --- a/bin/bbackupd/BackupClientContext.h +++ b/bin/bbackupd/BackupClientContext.h @@ -16,6 +16,7 @@ #include "BackupDaemonInterface.h" #include "BackupStoreFile.h" #include "ExcludeList.h" +#include "TcpNice.h" #include "Timer.h" class TLSContext; @@ -49,7 +50,8 @@ public: bool ExtendedLogging, bool ExtendedLogToFile, std::string ExtendedLogFile, - ProgressNotifier &rProgressNotifier + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode ); virtual ~BackupClientContext(); private: @@ -207,6 +209,16 @@ public: { return mrProgressNotifier; } + + void SetNiceMode(bool enabled) + { + if(mTcpNiceMode) + { + mapNice->SetEnabled(enabled); + } + } + + bool mExperimentalSnapshotMode; private: LocationResolver &mrResolver; @@ -214,8 +226,9 @@ private: std::string mHostname; int mPort; uint32_t mAccountNumber; - SocketStreamTLS *mpSocket; - BackupProtocolClient *mpConnection; + std::auto_ptr<SocketStream> mapSocket; + std::auto_ptr<NiceSocketStream> mapNice; + std::auto_ptr<BackupProtocolClient> mapConnection; bool mExtendedLogging; bool mExtendedLogToFile; std::string mExtendedLogFile; @@ -232,6 +245,7 @@ private: int mKeepAliveTime; int mMaximumDiffingTime; ProgressNotifier &mrProgressNotifier; + bool mTcpNiceMode; }; #endif // BACKUPCLIENTCONTEXT__H diff --git a/bin/bbackupd/BackupClientDeleteList.cpp b/bin/bbackupd/BackupClientDeleteList.cpp index b9b5b53e..c0414b78 100644 --- a/bin/bbackupd/BackupClientDeleteList.cpp +++ b/bin/bbackupd/BackupClientDeleteList.cpp @@ -13,7 +13,7 @@ #include "BackupClientDeleteList.h" #include "BackupClientContext.h" -#include "autogen_BackupProtocolClient.h" +#include "autogen_BackupProtocol.h" #include "MemLeakFindOn.h" diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp index 84c17dab..90caf2e7 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp @@ -17,21 +17,26 @@ #include <errno.h> #include <string.h> -#include "BackupClientDirectoryRecord.h" -#include "autogen_BackupProtocolClient.h" +#include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" +#include "autogen_ClientException.h" +#include "Archive.h" #include "BackupClientContext.h" -#include "IOStream.h" -#include "MemBlockStream.h" -#include "CommonException.h" -#include "CollectInBufferStream.h" -#include "BackupStoreFile.h" +#include "BackupClientDirectoryRecord.h" #include "BackupClientInodeToIDMap.h" -#include "FileModificationTime.h" #include "BackupDaemon.h" #include "BackupStoreException.h" -#include "Archive.h" -#include "PathUtils.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileEncodeStream.h" +#include "BufferedStream.h" +#include "CommonException.h" +#include "CollectInBufferStream.h" +#include "FileModificationTime.h" +#include "IOStream.h" #include "Logging.h" +#include "MemBlockStream.h" +#include "PathUtils.h" +#include "RateLimitingStream.h" #include "ReadLoggingStream.h" #include "MemLeakFindOn.h" @@ -51,6 +56,7 @@ BackupClientDirectoryRecord::BackupClientDirectoryRecord(int64_t ObjectID, const mSubDirName(rSubDirName), mInitialSyncDone(false), mSyncDone(false), + mSuppressMultipleLinksWarning(false), mpPendingEntries(0) { ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); @@ -98,6 +104,32 @@ void BackupClientDirectoryRecord::DeleteSubDirectories() mSubDirectories.clear(); } +std::string BackupClientDirectoryRecord::ConvertVssPathToRealPath( + const std::string &rVssPath, + const Location& rBackupLocation) +{ +#ifdef ENABLE_VSS + BOX_TRACE("VSS: ConvertVssPathToRealPath: mIsSnapshotCreated = " << + rBackupLocation.mIsSnapshotCreated); + BOX_TRACE("VSS: ConvertVssPathToRealPath: File/Directory Path = " << + rVssPath.substr(0, rBackupLocation.mSnapshotPath.length())); + BOX_TRACE("VSS: ConvertVssPathToRealPath: Snapshot Path = " << + rBackupLocation.mSnapshotPath); + if (rBackupLocation.mIsSnapshotCreated && + rVssPath.substr(0, rBackupLocation.mSnapshotPath.length()) == + rBackupLocation.mSnapshotPath) + { + std::string convertedPath = rBackupLocation.mPath + + rVssPath.substr(rBackupLocation.mSnapshotPath.length()); + BOX_TRACE("VSS: ConvertVssPathToRealPath: Converted Path = " << + convertedPath); + return convertedPath; + } +#endif + + return rVssPath; +} + // -------------------------------------------------------------------------- // // Function @@ -115,6 +147,7 @@ void BackupClientDirectoryRecord::SyncDirectory( int64_t ContainingDirectoryID, const std::string &rLocalPath, const std::string &rRemotePath, + const Location& rBackupLocation, bool ThisDirHasJustBeenCreated) { BackupClientContext& rContext(rParams.mrContext); @@ -160,10 +193,16 @@ void BackupClientDirectoryRecord::SyncDirectory( // just ignore this error. In a future scan, this // deletion will be noticed, deleted from server, // and this object deleted. - rNotifier.NotifyDirStatFailed(this, rLocalPath, + rNotifier.NotifyDirStatFailed(this, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation), strerror(errno)); return; } + + BOX_TRACE("Stat dir '" << rLocalPath << "' " + "found device/inode " << + dest_st.st_dev << "/" << dest_st.st_ino); + // Store inode number in map so directories are tracked // in case they're renamed { @@ -204,12 +243,12 @@ void BackupClientDirectoryRecord::SyncDirectory( { // Report the error (logs and // eventual email to administrator) - rNotifier.NotifyFileStatFailed(this, rLocalPath, + rNotifier.NotifyFileStatFailed(this, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation), strerror(errno)); // FIXME move to NotifyFileStatFailed() - SetErrorWhenReadingFilesystemObject(rParams, - rLocalPath.c_str()); + SetErrorWhenReadingFilesystemObject(rParams, rLocalPath); // This shouldn't happen, so we'd better not continue THROW_EXCEPTION(CommonException, OSFileError) @@ -221,7 +260,9 @@ void BackupClientDirectoryRecord::SyncDirectory( DIR *dirHandle = 0; try { - rNotifier.NotifyScanDirectory(this, rLocalPath); + std::string nonVssDirPath = ConvertVssPathToRealPath(rLocalPath, + rBackupLocation); + rNotifier.NotifyScanDirectory(this, nonVssDirPath); dirHandle = ::opendir(rLocalPath.c_str()); if(dirHandle == 0) @@ -231,18 +272,20 @@ void BackupClientDirectoryRecord::SyncDirectory( if (errno == EACCES) { rNotifier.NotifyDirListFailed(this, - rLocalPath, "Access denied"); + nonVssDirPath, + "Access denied"); } else { rNotifier.NotifyDirListFailed(this, - rLocalPath, strerror(errno)); + nonVssDirPath, + strerror(errno)); } // Report the error (logs and eventual email // to administrator) SetErrorWhenReadingFilesystemObject(rParams, - rLocalPath.c_str()); + nonVssDirPath); // Ignore this directory for now. return; } @@ -279,6 +322,8 @@ void BackupClientDirectoryRecord::SyncDirectory( // Stat file to get info filename = MakeFullPath(rLocalPath, en->d_name); + std::string realFileName = ConvertVssPathToRealPath(filename, + rBackupLocation); #ifdef WIN32 // Don't stat the file just yet, to ensure @@ -289,9 +334,18 @@ void BackupClientDirectoryRecord::SyncDirectory( // Our emulated readdir() abuses en->d_type, // which would normally contain DT_REG, // DT_DIR, etc, but we only use it here and - // prefer S_IFREG, S_IFDIR... - int type = en->d_type; - #else + // prefer to have the full file attributes. + int type; + if (en->d_type & FILE_ATTRIBUTE_DIRECTORY) + { + type = S_IFDIR; + } + else + { + type = S_IFREG; + } + + #else // !WIN32 if(EMU_LSTAT(filename.c_str(), &file_st) != 0) { if(!(rParams.mrContext.ExcludeDir( @@ -306,15 +360,52 @@ void BackupClientDirectoryRecord::SyncDirectory( // FIXME move to // NotifyFileStatFailed() - SetErrorWhenReadingFilesystemObject( - rParams, filename.c_str()); + SetErrorWhenReadingFilesystemObject(rParams, filename); } // Ignore this entry for now. continue; } - if(file_st.st_dev != dest_st.st_dev) + int type = file_st.st_mode & S_IFMT; + + // ecryptfs reports nlink > 1 for directories + // with contents, but no filesystem supports + // hardlinking directories? so we can ignore + // this if the entry is a directory. + if(file_st.st_nlink != 1 && type == S_IFDIR) + { + BOX_INFO("Ignoring apparent hard link " + "count on directory: " << + filename << ", nlink=" << + file_st.st_nlink); + } + else if(file_st.st_nlink > 1) + { + if(!mSuppressMultipleLinksWarning) + { + BOX_WARNING("File is hard linked, this may " + "cause rename tracking to fail and " + "move files incorrectly in your " + "backup! " << filename << + ", nlink=" << file_st.st_nlink << + " (suppressing further warnings)"); + mSuppressMultipleLinksWarning = true; + } + SetErrorWhenReadingFilesystemObject(rParams, filename); + } + + BOX_TRACE("Stat entry '" << filename << "' " + "found device/inode " << + file_st.st_dev << "/" << + file_st.st_ino); + + /* Workaround for apparent btrfs bug, where + symlinks appear to be on a different filesystem + than their containing directory, thanks to + Toke Hoiland-Jorgensen */ + if(type == S_IFDIR && + file_st.st_dev != dest_st.st_dev) { if(!(rParams.mrContext.ExcludeDir( filename))) @@ -324,8 +415,6 @@ void BackupClientDirectoryRecord::SyncDirectory( } continue; } - - int type = file_st.st_mode & S_IFMT; #endif if(type == S_IFREG || type == S_IFLNK) @@ -333,12 +422,9 @@ void BackupClientDirectoryRecord::SyncDirectory( // File or symbolic link // Exclude it? - if(rParams.mrContext.ExcludeFile(filename)) + if(rParams.mrContext.ExcludeFile(realFileName)) { - rNotifier.NotifyFileExcluded( - this, - filename); - + rNotifier.NotifyFileExcluded(this, realFileName); // Next item! continue; } @@ -351,38 +437,50 @@ void BackupClientDirectoryRecord::SyncDirectory( // Directory // Exclude it? - if(rParams.mrContext.ExcludeDir(filename)) + if(rParams.mrContext.ExcludeDir(realFileName)) { - rNotifier.NotifyDirExcluded( - this, - filename); + rNotifier.NotifyDirExcluded(this, realFileName); // Next item! continue; } + #ifdef WIN32 + // exclude reparse points, as Application Data points to the + // parent directory under Vista and later, and causes an + // infinite loop: + // http://social.msdn.microsoft.com/forums/en-US/windowscompatibility/thread/05d14368-25dd-41c8-bdba-5590bf762a68/ + if (en->d_type & FILE_ATTRIBUTE_REPARSE_POINT) + { + rNotifier.NotifyMountPointSkipped(this, realFileName); + continue; + } + #endif + // Store on list dirs.push_back(std::string(en->d_name)); } - else + else // not a file or directory, what is it? { - if (type == S_IFSOCK || type == S_IFIFO) + if (type == S_IFSOCK +# ifndef WIN32 + || type == S_IFIFO +# endif + ) { // removed notification for these types // see Debian bug 479145, no objections } - else if(rParams.mrContext.ExcludeFile(filename)) + else if(rParams.mrContext.ExcludeFile(realFileName)) { - rNotifier.NotifyFileExcluded( - this, - filename); + rNotifier.NotifyFileExcluded(this, realFileName); } else { - rNotifier.NotifyUnsupportedFileType( - this, filename); - SetErrorWhenReadingFilesystemObject( - rParams, filename.c_str()); + rNotifier.NotifyUnsupportedFileType(this, + realFileName); + SetErrorWhenReadingFilesystemObject(rParams, + realFileName); } continue; @@ -397,13 +495,12 @@ void BackupClientDirectoryRecord::SyncDirectory( if(emu_stat(filename.c_str(), &file_st) != 0) { rNotifier.NotifyFileStatFailed(this, - filename, + ConvertVssPathToRealPath(filename, rBackupLocation), strerror(errno)); // Report the error (logs and // eventual email to administrator) - SetErrorWhenReadingFilesystemObject( - rParams, filename.c_str()); + SetErrorWhenReadingFilesystemObject(rParams, filename); // Ignore this entry for now. continue; @@ -412,7 +509,7 @@ void BackupClientDirectoryRecord::SyncDirectory( if(file_st.st_dev != link_st.st_dev) { rNotifier.NotifyMountPointSkipped(this, - filename); + ConvertVssPathToRealPath(filename, rBackupLocation)); continue; } #endif @@ -432,8 +529,8 @@ void BackupClientDirectoryRecord::SyncDirectory( // Log that this has happened if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) { - rNotifier.NotifyFileModifiedInFuture( - this, filename); + rNotifier.NotifyFileModifiedInFuture(this, + ConvertVssPathToRealPath(filename, rBackupLocation)); rParams.mHaveLoggedWarningAboutFutureFileTimes = true; } } @@ -507,7 +604,7 @@ void BackupClientDirectoryRecord::SyncDirectory( // Do the directory reading bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, - rRemotePath, pdirOnStore, entriesLeftOver, files, dirs); + rRemotePath, rBackupLocation, pdirOnStore, entriesLeftOver, files, dirs); // LAST THING! (think exception safety) // Store the new checksum -- don't fetch things unnecessarily in the future @@ -564,10 +661,11 @@ BackupStoreDirectory *BackupClientDirectoryRecord::FetchDirectoryListing(BackupC BackupProtocolClient &connection(rParams.mrContext.GetConnection()); // Query the directory - std::auto_ptr<BackupProtocolClientSuccess> dirreply(connection.QueryListDirectory( + std::auto_ptr<BackupProtocolSuccess> dirreply(connection.QueryListDirectory( mObjectID, - BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, // both files and directories - BackupProtocolClientListDirectory::Flags_Deleted | BackupProtocolClientListDirectory::Flags_OldVersion, // exclude old/deleted stuff + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, // both files and directories + BackupProtocolListDirectory::Flags_Deleted | + BackupProtocolListDirectory::Flags_OldVersion, // exclude old/deleted stuff true /* want attributes */)); // Retrieve the directory from the stream following @@ -630,11 +728,38 @@ void BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord:: BackupProtocolClient &connection(rParams.mrContext.GetConnection()); // Exception thrown if this doesn't work - MemBlockStream attrStream(attr); + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); connection.QueryChangeDirAttributes(mObjectID, attrModTime, attrStream); } } +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath) +{ + BackupStoreFilenameClear fn(en->GetName()); + return DecryptFilename(fn, en->GetObjectID(), rRemoteDirectoryPath); +} + +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreFilenameClear fn, int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath) +{ + std::string filenameClear; + try + { + filenameClear = fn.GetClearFilename(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to decrypt filename for object " << + BOX_FORMAT_OBJECTID(filenameObjectID) << " in " + "directory " << BOX_FORMAT_OBJECTID(mObjectID) << + " (" << rRemoteDirectoryPath << ")"); + throw; + } + return filenameClear; +} // -------------------------------------------------------------------------- // @@ -649,6 +774,7 @@ bool BackupClientDirectoryRecord::UpdateItems( BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rLocalPath, const std::string &rRemotePath, + const Location& rBackupLocation, BackupStoreDirectory *pDirOnStore, std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, std::vector<std::string> &rFiles, @@ -673,7 +799,19 @@ bool BackupClientDirectoryRecord::UpdateItems( BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { - decryptedEntries[BackupStoreFilenameClear(en->GetName()).GetClearFilename()] = en; + std::string filenameClear; + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + decryptedEntries[filenameClear] = en; + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "pretending that the file doesn't " + "exist"); + } } } @@ -686,26 +824,26 @@ bool BackupClientDirectoryRecord::UpdateItems( // Filename of this file std::string filename(MakeFullPath(rLocalPath, *f)); + std::string nonVssFilePath = ConvertVssPathToRealPath(filename, + rBackupLocation); // Get relevant info about file box_time_t modTime = 0; uint64_t attributesHash = 0; int64_t fileSize = 0; InodeRefType inodeNum = 0; - bool hasMultipleHardLinks = true; // BLOCK { // Stat the file EMU_STRUCT_STAT st; if(EMU_LSTAT(filename.c_str(), &st) != 0) { - rNotifier.NotifyFileStatFailed(this, - filename, strerror(errno)); + rNotifier.NotifyFileStatFailed(this, nonVssFilePath, + strerror(errno)); // Report the error (logs and // eventual email to administrator) - SetErrorWhenReadingFilesystemObject(rParams, - filename.c_str()); + SetErrorWhenReadingFilesystemObject(rParams, nonVssFilePath); // Ignore this entry for now. continue; @@ -715,7 +853,6 @@ bool BackupClientDirectoryRecord::UpdateItems( modTime = FileModificationTime(st); fileSize = st.st_size; inodeNum = st.st_ino; - hasMultipleHardLinks = (st.st_nlink > 1); attributesHash = BackupClientFileAttributes::GenerateAttributeHash(st, filename, *f); } @@ -734,7 +871,7 @@ bool BackupClientDirectoryRecord::UpdateItems( } // Check that the entry which might have been found is in fact a file - if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == 0)) + if((en != 0) && !(en->IsFile())) { // Directory exists in the place of this file -- sort it out RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, @@ -760,7 +897,9 @@ bool BackupClientDirectoryRecord::UpdateItems( bool isCurrentVersion = false; box_time_t srvModTime = 0, srvAttributesHash = 0; BackupStoreFilenameClear oldLeafname; - if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname)) + if(rContext.FindFilename(renameObjectID, renameInDirectory, + localPotentialOldName, isDir, isCurrentVersion, + &srvModTime, &srvAttributesHash, &oldLeafname)) { // Only interested if it's a file and the latest version if(!isDir && isCurrentVersion) @@ -780,8 +919,11 @@ bool BackupClientDirectoryRecord::UpdateItems( if(!rContext.StorageLimitExceeded()) { // Rename the existing files (ie include old versions) on the server - connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */, - BackupProtocolClientMoveObject::Flags_MoveAllWithSameName | BackupProtocolClientMoveObject::Flags_AllowMoveOverDeletedObject, + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, storeFilename); // Stop the attempt to delete the file in the original location @@ -790,8 +932,11 @@ bool BackupClientDirectoryRecord::UpdateItems( // Create new entry in the directory for it // -- will be near enough what's actually on the server for the rest to work. - en = pDirOnStore->AddEntry(storeFilename, srvModTime, renameObjectID, 0 /* size in blocks unknown, but not needed */, - BackupStoreDirectory::Entry::Flags_File, srvAttributesHash); + en = pDirOnStore->AddEntry(storeFilename, + srvModTime, renameObjectID, + 0 /* size in blocks unknown, but not needed */, + BackupStoreDirectory::Entry::Flags_File, + srvAttributesHash); // Store the object ID for the inode lookup map later latestObjectID = renameObjectID; @@ -833,6 +978,7 @@ bool BackupClientDirectoryRecord::UpdateItems( // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store bool doUpload = false; + std::string decisionReason = "unknown"; // Only upload a file if the mod time locally is // different to that on the server. @@ -848,16 +994,12 @@ bool BackupClientDirectoryRecord::UpdateItems( if (pDirOnStore != 0 && en == 0) { doUpload = true; - BOX_TRACE("Upload decision: " << - filename << ": will upload " - "(not on server)"); + decisionReason = "not on server"; } else if (modTime >= rParams.mSyncPeriodStart) { doUpload = true; - BOX_TRACE("Upload decision: " << - filename << ": will upload " - "(modified since last sync)"); + decisionReason = "modified since last sync"; } } @@ -874,9 +1016,7 @@ bool BackupClientDirectoryRecord::UpdateItems( > rParams.mMaxUploadWait) { doUpload = true; - BOX_TRACE("Upload decision: " << - filename << ": will upload " - "(continually modified)"); + decisionReason = "continually modified"; } // Then make sure that if files are added with a @@ -892,9 +1032,7 @@ bool BackupClientDirectoryRecord::UpdateItems( en->GetModificationTime() != modTime) { doUpload = true; - BOX_TRACE("Upload decision: " << - filename << ": will upload " - "(mod time changed)"); + decisionReason = "mod time changed"; } // And just to catch really badly off clocks in @@ -905,17 +1043,14 @@ bool BackupClientDirectoryRecord::UpdateItems( rParams.mUploadAfterThisTimeInTheFuture) { doUpload = true; - BOX_TRACE("Upload decision: " << - filename << ": will upload " - "(mod time in the future)"); + decisionReason = "mod time in the future"; } } if (en != 0 && en->GetModificationTime() == modTime) { - BOX_TRACE("Upload decision: " << - filename << ": will not upload " - "(not modified since last upload)"); + doUpload = false; + decisionReason = "not modified since last upload"; } else if (!doUpload) { @@ -924,22 +1059,26 @@ bool BackupClientDirectoryRecord::UpdateItems( box_time_t now = GetCurrentBoxTime(); int age = BoxTimeToSeconds(now - modTime); - BOX_TRACE("Upload decision: " << - filename << ": will not upload " - "(modified too recently: " - "only " << age << " seconds ago)"); + std::ostringstream s; + s << "modified too recently: " + "only " << age << " seconds ago"; + decisionReason = s.str(); } else { - BOX_TRACE("Upload decision: " << - filename << ": will not upload " - "(mod time is " << modTime << + std::ostringstream s; + s << "mod time is " << modTime << " which is outside sync window, " << rParams.mSyncPeriodStart << " to " - << rParams.mSyncPeriodEnd << ")"); + << rParams.mSyncPeriodEnd; + decisionReason = s.str(); } } + BOX_TRACE("Upload decision: " << nonVssFilePath << ": " << + (doUpload ? "will upload" : "will not upload") << + " (" << decisionReason << ")"); + bool fileSynced = true; if (doUpload) @@ -968,7 +1107,9 @@ bool BackupClientDirectoryRecord::UpdateItems( try { latestObjectID = UploadFile(rParams, - filename, storeFilename, + filename, + nonVssFilePath, + storeFilename, fileSize, modTime, attributesHash, noPreviousVersionOnServer); @@ -992,7 +1133,7 @@ bool BackupClientDirectoryRecord::UpdateItems( // retries would probably just cause // more problems. rNotifier.NotifyFileUploadException( - this, filename, e); + this, nonVssFilePath, e); throw; } catch(BoxException &e) @@ -1009,9 +1150,10 @@ bool BackupClientDirectoryRecord::UpdateItems( // code false, to show error in directory allUpdatedSuccessfully = false; // Log it. - SetErrorWhenReadingFilesystemObject(rParams, filename.c_str()); - rNotifier.NotifyFileUploadException( - this, filename, e); + SetErrorWhenReadingFilesystemObject(rParams, + nonVssFilePath); + rNotifier.NotifyFileUploadException(this, + nonVssFilePath, e); } // Update structures if the file was uploaded @@ -1029,8 +1171,7 @@ bool BackupClientDirectoryRecord::UpdateItems( } else { - rNotifier.NotifyFileSkippedServerFull(this, - filename); + rNotifier.NotifyFileSkippedServerFull(this, nonVssFilePath); } } else if(en != 0 && en->GetAttributesHash() != attributesHash) @@ -1050,22 +1191,23 @@ bool BackupClientDirectoryRecord::UpdateItems( { try { - rNotifier.NotifyFileUploadingAttributes( - this, filename); + rNotifier.NotifyFileUploadingAttributes(this, + nonVssFilePath); // Update store BackupClientFileAttributes attr; - attr.ReadAttributes(filename.c_str(), false /* put mod times in the attributes, please */); - MemBlockStream attrStream(attr); + attr.ReadAttributes(filename, + false /* put mod times in the attributes, please */); + std::auto_ptr<IOStream> attrStream( + new MemBlockStream(attr)); connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream); fileSynced = true; } catch (BoxException &e) { - BOX_ERROR("Failed to read or store " - "file attributes for '" << - filename << "', will try " - "again later"); + BOX_ERROR("Failed to read or store file attributes " + "for '" << nonVssFilePath << "', will try again " + "later"); } } } @@ -1109,7 +1251,7 @@ bool BackupClientDirectoryRecord::UpdateItems( { // Use this one BOX_TRACE("Storing uploaded file ID " << - inodeNum << " (" << filename << ") " + inodeNum << " (" << nonVssFilePath << ") " "in ID map as object " << latestObjectID << " with parent " << mObjectID); @@ -1126,7 +1268,12 @@ bool BackupClientDirectoryRecord::UpdateItems( // Found if (dirid != mObjectID) { - BOX_WARNING("Found conflicting parent ID for file ID " << inodeNum << " (" << filename << "): expected " << mObjectID << " but found " << dirid << " (same directory used in two different locations?)"); + BOX_WARNING("Found conflicting parent ID for " + "file ID " << inodeNum << " (" << + nonVssFilePath << "): expected " << + mObjectID << " but found " << dirid << + " (same directory used in two different " + "locations?)"); } ASSERT(dirid == mObjectID); @@ -1136,11 +1283,9 @@ bool BackupClientDirectoryRecord::UpdateItems( // into it. However, in a long running process this may happen occasionally and // not indicate anything wrong. // Run the release version for real life use, where this check is not made. - BOX_TRACE("Storing found file ID " << - inodeNum << " (" << filename << - ") in ID map as object " << - objid << " with parent " << - mObjectID); + BOX_TRACE("Storing found file ID " << inodeNum << + " (" << nonVssFilePath << ") in ID map as " + "object " << objid << " with parent " << mObjectID); idMap.AddToMap(inodeNum, objid, mObjectID /* containing directory */); } @@ -1149,7 +1294,7 @@ bool BackupClientDirectoryRecord::UpdateItems( if (fileSynced) { - rNotifier.NotifyFileSynchronised(this, filename, + rNotifier.NotifyFileSynchronised(this, nonVssFilePath, fileSize); } } @@ -1175,6 +1320,8 @@ bool BackupClientDirectoryRecord::UpdateItems( // Get the local filename std::string dirname(MakeFullPath(rLocalPath, *d)); + std::string nonVssDirPath = ConvertVssPathToRealPath(dirname, + rBackupLocation); // See if it's in the listing (if we have one) BackupStoreFilenameClear storeFilename(*d); @@ -1189,14 +1336,17 @@ bool BackupClientDirectoryRecord::UpdateItems( } // Check that the entry which might have been found is in fact a directory - if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == 0)) + if((en != 0) && !(en->IsDir())) { // Entry exists, but is not a directory. Bad. // Get rid of it. BackupProtocolClient &connection(rContext.GetConnection()); connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); + + std::string filenameClear = DecryptFilename(en, + rRemotePath); rNotifier.NotifyFileDeleted(en->GetObjectID(), - storeFilename.GetClearFilename()); + filenameClear); // Nothing found en = 0; @@ -1266,7 +1416,7 @@ bool BackupClientDirectoryRecord::UpdateItems( BOX_WARNING("Failed to read attributes " "of directory, cannot check " "for rename, assuming new: '" - << dirname << "'"); + << nonVssDirPath << "'"); failedToReadAttributes = true; } @@ -1284,7 +1434,9 @@ bool BackupClientDirectoryRecord::UpdateItems( std::string localPotentialOldName; bool isDir = false; bool isCurrentVersion = false; - if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion)) + if(rContext.FindFilename(renameObjectID, + renameInDirectory, localPotentialOldName, + isDir, isCurrentVersion)) { // Only interested if it's a directory if(isDir && isCurrentVersion) @@ -1309,13 +1461,16 @@ bool BackupClientDirectoryRecord::UpdateItems( // in the else if(...) above will be correct. // Build attribute stream for sending - MemBlockStream attrStream(attr); + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); if(renameDir) { // Rename the existing directory on the server - connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */, - BackupProtocolClientMoveObject::Flags_MoveAllWithSameName | BackupProtocolClientMoveObject::Flags_AllowMoveOverDeletedObject, + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, storeFilename); // Put the latest attributes on it @@ -1332,12 +1487,22 @@ bool BackupClientDirectoryRecord::UpdateItems( else { // Create a new directory - std::auto_ptr<BackupProtocolClientSuccess> dirCreate(connection.QueryCreateDirectory( - mObjectID, attrModTime, storeFilename, attrStream)); + std::auto_ptr<BackupProtocolSuccess> dirCreate( + connection.QueryCreateDirectory( + mObjectID, attrModTime, + storeFilename, attrStream)); subDirObjectID = dirCreate->GetObjectID(); // Flag as having done this for optimisation later haveJustCreatedDirOnServer = true; + + std::string filenameClear = + DecryptFilename(storeFilename, + subDirObjectID, + rRemotePath); + rNotifier.NotifyDirectoryCreated( + subDirObjectID, filenameClear, + nonVssDirPath); } } @@ -1365,8 +1530,8 @@ bool BackupClientDirectoryRecord::UpdateItems( if(psubDirRecord) { // Sync this sub directory too - psubDirRecord->SyncDirectory(rParams, mObjectID, - dirname, rRemotePath + "/" + *d, + psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, + rRemotePath + "/" + *d, rBackupLocation, haveJustCreatedDirOnServer); } @@ -1397,10 +1562,26 @@ bool BackupClientDirectoryRecord::UpdateItems( // If there's an error during the process, it doesn't matter if things // aren't actually deleted, as the whole state will be reset anyway. BackupClientDeleteList &rdel(rContext.GetDeleteList()); + std::string filenameClear; + bool isCorruptFilename = false; + + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "scheduling that file for deletion"); + filenameClear = "<corrupt filename>"; + isCorruptFilename = true; + } - BackupStoreFilenameClear clear(en->GetName()); std::string localName = MakeFullPath(rLocalPath, - clear.GetClearFilename()); + filenameClear); + std::string nonVssLocalName = ConvertVssPathToRealPath(localName, + rBackupLocation); // Delete this entry -- file or directory? if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) @@ -1418,20 +1599,17 @@ bool BackupClientDirectoryRecord::UpdateItems( // If there's a directory record for it in // the sub directory map, delete it now BackupStoreFilenameClear dirname(en->GetName()); - std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(dirname.GetClearFilename())); - if(e != mSubDirectories.end()) + std::map<std::string, BackupClientDirectoryRecord *>::iterator + e(mSubDirectories.find(filenameClear)); + if(e != mSubDirectories.end() && !isCorruptFilename) { // Carefully delete the entry from the map BackupClientDirectoryRecord *rec = e->second; mSubDirectories.erase(e); delete rec; - std::string name = MakeFullPath( - rLocalPath, - dirname.GetClearFilename()); - - BOX_TRACE("Deleted directory record " - "for " << name); + BOX_TRACE("Deleted directory record for " << + nonVssLocalName); } } } @@ -1498,6 +1676,7 @@ void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile( int64_t BackupClientDirectoryRecord::UploadFile( BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rFilename, + const std::string &rNonVssFilePath, const BackupStoreFilename &rStoreFilename, int64_t FileSize, box_time_t ModificationTime, @@ -1512,11 +1691,14 @@ int64_t BackupClientDirectoryRecord::UploadFile( // Info int64_t objID = 0; - bool doNormalUpload = true; + int64_t uploadedSize = -1; // Use a try block to catch store full errors try { + std::auto_ptr<BackupStoreFileEncodeStream> apStreamToUpload; + int64_t diffFromID = 0; + // Might an old version be on the server, and is the file // size over the diffing threshold? if(!NoPreviousVersionOnServer && @@ -1524,14 +1706,14 @@ int64_t BackupClientDirectoryRecord::UploadFile( { // YES -- try to do diff, if possible // First, query the server to see if there's an old version available - std::auto_ptr<BackupProtocolClientSuccess> getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename)); - int64_t diffFromID = getBlockIndex->GetObjectID(); + std::auto_ptr<BackupProtocolSuccess> getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename)); + diffFromID = getBlockIndex->GetObjectID(); if(diffFromID != 0) { // Found an old version - rNotifier.NotifyFileUploadingPatch(this, - rFilename); + rNotifier.NotifyFileUploadingPatch(this, + rNonVssFilePath); // Get the index std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream()); @@ -1543,55 +1725,68 @@ int64_t BackupClientDirectoryRecord::UploadFile( rContext.ManageDiffProcess(); bool isCompletelyDifferent = false; - std::auto_ptr<IOStream> patchStream( - BackupStoreFile::EncodeFileDiff( - rFilename.c_str(), - mObjectID, /* containing directory */ - rStoreFilename, diffFromID, *blockIndexStream, - connection.GetTimeout(), - &rContext, // DiffTimer implementation - 0 /* not interested in the modification time */, - &isCompletelyDifferent)); - - rContext.UnManageDiffProcess(); - // - // Upload the patch to the store - // - std::auto_ptr<BackupProtocolClientSuccess> stored(connection.QueryStoreFile(mObjectID, ModificationTime, - AttributesHash, isCompletelyDifferent?(0):(diffFromID), rStoreFilename, *patchStream)); - - // Get object ID from the result - objID = stored->GetObjectID(); + apStreamToUpload = BackupStoreFile::EncodeFileDiff( + rFilename, + mObjectID, /* containing directory */ + rStoreFilename, diffFromID, *blockIndexStream, + connection.GetTimeout(), + &rContext, // DiffTimer implementation + 0 /* not interested in the modification time */, + &isCompletelyDifferent); - // Don't attempt to upload it again! - doNormalUpload = false; + if(isCompletelyDifferent) + { + diffFromID = 0; + } + + rContext.UnManageDiffProcess(); } } - if(doNormalUpload) + if(!apStreamToUpload.get()) // No patch upload, so do a normal upload { // below threshold or nothing to diff from, so upload whole - rNotifier.NotifyFileUploading(this, rFilename); + rNotifier.NotifyFileUploading(this, rNonVssFilePath); // Prepare to upload, getting a stream which will encode the file as we go along - std::auto_ptr<IOStream> upload( - BackupStoreFile::EncodeFile(rFilename.c_str(), - mObjectID, rStoreFilename, NULL, - &rParams, - &(rParams.mrRunStatusProvider))); - - // Send to store - std::auto_ptr<BackupProtocolClientSuccess> stored( - connection.QueryStoreFile( - mObjectID, ModificationTime, - AttributesHash, - 0 /* no diff from file ID */, - rStoreFilename, *upload)); - - // Get object ID from the result - objID = stored->GetObjectID(); + apStreamToUpload = BackupStoreFile::EncodeFile( + rFilename, mObjectID, /* containing directory */ + rStoreFilename, NULL, &rParams, + &(rParams.mrRunStatusProvider)); + } + + rContext.SetNiceMode(true); + std::auto_ptr<IOStream> apWrappedStream; + + if(rParams.mMaxUploadRate > 0) + { + apWrappedStream.reset(new RateLimitingStream( + *apStreamToUpload, rParams.mMaxUploadRate)); + } + else + { + // Wrap the stream in *something*, so that + // QueryStoreFile() doesn't delete the original + // stream (upload object) and we can retrieve + // the byte counter. + apWrappedStream.reset(new BufferedStream( + *apStreamToUpload)); } + + // Send to store + std::auto_ptr<BackupProtocolSuccess> stored( + connection.QueryStoreFile( + mObjectID, ModificationTime, + AttributesHash, + diffFromID, + rStoreFilename, apWrappedStream)); + + rContext.SetNiceMode(false); + + // Get object ID from the result + objID = stored->GetObjectID(); + uploadedSize = apStreamToUpload->GetTotalBytesSent(); } catch(BoxException &e) { @@ -1605,8 +1800,8 @@ int64_t BackupClientDirectoryRecord::UploadFile( int type, subtype; if(connection.GetLastError(type, subtype)) { - if(type == BackupProtocolClientError::ErrorType - && subtype == BackupProtocolClientError::Err_StorageLimitExceeded) + if(type == BackupProtocolError::ErrorType + && subtype == BackupProtocolError::Err_StorageLimitExceeded) { // The hard limit was exceeded on the server, notify! rParams.mrSysadminNotifier.NotifySysadmin( @@ -1617,7 +1812,7 @@ int64_t BackupClientDirectoryRecord::UploadFile( return 0; } rNotifier.NotifyFileUploadServerError(this, - rFilename, type, subtype); + rNonVssFilePath, type, subtype); } } @@ -1625,7 +1820,8 @@ int64_t BackupClientDirectoryRecord::UploadFile( throw; } - rNotifier.NotifyFileUploaded(this, rFilename, FileSize); + rNotifier.NotifyFileUploaded(this, rNonVssFilePath, FileSize, + uploadedSize); // Return the new object ID of this file return objID; @@ -1641,7 +1837,9 @@ int64_t BackupClientDirectoryRecord::UploadFile( // Created: 29/3/04 // // -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(BackupClientDirectoryRecord::SyncParams &rParams, const char *Filename) +void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string& rFilename) { // Zero hash, so it gets synced properly next time round. ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); @@ -1671,19 +1869,20 @@ BackupClientDirectoryRecord::SyncParams::SyncParams( SysadminNotifier &rSysadminNotifier, ProgressNotifier &rProgressNotifier, BackupClientContext &rContext) - : mSyncPeriodStart(0), - mSyncPeriodEnd(0), - mMaxUploadWait(0), - mMaxFileTimeInFuture(99999999999999999LL), - mFileTrackingSizeThreshold(16*1024), - mDiffingUploadSizeThreshold(16*1024), - mrRunStatusProvider(rRunStatusProvider), - mrSysadminNotifier(rSysadminNotifier), - mrProgressNotifier(rProgressNotifier), - mrContext(rContext), - mReadErrorsOnFilesystemObjects(false), - mUploadAfterThisTimeInTheFuture(99999999999999999LL), - mHaveLoggedWarningAboutFutureFileTimes(false) +: mSyncPeriodStart(0), + mSyncPeriodEnd(0), + mMaxUploadWait(0), + mMaxFileTimeInFuture(99999999999999999LL), + mFileTrackingSizeThreshold(16*1024), + mDiffingUploadSizeThreshold(16*1024), + mrRunStatusProvider(rRunStatusProvider), + mrSysadminNotifier(rSysadminNotifier), + mrProgressNotifier(rProgressNotifier), + mrContext(rContext), + mReadErrorsOnFilesystemObjects(false), + mMaxUploadRate(0), + mUploadAfterThisTimeInTheFuture(99999999999999999LL), + mHaveLoggedWarningAboutFutureFileTimes(false) { } @@ -1874,3 +2073,220 @@ void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const pSubItem->Serialize(rArchive); } } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Location() +// Purpose: Constructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +Location::Location() + : mIDMapIndex(0), + mpExcludeFiles(0), + mpExcludeDirs(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::~Location() +// Purpose: Destructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +Location::~Location() +{ + // Clean up exclude locations + if(mpExcludeDirs != 0) + { + delete mpExcludeDirs; + mpExcludeDirs = 0; + } + if(mpExcludeFiles != 0) + { + delete mpExcludeFiles; + mpExcludeFiles = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, +// using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void Location::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mName); + rArchive.Write(mPath); + rArchive.Write(mIDMapIndex); + + // + // + // + if(mpDirectoryRecord.get() == NULL) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpDirectoryRecord->Serialize(rArchive); + } + + // + // + // + if(!mpExcludeFiles) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpExcludeFiles->Serialize(rArchive); + } + + // + // + // + if(!mpExcludeDirs) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpExcludeDirs->Serialize(rArchive); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void Location::Deserialize(Archive &rArchive) +{ + // + // + // + mpDirectoryRecord.reset(NULL); + if(mpExcludeFiles) + { + delete mpExcludeFiles; + mpExcludeFiles = NULL; + } + if(mpExcludeDirs) + { + delete mpExcludeDirs; + mpExcludeDirs = NULL; + } + + // + // + // + rArchive.Read(mName); + rArchive.Read(mPath); + rArchive.Read(mIDMapIndex); + + // + // + // + int64_t aMagicMarker = 0; + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, ""); + if(!pSubRecord) + { + throw std::bad_alloc(); + } + + mpDirectoryRecord.reset(pSubRecord); + mpDirectoryRecord->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } + + // + // + // + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mpExcludeFiles = new ExcludeList; + if(!mpExcludeFiles) + { + throw std::bad_alloc(); + } + + mpExcludeFiles->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } + + // + // + // + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mpExcludeDirs = new ExcludeList; + if(!mpExcludeDirs) + { + throw std::bad_alloc(); + } + + mpExcludeDirs->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } +} diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/bin/bbackupd/BackupClientDirectoryRecord.h index fce3fc04..fb9d7fc8 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.h +++ b/bin/bbackupd/BackupClientDirectoryRecord.h @@ -12,6 +12,7 @@ #include <string> #include <map> +#include <memory> #include "BackupClientFileAttributes.h" #include "BackupDaemonInterface.h" @@ -21,9 +22,18 @@ #include "ReadLoggingStream.h" #include "RunStatusProvider.h" +#ifdef ENABLE_VSS +# include <comdef.h> +# include <Vss.h> +# include <VsWriter.h> +# include <VsBackup.h> +#endif + class Archive; class BackupClientContext; class BackupDaemon; +class ExcludeList; +class Location; // -------------------------------------------------------------------------- // @@ -86,6 +96,7 @@ public: ProgressNotifier &mrProgressNotifier; BackupClientContext &mrContext; bool mReadErrorsOnFilesystemObjects; + int64_t mMaxUploadRate; // Member variables modified by syncing process box_time_t mUploadAfterThisTimeInTheFuture; @@ -124,8 +135,14 @@ public: int64_t ContainingDirectoryID, const std::string &rLocalPath, const std::string &rRemotePath, + const Location& rBackupLocation, bool ThisDirHasJustBeenCreated = false); + std::string ConvertVssPathToRealPath(const std::string &rVssPath, + const Location& rBackupLocation); + + int64_t GetObjectID() const { return mObjectID; } + private: void DeleteSubDirectories(); BackupStoreDirectory *FetchDirectoryListing(SyncParams &rParams); @@ -134,27 +151,34 @@ private: const std::string &rLocalPath); bool UpdateItems(SyncParams &rParams, const std::string &rLocalPath, const std::string &rRemotePath, + const Location& rBackupLocation, BackupStoreDirectory *pDirOnStore, std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, std::vector<std::string> &rFiles, const std::vector<std::string> &rDirs); int64_t UploadFile(SyncParams &rParams, const std::string &rFilename, + const std::string &rNonVssFilePath, const BackupStoreFilename &rStoreFilename, int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer); void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, - const char *Filename); + const std::string& rFilename); void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory* pDirOnStore, BackupStoreDirectory::Entry* pEntry, const std::string &rFilename); + std::string DecryptFilename(BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath); + std::string DecryptFilename(BackupStoreFilenameClear fn, + int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath); -private: int64_t mObjectID; std::string mSubDirName; bool mInitialSyncDone; bool mSyncDone; + bool mSuppressMultipleLinksWarning; // Checksum of directory contents and attributes, used to detect changes uint8_t mStateChecksum[MD5Digest::DigestLength]; @@ -166,6 +190,32 @@ private: // waste a lot of memory because of STL allocation policies. }; +class Location +{ +public: + Location(); + ~Location(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; +private: + Location(const Location &); // copy not allowed + Location &operator=(const Location &); +public: + std::string mName; + std::string mPath; + std::auto_ptr<BackupClientDirectoryRecord> mpDirectoryRecord; + int mIDMapIndex; + ExcludeList *mpExcludeFiles; + ExcludeList *mpExcludeDirs; + +#ifdef ENABLE_VSS + bool mIsSnapshotCreated; + VSS_ID mSnapshotVolumeId; + std::string mSnapshotPath; +#endif +}; + #endif // BACKUPCLIENTDIRECTORYRECORD__H diff --git a/bin/bbackupd/BackupClientInodeToIDMap.cpp b/bin/bbackupd/BackupClientInodeToIDMap.cpp index b9f56c5a..8240d62c 100644 --- a/bin/bbackupd/BackupClientInodeToIDMap.cpp +++ b/bin/bbackupd/BackupClientInodeToIDMap.cpp @@ -9,32 +9,53 @@ #include "Box.h" -#ifdef HAVE_DB - // Include db headers and other OS files if they're needed for the disc implementation - #include <sys/types.h> - #include <fcntl.h> - #include <limits.h> - #include <db.h> - #include <sys/stat.h> -#endif +#include <stdlib.h> +#include <depot.h> #define BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION #include "BackupClientInodeToIDMap.h" +#undef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION #include "BackupStoreException.h" - #include "MemLeakFindOn.h" -// What type of Berkeley DB shall we use? -#define TABLE_DATABASE_TYPE DB_HASH - typedef struct { int64_t mObjectID; int64_t mInDirectory; } IDBRecord; +#define BOX_DBM_MESSAGE(stuff) stuff << " (qdbm): " << dperrmsg(dpecode) + +#define BOX_LOG_DBM_ERROR(stuff) \ + BOX_ERROR(BOX_DBM_MESSAGE(stuff)) + +#define THROW_DBM_ERROR(message, filename, exception, subtype) \ + BOX_LOG_DBM_ERROR(message << ": " << filename); \ + THROW_EXCEPTION_MESSAGE(exception, subtype, \ + BOX_DBM_MESSAGE(message << ": " << filename)); + +#define ASSERT_DBM_OK(operation, message, filename, exception, subtype) \ + if(!(operation)) \ + { \ + THROW_DBM_ERROR(message, filename, exception, subtype); \ + } + +#define ASSERT_DBM_OPEN() \ + if(mpDepot == 0) \ + { \ + THROW_EXCEPTION_MESSAGE(BackupStoreException, InodeMapNotOpen, \ + "Inode database not open"); \ + } + +#define ASSERT_DBM_CLOSED() \ + if(mpDepot != 0) \ + { \ + THROW_EXCEPTION_MESSAGE(CommonException, Internal, \ + "Inode database already open: " << mFilename); \ + } + // -------------------------------------------------------------------------- // // Function @@ -44,11 +65,9 @@ typedef struct // // -------------------------------------------------------------------------- BackupClientInodeToIDMap::BackupClientInodeToIDMap() -#ifndef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION : mReadOnly(true), mEmpty(false), - dbp(0) -#endif + mpDepot(0) { } @@ -62,19 +81,12 @@ BackupClientInodeToIDMap::BackupClientInodeToIDMap() // -------------------------------------------------------------------------- BackupClientInodeToIDMap::~BackupClientInodeToIDMap() { -#ifndef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION - if(dbp != 0) + if(mpDepot != 0) { -#if BDB_VERSION_MAJOR >= 3 - dbp->close(0); -#else - dbp->close(dbp); -#endif + Close(); } -#endif } - // -------------------------------------------------------------------------- // // Function @@ -83,56 +95,59 @@ BackupClientInodeToIDMap::~BackupClientInodeToIDMap() // Created: 20/11/03 // // -------------------------------------------------------------------------- -void BackupClientInodeToIDMap::Open(const char *Filename, bool ReadOnly, bool CreateNew) +void BackupClientInodeToIDMap::Open(const char *Filename, bool ReadOnly, + bool CreateNew) { -#ifndef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION + mFilename = Filename; + // Correct arguments? ASSERT(!(CreateNew && ReadOnly)); // Correct usage? - ASSERT(dbp == 0); + ASSERT_DBM_CLOSED(); ASSERT(!mEmpty); // Open the database file -#if BDB_VERSION_MAJOR >= 3 - dbp = new Db(0,0); - dbp->set_pagesize(1024); /* Page size: 1K. */ - dbp->set_cachesize(0, 32 * 1024, 0); - dbp->open(NULL, Filename, NULL, DB_HASH, DB_CREATE, 0664); -#else - dbp = dbopen(Filename, (CreateNew?O_CREAT:0) | (ReadOnly?O_RDONLY:O_RDWR), S_IRUSR | S_IWUSR | S_IRGRP, TABLE_DATABASE_TYPE, NULL); -#endif - if(dbp == NULL) + int mode = ReadOnly ? DP_OREADER : DP_OWRITER; + if(CreateNew) { - THROW_EXCEPTION(BackupStoreException, BerkelyDBFailure); + mode |= DP_OCREAT; + } + + mpDepot = dpopen(Filename, mode, 0); + + if(!mpDepot) + { + BOX_WARNING(BOX_DBM_MESSAGE("Failed to open inode " + "database: " << mFilename)); + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + BOX_DBM_MESSAGE("Failed to open inode database: " << + mFilename)); } // Read only flag mReadOnly = ReadOnly; -#endif } // -------------------------------------------------------------------------- // // Function // Name: BackupClientInodeToIDMap::OpenEmpty() -// Purpose: 'Open' this map. Not associated with a disc file. Useful for when a map -// is required, but is against an empty file on disc which shouldn't be created. -// Implies read only. +// Purpose: 'Open' this map. Not associated with a disc file. +// Useful for when a map is required, but is against +// an empty file on disc which shouldn't be created. +// Implies read only. // Created: 20/11/03 // // -------------------------------------------------------------------------- void BackupClientInodeToIDMap::OpenEmpty() { -#ifndef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION - ASSERT(dbp == 0); + ASSERT_DBM_CLOSED(); + ASSERT(mpDepot == 0); mEmpty = true; mReadOnly = true; -#endif } - - // -------------------------------------------------------------------------- // // Function @@ -143,75 +158,46 @@ void BackupClientInodeToIDMap::OpenEmpty() // -------------------------------------------------------------------------- void BackupClientInodeToIDMap::Close() { -#ifndef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION - if(dbp != 0) - { -#if BDB_VERSION_MAJOR >= 3 - if(dbp->close(0) != 0) -#else - if(dbp->close(dbp) != 0) -#endif - { - THROW_EXCEPTION(BackupStoreException, BerkelyDBFailure); - } - dbp = 0; - } -#endif + ASSERT_DBM_OPEN(); + ASSERT_DBM_OK(dpclose(mpDepot), "Failed to close inode database", + mFilename, BackupStoreException, BerkelyDBFailure); + mpDepot = 0; } - // -------------------------------------------------------------------------- // // Function -// Name: BackupClientInodeToIDMap::AddToMap(InodeRefType, int64_t, int64_t) -// Purpose: Adds an entry to the map. Overwrites any existing entry. +// Name: BackupClientInodeToIDMap::AddToMap(InodeRefType, +// int64_t, int64_t) +// Purpose: Adds an entry to the map. Overwrites any existing +// entry. // Created: 11/11/03 // // -------------------------------------------------------------------------- -void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, int64_t InDirectory) +void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, + int64_t InDirectory) { -#ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION - mMap[InodeRef] = std::pair<int64_t, int64_t>(ObjectID, InDirectory); -#else if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, InodeMapIsReadOnly); } - if(dbp == 0) + if(mpDepot == 0) { THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); } + ASSERT_DBM_OPEN(); + // Setup structures IDBRecord rec; rec.mObjectID = ObjectID; rec.mInDirectory = InDirectory; -#if BDB_VERSION_MAJOR >= 3 - Dbt key(&InodeRef, sizeof(InodeRef)); - Dbt data(&rec, sizeof(rec)); - - if (dbp->put(0, &key, &data, 0) != 0) { - THROW_EXCEPTION(BackupStoreException, BerkelyDBFailure); - } -#else - - DBT key; - key.data = &InodeRef; - key.size = sizeof(InodeRef); - - DBT data; - data.data = &rec; - data.size = sizeof(rec); - - // Add to map (or replace existing entry) - if(dbp->put(dbp, &key, &data, 0) != 0) - { - THROW_EXCEPTION(BackupStoreException, BerkelyDBFailure); - } -#endif -#endif + ASSERT_DBM_OK(dpput(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), + (const char *)&rec, sizeof(rec), DP_DOVER), + "Failed to add record to inode database", mFilename, + BackupStoreException, BerkelyDBFailure); } // -------------------------------------------------------------------------- @@ -228,100 +214,32 @@ void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const { -#ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION - std::map<InodeRefType, std::pair<int64_t, int64_t> >::const_iterator i(mMap.find(InodeRef)); - - // Found? - if(i == mMap.end()) - { - return false; - } - - // Yes. Return the details - rObjectIDOut = i->second.first; - rInDirectoryOut = i->second.second; - return true; -#else if(mEmpty) { // Map is empty return false; } - if(dbp == 0) + if(mpDepot == 0) { THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen); } - -#if BDB_VERSION_MAJOR >= 3 - Dbt key(&InodeRef, sizeof(InodeRef)); - Dbt data(0, 0); - switch(dbp->get(NULL, &key, &data, 0)) -#else - DBT key; - key.data = &InodeRef; - key.size = sizeof(InodeRef); - DBT data; - data.data = 0; - data.size = 0; + ASSERT_DBM_OPEN(); - switch(dbp->get(dbp, &key, &data, 0)) -#endif + IDBRecord rec; + if(dpgetwb(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), + 0, sizeof(IDBRecord), (char *)&rec) == -1) { - case 1: // key not in file + // key not in file return false; - - case -1: // error - default: // not specified in docs - THROW_EXCEPTION(BackupStoreException, BerkelyDBFailure); - return false; - - case 0: // success, found it - break; } - - // Check for sensible return -#if BDB_VERSION_MAJOR >= 3 - if(key.get_data() == 0 || data.get_size() != sizeof(IDBRecord)) - { - // Assert in debug version - ASSERT(key.get_data() == 0 || data.get_size() != sizeof(IDBRecord)); - // Invalid entries mean it wasn't found - return false; - } - - // Data alignment isn't guaranteed to be on a suitable boundary - IDBRecord rec; - - ::memcpy(&rec, data.get_data(), sizeof(rec)); -#else - if(key.data == 0 || data.size != sizeof(IDBRecord)) - { - // Assert in debug version - ASSERT(key.data == 0 || data.size != sizeof(IDBRecord)); - - // Invalid entries mean it wasn't found - return false; - } - - // Data alignment isn't guaranteed to be on a suitable boundary - IDBRecord rec; - - ::memcpy(&rec, data.data, sizeof(rec)); -#endif - // Return data rObjectIDOut = rec.mObjectID; rInDirectoryOut = rec.mInDirectory; - // Don't have to worry about freeing the returned data - // Found return true; -#endif } - - diff --git a/bin/bbackupd/BackupClientInodeToIDMap.h b/bin/bbackupd/BackupClientInodeToIDMap.h index 1dfef702..fbe45114 100644 --- a/bin/bbackupd/BackupClientInodeToIDMap.h +++ b/bin/bbackupd/BackupClientInodeToIDMap.h @@ -8,25 +8,16 @@ // -------------------------------------------------------------------------- #ifndef BACKUPCLIENTINODETOIDMAP_H -#define BACKUPCLIENTINODETOIDMAP__H +#define BACKUPCLIENTINODETOIDMAP_H #include <sys/types.h> #include <map> #include <utility> -// Use in memory implementation if there isn't access to the Berkely DB on this platform -#ifndef HAVE_DB - #define BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION -#endif - // avoid having to include the DB files when not necessary #ifndef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION -#ifdef BERKELY_V4 - class Db; -#else - class DB; -#endif + class DEPOT; #endif // -------------------------------------------------------------------------- @@ -55,19 +46,12 @@ public: void Close(); private: -#ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION - std::map<InodeRefType, std::pair<int64_t, int64_t> > mMap; -#else bool mReadOnly; bool mEmpty; -#ifdef BERKELY_V4 - Db *dbp; // c++ style implimentation -#else - DB *dbp; // C style interface, use notation from documentation -#endif // BERKELY_V4 -#endif // BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION + std::string mFilename; + DEPOT *mpDepot; }; -#endif // BACKUPCLIENTINODETOIDMAP__H +#endif // BACKUPCLIENTINODETOIDMAP_H diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp index b6f90cad..39bb98e3 100644 --- a/bin/bbackupd/BackupDaemon.cpp +++ b/bin/bbackupd/BackupDaemon.cpp @@ -40,6 +40,8 @@ #endif #include <iostream> +#include <set> +#include <sstream> #include "Configuration.h" #include "IOStream.h" @@ -49,7 +51,7 @@ #include "SSLLib.h" -#include "autogen_BackupProtocolClient.h" +#include "autogen_BackupProtocol.h" #include "autogen_ClientException.h" #include "autogen_ConversionException.h" #include "Archive.h" @@ -82,6 +84,85 @@ #include "Win32BackupService.h" extern Win32BackupService* gpDaemonService; + +# ifdef ENABLE_VSS +# include <comdef.h> +# include <Vss.h> +# include <VsWriter.h> +# include <VsBackup.h> + + // http://www.flounder.com/cstring.htm + std::string GetMsgForHresult(HRESULT hr) + { + std::ostringstream buf; + + if(hr == VSS_S_ASYNC_CANCELLED) + { + buf << "VSS async operation cancelled"; + } + else if(hr == VSS_S_ASYNC_FINISHED) + { + buf << "VSS async operation finished"; + } + else if(hr == VSS_S_ASYNC_PENDING) + { + buf << "VSS async operation pending"; + } + else + { + buf << _com_error(hr).ErrorMessage(); + } + + buf << " (" << BOX_FORMAT_HEX32(hr) << ")"; + return buf.str(); + } + + std::string WideStringToString(WCHAR *buf) + { + if (buf == NULL) + { + return "(null)"; + } + + char* pStr = ConvertFromWideString(buf, CP_UTF8); + + if(pStr == NULL) + { + return "(conversion failed)"; + } + + std::string result(pStr); + free(pStr); + return result; + } + + std::string GuidToString(GUID guid) + { + wchar_t buf[64]; + StringFromGUID2(guid, buf, sizeof(buf)); + return WideStringToString(buf); + } + + std::string BstrToString(const BSTR arg) + { + if(arg == NULL) + { + return std::string("(null)"); + } + else + { + // Extract the *long* before where the arg points to + long len = ((long *)arg)[-1] / 2; + std::wstring wstr((WCHAR *)arg, len); + std::string str; + if(!ConvertFromWideString(wstr, &str, CP_UTF8)) + { + throw std::exception("string conversion failed"); + } + return str; + } + } +# endif #endif #include "MemLeakFindOn.h" @@ -114,6 +195,9 @@ BackupDaemon::BackupDaemon() mUpdateStoreInterval(0), mDeleteStoreObjectInfoFile(false), mDoSyncForcedByPreviousSyncError(false), + mNumFilesUploaded(-1), + mNumDirsCreated(-1), + mMaxBandwidthFromSyncAllowScript(0), mLogAllFileAccess(false), mpProgressNotifier(this), mpLocationResolver(this), @@ -125,6 +209,9 @@ BackupDaemon::BackupDaemon() mRunAsService(false), mServiceName("bbackupd") #endif +#ifdef ENABLE_VSS + , mpVssBackupComponents(NULL) +#endif { // Only ever one instance of a daemon SSLLib::Initialise(); @@ -239,7 +326,7 @@ void BackupDaemon::SetupInInitialProcess() void BackupDaemon::DeleteAllLocations() { // Run through, and delete everything - for(std::vector<Location *>::iterator i = mLocations.begin(); + for(Locations::iterator i = mLocations.begin(); i != mLocations.end(); ++i) { delete *i; @@ -311,6 +398,16 @@ int BackupDaemon::Main(const std::string &rConfigFileName) return RemoveService(mServiceName); } +#ifdef ENABLE_VSS + HRESULT result = CoInitialize(NULL); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to initialize COM: " << + GetMsgForHresult(result)); + return 1; + } +#endif + int returnCode; if (mRunAsService) @@ -416,7 +513,7 @@ void BackupDaemon::InitCrypto() keyFile.c_str(), caFile.c_str()); // Set up the keys for various things - BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str()); + BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile")); } // -------------------------------------------------------------------------- @@ -573,15 +670,14 @@ void BackupDaemon::Run2() void BackupDaemon::RunSyncNowWithExceptionHandling() { - OnBackupStart(); - - // Do sync bool errorOccurred = false; int errorCode = 0, errorSubCode = 0; const char* errorString = "unknown"; try { + OnBackupStart(); + // Do sync RunSyncNow(); } catch(BoxException &e) @@ -607,6 +703,7 @@ void BackupDaemon::RunSyncNowWithExceptionHandling() // do not retry immediately without a good reason mDoSyncForcedByPreviousSyncError = false; + // Notify system administrator about the final state of the backup if(errorOccurred) { // Is it a berkely db failure? @@ -624,12 +721,7 @@ void BackupDaemon::RunSyncNowWithExceptionHandling() DeleteCorruptBerkelyDbFiles(); } - // Clear state data - // Go back to beginning of time - mLastSyncTime = 0; - mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything - DeleteAllLocations(); - DeleteAllIDMaps(); + ResetCachedState(); // Handle restart? if(StopRun()) @@ -668,16 +760,19 @@ void BackupDaemon::RunSyncNowWithExceptionHandling() SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); } } - // Notify system administrator about the final state of the backup - else if(mReadErrorsOnFilesystemObjects) + + if(mReadErrorsOnFilesystemObjects) { NotifySysadmin(SysadminNotifier::ReadError); } - else if(mStorageLimitExceeded) + + if(mStorageLimitExceeded) { NotifySysadmin(SysadminNotifier::StoreFull); } - else + + if (!errorOccurred && !mReadErrorsOnFilesystemObjects && + !mStorageLimitExceeded) { NotifySysadmin(SysadminNotifier::BackupOK); } @@ -689,6 +784,16 @@ void BackupDaemon::RunSyncNowWithExceptionHandling() OnBackupFinish(); } +void BackupDaemon::ResetCachedState() +{ + // Clear state data + // Go back to beginning of time + mLastSyncTime = 0; + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything + DeleteAllLocations(); + DeleteAllIDMaps(); +} + void BackupDaemon::RunSyncNow() { // Delete the serialised store object file, @@ -746,7 +851,9 @@ void BackupDaemon::RunSyncNow() conf.GetKeyValueUint32("AccountNumber"), conf.GetKeyValueBool("ExtendedLogging"), conf.KeyExists("ExtendedLogFile"), - extendedLogFile, *mpProgressNotifier + extendedLogFile, + *mpProgressNotifier, + conf.GetKeyValueBool("TcpNice") ); // The minimum age a file needs to be before it will be @@ -830,6 +937,19 @@ void BackupDaemon::RunSyncNow() conf.GetKeyValueInt("DiffingUploadSizeThreshold"); params.mMaxFileTimeInFuture = SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture")); + mNumFilesUploaded = 0; + mNumDirsCreated = 0; + + if(conf.KeyExists("MaxUploadRate")) + { + params.mMaxUploadRate = conf.GetKeyValueInt("MaxUploadRate"); + } + + if(mMaxBandwidthFromSyncAllowScript != 0) + { + params.mMaxUploadRate = mMaxBandwidthFromSyncAllowScript; + } + mDeleteRedundantLocationsAfter = conf.GetKeyValueInt("DeleteRedundantLocationsAfter"); mStorageLimitExceeded = false; @@ -858,7 +978,6 @@ void BackupDaemon::RunSyncNow() // Set up the locations, if necessary -- // need to do it here so we have a // (potential) connection to use - if(mLocations.empty()) { const Configuration &locations( conf.GetSubConfiguration( @@ -876,9 +995,13 @@ void BackupDaemon::RunSyncNow() // Delete any unused directories? DeleteUnusedRootDirEntries(clientContext); + +#ifdef ENABLE_VSS + CreateVssBackupComponents(); +#endif // Go through the records, syncing them - for(std::vector<Location *>::const_iterator + for(Locations::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i) { @@ -894,10 +1017,17 @@ void BackupDaemon::RunSyncNow() (*i)->mpExcludeDirs); // Sync the directory - (*i)->mpDirectoryRecord->SyncDirectory( - params, - BackupProtocolClientListDirectory::RootDirectory, - (*i)->mPath, std::string("/") + (*i)->mName); + std::string locationPath = (*i)->mPath; +#ifdef ENABLE_VSS + if((*i)->mIsSnapshotCreated) + { + locationPath = (*i)->mSnapshotPath; + } +#endif + + (*i)->mpDirectoryRecord->SyncDirectory(params, + BackupProtocolListDirectory::RootDirectory, + locationPath, std::string("/") + (*i)->mName, **i); // Unset exclude lists (just in case) clientContext.SetExcludeLists(0, 0); @@ -910,11 +1040,15 @@ void BackupDaemon::RunSyncNow() // Close any open connection clientContext.CloseAnyOpenConnection(); - + +#ifdef ENABLE_VSS + CleanupVssBackupComponents(); +#endif + // Get the new store marker mClientStoreMarker = clientContext.GetClientStoreMarker(); mStorageLimitExceeded = clientContext.StorageLimitExceeded(); - mReadErrorsOnFilesystemObjects = + mReadErrorsOnFilesystemObjects |= params.mReadErrorsOnFilesystemObjects; if(!mStorageLimitExceeded) @@ -948,6 +1082,562 @@ void BackupDaemon::RunSyncNow() // -------------------------------------------------------------------------------------------- } +#ifdef ENABLE_VSS +bool BackupDaemon::WaitForAsync(IVssAsync *pAsync, + const std::string& description) +{ + BOX_INFO("VSS: waiting for " << description << " to complete"); + HRESULT result; + + do + { + result = pAsync->Wait(1000); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to wait for " << description << + " to complete: " << GetMsgForHresult(result)); + break; + } + + HRESULT result2; + result = pAsync->QueryStatus(&result2, NULL); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query " << description << + " status: " << GetMsgForHresult(result)); + break; + } + + result = result2; + BOX_INFO("VSS: " << description << " status: " << + GetMsgForHresult(result)); + } + while(result == VSS_S_ASYNC_PENDING); + + pAsync->Release(); + + return (result == VSS_S_ASYNC_FINISHED); +} + +#define CALL_MEMBER_FN(object, method) ((object).*(method)) + +bool BackupDaemon::CallAndWaitForAsync(AsyncMethod method, + const std::string& description) +{ + IVssAsync *pAsync; + HRESULT result = CALL_MEMBER_FN(*mpVssBackupComponents, method)(&pAsync); + if(result != S_OK) + { + BOX_ERROR("VSS: " << description << " failed: " << + GetMsgForHresult(result)); + return false; + } + + return WaitForAsync(pAsync, description); +} + +void FreeSnapshotProp(VSS_SNAPSHOT_PROP *pSnap) +{ + CoTaskMemFree(pSnap->m_pwszSnapshotDeviceObject); + CoTaskMemFree(pSnap->m_pwszOriginalVolumeName); + CoTaskMemFree(pSnap->m_pwszOriginatingMachine); + CoTaskMemFree(pSnap->m_pwszServiceMachine); + CoTaskMemFree(pSnap->m_pwszExposedName); + CoTaskMemFree(pSnap->m_pwszExposedPath); +} + +void BackupDaemon::CreateVssBackupComponents() +{ + std::map<char, VSS_ID> volumesIncluded; + + HRESULT result = ::CreateVssBackupComponents(&mpVssBackupComponents); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to create backup components: " << + GetMsgForHresult(result)); + return; + } + + result = mpVssBackupComponents->InitializeForBackup(NULL); + if(result != S_OK) + { + std::string message = GetMsgForHresult(result); + + if (result == VSS_E_UNEXPECTED) + { + message = "Check the Application Log for details, and ensure " + "that the Volume Shadow Copy, COM+ System Application, " + "and Distributed Transaction Coordinator services " + "are running"; + } + + BOX_ERROR("VSS: Failed to initialize for backup: " << message); + return; + } + + result = mpVssBackupComponents->SetContext(VSS_CTX_BACKUP); + if(result == E_NOTIMPL) + { + BOX_INFO("VSS: Failed to set context to VSS_CTX_BACKUP: " + "not implemented, probably Windows XP, ignored."); + } + else if(result != S_OK) + { + BOX_ERROR("VSS: Failed to set context to VSS_CTX_BACKUP: " << + GetMsgForHresult(result)); + return; + } + + result = mpVssBackupComponents->SetBackupState( + false, /* no components for now */ + true, /* might as well ask for a bootable backup */ + VSS_BT_FULL, + false /* what is Partial File Support? */); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to set backup state: " << + GetMsgForHresult(result)); + return; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterMetadata, + "GatherWriterMetadata()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + UINT writerCount; + result = mpVssBackupComponents->GetWriterMetadataCount(&writerCount); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get writer count: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + for(UINT iWriter = 0; iWriter < writerCount; iWriter++) + { + BOX_INFO("VSS: Getting metadata from writer " << iWriter); + VSS_ID writerInstance; + IVssExamineWriterMetadata* pMetadata; + result = mpVssBackupComponents->GetWriterMetadata(iWriter, + &writerInstance, &pMetadata); + if(result != S_OK) + { + BOX_ERROR("Failed to get VSS metadata from writer " << iWriter << + ": " << GetMsgForHresult(result)); + continue; + } + + UINT includeFiles, excludeFiles, numComponents; + result = pMetadata->GetFileCounts(&includeFiles, &excludeFiles, + &numComponents); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata file counts from " + "writer " << iWriter << ": " << + GetMsgForHresult(result)); + pMetadata->Release(); + continue; + } + + for(UINT iComponent = 0; iComponent < numComponents; iComponent++) + { + IVssWMComponent* pComponent; + result = pMetadata->GetComponent(iComponent, &pComponent); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata component " << + iComponent << " from writer " << iWriter << ": " << + GetMsgForHresult(result)); + continue; + } + + PVSSCOMPONENTINFO pComponentInfo; + result = pComponent->GetComponentInfo(&pComponentInfo); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get metadata component " << + iComponent << " info from writer " << iWriter << ": " << + GetMsgForHresult(result)); + pComponent->Release(); + continue; + } + + BOX_TRACE("VSS: writer " << iWriter << " component " << + iComponent << " info:"); + switch(pComponentInfo->type) + { + case VSS_CT_UNDEFINED: BOX_TRACE("VSS: type: undefined"); break; + case VSS_CT_DATABASE: BOX_TRACE("VSS: type: database"); break; + case VSS_CT_FILEGROUP: BOX_TRACE("VSS: type: filegroup"); break; + default: + BOX_WARNING("VSS: type: unknown (" << pComponentInfo->type << ")"); + } + + BOX_TRACE("VSS: logical path: " << + BstrToString(pComponentInfo->bstrLogicalPath)); + BOX_TRACE("VSS: component name: " << + BstrToString(pComponentInfo->bstrComponentName)); + BOX_TRACE("VSS: caption: " << + BstrToString(pComponentInfo->bstrCaption)); + BOX_TRACE("VSS: restore metadata: " << + pComponentInfo->bRestoreMetadata); + BOX_TRACE("VSS: notify on complete: " << + pComponentInfo->bRestoreMetadata); + BOX_TRACE("VSS: selectable: " << + pComponentInfo->bSelectable); + BOX_TRACE("VSS: selectable for restore: " << + pComponentInfo->bSelectableForRestore); + BOX_TRACE("VSS: component flags: " << + BOX_FORMAT_HEX32(pComponentInfo->dwComponentFlags)); + BOX_TRACE("VSS: file count: " << + pComponentInfo->cFileCount); + BOX_TRACE("VSS: databases: " << + pComponentInfo->cDatabases); + BOX_TRACE("VSS: log files: " << + pComponentInfo->cLogFiles); + BOX_TRACE("VSS: dependencies: " << + pComponentInfo->cDependencies); + + pComponent->FreeComponentInfo(pComponentInfo); + pComponent->Release(); + } + + pMetadata->Release(); + } + + VSS_ID snapshotSetId; + result = mpVssBackupComponents->StartSnapshotSet(&snapshotSetId); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to start snapshot set: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + // Add all volumes included as backup locations to the snapshot set + for(Locations::iterator + iLocation = mLocations.begin(); + iLocation != mLocations.end(); + iLocation++) + { + Location& rLocation(**iLocation); + std::string path = rLocation.mPath; + // convert to absolute and remove Unicode prefix + path = ConvertPathToAbsoluteUnicode(path.c_str()).substr(4); + + if(path.length() >= 3 && path[1] == ':' && path[2] == '\\') + { + std::string volumeRoot = path.substr(0, 3); + + std::map<char, VSS_ID>::iterator i = + volumesIncluded.find(path[0]); + + if(i == volumesIncluded.end()) + { + std::wstring volumeRootWide; + volumeRootWide.push_back((WCHAR) path[0]); + volumeRootWide.push_back((WCHAR) ':'); + volumeRootWide.push_back((WCHAR) '\\'); + VSS_ID newVolumeId; + result = mpVssBackupComponents->AddToSnapshotSet( + (VSS_PWSZ)(volumeRootWide.c_str()), GUID_NULL, + &newVolumeId); + if(result == S_OK) + { + BOX_TRACE("VSS: Added volume " << volumeRoot << + " for backup location " << path << + " to snapshot set"); + volumesIncluded[path[0]] = newVolumeId; + rLocation.mSnapshotVolumeId = newVolumeId; + rLocation.mIsSnapshotCreated = true; + + // If the snapshot path starts with the volume root + // (drive letter), because the path is absolute (as + // it should be), then remove it so that the + // resulting snapshot path can be appended to the + // snapshot device object to make a real path, + // without a spurious drive letter in it. + + if (path.substr(0, volumeRoot.length()) == volumeRoot) + { + path = path.substr(volumeRoot.length()); + } + + rLocation.mSnapshotPath = path; + } + else + { + BOX_ERROR("VSS: Failed to add volume " << + volumeRoot << " to snapshot set: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + } + else + { + BOX_TRACE("VSS: Skipping already included volume " << + volumeRoot << " for backup location " << path); + rLocation.mSnapshotVolumeId = i->second; + rLocation.mIsSnapshotCreated = true; + } + } + else + { + BOX_WARNING("VSS: Skipping backup location " << path << + " which does not start with a volume specification"); + } + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::PrepareForBackup, + "PrepareForBackup()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::DoSnapshotSet, + "DoSnapshotSet()")) + { + goto CreateVssBackupComponents_cleanup_WriterMetadata; + } + + if(!CallAndWaitForAsync(&IVssBackupComponents::GatherWriterStatus, + "GatherWriterStatus()")) + { + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + result = mpVssBackupComponents->GetWriterStatusCount(&writerCount); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get writer status count: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + for(UINT iWriter = 0; iWriter < writerCount; iWriter++) + { + VSS_ID instance, writer; + BSTR writerNameBstr; + VSS_WRITER_STATE writerState; + HRESULT writerResult; + + result = mpVssBackupComponents->GetWriterStatus(iWriter, + &instance, &writer, &writerNameBstr, &writerState, + &writerResult); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query writer " << iWriter << + " status: " << GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + std::string writerName = BstrToString(writerNameBstr); + ::SysFreeString(writerNameBstr); + + if(writerResult != S_OK) + { + BOX_ERROR("VSS: Writer " << iWriter << " (" << + writerName << ") failed: " << + GetMsgForHresult(writerResult)); + continue; + } + + std::string stateName; + + switch(writerState) + { +#define WRITER_STATE(code) \ + case code: stateName = #code; break; + WRITER_STATE(VSS_WS_UNKNOWN); + WRITER_STATE(VSS_WS_STABLE); + WRITER_STATE(VSS_WS_WAITING_FOR_FREEZE); + WRITER_STATE(VSS_WS_WAITING_FOR_THAW); + WRITER_STATE(VSS_WS_WAITING_FOR_POST_SNAPSHOT); + WRITER_STATE(VSS_WS_WAITING_FOR_BACKUP_COMPLETE); + WRITER_STATE(VSS_WS_FAILED_AT_IDENTIFY); + WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_BACKUP); + WRITER_STATE(VSS_WS_FAILED_AT_PREPARE_SNAPSHOT); + WRITER_STATE(VSS_WS_FAILED_AT_FREEZE); + WRITER_STATE(VSS_WS_FAILED_AT_THAW); + WRITER_STATE(VSS_WS_FAILED_AT_POST_SNAPSHOT); + WRITER_STATE(VSS_WS_FAILED_AT_BACKUP_COMPLETE); + WRITER_STATE(VSS_WS_FAILED_AT_PRE_RESTORE); + WRITER_STATE(VSS_WS_FAILED_AT_POST_RESTORE); + WRITER_STATE(VSS_WS_FAILED_AT_BACKUPSHUTDOWN); +#undef WRITER_STATE + default: + std::ostringstream o; + o << "unknown (" << writerState << ")"; + stateName = o.str(); + } + + BOX_TRACE("VSS: Writer " << iWriter << " (" << + writerName << ") is in state " << stateName); + } + + // lookup new snapshot volume for each location that has a snapshot + for(Locations::iterator + iLocation = mLocations.begin(); + iLocation != mLocations.end(); + iLocation++) + { + Location& rLocation(**iLocation); + if(rLocation.mIsSnapshotCreated) + { + VSS_SNAPSHOT_PROP prop; + result = mpVssBackupComponents->GetSnapshotProperties( + rLocation.mSnapshotVolumeId, &prop); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to get snapshot properties " + "for volume " << GuidToString(rLocation.mSnapshotVolumeId) << + " for location " << rLocation.mPath << ": " << + GetMsgForHresult(result)); + rLocation.mIsSnapshotCreated = false; + continue; + } + + rLocation.mSnapshotPath = + WideStringToString(prop.m_pwszSnapshotDeviceObject) + + DIRECTORY_SEPARATOR + rLocation.mSnapshotPath; + FreeSnapshotProp(&prop); + + BOX_INFO("VSS: Location " << rLocation.mPath << " using " + "snapshot path " << rLocation.mSnapshotPath); + } + } + + IVssEnumObject *pEnum; + result = mpVssBackupComponents->Query(GUID_NULL, VSS_OBJECT_NONE, + VSS_OBJECT_SNAPSHOT, &pEnum); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to query snapshot list: " << + GetMsgForHresult(result)); + goto CreateVssBackupComponents_cleanup_WriterStatus; + } + + while(result == S_OK) + { + VSS_OBJECT_PROP rgelt; + ULONG count; + result = pEnum->Next(1, &rgelt, &count); + + if(result == S_FALSE) + { + // end of list, break out of the loop + break; + } + else if(result != S_OK) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + GetMsgForHresult(result)); + } + else if(count != 1) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + "Next() returned " << count << " objects instead of 1"); + } + else if(rgelt.Type != VSS_OBJECT_SNAPSHOT) + { + BOX_ERROR("VSS: Failed to enumerate snapshot: " << + "Next() returned a type " << rgelt.Type << " object " + "instead of VSS_OBJECT_SNAPSHOT"); + } + else + { + VSS_SNAPSHOT_PROP *pSnap = &rgelt.Obj.Snap; + BOX_TRACE("VSS: Snapshot ID: " << + GuidToString(pSnap->m_SnapshotId)); + BOX_TRACE("VSS: Snapshot set ID: " << + GuidToString(pSnap->m_SnapshotSetId)); + BOX_TRACE("VSS: Number of volumes: " << + pSnap->m_lSnapshotsCount); + BOX_TRACE("VSS: Snapshot device object: " << + WideStringToString(pSnap->m_pwszSnapshotDeviceObject)); + BOX_TRACE("VSS: Original volume name: " << + WideStringToString(pSnap->m_pwszOriginalVolumeName)); + BOX_TRACE("VSS: Originating machine: " << + WideStringToString(pSnap->m_pwszOriginatingMachine)); + BOX_TRACE("VSS: Service machine: " << + WideStringToString(pSnap->m_pwszServiceMachine)); + BOX_TRACE("VSS: Exposed name: " << + WideStringToString(pSnap->m_pwszExposedName)); + BOX_TRACE("VSS: Exposed path: " << + WideStringToString(pSnap->m_pwszExposedPath)); + BOX_TRACE("VSS: Provider ID: " << + GuidToString(pSnap->m_ProviderId)); + BOX_TRACE("VSS: Snapshot attributes: " << + BOX_FORMAT_HEX32(pSnap->m_lSnapshotAttributes)); + BOX_TRACE("VSS: Snapshot creation time: " << + BOX_FORMAT_HEX32(pSnap->m_tsCreationTimestamp)); + + std::string status; + switch(pSnap->m_eStatus) + { + case VSS_SS_UNKNOWN: status = "Unknown (error)"; break; + case VSS_SS_PREPARING: status = "Preparing"; break; + case VSS_SS_PROCESSING_PREPARE: status = "Preparing (processing)"; break; + case VSS_SS_PREPARED: status = "Prepared"; break; + case VSS_SS_PROCESSING_PRECOMMIT: status = "Precommitting"; break; + case VSS_SS_PRECOMMITTED: status = "Precommitted"; break; + case VSS_SS_PROCESSING_COMMIT: status = "Commiting"; break; + case VSS_SS_COMMITTED: status = "Committed"; break; + case VSS_SS_PROCESSING_POSTCOMMIT: status = "Postcommitting"; break; + case VSS_SS_PROCESSING_PREFINALCOMMIT: status = "Pre final committing"; break; + case VSS_SS_PREFINALCOMMITTED: status = "Pre final committed"; break; + case VSS_SS_PROCESSING_POSTFINALCOMMIT: status = "Post final committing"; break; + case VSS_SS_CREATED: status = "Created"; break; + case VSS_SS_ABORTED: status = "Aborted"; break; + case VSS_SS_DELETED: status = "Deleted"; break; + case VSS_SS_POSTCOMMITTED: status = "Postcommitted"; break; + default: + std::ostringstream buf; + buf << "Unknown code: " << pSnap->m_eStatus; + status = buf.str(); + } + + BOX_TRACE("VSS: Snapshot status: " << status); + FreeSnapshotProp(pSnap); + } + } + + pEnum->Release(); + +CreateVssBackupComponents_cleanup_WriterStatus: + result = mpVssBackupComponents->FreeWriterStatus(); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to free writer status: " << + GetMsgForHresult(result)); + } + +CreateVssBackupComponents_cleanup_WriterMetadata: + result = mpVssBackupComponents->FreeWriterMetadata(); + if(result != S_OK) + { + BOX_ERROR("VSS: Failed to free writer metadata: " << + GetMsgForHresult(result)); + } +} + +void BackupDaemon::CleanupVssBackupComponents() +{ + if(mpVssBackupComponents == NULL) + { + return; + } + + CallAndWaitForAsync(&IVssBackupComponents::BackupComplete, + "BackupComplete()"); + + mpVssBackupComponents->Release(); + mpVssBackupComponents = NULL; +} +#endif + void BackupDaemon::OnBackupStart() { // Touch a file to record times in filesystem @@ -969,28 +1659,37 @@ void BackupDaemon::OnBackupStart() void BackupDaemon::OnBackupFinish() { - // Log - BOX_NOTICE("Finished scan of local files"); - - // Log the stats - BOX_NOTICE("File statistics: total file size uploaded " - << BackupStoreFile::msStats.mBytesInEncodedFiles - << ", bytes already on server " - << BackupStoreFile::msStats.mBytesAlreadyOnServer - << ", encoded size " - << BackupStoreFile::msStats.mTotalFileStreamSize); - - // Reset statistics again - BackupStoreFile::ResetStats(); + try + { + // Log + BOX_NOTICE("Finished scan of local files"); - // Notify administrator - NotifySysadmin(SysadminNotifier::BackupFinish); + // Log the stats + BOX_NOTICE("File statistics: total file size uploaded " + << BackupStoreFile::msStats.mBytesInEncodedFiles + << ", bytes already on server " + << BackupStoreFile::msStats.mBytesAlreadyOnServer + << ", encoded size " + << BackupStoreFile::msStats.mTotalFileStreamSize + << ", " << mNumFilesUploaded << " files uploaded, " + << mNumDirsCreated << " dirs created"); - // Tell anything connected to the command socket - SendSyncStartOrFinish(false /* finish */); + // Reset statistics again + BackupStoreFile::ResetStats(); - // Touch a file to record times in filesystem - TouchFileInWorkingDir("last_sync_finish"); + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupFinish); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(false /* finish */); + + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_finish"); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to perform backup finish actions: " << e.what()); + } } // -------------------------------------------------------------------------- @@ -1032,33 +1731,14 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() std::string line; if(getLine.GetLine(line, true, 30000)) // 30 seconds should be enough { - // Got a string, interpret - if(line == "now") - { - // Script says do it now. Obey. - waitInSeconds = -1; - } - else - { - try - { - // How many seconds to wait? - waitInSeconds = BoxConvert::Convert<int32_t, const std::string&>(line); - } - catch(ConversionException &e) - { - BOX_ERROR("Invalid output from " - "SyncAllowScript: '" << - line << "' (" << script << ")"); - throw; - } - - BOX_NOTICE("Delaying sync by " << waitInSeconds - << " seconds due to SyncAllowScript " - << "(" << script << ")"); - } + waitInSeconds = BackupDaemon::ParseSyncAllowScriptOutput(script, line); + } + else + { + BOX_ERROR("SyncAllowScript output nothing within " + "30 seconds, waiting 5 minutes to try again" + " (" << script << ")"); } - } catch(std::exception &e) { @@ -1083,6 +1763,83 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() return waitInSeconds; } +int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, + const std::string& output) +{ + int waitInSeconds = (60*5); + std::istringstream iss(output); + + std::string delay; + iss >> delay; + + if(delay == "") + { + BOX_ERROR("SyncAllowScript output an empty line"); + return waitInSeconds; + } + + // Got a string, interpret + if(delay == "now") + { + // Script says do it now. Obey. + waitInSeconds = -1; + + BOX_NOTICE("SyncAllowScript requested a backup now " + << "(" << script << ")"); + } + else + { + try + { + // How many seconds to wait? + waitInSeconds = BoxConvert::Convert<int32_t, const std::string&>(delay); + } + catch(ConversionException &e) + { + BOX_ERROR("SyncAllowScript output an invalid " + "number: '" << output << "' (" << + script << ")"); + throw; + } + + BOX_NOTICE("SyncAllowScript requested a delay of " << + waitInSeconds << " seconds due to SyncAllowScript " + << "(" << script << ")"); + } + + if(iss.eof()) + { + // No bandwidth limit requested + mMaxBandwidthFromSyncAllowScript = 0; + BOX_NOTICE("SyncAllowScript did not set a maximum bandwidth " + "(" << script << ")"); + } + else + { + std::string maxBandwidth; + iss >> maxBandwidth; + + try + { + // How many seconds to wait? + mMaxBandwidthFromSyncAllowScript = + BoxConvert::Convert<int32_t, const std::string&>(maxBandwidth); + } + catch(ConversionException &e) + { + BOX_ERROR("Invalid maximum bandwidth from " + "SyncAllowScript: '" << + output << "' (" << script << ")"); + throw; + } + + BOX_NOTICE("SyncAllowScript set maximum bandwidth to " << + mMaxBandwidthFromSyncAllowScript << " kB/s (" << + script << ")"); + } + + return waitInSeconds; +} // -------------------------------------------------------------------------- @@ -1418,33 +2175,20 @@ void BackupDaemon::SendSyncStartOrFinish(bool SendStart) // -------------------------------------------------------------------------- void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf) { - if(!mLocations.empty()) - { - // Looks correctly set up - return; - } - - // Make sure that if a directory is reinstated, then it doesn't get deleted - mDeleteUnusedRootDirEntriesAfter = 0; - mUnusedRootDirEntries.clear(); - - // Just a check to make sure it's right. - DeleteAllLocations(); - // Going to need a copy of the root directory. Get a connection, // and fetch it. - BackupProtocolClient &connection(rClientContext.GetConnection()); + BackupProtocolCallable& connection(rClientContext.GetConnection()); // Ask server for a list of everything in the root directory, // which is a directory itself - std::auto_ptr<BackupProtocolClientSuccess> dirreply( + std::auto_ptr<BackupProtocolSuccess> dirreply( connection.QueryListDirectory( - BackupProtocolClientListDirectory::RootDirectory, + BackupProtocolListDirectory::RootDirectory, // only directories - BackupProtocolClientListDirectory::Flags_Dir, + BackupProtocolListDirectory::Flags_Dir, // exclude old/deleted stuff - BackupProtocolClientListDirectory::Flags_Deleted | - BackupProtocolClientListDirectory::Flags_OldVersion, + BackupProtocolListDirectory::Flags_Deleted | + BackupProtocolListDirectory::Flags_OldVersion, false /* no attributes */)); // Retrieve the directory from the stream following @@ -1537,35 +2281,68 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // making sure there's a directory created for it. std::vector<std::string> locNames = rLocationsConf.GetSubConfigurationNames(); - + + // We only want completely configured locations to be in the list + // when this function exits, so move them all to a temporary list. + // Entries matching a properly configured location will be moved + // back to mLocations. Anything left in this list after the loop + // finishes will be deleted. + Locations tmpLocations = mLocations; + mLocations.clear(); + + // The ID map list will be repopulated automatically by this loop + mIDMapMounts.clear(); + for(std::vector<std::string>::iterator pLocName = locNames.begin(); pLocName != locNames.end(); pLocName++) { + Location* pLoc = NULL; + + // Try to find and reuse an existing Location object + for(Locations::const_iterator + i = tmpLocations.begin(); + i != tmpLocations.end(); i++) + { + if ((*i)->mName == *pLocName) + { + BOX_TRACE("Location already configured: " << *pLocName); + pLoc = *i; + break; + } + } + const Configuration& rConfig( rLocationsConf.GetSubConfiguration(*pLocName)); - BOX_TRACE("new location: " << *pLocName); - - // Create a record for it - std::auto_ptr<Location> apLoc(new Location); + std::auto_ptr<Location> apLoc; try { - // Setup names in the location record - apLoc->mName = *pLocName; - apLoc->mPath = rConfig.GetKeyValue("Path"); - - // Read the exclude lists from the Configuration - apLoc->mpExcludeFiles = BackupClientMakeExcludeList_Files(rConfig); - apLoc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(rConfig); + if(pLoc == NULL) + { + // Create a record for it + BOX_TRACE("New location: " << *pLocName); + pLoc = new Location; + + // ensure deletion if setup fails + apLoc.reset(pLoc); + + // Setup names in the location record + pLoc->mName = *pLocName; + pLoc->mPath = rConfig.GetKeyValue("Path"); + + // Read the exclude lists from the Configuration + pLoc->mpExcludeFiles = BackupClientMakeExcludeList_Files(rConfig); + pLoc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(rConfig); + } // Does this exist on the server? // Remove from dir object early, so that if we fail // to stat the local directory, we still don't // consider to remote one for deletion. BackupStoreDirectory::Iterator iter(dir); - BackupStoreFilenameClear dirname(apLoc->mName); // generate the filename + BackupStoreFilenameClear dirname(pLoc->mName); // generate the filename BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname); int64_t oid = 0; if(en != 0) @@ -1585,17 +2362,17 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // BSD style statfs -- includes mount point, which is nice. #ifdef HAVE_STRUCT_STATVFS_F_MNTONNAME struct statvfs s; - if(::statvfs(apLoc->mPath.c_str(), &s) != 0) + if(::statvfs(pLoc->mPath.c_str(), &s) != 0) #else // HAVE_STRUCT_STATVFS_F_MNTONNAME struct statfs s; - if(::statfs(apLoc->mPath.c_str(), &s) != 0) + if(::statfs(pLoc->mPath.c_str(), &s) != 0) #endif // HAVE_STRUCT_STATVFS_F_MNTONNAME { - BOX_LOG_SYS_WARNING("Failed to stat location " - "path '" << apLoc->mPath << - "', skipping location '" << - apLoc->mName << "'"); - continue; + THROW_SYS_ERROR("Failed to stat path " + "'" << pLoc->mPath << "' " + "for location " + "'" << pLoc->mName << "'", + CommonException, OSFileError); } // Where the filesystem is mounted @@ -1604,10 +2381,10 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con #else // !HAVE_STRUCT_STATFS_F_MNTONNAME && !WIN32 // Warn in logs if the directory isn't absolute - if(apLoc->mPath[0] != '/') + if(pLoc->mPath[0] != '/') { BOX_WARNING("Location path '" - << apLoc->mPath + << pLoc->mPath << "' is not absolute"); } // Go through the mount points found, and find a suitable one @@ -1622,7 +2399,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // If it matches, the file belongs in that mount point // (sorting order ensures this) BOX_TRACE("checking against mount point " << *i); - if(::strncmp(i->c_str(), apLoc->mPath.c_str(), i->size()) == 0) + if(::strncmp(i->c_str(), pLoc->mPath.c_str(), i->size()) == 0) { // Match mountName = *i; @@ -1630,7 +2407,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con } } BOX_TRACE("mount point chosen for " - << apLoc->mPath << " is " + << pLoc->mPath << " is " << mountName); } @@ -1641,12 +2418,12 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con if(f != mounts.end()) { // Yes -- store the index - apLoc->mIDMapIndex = f->second; + pLoc->mIDMapIndex = f->second; } else { // No -- new index - apLoc->mIDMapIndex = numIDMaps; + pLoc->mIDMapIndex = numIDMaps; mounts[mountName] = numIDMaps; // Store the mount name @@ -1666,7 +2443,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con BackupClientFileAttributes attr; try { - attr.ReadAttributes(apLoc->mPath.c_str(), + attr.ReadAttributes(pLoc->mPath.c_str(), true /* directories have zero mod times */, 0 /* not interested in mod time */, &attrModTime /* get the attribute modification time */); @@ -1674,19 +2451,20 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con catch (BoxException &e) { BOX_ERROR("Failed to get attributes " - "for path '" << apLoc->mPath + "for path '" << pLoc->mPath << "', skipping location '" << - apLoc->mName << "'"); - continue; + pLoc->mName << "'"); + throw; } // Execute create directory command try { - MemBlockStream attrStream(attr); - std::auto_ptr<BackupProtocolClientSuccess> + std::auto_ptr<IOStream> attrStream( + new MemBlockStream(attr)); + std::auto_ptr<BackupProtocolSuccess> dirCreate(connection.QueryCreateDirectory( - BackupProtocolClientListDirectory::RootDirectory, + BackupProtocolListDirectory::RootDirectory, attrModTime, dirname, attrStream)); // Object ID for later creation @@ -1695,40 +2473,64 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con catch (BoxException &e) { BOX_ERROR("Failed to create remote " - "directory '/" << apLoc->mName << + "directory '/" << pLoc->mName << "', skipping location '" << - apLoc->mName << "'"); - continue; + pLoc->mName << "'"); + throw; } } // Create and store the directory object for the root of this location ASSERT(oid != 0); - BackupClientDirectoryRecord *precord = - new BackupClientDirectoryRecord(oid, *pLocName); - apLoc->mpDirectoryRecord.reset(precord); + if(pLoc->mpDirectoryRecord.get() == NULL) + { + BackupClientDirectoryRecord *precord = + new BackupClientDirectoryRecord(oid, *pLocName); + pLoc->mpDirectoryRecord.reset(precord); + } + // Remove it from the temporary list to avoid deletion + tmpLocations.remove(pLoc); + // Push it back on the vector of locations - mLocations.push_back(apLoc.release()); + mLocations.push_back(pLoc); + + if(apLoc.get() != NULL) + { + // Don't delete it now! + apLoc.release(); + } } catch (std::exception &e) { BOX_ERROR("Failed to configure location '" - << apLoc->mName << "' path '" - << apLoc->mPath << "': " << e.what() << + << pLoc->mName << "' path '" + << pLoc->mPath << "': " << e.what() << ": please check for previous errors"); - throw; + mReadErrorsOnFilesystemObjects = true; } catch(...) { BOX_ERROR("Failed to configure location '" - << apLoc->mName << "' path '" - << apLoc->mPath << "': please check for " + << pLoc->mName << "' path '" + << pLoc->mPath << "': please check for " "previous errors"); - throw; + mReadErrorsOnFilesystemObjects = true; } } + + // Now remove any leftovers + for(BackupDaemon::Locations::iterator + i = tmpLocations.begin(); + i != tmpLocations.end(); i++) + { + BOX_INFO("Removing obsolete location from memory: " << + (*i)->mName); + delete *i; + } + + tmpLocations.clear(); // Any entries in the root directory which need deleting? if(dir.GetNumberOfEntries() > 0 && @@ -1791,31 +2593,11 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // -------------------------------------------------------------------------- void BackupDaemon::SetupIDMapsForSync() { - // Need to do different things depending on whether it's an - // in memory implementation, or whether it's all stored on disc. - -#ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION - - // Make sure we have some blank, empty ID maps - DeleteIDMapVector(mNewIDMaps); - FillIDMapVector(mNewIDMaps, true /* new maps */); - - // Then make sure that the current maps have objects, - // even if they are empty (for the very first run) - if(mCurrentIDMaps.empty()) - { - FillIDMapVector(mCurrentIDMaps, false /* current maps */); - } - -#else - // Make sure we have some blank, empty ID maps DeleteIDMapVector(mNewIDMaps); FillIDMapVector(mNewIDMaps, true /* new maps */); DeleteIDMapVector(mCurrentIDMaps); FillIDMapVector(mCurrentIDMaps, false /* new maps */); - -#endif } @@ -1848,6 +2630,24 @@ void BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVec filename += ".n"; } + // The new map file should not exist yet. If there's + // one left over from a previous failed run, it's not + // useful to us because we never read from it and will + // overwrite the entries of all files that still + // exist, so we should just delete it and start afresh. + if(NewMaps && FileExists(filename.c_str())) + { + BOX_NOTICE("Found an incomplete ID map " + "database, deleting it to start " + "afresh: " << filename); + if(unlink(filename.c_str()) != 0) + { + BOX_LOG_NATIVE_ERROR(BOX_FILE_MESSAGE( + filename, "Failed to delete " + "incomplete ID map database")); + } + } + // If it's not a new map, it may not exist in which case an empty map should be created if(!NewMaps && !FileExists(filename.c_str())) { @@ -1942,21 +2742,6 @@ void BackupDaemon::MakeMapBaseName(unsigned int MountNumber, std::string &rNameO // -------------------------------------------------------------------------- void BackupDaemon::CommitIDMapsAfterSync() { - // Need to do different things depending on whether it's an in memory implementation, - // or whether it's all stored on disc. - -#ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION - // Remove the current ID maps - DeleteIDMapVector(mCurrentIDMaps); - - // Copy the (pointers to) "new" maps over to be the new "current" maps - mCurrentIDMaps = mNewIDMaps; - - // Clear the new ID maps vector (not delete them!) - mNewIDMaps.clear(); - -#else - // Get rid of the maps in memory (leaving them on disc of course) DeleteIDMapVector(mCurrentIDMaps); DeleteIDMapVector(mNewIDMaps); @@ -1980,8 +2765,6 @@ void BackupDaemon::CommitIDMapsAfterSync() THROW_EXCEPTION(CommonException, OSFileError) } } - -#endif } @@ -2003,7 +2786,6 @@ void BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rV rVector.pop_back(); // Close and delete - toDel->Close(); delete toDel; } ASSERT(rVector.size() == 0); @@ -2022,7 +2804,7 @@ void BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rV bool BackupDaemon::FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const { // Search for the location - for(std::vector<Location *>::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i) + for(Locations::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i) { if((*i)->mName == rLocationName) { @@ -2120,7 +2902,16 @@ void BackupDaemon::TouchFileInWorkingDir(const char *Filename) fn += Filename; // Open and close it to update the timestamp - FileStream touch(fn.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + try + { + FileStream touch(fn, O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to write to timestamp file: " << fn << ": " << + e.what()); + } } @@ -2259,7 +3050,7 @@ void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) // Entries to delete, and it's the right time to do so... BOX_NOTICE("Deleting unused locations from store root..."); - BackupProtocolClient &connection(rContext.GetConnection()); + BackupProtocolCallable &connection(rContext.GetConnection()); for(std::vector<std::pair<int64_t,std::string> >::iterator i(mUnusedRootDirEntries.begin()); i != mUnusedRootDirEntries.end(); ++i) @@ -2290,222 +3081,6 @@ typedef struct // -------------------------------------------------------------------------- // // Function -// Name: BackupDaemon::Location::Location() -// Purpose: Constructor -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -BackupDaemon::Location::Location() - : mIDMapIndex(0), - mpExcludeFiles(0), - mpExcludeDirs(0) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::Location::~Location() -// Purpose: Destructor -// Created: 11/11/03 -// -// -------------------------------------------------------------------------- -BackupDaemon::Location::~Location() -{ - // Clean up exclude locations - if(mpExcludeDirs != 0) - { - delete mpExcludeDirs; - mpExcludeDirs = 0; - } - if(mpExcludeFiles != 0) - { - delete mpExcludeFiles; - mpExcludeFiles = 0; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::Location::Deserialize(Archive & rArchive) -// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. -// -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -void BackupDaemon::Location::Deserialize(Archive &rArchive) -{ - // - // - // - mpDirectoryRecord.reset(NULL); - if(mpExcludeFiles) - { - delete mpExcludeFiles; - mpExcludeFiles = NULL; - } - if(mpExcludeDirs) - { - delete mpExcludeDirs; - mpExcludeDirs = NULL; - } - - // - // - // - rArchive.Read(mName); - rArchive.Read(mPath); - rArchive.Read(mIDMapIndex); - - // - // - // - int64_t aMagicMarker = 0; - rArchive.Read(aMagicMarker); - - if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) - { - // NOOP - } - else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) - { - BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, ""); - if(!pSubRecord) - { - throw std::bad_alloc(); - } - - mpDirectoryRecord.reset(pSubRecord); - mpDirectoryRecord->Deserialize(rArchive); - } - else - { - // there is something going on here - THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); - } - - // - // - // - rArchive.Read(aMagicMarker); - - if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) - { - // NOOP - } - else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) - { - mpExcludeFiles = new ExcludeList; - if(!mpExcludeFiles) - { - throw std::bad_alloc(); - } - - mpExcludeFiles->Deserialize(rArchive); - } - else - { - // there is something going on here - THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); - } - - // - // - // - rArchive.Read(aMagicMarker); - - if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) - { - // NOOP - } - else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) - { - mpExcludeDirs = new ExcludeList; - if(!mpExcludeDirs) - { - throw std::bad_alloc(); - } - - mpExcludeDirs->Deserialize(rArchive); - } - else - { - // there is something going on here - THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::Location::Serialize(Archive & rArchive) -// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. -// -// Created: 2005/04/11 -// -// -------------------------------------------------------------------------- -void BackupDaemon::Location::Serialize(Archive & rArchive) const -{ - // - // - // - rArchive.Write(mName); - rArchive.Write(mPath); - rArchive.Write(mIDMapIndex); - - // - // - // - if(mpDirectoryRecord.get() == NULL) - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; - rArchive.Write(aMagicMarker); - } - else - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows - rArchive.Write(aMagicMarker); - - mpDirectoryRecord->Serialize(rArchive); - } - - // - // - // - if(!mpExcludeFiles) - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; - rArchive.Write(aMagicMarker); - } - else - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows - rArchive.Write(aMagicMarker); - - mpExcludeFiles->Serialize(rArchive); - } - - // - // - // - if(!mpExcludeDirs) - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; - rArchive.Write(aMagicMarker); - } - else - { - int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows - rArchive.Write(aMagicMarker); - - mpExcludeDirs->Serialize(rArchive); - } -} - -// -------------------------------------------------------------------------- -// -// Function // Name: BackupDaemon::CommandSocketInfo::CommandSocketInfo() // Purpose: Constructor // Created: 18/2/04 @@ -2591,10 +3166,11 @@ bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime, int64_t iCount = mLocations.size(); anArchive.Write(iCount); - for(int v = 0; v < iCount; v++) + for(Locations::const_iterator i = mLocations.begin(); + i != mLocations.end(); i++) { - ASSERT(mLocations[v]); - mLocations[v]->Serialize(anArchive); + ASSERT(*i); + (*i)->Serialize(anArchive); } // diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h index b41c6508..1d3c991e 100644 --- a/bin/bbackupd/BackupDaemon.h +++ b/bin/bbackupd/BackupDaemon.h @@ -24,13 +24,20 @@ #include "SocketStream.h" #include "TLSContext.h" -#include "autogen_BackupProtocolClient.h" +#include "autogen_BackupProtocol.h" #ifdef WIN32 #include "WinNamedPipeListener.h" #include "WinNamedPipeStream.h" #endif +#ifdef ENABLE_VSS +# include <comdef.h> +# include <Vss.h> +# include <VsWriter.h> +# include <VsBackup.h> +#endif + class BackupClientDirectoryRecord; class BackupClientContext; class Configuration; @@ -110,6 +117,7 @@ public: void InitCrypto(); void RunSyncNowWithExceptionHandling(); void RunSyncNow(); + void ResetCachedState(); void OnBackupStart(); void OnBackupFinish(); // TouchFileInWorkingDir is only here for use by Boxi. @@ -150,33 +158,15 @@ private: int UseScriptToSeeIfSyncAllowed(); public: - class Location - { - public: - Location(); - ~Location(); - - void Deserialize(Archive & rArchive); - void Serialize(Archive & rArchive) const; - private: - Location(const Location &); // copy not allowed - Location &operator=(const Location &); - public: - std::string mName; - std::string mPath; - std::auto_ptr<BackupClientDirectoryRecord> mpDirectoryRecord; - int mIDMapIndex; - ExcludeList *mpExcludeFiles; - ExcludeList *mpExcludeDirs; - }; - - typedef const std::vector<Location *> Locations; + int ParseSyncAllowScriptOutput(const std::string& script, + const std::string& output); + typedef std::list<Location *> Locations; Locations GetLocations() { return mLocations; } private: int mState; // what the daemon is currently doing - std::vector<Location *> mLocations; + Locations mLocations; std::vector<std::string> mIDMapMounts; std::vector<BackupClientInodeToIDMap *> mCurrentIDMaps; @@ -222,8 +212,11 @@ private: TLSContext mTlsContext; bool mDeleteStoreObjectInfoFile; bool mDoSyncForcedByPreviousSyncError; + int64_t mNumFilesUploaded, mNumDirsCreated; + int mMaxBandwidthFromSyncAllowScript; public: + int GetMaxBandwidthFromSyncAllowScript() { return mMaxBandwidthFromSyncAllowScript; } bool StopRun() { return this->Daemon::StopRun(); } bool StorageLimitExceeded() { return mStorageLimitExceeded; } @@ -368,7 +361,7 @@ public: int type, int subtype) { std::ostringstream msgs; - if (type != BackupProtocolClientError::ErrorType) + if (type != BackupProtocolError::ErrorType) { msgs << "unknown error type " << type; } @@ -376,46 +369,46 @@ public: { switch(subtype) { - case BackupProtocolClientError::Err_WrongVersion: + case BackupProtocolError::Err_WrongVersion: msgs << "WrongVersion"; break; - case BackupProtocolClientError::Err_NotInRightProtocolPhase: + case BackupProtocolError::Err_NotInRightProtocolPhase: msgs << "NotInRightProtocolPhase"; break; - case BackupProtocolClientError::Err_BadLogin: + case BackupProtocolError::Err_BadLogin: msgs << "BadLogin"; break; - case BackupProtocolClientError::Err_CannotLockStoreForWriting: + case BackupProtocolError::Err_CannotLockStoreForWriting: msgs << "CannotLockStoreForWriting"; break; - case BackupProtocolClientError::Err_SessionReadOnly: + case BackupProtocolError::Err_SessionReadOnly: msgs << "SessionReadOnly"; break; - case BackupProtocolClientError::Err_FileDoesNotVerify: + case BackupProtocolError::Err_FileDoesNotVerify: msgs << "FileDoesNotVerify"; break; - case BackupProtocolClientError::Err_DoesNotExist: + case BackupProtocolError::Err_DoesNotExist: msgs << "DoesNotExist"; break; - case BackupProtocolClientError::Err_DirectoryAlreadyExists: + case BackupProtocolError::Err_DirectoryAlreadyExists: msgs << "DirectoryAlreadyExists"; break; - case BackupProtocolClientError::Err_CannotDeleteRoot: + case BackupProtocolError::Err_CannotDeleteRoot: msgs << "CannotDeleteRoot"; break; - case BackupProtocolClientError::Err_TargetNameExists: + case BackupProtocolError::Err_TargetNameExists: msgs << "TargetNameExists"; break; - case BackupProtocolClientError::Err_StorageLimitExceeded: + case BackupProtocolError::Err_StorageLimitExceeded: msgs << "StorageLimitExceeded"; break; - case BackupProtocolClientError::Err_DiffFromFileDoesNotExist: + case BackupProtocolError::Err_DiffFromFileDoesNotExist: msgs << "DiffFromFileDoesNotExist"; break; - case BackupProtocolClientError::Err_DoesNotExistInDirectory: + case BackupProtocolError::Err_DoesNotExistInDirectory: msgs << "DoesNotExistInDirectory"; break; - case BackupProtocolClientError::Err_PatchConsistencyError: + case BackupProtocolError::Err_PatchConsistencyError: msgs << "PatchConsistencyError"; break; default: @@ -457,12 +450,15 @@ public: virtual void NotifyFileUploaded( const BackupClientDirectoryRecord* pDirRecord, const std::string& rLocalPath, - int64_t FileSize) + int64_t FileSize, int64_t UploadedSize) { if (mLogAllFileAccess) { - BOX_NOTICE("Uploaded file: " << rLocalPath); - } + BOX_NOTICE("Uploaded file: " << rLocalPath << ", " + "total size = " << FileSize << ", " + "uploaded size = " << UploadedSize); + } + mNumFilesUploaded++; } virtual void NotifyFileSynchronised( const BackupClientDirectoryRecord* pDirRecord, @@ -474,6 +470,19 @@ public: BOX_INFO("Synchronised file: " << rLocalPath); } } + virtual void NotifyDirectoryCreated( + int64_t ObjectID, + const std::string& rLocalPath, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Created directory: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + mNumDirsCreated++; + } virtual void NotifyDirectoryDeleted( int64_t ObjectID, const std::string& rRemotePath) @@ -520,6 +529,16 @@ public: bool mInstallService, mRemoveService, mRunAsService; std::string mServiceName; #endif + +#ifdef ENABLE_VSS + IVssBackupComponents* mpVssBackupComponents; + void CreateVssBackupComponents(); + bool WaitForAsync(IVssAsync *pAsync, const std::string& description); + typedef HRESULT (__stdcall IVssBackupComponents::*AsyncMethod)(IVssAsync**); + bool CallAndWaitForAsync(AsyncMethod method, + const std::string& description); + void CleanupVssBackupComponents(); +#endif }; #endif // BACKUPDAEMON__H diff --git a/bin/bbackupd/BackupDaemonInterface.h b/bin/bbackupd/BackupDaemonInterface.h index 2a2d8d4b..a847b264 100644 --- a/bin/bbackupd/BackupDaemonInterface.h +++ b/bin/bbackupd/BackupDaemonInterface.h @@ -129,11 +129,15 @@ class ProgressNotifier virtual void NotifyFileUploaded( const BackupClientDirectoryRecord* pDirRecord, const std::string& rLocalPath, - int64_t FileSize) = 0; + int64_t FileSize, int64_t UploadedSize) = 0; virtual void NotifyFileSynchronised( const BackupClientDirectoryRecord* pDirRecord, const std::string& rLocalPath, int64_t FileSize) = 0; + virtual void NotifyDirectoryCreated( + int64_t ObjectID, + const std::string& rLocalPath, + const std::string& rRemotePath) = 0; virtual void NotifyDirectoryDeleted( int64_t ObjectID, const std::string& rRemotePath) = 0; diff --git a/bin/bbackupd/bbackupd-config.in b/bin/bbackupd/bbackupd-config.in index 98dc8b6e..1fc224c2 100755 --- a/bin/bbackupd/bbackupd-config.in +++ b/bin/bbackupd/bbackupd-config.in @@ -227,7 +227,7 @@ SUBJECT="BACKUP PROBLEM on host $hostname" SENDTO="$current_username" if [ "\$1" = "" ]; then - echo "Usage: \$0 <store-full|read-error|backup-error|backup-start|backup-finish>" >&2 + echo "Usage: \$0 <store-full|read-error|backup-ok|backup-error|backup-start|backup-finish>" >&2 exit 2 elif [ "\$1" = store-full ]; then $sendmail \$SENDTO <<EOM @@ -262,7 +262,7 @@ these errors, and take appropriate action. Other files are being backed up. EOM -elif [ "\$1" = backup-start -o "\$1" = backup-finish ]; then +elif [ "\$1" = backup-start -o "\$1" = backup-finish -o "\$1" = backup-ok ]; then # do nothing by default true else diff --git a/bin/bbackupd/bbackupd.cpp b/bin/bbackupd/bbackupd.cpp index d334a2df..bb64f745 100644 --- a/bin/bbackupd/bbackupd.cpp +++ b/bin/bbackupd/bbackupd.cpp @@ -41,12 +41,13 @@ int main(int argc, const char *argv[]) ExitCode = gpDaemonService->Daemon::Main( BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE, argc, argv); - delete gpDaemonService; + delete gpDaemonService; #else // !WIN32 BackupDaemon daemon; - ExitCode = daemon.Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); + ExitCode = daemon.Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE, + argc, argv); #endif // WIN32 diff --git a/bin/bbackupd/win32/installer.iss b/bin/bbackupd/win32/installer.iss index 20e3addb..e69de29b 100644 --- a/bin/bbackupd/win32/installer.iss +++ b/bin/bbackupd/win32/installer.iss @@ -1,51 +0,0 @@ -; Script to generate output file for Box Backup client for the Windows Platform
-;
-; Very important - this is the release process
-;
-; 1/ Upgrade BOX_VERSION in the file emu.h to the current version for example 0.09eWin32 - then perform a full rebuild
-;
-; 2/ Upgrade the AppVerName below to reflect the version
-;
-; 3/ Generate the output file, then rename it to the relevent filename to reflect the version
-
-[Setup]
-AppName=Box Backup
-AppVerName=BoxWin32 0.09h
-AppPublisher=Fluffy & Omniis
-AppPublisherURL=http://www.omniis.com
-AppSupportURL=http://www.omniis.com
-AppUpdatesURL=http://www.omniis.com
-DefaultDirName={pf}\Box Backup
-DefaultGroupName=Box Backup
-Compression=lzma
-SolidCompression=yes
-PrivilegesRequired=admin
-
-[Files]
-Source: "..\..\Release\bbackupd.exe"; DestDir: "{app}"; Flags: ignoreversion restartreplace
-Source: "..\..\Release\bbackupctl.exe"; DestDir: "{app}"; Flags: ignoreversion restartreplace
-Source: "..\..\Release\bbackupquery.exe"; DestDir: "{app}"; Flags: ignoreversion restartreplace
-Source: "..\..\ExceptionCodes.txt"; DestDir: "{app}"; Flags: ignoreversion restartreplace
-Source: "icon.ico"; DestDir: "{app}\"; Flags: ignoreversion restartreplace
-Source: "msvcr71.dll"; DestDir: "{app}\"; Flags: restartreplace
-Source: "bbackupd.conf"; DestDir: "{app}"; Flags: confirmoverwrite
-Source: "..\..\..\zlib\zlib1.dll"; DestDir: "{app}"; Flags: ignoreversion restartreplace
-Source: "..\..\..\openssl\bin\libeay32.dll"; DestDir: "{app}"; Flags: ignoreversion restartreplace
-Source: "..\..\..\openssl\bin\ssleay32.dll"; DestDir: "{app}"; Flags: ignoreversion restartreplace
-Source: "ReadMe.txt"; DestDir: "{app}"; Flags: ignoreversion restartreplace
-
-; NOTE: Don't use "Flags: ignoreversion" on any shared system files
-
-[Icons]
-Name: "{group}\Box Backup Query"; Filename: "{app}\bbackupquery.exe"; IconFilename: "{app}\icon.ico" ;Parameters: "-c bbackupd.conf"; WorkingDir: "{app}"
-Name: "{group}\Service\Install Service"; Filename: "{app}\bbackupd.exe"; IconFilename: "{app}\icon.ico" ;Parameters: "-i"; WorkingDir: "{app}"
-Name: "{group}\Service\Remove Service"; Filename: "{app}\bbackupd.exe"; IconFilename: "{app}\icon.ico" ;Parameters: "-r"; WorkingDir: "{app}"
-Name: "{group}\Initiate Backup Now"; Filename: "{app}\bbackupctl.exe"; IconFilename: "{app}\icon.ico" ;Parameters: "-c bbackupd.conf sync"; WorkingDir: "{app}"
-
-[Dirs]
-Name: "{app}\bbackupd"
-
-[Run]
-Filename: "{app}\bbackupd.exe"; Description: "Install Boxbackup as service"; Parameters: "-i"; Flags: postinstall
-Filename: "{app}\Readme.txt"; Description: "View upgrade notes"; Flags: postinstall shellexec skipifsilent
-
diff --git a/bin/bbackupquery/BackupQueries.cpp b/bin/bbackupquery/BackupQueries.cpp index 60724800..b8b9525b 100644 --- a/bin/bbackupquery/BackupQueries.cpp +++ b/bin/bbackupquery/BackupQueries.cpp @@ -48,9 +48,9 @@ #include "Logging.h" #include "PathUtils.h" #include "SelfFlushingStream.h" -#include "TemporaryDirectory.h" #include "Utils.h" -#include "autogen_BackupProtocolClient.h" +#include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" #include "MemLeakFindOn.h" @@ -100,12 +100,6 @@ BackupQueries::~BackupQueries() { } -typedef struct -{ - const char* name; - const char* opts; -} QueryCommandSpecification; - // -------------------------------------------------------------------------- // // Function @@ -114,173 +108,46 @@ typedef struct // Created: 2003/10/10 // // -------------------------------------------------------------------------- -void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) +void BackupQueries::DoCommand(ParsedCommand& rCommand) { - // is the command a shell command? - if(Command[0] == 's' && Command[1] == 'h' && Command[2] == ' ' && Command[3] != '\0') + // Check... + + if(rCommand.mFailed) { - // Yes, run shell command - int result = ::system(Command + 3); - if(result != 0) - { - BOX_WARNING("System command returned error code " << - result); - SetReturnCode(ReturnCode::Command_Error); - } + BOX_ERROR("Parse failed"); return; } - // split command into components - std::vector<std::string> cmdElements; - std::string options; + if(rCommand.mCmdElements.size() < 1) { - const char *c = Command; - bool inQuoted = false; - bool inOptions = false; - - std::string s; - while(*c != 0) - { - // Terminating char? - if(*c == ((inQuoted)?'"':' ')) - { - if(!s.empty()) cmdElements.push_back(s); - s.resize(0); - inQuoted = false; - inOptions = false; - } - else - { - // No. Start of quoted parameter? - if(s.empty() && *c == '"') - { - inQuoted = true; - } - // Start of options? - else if(s.empty() && *c == '-') - { - inOptions = true; - } - else - { - if(inOptions) - { - // Option char - options += *c; - } - else - { - // Normal string char - s += *c; - } - } - } - - ++c; - } - if(!s.empty()) cmdElements.push_back(s); + // blank command + return; } - - #ifdef WIN32 - if (isFromCommandLine) + + if(rCommand.pSpec->type == Command_sh && + rCommand.mCmdElements.size() == 2) { - for (std::vector<std::string>::iterator - i = cmdElements.begin(); - i != cmdElements.end(); i++) + // Yes, run shell command + int result = ::system(rCommand.mCmdElements[1].c_str()); + if(result != 0) { - std::string converted; - if (!ConvertEncoding(*i, CP_ACP, converted, - GetConsoleCP())) - { - BOX_ERROR("Failed to convert encoding"); - return; - } - *i = converted; + BOX_WARNING("System command returned error code " << + result); + SetReturnCode(ReturnCode::Command_Error); } - } - #endif - - // Check... - if(cmdElements.size() < 1) - { - // blank command return; } - - // Data about commands - static QueryCommandSpecification commands[] = - { - { "quit", "" }, - { "exit", "" }, - { "list", "rodIFtTash", }, - { "pwd", "" }, - { "cd", "od" }, - { "lcd", "" }, - { "sh", "" }, - { "getobject", "" }, - { "get", "i" }, - { "compare", "alcqAEQ" }, - { "restore", "drif" }, - { "help", "" }, - { "usage", "m" }, - { "undelete", "i" }, - { "delete", "i" }, - { NULL, NULL } - }; - - typedef enum - { - Command_Quit = 0, - Command_Exit, - Command_List, - Command_pwd, - Command_cd, - Command_lcd, - Command_sh, - Command_GetObject, - Command_Get, - Command_Compare, - Command_Restore, - Command_Help, - Command_Usage, - Command_Undelete, - Command_Delete, - } - CommandType; - - static const char *alias[] = {"ls", 0}; - static const int aliasIs[] = {Command_List, 0}; - - // Work out which command it is... - int cmd = 0; - while(commands[cmd].name != 0 && ::strcmp(cmdElements[0].c_str(), commands[cmd].name) != 0) - { - cmd++; - } - if(commands[cmd].name == 0) + + if(rCommand.pSpec->type == Command_Unknown) { - // Check for aliases - int a; - for(a = 0; alias[a] != 0; ++a) - { - if(::strcmp(cmdElements[0].c_str(), alias[a]) == 0) - { - // Found an alias - cmd = aliasIs[a]; - break; - } - } - // No such command - if(alias[a] == 0) - { - BOX_ERROR("Unrecognised command: " << Command); - return; - } + BOX_ERROR("Unrecognised command: " << rCommand.mCmdElements[0]); + return; } // Arguments - std::vector<std::string> args(cmdElements.begin() + 1, cmdElements.end()); + std::vector<std::string> args(rCommand.mCmdElements.begin() + 1, + rCommand.mCmdElements.end()); // Set up options bool opts[256]; @@ -288,14 +155,14 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) // BLOCK { // options - const char *c = options.c_str(); + const char *c = rCommand.mOptions.c_str(); while(*c != 0) { // Valid option? - if(::strchr(commands[cmd].opts, *c) == NULL) + if(::strchr(rCommand.pSpec->opts, *c) == NULL) { BOX_ERROR("Invalid option '" << *c << "' for " - "command " << commands[cmd].name); + "command " << rCommand.pSpec->name); return; } opts[(int)*c] = true; @@ -303,17 +170,16 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) } } - if(cmd != Command_Quit && cmd != Command_Exit) + if(rCommand.pSpec->type != Command_Quit) { // If not a quit command, set the return code to zero SetReturnCode(ReturnCode::Command_OK); } // Handle command - switch(cmd) + switch(rCommand.pSpec->type) { case Command_Quit: - case Command_Exit: mQuitNow = true; break; @@ -375,7 +241,7 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) break; default: - BOX_ERROR("Unknown command: " << Command); + BOX_ERROR("Unknown command: " << rCommand.mCmdElements[0]); break; } } @@ -392,8 +258,6 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) void BackupQueries::CommandList(const std::vector<std::string> &args, const bool *opts) { #define LIST_OPTION_RECURSIVE 'r' - #define LIST_OPTION_ALLOWOLD 'o' - #define LIST_OPTION_ALLOWDELETED 'd' #define LIST_OPTION_NOOBJECTID 'I' #define LIST_OPTION_NOFLAGS 'F' #define LIST_OPTION_TIMES_LOCAL 't' @@ -492,22 +356,28 @@ static std::string GetTimeString(BackupStoreDirectory::Entry& en, // Created: 2003/10/10 // // -------------------------------------------------------------------------- -void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool *opts, bool FirstLevel) +void BackupQueries::List(int64_t DirID, const std::string &rListRoot, + const bool *opts, bool FirstLevel, std::ostream* pOut) { +#ifdef WIN32 + DWORD n_chars; + HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE); +#endif + // Generate exclude flags - int16_t excludeFlags = BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING; - if(!opts[LIST_OPTION_ALLOWOLD]) excludeFlags |= BackupProtocolClientListDirectory::Flags_OldVersion; - if(!opts[LIST_OPTION_ALLOWDELETED]) excludeFlags |= BackupProtocolClientListDirectory::Flags_Deleted; + int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + if(!opts[LIST_OPTION_ALLOWOLD]) excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + if(!opts[LIST_OPTION_ALLOWDELETED]) excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; // Do communication try { mrConnection.QueryListDirectory( - DirID, - BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, - // both files and directories - excludeFlags, - true /* want attributes */); + DirID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // both files and directories + excludeFlags, + true /* want attributes */); } catch (std::exception &e) { @@ -522,7 +392,6 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool return; } - // Retrieve the directory from the stream following BackupStoreDirectory dir; std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); @@ -533,6 +402,8 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { + std::ostringstream buf; + // Display this entry BackupStoreFilenameClear clear(en->GetName()); @@ -540,11 +411,9 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool if(!opts[LIST_OPTION_NOOBJECTID]) { // add object ID to line -#ifdef _MSC_VER - printf("%08I64x ", (int64_t)en->GetObjectID()); -#else - printf("%08llx ", (long long)en->GetObjectID()); -#endif + buf << std::hex << std::internal << std::setw(8) << + std::setfill('0') << en->GetObjectID() << + std::dec << " "; } // Flags? @@ -571,44 +440,40 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool // terminate *(f++) = ' '; *(f++) = '\0'; - printf(displayflags); + buf << displayflags; if(en_flags != 0) { - printf("[ERROR: Entry has additional flags set] "); + buf << "[ERROR: Entry has additional flags set] "; } } if(opts[LIST_OPTION_TIMES_UTC]) { // Show UTC times... - printf("%s ", GetTimeString(*en, false, - opts[LIST_OPTION_TIMES_ATTRIBS]).c_str()); + buf << GetTimeString(*en, false, + opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; } if(opts[LIST_OPTION_TIMES_LOCAL]) { // Show local times... - printf("%s ", GetTimeString(*en, true, - opts[LIST_OPTION_TIMES_ATTRIBS]).c_str()); + buf << GetTimeString(*en, true, + opts[LIST_OPTION_TIMES_ATTRIBS]) << " "; } if(opts[LIST_OPTION_DISPLAY_HASH]) { -#ifdef _MSC_VER - printf("%016I64x ", (int64_t)en->GetAttributesHash()); -#else - printf("%016llx ", (long long)en->GetAttributesHash()); -#endif + buf << std::hex << std::internal << std::setw(16) << + std::setfill('0') << en->GetAttributesHash() << + std::dec; } if(opts[LIST_OPTION_SIZEINBLOCKS]) { -#ifdef _MSC_VER - printf("%05I64d ", (int64_t)en->GetSizeInBlocks()); -#else - printf("%05lld ", (long long)en->GetSizeInBlocks()); -#endif + buf << std::internal << std::setw(5) << + std::setfill('0') << en->GetSizeInBlocks() << + " "; } // add name @@ -618,30 +483,60 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool std::string listRootDecoded; if(!ConvertUtf8ToConsole(rListRoot.c_str(), listRootDecoded)) return; - printf("%s/", listRootDecoded.c_str()); + listRootDecoded += "/"; + buf << listRootDecoded; + WriteConsole(hOut, listRootDecoded.c_str(), + strlen(listRootDecoded.c_str()), &n_chars, NULL); #else - printf("%s/", rListRoot.c_str()); + buf << rListRoot << "/"; #endif } + std::string fileName; + try + { + fileName = clear.GetClearFilename(); + } + catch(CipherException &e) + { + fileName = "<decrypt failed>"; + } + #ifdef WIN32 + std::string fileNameUtf8 = fileName; + if(!ConvertUtf8ToConsole(fileNameUtf8, fileName)) { - std::string fileName; - if(!ConvertUtf8ToConsole( - clear.GetClearFilename().c_str(), fileName)) - return; - printf("%s", fileName.c_str()); + fileName = fileNameUtf8 + " [convert encoding failed]"; } -#else - printf("%s", clear.GetClearFilename().c_str()); #endif - + + buf << fileName; + if(!en->GetName().IsEncrypted()) { - printf("[FILENAME NOT ENCRYPTED]"); + buf << " [FILENAME NOT ENCRYPTED]"; } - printf("\n"); + buf << std::endl; + + if(pOut) + { + (*pOut) << buf.str(); + } + else + { +#ifdef WIN32 + std::string line = buf.str(); + if (!WriteConsole(hOut, line.c_str(), line.size(), + &n_chars, NULL)) + { + // WriteConsole failed, try standard method + std::cout << buf.str(); + } +#else + std::cout << buf.str(); +#endif + } // Directory? if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) @@ -652,7 +547,9 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool std::string subroot(rListRoot); if(!FirstLevel) subroot += '/'; subroot += clear.GetClearFilename(); - List(en->GetObjectID(), subroot, opts, false /* not the first level to list */); + List(en->GetObjectID(), subroot, opts, + false /* not the first level to list */, + pOut); } } } @@ -681,7 +578,7 @@ int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, // Start from current stack, or root, whichever is required std::vector<std::pair<std::string, int64_t> > stack; - int64_t dirID = BackupProtocolClientListDirectory::RootDirectory; + int64_t dirID = BackupProtocolListDirectory::RootDirectory; if(rDirName.size() > 0 && rDirName[0] == '/') { // Root, do nothing @@ -697,9 +594,9 @@ int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, } // Generate exclude flags - int16_t excludeFlags = BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING; - if(!AllowOldVersion) excludeFlags |= BackupProtocolClientListDirectory::Flags_OldVersion; - if(!AllowDeletedDirs) excludeFlags |= BackupProtocolClientListDirectory::Flags_Deleted; + int16_t excludeFlags = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; + if(!AllowOldVersion) excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + if(!AllowDeletedDirs) excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; // Read directories for(unsigned int e = 0; e < dirElements.size(); ++e) @@ -719,20 +616,20 @@ int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, stack.pop_back(); // New dir ID - dirID = (stack.size() > 0)?(stack[stack.size() - 1].second):BackupProtocolClientListDirectory::RootDirectory; + dirID = (stack.size() > 0)?(stack[stack.size() - 1].second):BackupProtocolListDirectory::RootDirectory; } else { // At root anyway - dirID = BackupProtocolClientListDirectory::RootDirectory; + dirID = BackupProtocolListDirectory::RootDirectory; } } else { // Not blank element. Read current directory. - std::auto_ptr<BackupProtocolClientSuccess> dirreply(mrConnection.QueryListDirectory( + std::auto_ptr<BackupProtocolSuccess> dirreply(mrConnection.QueryListDirectory( dirID, - BackupProtocolClientListDirectory::Flags_Dir, // just directories + BackupProtocolListDirectory::Flags_Dir, // just directories excludeFlags, true /* want attributes */)); @@ -783,7 +680,7 @@ int64_t BackupQueries::GetCurrentDirectoryID() // Special case for root if(mDirStack.size() == 0) { - return BackupProtocolClientListDirectory::RootDirectory; + return BackupProtocolListDirectory::RootDirectory; } // Otherwise, get from the last entry on the stack @@ -955,7 +852,8 @@ void BackupQueries::CommandGetObject(const std::vector<std::string> &args, const int64_t id = ::strtoll(args[0].c_str(), 0, 16); if(id == std::numeric_limits<long long>::min() || id == std::numeric_limits<long long>::max() || id == 0) { - BOX_ERROR("Not a valid object ID (specified in hex)."); + BOX_ERROR("Not a valid object ID (specified in hex): " << + args[0]); return; } @@ -974,8 +872,8 @@ void BackupQueries::CommandGetObject(const std::vector<std::string> &args, const try { // Request object - std::auto_ptr<BackupProtocolClientSuccess> getobj(mrConnection.QueryGetObject(id)); - if(getobj->GetObjectID() != BackupProtocolClientGetObject::NoObject) + std::auto_ptr<BackupProtocolSuccess> getobj(mrConnection.QueryGetObject(id)); + if(getobj->GetObjectID() != BackupProtocolGetObject::NoObject) { // Stream that object out to the file std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); @@ -1062,7 +960,8 @@ int64_t BackupQueries::FindFileID(const std::string& rNameOrIdString, fileId == std::numeric_limits<long long>::max() || fileId == 0) { - BOX_ERROR("Not a valid object ID (specified in hex)."); + BOX_ERROR("Not a valid object ID (specified in hex): " + << rNameOrIdString); return 0; } @@ -1154,19 +1053,19 @@ void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) if(opts['i']) { // can retrieve anything by ID - flagsExclude = BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING; + flagsExclude = BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING; } else { // only current versions by name flagsExclude = - BackupProtocolClientListDirectory::Flags_OldVersion | - BackupProtocolClientListDirectory::Flags_Deleted; + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted; } fileId = FindFileID(args[0], opts, &dirId, &localName, - BackupProtocolClientListDirectory::Flags_File, // just files + BackupProtocolListDirectory::Flags_File, // just files flagsExclude, NULL /* don't care about flags found */); if (fileId == 0) @@ -1469,6 +1368,159 @@ void BackupQueries::Compare(const std::string &rStoreDir, Compare(dirID, storeDirEncoded, localDirEncoded, rParams); } +void BackupQueries::CompareOneFile(int64_t DirID, + BackupStoreDirectory::Entry *pEntry, + const std::string& rLocalPath, + const std::string& rStorePath, + BoxBackupCompareParams &rParams) +{ + int64_t fileId = pEntry->GetObjectID(); + int64_t fileSize = 0; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalPath.c_str(), &st) == 0) + { + fileSize = st.st_size; + } + + try + { + // Files the same flag? + bool equal = true; + + // File modified after last sync flag + bool modifiedAfterLastSync = false; + + bool hasDifferentAttribs = false; + + bool alreadyReported = false; + + if(rParams.QuickCompare()) + { + // Compare file -- fetch it + mrConnection.QueryGetBlockIndexByID(fileId); + + // Stream containing block index + std::auto_ptr<IOStream> blockIndexStream(mrConnection.ReceiveStream()); + + // Compare + equal = BackupStoreFile::CompareFileContentsAgainstBlockIndex( + rLocalPath.c_str(), *blockIndexStream, + mrConnection.GetTimeout()); + } + else + { + // Compare file -- fetch it + mrConnection.QueryGetFile(DirID, pEntry->GetObjectID()); + + // Stream containing encoded file + std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); + + // Decode it + std::auto_ptr<BackupStoreFile::DecodedStream> fileOnServerStream; + + // Got additional attributes? + if(pEntry->HasAttributes()) + { + // Use these attributes + const StreamableMemBlock &storeAttr(pEntry->GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + fileOnServerStream.reset( + BackupStoreFile::DecodeFileStream( + *objectStream, + mrConnection.GetTimeout(), + &attr).release()); + } + else + { + // Use attributes stored in file + fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout()).release()); + } + + // Should always be something in the auto_ptr, it's how the interface is defined. But be paranoid. + if(!fileOnServerStream.get()) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Compare attributes + BackupClientFileAttributes localAttr; + box_time_t fileModTime = 0; + localAttr.ReadAttributes(rLocalPath.c_str(), false /* don't zero mod times */, &fileModTime); + modifiedAfterLastSync = (fileModTime > rParams.LatestFileUploadTime()); + bool ignoreAttrModTime = true; + + #ifdef WIN32 + // attr mod time is really + // creation time, so check it + ignoreAttrModTime = false; + #endif + + if(!rParams.IgnoreAttributes() && + #ifdef PLATFORM_DISABLE_SYMLINK_ATTRIB_COMPARE + !fileOnServerStream->IsSymLink() && + #endif + !localAttr.Compare(fileOnServerStream->GetAttributes(), + ignoreAttrModTime, + fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */)) + { + hasDifferentAttribs = true; + } + + // Compare contents, if it's a regular file not a link + // Remember, we MUST read the entire stream from the server. + SelfFlushingStream flushObject(*objectStream); + + if(!fileOnServerStream->IsSymLink()) + { + SelfFlushingStream flushFile(*fileOnServerStream); + // Open the local file + std::auto_ptr<FileStream> apLocalFile; + + try + { + apLocalFile.reset(new FileStream(rLocalPath.c_str())); + } + catch(std::exception &e) + { + rParams.NotifyLocalFileReadFailed(rLocalPath, + rStorePath, fileSize, e); + alreadyReported = true; + } + catch(...) + { + rParams.NotifyLocalFileReadFailed(rLocalPath, + rStorePath, fileSize); + alreadyReported = true; + } + + if(apLocalFile.get()) + { + equal = apLocalFile->CompareWith(*fileOnServerStream, + mrConnection.GetTimeout()); + } + } + } + + rParams.NotifyFileCompared(rLocalPath, rStorePath, fileSize, + hasDifferentAttribs, !equal, modifiedAfterLastSync, + pEntry->HasAttributes()); + } + catch(BoxException &e) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize, + e); + } + catch(std::exception &e) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize, + e); + } + catch(...) + { + rParams.NotifyDownloadFailed(rLocalPath, rStorePath, fileSize); + } +} // -------------------------------------------------------------------------- // @@ -1503,10 +1555,10 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, // Get the directory listing from the store mrConnection.QueryListDirectory( DirID, - BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, // get everything - BackupProtocolClientListDirectory::Flags_OldVersion | - BackupProtocolClientListDirectory::Flags_Deleted, + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted, // except for old versions and deleted files true /* want attributes */); @@ -1700,124 +1752,8 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, } else { - int64_t fileSize = 0; - - EMU_STRUCT_STAT st; - if(EMU_STAT(localPath.c_str(), &st) == 0) - { - fileSize = st.st_size; - } - - try - { - // Files the same flag? - bool equal = true; - - // File modified after last sync flag - bool modifiedAfterLastSync = false; - - bool hasDifferentAttribs = false; - - if(rParams.QuickCompare()) - { - // Compare file -- fetch it - mrConnection.QueryGetBlockIndexByID(i->second->GetObjectID()); - - // Stream containing block index - std::auto_ptr<IOStream> blockIndexStream(mrConnection.ReceiveStream()); - - // Compare - equal = BackupStoreFile::CompareFileContentsAgainstBlockIndex(localPath.c_str(), *blockIndexStream, mrConnection.GetTimeout()); - } - else - { - // Compare file -- fetch it - mrConnection.QueryGetFile(DirID, i->second->GetObjectID()); - - // Stream containing encoded file - std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); - - // Decode it - std::auto_ptr<BackupStoreFile::DecodedStream> fileOnServerStream; - // Got additional attributes? - if(i->second->HasAttributes()) - { - // Use these attributes - const StreamableMemBlock &storeAttr(i->second->GetAttributes()); - BackupClientFileAttributes attr(storeAttr); - fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout(), &attr).release()); - } - else - { - // Use attributes stored in file - fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout()).release()); - } - - // Should always be something in the auto_ptr, it's how the interface is defined. But be paranoid. - if(!fileOnServerStream.get()) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Compare attributes - BackupClientFileAttributes localAttr; - box_time_t fileModTime = 0; - localAttr.ReadAttributes(localPath.c_str(), false /* don't zero mod times */, &fileModTime); - modifiedAfterLastSync = (fileModTime > rParams.LatestFileUploadTime()); - bool ignoreAttrModTime = true; - - #ifdef WIN32 - // attr mod time is really - // creation time, so check it - ignoreAttrModTime = false; - #endif - - if(!rParams.IgnoreAttributes() && - #ifdef PLATFORM_DISABLE_SYMLINK_ATTRIB_COMPARE - !fileOnServerStream->IsSymLink() && - #endif - !localAttr.Compare(fileOnServerStream->GetAttributes(), - ignoreAttrModTime, - fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */)) - { - hasDifferentAttribs = true; - } - - // Compare contents, if it's a regular file not a link - // Remember, we MUST read the entire stream from the server. - SelfFlushingStream flushObject(*objectStream); - - if(!fileOnServerStream->IsSymLink()) - { - SelfFlushingStream flushFile(*fileOnServerStream); - // Open the local file - FileStream l(localPath.c_str()); - equal = l.CompareWith(*fileOnServerStream, - mrConnection.GetTimeout()); - } - } - - rParams.NotifyFileCompared(localPath, - storePath, fileSize, - hasDifferentAttribs, !equal, - modifiedAfterLastSync, - i->second->HasAttributes()); - } - catch(BoxException &e) - { - rParams.NotifyDownloadFailed(localPath, - storePath, fileSize, e); - } - catch(std::exception &e) - { - rParams.NotifyDownloadFailed(localPath, - storePath, fileSize, e); - } - catch(...) - { - rParams.NotifyDownloadFailed(localPath, - storePath, fileSize); - } + CompareOneFile(DirID, i->second, localPath, + storePath, rParams); // Remove from set so that we know it's been compared localFiles.erase(local); @@ -1947,9 +1883,10 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, void BackupQueries::CommandRestore(const std::vector<std::string> &args, const bool *opts) { // Check arguments - if(args.size() != 2) + if(args.size() < 1 || args.size() > 2) { - BOX_ERROR("Incorrect usage. restore [-drif] <remote-name> <local-name>"); + BOX_ERROR("Incorrect usage. restore [-drif] <remote-name> " + "[<local-name>]"); return; } @@ -1966,7 +1903,8 @@ void BackupQueries::CommandRestore(const std::vector<std::string> &args, const b dirID = ::strtoll(args[0].c_str(), 0, 16); if(dirID == std::numeric_limits<long long>::min() || dirID == std::numeric_limits<long long>::max() || dirID == 0) { - BOX_ERROR("Not a valid object ID (specified in hex)"); + BOX_ERROR("Not a valid object ID (specified in hex): " + << args[0]); return; } std::ostringstream oss; @@ -1994,18 +1932,30 @@ void BackupQueries::CommandRestore(const std::vector<std::string> &args, const b BOX_ERROR("Directory '" << args[0] << "' not found on server"); return; } - if(dirID == BackupProtocolClientListDirectory::RootDirectory) + + if(dirID == BackupProtocolListDirectory::RootDirectory) { BOX_ERROR("Cannot restore the root directory -- restore locations individually."); return; } - -#ifdef WIN32 + std::string localName; - if(!ConvertConsoleToUtf8(args[1].c_str(), localName)) return; -#else - std::string localName(args[1]); -#endif + + if(args.size() == 2) + { + #ifdef WIN32 + if(!ConvertConsoleToUtf8(args[1].c_str(), localName)) + { + return; + } + #else + localName = args[1]; + #endif + } + else + { + localName = args[0]; + } // Go and restore... int result; @@ -2082,8 +2032,8 @@ void BackupQueries::CommandRestore(const std::vector<std::string> &args, const b // These are autogenerated by a script. -extern char *help_commands[]; -extern char *help_text[]; +extern const char *help_commands[]; +extern const char *help_text[]; // -------------------------------------------------------------------------- @@ -2140,7 +2090,7 @@ void BackupQueries::CommandUsage(const bool *opts) bool MachineReadable = opts['m']; // Request full details from the server - std::auto_ptr<BackupProtocolClientAccountUsage> usage(mrConnection.QueryGetAccountUsage()); + std::auto_ptr<BackupProtocolAccountUsage> usage(mrConnection.QueryGetAccountUsage()); // Display each entry in turn int64_t hardLimit = usage->GetBlocksHardLimit(); @@ -2216,9 +2166,9 @@ void BackupQueries::CommandUndelete(const std::vector<std::string> &args, const fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, /* include files and directories */ - BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, /* include old and deleted files */ - BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, &flagsOut); if (fileId == 0) @@ -2231,7 +2181,7 @@ void BackupQueries::CommandUndelete(const std::vector<std::string> &args, const try { // Undelete object - if(flagsOut & BackupProtocolClientListDirectory::Flags_File) + if(flagsOut & BackupProtocolListDirectory::Flags_File) { mrConnection.QueryUndeleteFile(parentId, fileId); } @@ -2296,10 +2246,10 @@ void BackupQueries::CommandDelete(const std::vector<std::string> &args, fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, /* include files and directories */ - BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, /* exclude old and deleted files */ - BackupProtocolClientListDirectory::Flags_OldVersion | - BackupProtocolClientListDirectory::Flags_Deleted, + BackupProtocolListDirectory::Flags_OldVersion | + BackupProtocolListDirectory::Flags_Deleted, &flagsOut); if (fileId == 0) @@ -2314,7 +2264,7 @@ void BackupQueries::CommandDelete(const std::vector<std::string> &args, try { // Delete object - if(flagsOut & BackupProtocolClientListDirectory::Flags_File) + if(flagsOut & BackupProtocolListDirectory::Flags_File) { mrConnection.QueryDeleteFile(parentId, fn); } diff --git a/bin/bbackupquery/BackupQueries.h b/bin/bbackupquery/BackupQueries.h index 392aa428..62ff231d 100644 --- a/bin/bbackupquery/BackupQueries.h +++ b/bin/bbackupquery/BackupQueries.h @@ -10,16 +10,40 @@ #ifndef BACKUPQUERIES__H #define BACKUPQUERIES__H -#include <vector> +#include <iostream> #include <string> +#include <vector> #include "BoxTime.h" #include "BoxBackupCompareParams.h" +#include "BackupStoreDirectory.h" class BackupProtocolClient; class Configuration; class ExcludeList; +typedef enum +{ + Command_Unknown = 0, + Command_Quit, + Command_List, + Command_pwd, + Command_cd, + Command_lcd, + Command_sh, + Command_GetObject, + Command_Get, + Command_Compare, + Command_Restore, + Command_Help, + Command_Usage, + Command_Undelete, + Command_Delete, +} +CommandType; + +struct QueryCommandSpecification; + // -------------------------------------------------------------------------- // // Class @@ -38,8 +62,24 @@ public: private: BackupQueries(const BackupQueries &); public: + struct ParsedCommand + { + std::vector<std::string> mCmdElements; + std::string mOptions; + std::string mCompleteCommand; + bool mInOptions, mFailed; + QueryCommandSpecification* pSpec; + // mArgCount is the same as mCmdElements.size() for a complete + // command, but if the command line ends in a space, + // e.g. during readline parsing, it can be one greater, + // to indicate that we should complete the next item instead. + size_t mCompleteArgCount; + ParsedCommand(const std::string& Command, + bool isFromCommandLine); + bool IsEmpty() { return mCmdElements.empty(); } + }; - void DoCommand(const char *Command, bool isFromCommandLine); + void DoCommand(ParsedCommand& rCommand); // Ready to stop? bool Stop() {return mQuitNow;} @@ -47,9 +87,11 @@ public: // Return code? int GetReturnCode() {return mReturnCode;} -private: - // Commands + void List(int64_t DirID, const std::string &rListRoot, const bool *opts, + bool FirstLevel, std::ostream* pOut = NULL); void CommandList(const std::vector<std::string> &args, const bool *opts); + + // Commands void CommandChangeDir(const std::vector<std::string> &args, const bool *opts); void CommandChangeLocalDir(const std::vector<std::string> &args); void CommandGetObject(const std::vector<std::string> &args, const bool *opts); @@ -64,11 +106,6 @@ private: int64_t HardLimit, int32_t BlockSize, bool MachineReadable); void CommandHelp(const std::vector<std::string> &args); - // Implementations - void List(int64_t DirID, const std::string &rListRoot, const bool *opts, - bool FirstLevel); - -public: class CompareParams : public BoxBackupCompareParams { public: @@ -201,6 +238,24 @@ public: mUncheckedFiles ++; } + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) + { + BOX_ERROR("Failed to read local file '" << + ConvertForConsole(rLocalPath) << "': " << + rException.what()); + mUncheckedFiles ++; + } + + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) + { + BOX_ERROR("Failed to read local file '" << + ConvertForConsole(rLocalPath)); + mUncheckedFiles ++; + } + virtual void NotifyExcludedFile(const std::string& rLocalPath, const std::string& rRemotePath) { @@ -302,13 +357,16 @@ public: const std::string &rLocalDir, BoxBackupCompareParams &rParams); void Compare(int64_t DirID, const std::string &rStoreDir, const std::string &rLocalDir, BoxBackupCompareParams &rParams); + void CompareOneFile(int64_t DirID, BackupStoreDirectory::Entry *pEntry, + const std::string& rLocalPath, const std::string& rStorePath, + BoxBackupCompareParams &rParams); public: class ReturnCode { public: - enum { + typedef enum { Command_OK = 0, Compare_Same = 1, Compare_Different, @@ -317,17 +375,19 @@ public: } Type; }; -private: - - // Utility functions + // Were private, but needed by completion functions: + int64_t GetCurrentDirectoryID(); int64_t FindDirectoryObjectID(const std::string &rDirName, bool AllowOldVersion = false, bool AllowDeletedDirs = false, std::vector<std::pair<std::string, int64_t> > *pStack = 0); + +private: + + // Utility functions int64_t FindFileID(const std::string& rNameOrIdString, const bool *opts, int64_t *pDirIdOut, std::string* pFileNameOut, int16_t flagsInclude, int16_t flagsExclude, int16_t* pFlagsOut); - int64_t GetCurrentDirectoryID(); std::string GetCurrentDirectoryName(); void SetReturnCode(int code) {mReturnCode = code;} @@ -342,5 +402,36 @@ private: int mReturnCode; }; +typedef std::vector<std::string> (*CompletionHandler) + (BackupQueries::ParsedCommand& rCommand, const std::string& prefix, + BackupProtocolClient& rProtocol, const Configuration& rConfig, + BackupQueries& rQueries); + +std::vector<std::string> CompleteCommand(BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolClient& rProtocol, + const Configuration& rConfig, BackupQueries& rQueries); +std::vector<std::string> CompleteOptions(BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolClient& rProtocol, + const Configuration& rConfig, BackupQueries& rQueries); + +#define MAX_COMPLETION_HANDLERS 4 + +struct QueryCommandSpecification +{ + const char* name; + const char* opts; + CommandType type; + CompletionHandler complete[MAX_COMPLETION_HANDLERS]; +}; + +// Data about commands +extern QueryCommandSpecification commands[]; + +extern const char *alias[]; +extern const int aliasIs[]; + +#define LIST_OPTION_ALLOWOLD 'o' +#define LIST_OPTION_ALLOWDELETED 'd' + #endif // BACKUPQUERIES__H diff --git a/bin/bbackupquery/BoxBackupCompareParams.h b/bin/bbackupquery/BoxBackupCompareParams.h index c58759a2..655df947 100644 --- a/bin/bbackupquery/BoxBackupCompareParams.h +++ b/bin/bbackupquery/BoxBackupCompareParams.h @@ -82,6 +82,11 @@ public: virtual void NotifyDownloadFailed(const std::string& rLocalPath, const std::string& rRemotePath, int64_t NumBytes, BoxException& rException) = 0; + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) = 0; + virtual void NotifyLocalFileReadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) = 0; virtual void NotifyDownloadFailed(const std::string& rLocalPath, const std::string& rRemotePath, int64_t NumBytes, std::exception& rException) = 0; diff --git a/bin/bbackupquery/CommandCompletion.cpp b/bin/bbackupquery/CommandCompletion.cpp new file mode 100644 index 00000000..93c4d3fd --- /dev/null +++ b/bin/bbackupquery/CommandCompletion.cpp @@ -0,0 +1,602 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: CommandCompletion.cpp +// Purpose: Parts of BackupQueries that depend on readline +// Created: 2011/01/21 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_LIBREADLINE + #ifdef HAVE_READLINE_READLINE_H + #include <readline/readline.h> + #elif defined(HAVE_EDITLINE_READLINE_H) + #include <editline/readline.h> + #elif defined(HAVE_READLINE_H) + #include <readline.h> + #endif +#endif + +#ifdef HAVE_READLINE_HISTORY + #ifdef HAVE_READLINE_HISTORY_H + #include <readline/history.h> + #elif defined(HAVE_HISTORY_H) + #include <history.h> + #endif +#endif + +#include <cstring> + +#include "BackupQueries.h" +#include "Configuration.h" + +#include "autogen_BackupProtocol.h" + +#include "MemLeakFindOn.h" + +#define COMPARE_RETURN_SAME 1 +#define COMPARE_RETURN_DIFFERENT 2 +#define COMPARE_RETURN_ERROR 3 +#define COMMAND_RETURN_ERROR 4 + +#define COMPLETION_FUNCTION(name, code) \ +std::vector<std::string> Complete ## name( \ + BackupQueries::ParsedCommand& rCommand, \ + const std::string& prefix, \ + BackupProtocolClient& rProtocol, const Configuration& rConfig, \ + BackupQueries& rQueries) \ +{ \ + std::vector<std::string> completions; \ + \ + try \ + { \ + code \ + } \ + catch(std::exception &e) \ + { \ + BOX_TRACE("Failed to complete " << prefix << ": " << e.what()); \ + } \ + catch(...) \ + { \ + BOX_TRACE("Failed to complete " << prefix << ": " \ + "unknown error"); \ + } \ + \ + return completions; \ +} + +#define DELEGATE_COMPLETION(name) \ + completions = Complete ## name(rCommand, prefix, rProtocol, rConfig, \ + rQueries); + +COMPLETION_FUNCTION(None,) + +#ifdef HAVE_RL_FILENAME_COMPLETION_FUNCTION + #define RL_FILENAME_COMPLETION_FUNCTION rl_filename_completion_function + #define HAVE_A_FILENAME_COMPLETION_FUNCTION 1 +#elif defined HAVE_FILENAME_COMPLETION_FUNCTION + #define RL_FILENAME_COMPLETION_FUNCTION filename_completion_function + #define HAVE_A_FILENAME_COMPLETION_FUNCTION 1 +#endif + +#ifdef HAVE_A_FILENAME_COMPLETION_FUNCTION +COMPLETION_FUNCTION(Default, + int i = 0; + + while (const char *match = RL_FILENAME_COMPLETION_FUNCTION(prefix.c_str(), i)) + { + completions.push_back(match); + ++i; + } +) +#else // !HAVE_A_FILENAME_COMPLETION_FUNCTION +COMPLETION_FUNCTION(Default,) +#endif // HAVE_A_FILENAME_COMPLETION_FUNCTION + +COMPLETION_FUNCTION(Command, + int len = prefix.length(); + + for(int i = 0; commands[i].name != NULL; i++) + { + if(::strncmp(commands[i].name, prefix.c_str(), len) == 0) + { + completions.push_back(commands[i].name); + } + } +) + +void CompleteOptionsInternal(const std::string& prefix, + BackupQueries::ParsedCommand& rCommand, + std::vector<std::string>& completions) +{ + std::string availableOptions = rCommand.pSpec->opts; + + for(std::string::iterator + opt = availableOptions.begin(); + opt != availableOptions.end(); opt++) + { + if(rCommand.mOptions.find(*opt) == std::string::npos) + { + if(prefix == "") + { + // complete with possible option strings + completions.push_back(std::string("-") + *opt); + } + else + { + // complete with possible additional options + completions.push_back(prefix + *opt); + } + } + } +} + +COMPLETION_FUNCTION(Options, + CompleteOptionsInternal(prefix, rCommand, completions); +) + +std::string EncodeFileName(const std::string &rUnEncodedName) +{ +#ifdef WIN32 + std::string encodedName; + if(!ConvertConsoleToUtf8(rUnEncodedName, encodedName)) + { + return std::string(); + } + return encodedName; +#else + return rUnEncodedName; +#endif +} + +int16_t GetExcludeFlags(BackupQueries::ParsedCommand& rCommand) +{ + int16_t excludeFlags = 0; + + if (rCommand.mOptions.find(LIST_OPTION_ALLOWOLD) == std::string::npos) + { + excludeFlags |= BackupProtocolListDirectory::Flags_OldVersion; + } + + if (rCommand.mOptions.find(LIST_OPTION_ALLOWDELETED) == std::string::npos) + { + excludeFlags |= BackupProtocolListDirectory::Flags_Deleted; + } + + return excludeFlags; +} + +std::vector<std::string> CompleteRemoteFileOrDirectory( + BackupQueries::ParsedCommand& rCommand, + const std::string& prefix, BackupProtocolClient& rProtocol, + BackupQueries& rQueries, int16_t includeFlags) +{ + std::vector<std::string> completions; + + // default to using the current directory + int64_t listDirId = rQueries.GetCurrentDirectoryID(); + std::string searchPrefix; + std::string listDir = prefix; + + if(rCommand.mCompleteArgCount == rCommand.mCmdElements.size()) + { + // completing an empty name, from the current directory + // nothing to change + } + else + { + // completing a partially-completed subdirectory name + searchPrefix = prefix; + listDir = ""; + + // do we need to list a subdirectory to complete? + size_t lastSlash = searchPrefix.rfind('/'); + if(lastSlash == std::string::npos) + { + // no slashes, so the whole name is the prefix + // nothing to change + } + else + { + // listing a partially-completed subdirectory name + listDir = searchPrefix.substr(0, lastSlash); + + listDirId = rQueries.FindDirectoryObjectID(listDir, + rCommand.mOptions.find(LIST_OPTION_ALLOWOLD) + != std::string::npos, + rCommand.mOptions.find(LIST_OPTION_ALLOWDELETED) + != std::string::npos); + + if(listDirId == 0) + { + // no matches for subdir to list, + // return empty-handed. + return completions; + } + + // matched, and updated listDir and listDirId already + searchPrefix = searchPrefix.substr(lastSlash + 1); + } + } + + // Always include directories, because they contain files. + // We will append a slash later for each directory if we're + // actually looking for files. + // + // If we're looking for directories, then only list directories. + + bool completeFiles = includeFlags & + BackupProtocolListDirectory::Flags_File; + bool completeDirs = includeFlags & + BackupProtocolListDirectory::Flags_Dir; + int16_t listFlags = 0; + + if(completeFiles) + { + listFlags = BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING; + } + else if(completeDirs) + { + listFlags = BackupProtocolListDirectory::Flags_Dir; + } + + rProtocol.QueryListDirectory(listDirId, + listFlags, GetExcludeFlags(rCommand), + false /* no attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream()); + dir.ReadFromStream(*dirstream, rProtocol.GetTimeout()); + + // Then... display everything + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + BackupStoreFilenameClear clear(en->GetName()); + std::string name = clear.GetClearFilename().c_str(); + if(name.compare(0, searchPrefix.length(), searchPrefix) == 0) + { + bool dir_added = false; + + if(en->IsDir() && + (includeFlags & BackupProtocolListDirectory::Flags_Dir) == 0) + { + // Was looking for a file, but this is a + // directory, so append a slash to the name + name += "/"; + } + + #ifdef HAVE_LIBREADLINE + if(strchr(name.c_str(), ' ')) + { + int n_quote = 0; + + for(int k = strlen(rl_line_buffer); k >= 0; k--) + { + if (rl_line_buffer[k] == '\"') { + ++n_quote; + } + } + + dir_added = false; + + if (!(n_quote % 2)) + { + name = "\"" + (listDir == "" ? name : listDir + "/" + name); + dir_added = true; + } + + name = name + "\""; + } + #endif + + if(listDir == "" || dir_added) + { + completions.push_back(name); + } + else + { + completions.push_back(listDir + "/" + name); + } + } + } + + return completions; +} + +COMPLETION_FUNCTION(RemoteDir, + completions = CompleteRemoteFileOrDirectory(rCommand, prefix, + rProtocol, rQueries, + BackupProtocolListDirectory::Flags_Dir); +) + +COMPLETION_FUNCTION(RemoteFile, + completions = CompleteRemoteFileOrDirectory(rCommand, prefix, + rProtocol, rQueries, + BackupProtocolListDirectory::Flags_File); +) + +COMPLETION_FUNCTION(LocalDir, + DELEGATE_COMPLETION(Default); +) + +COMPLETION_FUNCTION(LocalFile, + DELEGATE_COMPLETION(Default); +) + +COMPLETION_FUNCTION(LocationName, + const Configuration &locations(rConfig.GetSubConfiguration( + "BackupLocations")); + + std::vector<std::string> locNames = + locations.GetSubConfigurationNames(); + + for(std::vector<std::string>::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + if(pLocName->compare(0, pLocName->length(), prefix) == 0) + { + completions.push_back(*pLocName); + } + } +) + +COMPLETION_FUNCTION(RemoteFileIdInCurrentDir, + int64_t listDirId = rQueries.GetCurrentDirectoryID(); + int16_t excludeFlags = GetExcludeFlags(rCommand); + + rProtocol.QueryListDirectory( + listDirId, + BackupProtocolListDirectory::Flags_File, + excludeFlags, false /* no attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream()); + dir.ReadFromStream(*dirstream, rProtocol.GetTimeout()); + + // Then... compare each item + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + std::ostringstream hexId; + hexId << std::hex << en->GetObjectID(); + if(hexId.str().compare(0, prefix.length(), prefix) == 0) + { + completions.push_back(hexId.str()); + } + } +) + +// TODO implement completion of hex IDs up to the maximum according to Usage +COMPLETION_FUNCTION(RemoteId,) + +COMPLETION_FUNCTION(GetFileOrId, + if(rCommand.mOptions.find('i') != std::string::npos) + { + DELEGATE_COMPLETION(RemoteFileIdInCurrentDir); + } + else + { + DELEGATE_COMPLETION(RemoteFile); + } +) + +COMPLETION_FUNCTION(CompareLocationOrRemoteDir, + if(rCommand.mOptions.find('l') != std::string::npos) + { + DELEGATE_COMPLETION(LocationName); + } + else + { + DELEGATE_COMPLETION(RemoteDir); + } +) + +COMPLETION_FUNCTION(CompareNoneOrLocalDir, + if(rCommand.mOptions.find('l') != std::string::npos) + { + // no completions + DELEGATE_COMPLETION(None); + } + else + { + DELEGATE_COMPLETION(LocalDir); + } +) + +COMPLETION_FUNCTION(RestoreRemoteDirOrId, + if(rCommand.mOptions.find('i') != std::string::npos) + { + DELEGATE_COMPLETION(RemoteId); + } + else + { + DELEGATE_COMPLETION(RemoteDir); + } +) + +// Data about commands +QueryCommandSpecification commands[] = +{ + { "quit", "", Command_Quit, {} }, + { "exit", "", Command_Quit, {} }, + { "list", "rodIFtTash", Command_List, {CompleteRemoteDir} }, + { "pwd", "", Command_pwd, {} }, + { "cd", "od", Command_cd, {CompleteRemoteDir} }, + { "lcd", "", Command_lcd, {CompleteLocalDir} }, + { "sh", "", Command_sh, {CompleteDefault} }, + { "getobject", "", Command_GetObject, + {CompleteRemoteId, CompleteLocalDir} }, + { "get", "i", Command_Get, + {CompleteGetFileOrId, CompleteLocalDir} }, + { "compare", "alcqAEQ", Command_Compare, + {CompleteCompareLocationOrRemoteDir, CompleteCompareNoneOrLocalDir} }, + { "restore", "drif", Command_Restore, + {CompleteRestoreRemoteDirOrId, CompleteLocalDir} }, + { "help", "", Command_Help, {} }, + { "usage", "m", Command_Usage, {} }, + { "undelete", "i", Command_Undelete, + {CompleteGetFileOrId} }, + { "delete", "i", Command_Delete, {CompleteGetFileOrId} }, + { NULL, NULL, Command_Unknown, {} } +}; + +const char *alias[] = {"ls", 0}; +const int aliasIs[] = {Command_List, 0}; + +BackupQueries::ParsedCommand::ParsedCommand(const std::string& Command, + bool isFromCommandLine) +: mInOptions(false), + mFailed(false), + pSpec(NULL), + mCompleteArgCount(0) +{ + mCompleteCommand = Command; + + // is the command a shell command? + if(Command[0] == 's' && Command[1] == 'h' && Command[2] == ' ' && Command[3] != '\0') + { + // Yes, run shell command + for(int i = 0; commands[i].type != Command_Unknown; i++) + { + if(commands[i].type == Command_sh) + { + pSpec = &(commands[i]); + break; + } + } + + mCmdElements[0] = "sh"; + mCmdElements[1] = Command.c_str() + 3; + return; + } + + // split command into components + bool inQuoted = false; + mInOptions = false; + + std::string currentArg; + for (std::string::const_iterator c = Command.begin(); + c != Command.end(); c++) + { + // Terminating char? + if(*c == ((inQuoted)?'"':' ')) + { + if(!currentArg.empty()) + { + mCmdElements.push_back(currentArg); + + // Because we just found a space, and the last + // word was not options (otherwise currentArg + // would be empty), we've received a complete + // command or non-option argument. + mCompleteArgCount++; + } + + currentArg.resize(0); + inQuoted = false; + mInOptions = false; + } + // Start of quoted parameter? + else if(currentArg.empty() && *c == '"') + { + inQuoted = true; + } + // Start of options? + else if(currentArg.empty() && *c == '-') + { + mInOptions = true; + } + else if(mInOptions) + { + // Option char + mOptions += *c; + } + else + { + // Normal string char, part of current arg + currentArg += *c; + } + } + + if(!currentArg.empty()) + { + mCmdElements.push_back(currentArg); + } + + // If there are no commands then there's nothing to do except return + if(mCmdElements.empty()) + { + return; + } + + // Work out which command it is... + int cmd = 0; + while(commands[cmd].name != 0 && + mCmdElements[0] != commands[cmd].name) + { + cmd++; + } + + if(commands[cmd].name == 0) + { + // Check for aliases + int a; + for(a = 0; alias[a] != 0; ++a) + { + if(mCmdElements[0] == alias[a]) + { + // Found an alias + cmd = aliasIs[a]; + break; + } + } + } + + if(commands[cmd].name == 0) + { + mFailed = true; + return; + } + + pSpec = &(commands[cmd]); + + #ifdef WIN32 + if(isFromCommandLine) + { + std::string converted; + + if(!ConvertEncoding(mCompleteCommand, CP_ACP, converted, + GetConsoleCP())) + { + BOX_ERROR("Failed to convert encoding"); + mFailed = true; + } + + mCompleteCommand = converted; + + for(std::vector<std::string>::iterator + i = mCmdElements.begin(); + i != mCmdElements.end(); i++) + { + if(!ConvertEncoding(*i, CP_ACP, converted, + GetConsoleCP())) + { + BOX_ERROR("Failed to convert encoding"); + mFailed = true; + } + + *i = converted; + } + } + #endif +} + diff --git a/bin/bbackupquery/bbackupquery.cpp b/bin/bbackupquery/bbackupquery.cpp index 5aa7e97e..5493f49c 100644 --- a/bin/bbackupquery/bbackupquery.cpp +++ b/bin/bbackupquery/bbackupquery.cpp @@ -30,6 +30,7 @@ #include <readline.h> #endif #endif + #ifdef HAVE_READLINE_HISTORY #ifdef HAVE_READLINE_HISTORY_H #include <readline/history.h> @@ -49,7 +50,7 @@ #include "SSLLib.h" #include "BackupStoreConstants.h" #include "BackupStoreException.h" -#include "autogen_BackupProtocolClient.h" +#include "autogen_BackupProtocol.h" #include "BackupQueries.h" #include "FdGetLine.h" #include "BackupClientCryptoKeys.h" @@ -60,20 +61,128 @@ void PrintUsageAndExit() { - printf("Usage: bbackupquery [-q*|v*|V|W<level>] [-w] " + std::ostringstream out; + out << + "Usage: bbackupquery [options] [command]...\n" + "\n" + "Options:\n" + " -q Run more quietly, reduce verbosity level by one, can repeat\n" + " -Q Run at minimum verbosity, log nothing\n" + " -v Run more verbosely, increase verbosity level by one, can repeat\n" + " -V Run at maximum verbosity, log everything\n" + " -W <level> Set verbosity to error/warning/notice/info/trace/everything\n" + " -w Read/write mode, allow changes to store\n" #ifdef WIN32 - "[-u] " + " -u Enable Unicode console, requires font change to Lucida Console\n" +#endif +#ifdef HAVE_LIBREADLINE + " -E Disable interactive command editing, may fix entering intl chars\n" #endif - "\n" - "\t[-c config_file] [-o log_file] [-O log_file_level]\n" - "\t[-l protocol_log_file] [commands]\n" - "\n" - "As many commands as you require.\n" - "If commands are multiple words, remember to enclose the command in quotes.\n" - "Remember to use the quit command unless you want to end up in interactive mode.\n"); + " -c <file> Use the specified configuration file. If -c is omitted, the last\n" + " argument is the configuration file, or else the default \n" + " [" << BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE << + "]\n" + " -o <file> Write logging output to specified file as well as console\n" + " -O <level> Set file verbosity to error/warning/notice/info/trace/everything\n" + " -l <file> Write protocol debugging logs to specified file\n" + "\n" + "Parameters: as many commands as you like. If commands are multiple words,\n" + "remember to enclose the command in quotes. Remember to use the quit command\n" + "unless you want to end up in interactive mode.\n"; + printf("%s", out.str().c_str()); exit(1); } +#ifdef HAVE_LIBREADLINE +static BackupProtocolClient* pProtocol; +static const Configuration* pConfig; +static BackupQueries* pQueries; +static std::vector<std::string> completions; +static std::auto_ptr<BackupQueries::ParsedCommand> sapCmd; + +char * completion_generator(const char *text, int state) +{ + if(state == 0) + { + completions.clear(); + + std::string partialCommand(rl_line_buffer, rl_point); + sapCmd.reset(new BackupQueries::ParsedCommand(partialCommand, + false)); + int currentArg = sapCmd->mCompleteArgCount; + + if(currentArg == 0) // incomplete command + { + completions = CompleteCommand(*sapCmd, text, *pProtocol, + *pConfig, *pQueries); + } + else if(sapCmd->mInOptions) + { + completions = CompleteOptions(*sapCmd, text, *pProtocol, + *pConfig, *pQueries); + } + else if(currentArg - 1 < MAX_COMPLETION_HANDLERS) + // currentArg must be at least 1 if we're here + { + CompletionHandler handler = + sapCmd->pSpec->complete[currentArg - 1]; + + if(handler != NULL) + { + completions = handler(*sapCmd, text, *pProtocol, + *pConfig, *pQueries); + } + + if(std::string(text) == "") + { + // additional options are also allowed here + std::vector<std::string> addOpts = + CompleteOptions(*sapCmd, text, + *pProtocol, *pConfig, + *pQueries); + + for(std::vector<std::string>::iterator + i = addOpts.begin(); + i != addOpts.end(); i++) + { + completions.push_back(*i); + } + } + } + } + + if(state < 0 || state >= (int) completions.size()) + { + rl_attempted_completion_over = 1; + return NULL; + } + + return strdup(completions[state].c_str()); + // string must be allocated with malloc() and will be freed + // by rl_completion_matches(). +} + +#ifdef HAVE_RL_COMPLETION_MATCHES + #define RL_COMPLETION_MATCHES rl_completion_matches +#elif defined HAVE_COMPLETION_MATCHES + #define RL_COMPLETION_MATCHES completion_matches +#else + char* no_matches[] = {NULL}; + char** bbackupquery_completion_dummy(const char *text, + char * (completion_generator)(const char *text, int state)) + { + return no_matches; + } + #define RL_COMPLETION_MATCHES bbackupquery_completion_dummy +#endif + +char ** bbackupquery_completion(const char *text, int start, int end) +{ + return RL_COMPLETION_MATCHES(text, completion_generator); +} + +#endif // HAVE_LIBREADLINE + int main(int argc, const char *argv[]) { int returnCode = 0; @@ -103,13 +212,7 @@ int main(int argc, const char *argv[]) FILE *logFile = 0; // Filename for configuration file? - std::string configFilename; - - #ifdef WIN32 - configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; - #else - configFilename = BOX_FILE_BBACKUPD_DEFAULT_CONFIG; - #endif + std::string configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; // Flags bool readWrite = false; @@ -123,12 +226,21 @@ int main(int argc, const char *argv[]) #endif #ifdef WIN32 - const char* validOpts = "qvVwuc:l:o:O:W:"; + #define WIN32_OPTIONS "u" bool unicodeConsole = false; #else - const char* validOpts = "qvVwc:l:o:O:W:"; + #define WIN32_OPTIONS #endif +#ifdef HAVE_LIBREADLINE + #define READLINE_OPTIONS "E" + bool useReadline = true; +#else + #define READLINE_OPTIONS +#endif + + const char* validOpts = "qvVwc:l:o:O:W:" WIN32_OPTIONS READLINE_OPTIONS; + std::string fileLogFile; Log::Level fileLogLevel = Log::INVALID; @@ -222,6 +334,12 @@ int main(int argc, const char *argv[]) unicodeConsole = true; break; #endif + +#ifdef HAVE_LIBREADLINE + case 'E': + useReadline = false; + break; +#endif case '?': default: @@ -317,7 +435,9 @@ int main(int argc, const char *argv[]) // 3. Make a protocol, and handshake if(!quiet) BOX_INFO("Handshake with store..."); - BackupProtocolClient connection(socket); + std::auto_ptr<BackupProtocolClient> + apConnection(new BackupProtocolClient(socket)); + BackupProtocolClient& connection(*(apConnection.get())); connection.Handshake(); // logging? @@ -330,15 +450,15 @@ int main(int argc, const char *argv[]) if(!quiet) BOX_INFO("Login to store..."); // Check the version of the server { - std::auto_ptr<BackupProtocolClientVersion> serverVersion(connection.QueryVersion(BACKUP_STORE_SERVER_VERSION)); + std::auto_ptr<BackupProtocolVersion> serverVersion(connection.QueryVersion(BACKUP_STORE_SERVER_VERSION)); if(serverVersion->GetVersion() != BACKUP_STORE_SERVER_VERSION) { THROW_EXCEPTION(BackupStoreException, WrongServerVersion) } } // Login -- if this fails, the Protocol will exception - connection.QueryLogin(conf.GetKeyValueInt("AccountNumber"), - (readWrite)?0:(BackupProtocolClientLogin::Flags_ReadOnly)); + connection.QueryLogin(conf.GetKeyValueUint32("AccountNumber"), + (readWrite)?0:(BackupProtocolLogin::Flags_ReadOnly)); // 5. Tell user. if(!quiet) printf("Login complete.\n\nType \"help\" for a list of commands.\n\n"); @@ -351,66 +471,102 @@ int main(int argc, const char *argv[]) int c = 0; while(c < argc && !context.Stop()) { - context.DoCommand(argv[c++], true); + BackupQueries::ParsedCommand cmd(argv[c++], true); + context.DoCommand(cmd); } } // Get commands from input #ifdef HAVE_LIBREADLINE - // Must initialise the locale before using editline's readline(), - // otherwise cannot enter international characters. - if (setlocale(LC_ALL, "") == NULL) + if(useReadline) { - BOX_ERROR("Failed to initialise locale. International " - "character support may not work."); + // Must initialise the locale before using editline's + // readline(), otherwise cannot enter international characters. + if (setlocale(LC_ALL, "") == NULL) + { + BOX_ERROR("Failed to initialise locale. International " + "character support may not work."); + } + + #ifdef HAVE_READLINE_HISTORY + using_history(); + #endif + + /* Allow conditional parsing of the ~/.inputrc file. */ + rl_readline_name = strdup("bbackupquery"); + + /* Tell the completer that we want a crack first. */ + rl_attempted_completion_function = bbackupquery_completion; + + pProtocol = &connection; + pConfig = &conf; + pQueries = &context; } -#ifdef HAVE_READLINE_HISTORY - using_history(); + std::string last_cmd; #endif - char *last_cmd = 0; - while(!context.Stop()) + + std::auto_ptr<FdGetLine> apGetLine; + if(fileno(stdin) >= 0) + { + apGetLine.reset(new FdGetLine(fileno(stdin))); + } + + while(!context.Stop() && fileno(stdin) >= 0) { - char *command = readline("query > "); - if(command == NULL) + std::string cmd_str; + + #ifdef HAVE_LIBREADLINE + if(useReadline) { - // Ctrl-D pressed -- terminate now - break; + char *cmd_ptr = readline("query > "); + + if(cmd_ptr == NULL) + { + // Ctrl-D pressed -- terminate now + break; + } + + cmd_str = cmd_ptr; + free(cmd_ptr); } - context.DoCommand(command, false); - if(last_cmd != 0 && ::strcmp(last_cmd, command) == 0) + else + #endif // HAVE_LIBREADLINE { - free(command); + printf("query > "); + fflush(stdout); + + try + { + cmd_str = apGetLine->GetLine(); + } + catch(CommonException &e) + { + if(e.GetSubType() == CommonException::GetLineEOF) + { + break; + } + throw; + } } - else + + BackupQueries::ParsedCommand cmd_parsed(cmd_str, false); + if (cmd_parsed.IsEmpty()) { -#ifdef HAVE_READLINE_HISTORY - add_history(command); -#else - free(last_cmd); -#endif - last_cmd = command; + continue; } - } -#ifndef HAVE_READLINE_HISTORY - free(last_cmd); - last_cmd = 0; -#endif -#else - // Version for platforms which don't have readline by default - if(fileno(stdin) >= 0) - { - FdGetLine getLine(fileno(stdin)); - while(!context.Stop()) + + context.DoCommand(cmd_parsed); + + #ifdef HAVE_READLINE_HISTORY + if(last_cmd != cmd_str) { - printf("query > "); - fflush(stdout); - std::string command(getLine.GetLine()); - context.DoCommand(command.c_str(), false); + add_history(cmd_str.c_str()); + last_cmd = cmd_str; } + #endif // HAVE_READLINE_HISTORY } -#endif // Done... stop nicely if(!quiet) BOX_INFO("Logging off..."); diff --git a/bin/bbackupquery/documentation.txt b/bin/bbackupquery/documentation.txt index 96eda158..214fe218 100644 --- a/bin/bbackupquery/documentation.txt +++ b/bin/bbackupquery/documentation.txt @@ -119,10 +119,13 @@ compare <store-dir-name> <local-dir-name> This can be used for automated tests. < -> restore [-drif] <directory-name> <local-directory-name> +> restore [-drif] <directory-name> [<local-directory-name>] Restores a directory to the local disc. The local directory specified - must not exist (unless a previous restore is being restarted). + must not exist (unless a previous restore is being restarted). If the + local directory is omitted, the default is to restore to the same + directory name and path, relative to the current local directory, + as set with the "lcd" command. The root cannot be restored -- restore locations individually. diff --git a/bin/bbstoreaccounts/bbstoreaccounts.cpp b/bin/bbstoreaccounts/bbstoreaccounts.cpp index a9d2b0af..6a5c2e33 100644 --- a/bin/bbstoreaccounts/bbstoreaccounts.cpp +++ b/bin/bbstoreaccounts/bbstoreaccounts.cpp @@ -11,7 +11,10 @@ #include <limits.h> #include <stdio.h> -#include <unistd.h> + +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif #include <sys/types.h> @@ -21,17 +24,18 @@ #include <ostream> #include <vector> -#include "BoxPortsAndFiles.h" -#include "BackupStoreConfigVerify.h" -#include "RaidFileController.h" #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" -#include "MainHelper.h" +#include "BackupStoreCheck.h" +#include "BackupStoreConfigVerify.h" #include "BackupStoreInfo.h" -#include "StoreStructure.h" +#include "BoxPortsAndFiles.h" +#include "HousekeepStoreAccount.h" +#include "MainHelper.h" #include "NamedLock.h" +#include "RaidFileController.h" +#include "StoreStructure.h" #include "UnixUser.h" -#include "BackupStoreCheck.h" #include "Utils.h" #include "MemLeakFindOn.h" @@ -59,31 +63,31 @@ void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit) } } -int BlockSizeOfDiscSet(int DiscSet) +int BlockSizeOfDiscSet(int discSetNum) { // Get controller, check disc set number RaidFileController &controller(RaidFileController::GetController()); - if(DiscSet < 0 || DiscSet >= controller.GetNumDiscSets()) + if(discSetNum < 0 || discSetNum >= controller.GetNumDiscSets()) { - BOX_FATAL("Disc set " << DiscSet << " does not exist."); + BOX_FATAL("Disc set " << discSetNum << " does not exist."); exit(1); } // Return block size - return controller.GetDiscSet(DiscSet).GetBlockSize(); + return controller.GetDiscSet(discSetNum).GetBlockSize(); } -std::string BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int DiscSet) +std::string BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int discSetNum) { - return FormatUsageBar(Blocks, Blocks * BlockSizeOfDiscSet(DiscSet), - MaxBlocks * BlockSizeOfDiscSet(DiscSet), + return FormatUsageBar(Blocks, Blocks * BlockSizeOfDiscSet(discSetNum), + MaxBlocks * BlockSizeOfDiscSet(discSetNum), sMachineReadableOutput); } -int64_t SizeStringToBlocks(const char *string, int DiscSet) +int64_t SizeStringToBlocks(const char *string, int discSetNum) { // Find block size - int blockSize = BlockSizeOfDiscSet(DiscSet); + int blockSize = BlockSizeOfDiscSet(discSetNum); // Get number char *endptr = (char*)string; @@ -124,144 +128,174 @@ int64_t SizeStringToBlocks(const char *string, int DiscSet) } } -bool GetWriteLockOnAccount(NamedLock &rLock, const std::string rRootDir, int DiscSetNum) -{ - std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(rRootDir, DiscSetNum, writeLockFilename); +bool OpenAccount(Configuration &rConfig, int32_t ID, std::string &rRootDirOut, + int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock); - bool gotLock = false; - int triesLeft = 8; - do - { - gotLock = rLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */); - - if(!gotLock) - { - --triesLeft; - ::sleep(1); - } - } while(!gotLock && triesLeft > 0); +int SetLimit(Configuration &rConfig, int32_t ID, const char *SoftLimitStr, + const char *HardLimitStr) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; - if(!gotLock) + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) { - // Couldn't lock the account -- just stop now - BOX_ERROR("Failed to lock the account, did not change limits. " - "Try again later."); + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change limits."); + return 1; } + + // Load the info + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, + discSetNum, false /* Read/Write */)); - return gotLock; + // Change the limits + int64_t softlimit = SizeStringToBlocks(SoftLimitStr, discSetNum); + int64_t hardlimit = SizeStringToBlocks(HardLimitStr, discSetNum); + CheckSoftHardLimits(softlimit, hardlimit); + info->ChangeLimits(softlimit, hardlimit); + + // Save + info->Save(); + + BOX_NOTICE("Limits on account " << BOX_FORMAT_ACCOUNT(ID) << + " changed to " << softlimit << " soft, " << + hardlimit << " hard."); + + return 0; } -int SetLimit(Configuration &rConfig, const std::string &rUsername, int32_t ID, const char *SoftLimitStr, const char *HardLimitStr) +int SetAccountName(Configuration &rConfig, int32_t ID, + const std::string& rNewAccountName) { - // Become the user specified in the config file? - std::auto_ptr<UnixUser> user; - if(!rUsername.empty()) - { - // Username specified, change... - user.reset(new UnixUser(rUsername.c_str())); - user->ChangeProcessUser(true /* temporary */); - // Change will be undone at the end of this function - } - - // Load in the account database - std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str())); - - // Already exists? - if(!db->EntryExists(ID)) - { - BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << - " does not exist."); - return 1; - } - - // Load it in - BackupStoreAccounts acc(*db); std::string rootDir; - int discSet; - acc.GetAccountRoot(ID, rootDir, discSet); - - // Attempt to lock + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return NamedLock writeLock; - if(!GetWriteLockOnAccount(writeLock, rootDir, discSet)) + + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) { - // Failed to get lock + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change name."); return 1; } // Load the info - std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, discSet, false /* Read/Write */)); + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, false /* Read/Write */)); - // Change the limits - int64_t softlimit = SizeStringToBlocks(SoftLimitStr, discSet); - int64_t hardlimit = SizeStringToBlocks(HardLimitStr, discSet); - CheckSoftHardLimits(softlimit, hardlimit); - info->ChangeLimits(softlimit, hardlimit); + info->SetAccountName(rNewAccountName); // Save info->Save(); - BOX_NOTICE("Limits on account " << BOX_FORMAT_ACCOUNT(ID) << - " changed to " << softlimit << " soft, " << - hardlimit << " hard."); + BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << + " name changed to " << rNewAccountName); return 0; } int AccountInfo(Configuration &rConfig, int32_t ID) { - // Load in the account database - std::auto_ptr<BackupStoreAccountDatabase> db( - BackupStoreAccountDatabase::Read( - rConfig.GetKeyValue("AccountDatabase").c_str())); - - // Exists? - if(!db->EntryExists(ID)) + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, + NULL /* no write lock needed for this read-only operation */)) { - BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << - " does not exist."); + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to display info."); return 1; } // Load it in - BackupStoreAccounts acc(*db); - std::string rootDir; - int discSet; - acc.GetAccountRoot(ID, rootDir, discSet); std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, - rootDir, discSet, true /* ReadOnly */)); + rootDir, discSetNum, true /* ReadOnly */)); // Then print out lots of info std::cout << FormatUsageLineStart("Account ID", sMachineReadableOutput) << BOX_FORMAT_ACCOUNT(ID) << std::endl; + std::cout << FormatUsageLineStart("Account Name", sMachineReadableOutput) << + info->GetAccountName() << std::endl; std::cout << FormatUsageLineStart("Last object ID", sMachineReadableOutput) << BOX_FORMAT_OBJECTID(info->GetLastObjectIDUsed()) << std::endl; std::cout << FormatUsageLineStart("Used", sMachineReadableOutput) << BlockSizeToString(info->GetBlocksUsed(), - info->GetBlocksHardLimit(), discSet) << std::endl; + info->GetBlocksHardLimit(), discSetNum) << std::endl; + std::cout << FormatUsageLineStart("Current files", + sMachineReadableOutput) << + BlockSizeToString(info->GetBlocksInCurrentFiles(), + info->GetBlocksHardLimit(), discSetNum) << std::endl; std::cout << FormatUsageLineStart("Old files", sMachineReadableOutput) << BlockSizeToString(info->GetBlocksInOldFiles(), - info->GetBlocksHardLimit(), discSet) << std::endl; + info->GetBlocksHardLimit(), discSetNum) << std::endl; std::cout << FormatUsageLineStart("Deleted files", sMachineReadableOutput) << BlockSizeToString(info->GetBlocksInDeletedFiles(), - info->GetBlocksHardLimit(), discSet) << std::endl; + info->GetBlocksHardLimit(), discSetNum) << std::endl; std::cout << FormatUsageLineStart("Directories", sMachineReadableOutput) << BlockSizeToString(info->GetBlocksInDirectories(), - info->GetBlocksHardLimit(), discSet) << std::endl; + info->GetBlocksHardLimit(), discSetNum) << std::endl; std::cout << FormatUsageLineStart("Soft limit", sMachineReadableOutput) << BlockSizeToString(info->GetBlocksSoftLimit(), - info->GetBlocksHardLimit(), discSet) << std::endl; + info->GetBlocksHardLimit(), discSetNum) << std::endl; std::cout << FormatUsageLineStart("Hard limit", sMachineReadableOutput) << BlockSizeToString(info->GetBlocksHardLimit(), - info->GetBlocksHardLimit(), discSet) << std::endl; + info->GetBlocksHardLimit(), discSetNum) << std::endl; std::cout << FormatUsageLineStart("Client store marker", sMachineReadableOutput) << info->GetLastObjectIDUsed() << std::endl; + std::cout << FormatUsageLineStart("Live Files", sMachineReadableOutput) << + info->GetNumFiles() << std::endl; + std::cout << FormatUsageLineStart("Old Files", sMachineReadableOutput) << + info->GetNumOldFiles() << std::endl; + std::cout << FormatUsageLineStart("Deleted Files", sMachineReadableOutput) << + info->GetNumDeletedFiles() << std::endl; + std::cout << FormatUsageLineStart("Directories", sMachineReadableOutput) << + info->GetNumDirectories() << std::endl; + std::cout << FormatUsageLineStart("Enabled", sMachineReadableOutput) << + (info->IsAccountEnabled() ? "yes" : "no") << std::endl; return 0; } -int DeleteAccount(Configuration &rConfig, const std::string &rUsername, int32_t ID, bool AskForConfirmation) +int SetAccountEnabled(Configuration &rConfig, int32_t ID, bool enabled) { + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change enabled flag."); + return 1; + } + + // Load it in + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, false /* ReadOnly */)); + info->SetAccountEnabled(enabled); + info->Save(); + return 0; +} + +int DeleteAccount(Configuration &rConfig, int32_t ID, bool AskForConfirmation) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + // Obtain a write lock, as the daemon user + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for deletion."); + return 1; + } + // Check user really wants to do this if(AskForConfirmation) { @@ -275,45 +309,10 @@ int DeleteAccount(Configuration &rConfig, const std::string &rUsername, int32_t } } - // Load in the account database + // Back to original user, but write lock is maintained + user.reset(); + std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str())); - - // Exists? - if(!db->EntryExists(ID)) - { - BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << - " does not exist."); - return 1; - } - - // Get info from the database - BackupStoreAccounts acc(*db); - std::string rootDir; - int discSetNum; - acc.GetAccountRoot(ID, rootDir, discSetNum); - - // Obtain a write lock, as the daemon user - NamedLock writeLock; - { - // Bbecome the user specified in the config file - std::auto_ptr<UnixUser> user; - if(!rUsername.empty()) - { - // Username specified, change... - user.reset(new UnixUser(rUsername.c_str())); - user->ChangeProcessUser(true /* temporary */); - // Change will be undone at the end of this function - } - - // Get a write lock - if(!GetWriteLockOnAccount(writeLock, rootDir, discSetNum)) - { - // Failed to get lock - return 1; - } - - // Back to original user, but write is maintained - } // Delete from account database db->DeleteEntry(ID); @@ -324,15 +323,24 @@ int DeleteAccount(Configuration &rConfig, const std::string &rUsername, int32_t // Remove the store files... // First, become the user specified in the config file - std::auto_ptr<UnixUser> user; - if(!rUsername.empty()) + std::string username; + { + const Configuration &rserverConfig(rConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Become the right user + if(!username.empty()) { // Username specified, change... - user.reset(new UnixUser(rUsername.c_str())); + user.reset(new UnixUser(username)); user->ChangeProcessUser(true /* temporary */); - // Change will be undone at the end of this function + // Change will be undone when user goes out of scope } - + // Secondly, work out which directories need wiping std::vector<std::string> toDelete; RaidFileController &rcontroller(RaidFileController::GetController()); @@ -345,6 +353,11 @@ int DeleteAccount(Configuration &rConfig, const std::string &rUsername, int32_t } } +#ifdef WIN32 + // Cannot remove files while holding a lock on them + writeLock.ReleaseLock(); +#endif + int retcode = 0; // Thirdly, delete the directories... @@ -367,7 +380,8 @@ int DeleteAccount(Configuration &rConfig, const std::string &rUsername, int32_t return retcode; } -int CheckAccount(Configuration &rConfig, const std::string &rUsername, int32_t ID, bool FixErrors, bool Quiet) +bool OpenAccount(Configuration &rConfig, int32_t ID, std::string &rRootDirOut, + int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock) { // Load in the account database std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str())); @@ -377,23 +391,53 @@ int CheckAccount(Configuration &rConfig, const std::string &rUsername, int32_t I { BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << " does not exist."); - return 1; + return false; } // Get info from the database BackupStoreAccounts acc(*db); - std::string rootDir; - int discSetNum; - acc.GetAccountRoot(ID, rootDir, discSetNum); - + acc.GetAccountRoot(ID, rRootDirOut, rDiscSetOut); + + // Get the user under which the daemon runs + std::string username; + { + const Configuration &rserverConfig(rConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + // Become the right user - std::auto_ptr<UnixUser> user; - if(!rUsername.empty()) + if(!username.empty()) { // Username specified, change... - user.reset(new UnixUser(rUsername.c_str())); - user->ChangeProcessUser(true /* temporary */); - // Change will be undone at the end of this function + apUser.reset(new UnixUser(username)); + apUser->ChangeProcessUser(true /* temporary */); + // Change will be undone when apUser goes out of scope + // in the caller. + } + + if(pLock) + { + acc.LockAccount(ID, *pLock); + } + + return true; +} + +int CheckAccount(Configuration &rConfig, int32_t ID, bool FixErrors, bool Quiet) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for checking."); + return 1; } // Check it @@ -403,7 +447,8 @@ int CheckAccount(Configuration &rConfig, const std::string &rUsername, int32_t I return check.ErrorsFound()?1:0; } -int CreateAccount(Configuration &rConfig, const std::string &rUsername, int32_t ID, int32_t DiscNumber, int32_t SoftLimit, int32_t HardLimit) +int CreateAccount(Configuration &rConfig, int32_t ID, int32_t DiscNumber, + int32_t SoftLimit, int32_t HardLimit) { // Load in the account database std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str())); @@ -415,16 +460,58 @@ int CreateAccount(Configuration &rConfig, const std::string &rUsername, int32_t " already exists."); return 1; } + + // Get the user under which the daemon runs + std::string username; + { + const Configuration &rserverConfig(rConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } // Create it. BackupStoreAccounts acc(*db); - acc.Create(ID, DiscNumber, SoftLimit, HardLimit, rUsername); + acc.Create(ID, DiscNumber, SoftLimit, HardLimit, username); BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << " created."); return 0; } +int HousekeepAccountNow(Configuration &rConfig, int32_t ID) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + + if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, + NULL /* housekeeping locks the account itself */)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for housekeeping."); + return 1; + } + + HousekeepStoreAccount housekeeping(ID, rootDir, discSetNum, NULL); + bool success = housekeeping.DoHousekeeping(); + + if(!success) + { + BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(ID) + << " for housekeeping: perhaps a client is " + "still connected?"); + return 1; + } + else + { + BOX_TRACE("Finished housekeeping on account " << + BOX_FORMAT_ACCOUNT(ID)); + return 0; + } +} + void PrintUsageAndExit() { printf( @@ -440,6 +527,8 @@ void PrintUsageAndExit() " info [-m] <account>\n" " Prints information about the specified account including number\n" " of blocks used. The -m option enable machine-readable output.\n" +" enabled <accounts> <yes|no>\n" +" Sets the account as enabled or disabled for new logins.\n" " setlimit <accounts> <softlimit> <hardlimit>\n" " Changes the limits of the account as specified. Numbers are\n" " interpreted as for the 'create' command (suffixed with B, M or G)\n" @@ -451,6 +540,14 @@ void PrintUsageAndExit() " provided, any errors discovered that can be fixed automatically\n" " will be fixed. If the 'quiet' option is provided, less output is\n" " produced.\n" +" name <account> <new name>\n" +" Changes the \"name\" of the account to the specified string.\n" +" The name is purely cosmetic and intended to make it easier to\n" +" identify your accounts.\n" +" housekeep <account>\n" +" Runs housekeeping immediately on the account. If it cannot be locked,\n" +" bbstoreaccounts returns an error status code (1), otherwise success\n" +" (0) even if any errors were fixed by housekeeping.\n" ); exit(2); } @@ -465,17 +562,12 @@ int main(int argc, const char *argv[]) Logging::SetProgramName("bbstoreaccounts"); // Filename for configuration file? - std::string configFilename; - - #ifdef WIN32 - configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; - #else - configFilename = BOX_FILE_BBSTORED_DEFAULT_CONFIG; - #endif + std::string configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; + int logLevel = Log::EVERYTHING; // See if there's another entry on the command line int c; - while((c = getopt(argc, (char * const *)argv, "c:m")) != -1) + while((c = getopt(argc, (char * const *)argv, "c:W:m")) != -1) { switch(c) { @@ -484,6 +576,15 @@ int main(int argc, const char *argv[]) configFilename = optarg; break; + case 'W': + logLevel = Logging::GetNamedLevel(optarg); + if(logLevel == Log::INVALID) + { + BOX_FATAL("Invalid logging level: " << optarg); + return 2; + } + break; + case 'm': // enable machine readable output sMachineReadableOutput = true; @@ -494,6 +595,10 @@ int main(int argc, const char *argv[]) PrintUsageAndExit(); } } + + Logging::FilterConsole((Log::Level) logLevel); + Logging::FilterSyslog (Log::NOTHING); + // Adjust arguments argc -= optind; argv += optind; @@ -510,16 +615,6 @@ int main(int argc, const char *argv[]) ":" << errs); } - // Get the user under which the daemon runs - std::string username; - { - const Configuration &rserverConfig(config->GetSubConfiguration("Server")); - if(rserverConfig.KeyExists("User")) - { - username = rserverConfig.GetKeyValue("User"); - } - } - // Initialise the raid file controller RaidFileController &rcontroller(RaidFileController::GetController()); rcontroller.Initialise(config->GetKeyValue("RaidFileConf").c_str()); @@ -537,8 +632,10 @@ int main(int argc, const char *argv[]) PrintUsageAndExit(); } + std::string command = argv[0]; + // Now do the command. - if(::strcmp(argv[0], "create") == 0) + if(command == "create") { // which disc? int32_t discnum; @@ -558,14 +655,39 @@ int main(int argc, const char *argv[]) CheckSoftHardLimits(softlimit, hardlimit); // Create the account... - return CreateAccount(*config, username, id, discnum, softlimit, hardlimit); + return CreateAccount(*config, id, discnum, softlimit, hardlimit); } - else if(::strcmp(argv[0], "info") == 0) + else if(command == "info") { // Print information on this account return AccountInfo(*config, id); } - else if(::strcmp(argv[0], "setlimit") == 0) + else if(command == "enabled") + { + // Change the AccountEnabled flag on this account + if(argc != 3) + { + PrintUsageAndExit(); + } + + bool enabled = true; + std::string enabled_string = argv[2]; + if(enabled_string == "yes") + { + enabled = true; + } + else if(enabled_string == "no") + { + enabled = false; + } + else + { + PrintUsageAndExit(); + } + + return SetAccountEnabled(*config, id, enabled); + } + else if(command == "setlimit") { // Change the limits on this account if(argc < 4) @@ -574,9 +696,20 @@ int main(int argc, const char *argv[]) return 1; } - return SetLimit(*config, username, id, argv[2], argv[3]); + return SetLimit(*config, id, argv[2], argv[3]); + } + else if(command == "name") + { + // Change the limits on this account + if(argc != 3) + { + BOX_ERROR("name command requires a new name."); + return 1; + } + + return SetAccountName(*config, id, argv[2]); } - else if(::strcmp(argv[0], "delete") == 0) + else if(command == "delete") { // Delete an account bool askForConfirmation = true; @@ -584,9 +717,9 @@ int main(int argc, const char *argv[]) { askForConfirmation = false; } - return DeleteAccount(*config, username, id, askForConfirmation); + return DeleteAccount(*config, id, askForConfirmation); } - else if(::strcmp(argv[0], "check") == 0) + else if(command == "check") { bool fixErrors = false; bool quiet = false; @@ -610,11 +743,15 @@ int main(int argc, const char *argv[]) } // Check the account - return CheckAccount(*config, username, id, fixErrors, quiet); + return CheckAccount(*config, id, fixErrors, quiet); + } + else if(command == "housekeep") + { + return HousekeepAccountNow(*config, id); } else { - BOX_ERROR("Unknown command '" << argv[0] << "'."); + BOX_ERROR("Unknown command '" << command << "'."); return 1; } @@ -623,4 +760,3 @@ int main(int argc, const char *argv[]) MAINHELPER_END } - diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index 7f799008..86d6409c 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -79,8 +79,19 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() // Do housekeeping if the time interval has elapsed since the last check if((timeNow - mLastHousekeepingRun) < housekeepingInterval) { + BOX_TRACE("No need for housekeeping, " << + BoxTimeToSeconds(timeNow - mLastHousekeepingRun) << + " seconds since last run is less than " << + BoxTimeToSeconds(housekeepingInterval)); return; } + else + { + BOX_TRACE("Running housekeeping now, because " << + BoxTimeToSeconds(timeNow - mLastHousekeepingRun) << + " seconds since last run is more than " << + BoxTimeToSeconds(housekeepingInterval)); + } // Store the time mLastHousekeepingRun = timeNow; @@ -100,18 +111,28 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() { try { - if(mpAccounts) + std::string rootDir; + int discSet = 0; + { + // Tag log output to identify account + std::ostringstream tag; + tag << "hk/" << BOX_FORMAT_ACCOUNT(*i); + Logging::Tagger tagWithClientID(tag.str()); + // Get the account root - std::string rootDir; - int discSet = 0; mpAccounts->GetAccountRoot(*i, rootDir, discSet); - - // Do housekeeping on this account - HousekeepStoreAccount housekeeping(*i, rootDir, - discSet, this); - housekeeping.DoHousekeeping(); + + // Reset tagging as HousekeepStoreAccount will + // do that itself, to avoid duplicate tagging. + // Happens automatically when tagWithClientID + // goes out of scope. } + + // Do housekeeping on this account + HousekeepStoreAccount housekeeping(*i, rootDir, + discSet, this); + housekeeping.DoHousekeeping(); } catch(BoxException &e) { diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 4de0a078..70ca1d67 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -20,7 +20,7 @@ #include "BackupStoreContext.h" #include "BackupStoreDaemon.h" #include "BackupStoreConfigVerify.h" -#include "autogen_BackupProtocolServer.h" +#include "autogen_BackupProtocol.h" #include "RaidFileController.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreAccounts.h" @@ -317,6 +317,8 @@ void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) if(::sscanf(clientCommonName.c_str(), "BACKUP-%x", &id) != 1) { // Bad! Disconnect immediately + BOX_WARNING("Failed login: invalid client common name: " << + clientCommonName); return; } @@ -327,7 +329,7 @@ void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) Logging::Tagger tagWithClientID(tag.str()); // Create a context, using this ID - BackupStoreContext context(id, *this); + BackupStoreContext context(id, *this, GetConnectionDetails()); if (mpTestHook) { @@ -353,19 +355,22 @@ void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) } catch(...) { - LogConnectionStats(clientCommonName.c_str(), rStream); + LogConnectionStats(id, context.GetAccountName(), rStream); throw; } - LogConnectionStats(clientCommonName.c_str(), rStream); + LogConnectionStats(id, context.GetAccountName(), rStream); context.CleanUp(); } -void BackupStoreDaemon::LogConnectionStats(const char *commonName, - const SocketStreamTLS &s) +void BackupStoreDaemon::LogConnectionStats(uint32_t accountId, + const std::string& accountName, const SocketStreamTLS &s) { // Log the amount of data transferred - BOX_NOTICE("Connection statistics for " << commonName << ":" + BOX_NOTICE("Connection statistics for " << + BOX_FORMAT_ACCOUNT(accountId) << " " + "(name=" << accountName << "):" " IN=" << s.GetBytesRead() << " OUT=" << s.GetBytesWritten() << + " NET_IN=" << (s.GetBytesRead() - s.GetBytesWritten()) << " TOTAL=" << (s.GetBytesRead() + s.GetBytesWritten())); } diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 49af5b81..ce538477 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -63,11 +63,13 @@ protected: // Housekeeping functions void HousekeepingProcess(); - void LogConnectionStats(const char *commonName, const SocketStreamTLS &s); + void LogConnectionStats(uint32_t accountId, + const std::string& accountName, const SocketStreamTLS &s); public: // HousekeepingInterface implementation virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0); + void RunHousekeepingIfNeeded(); private: BackupStoreAccountDatabase *mpAccountDatabase; @@ -82,7 +84,6 @@ private: virtual void OnIdle(); void HousekeepingInit(); - void RunHousekeepingIfNeeded(); int64_t mLastHousekeepingRun; public: diff --git a/bin/bbstored/bbstored.cpp b/bin/bbstored/bbstored.cpp index 21a9e5f1..2c18b4fc 100644 --- a/bin/bbstored/bbstored.cpp +++ b/bin/bbstored/bbstored.cpp @@ -24,13 +24,7 @@ int main(int argc, const char *argv[]) BackupStoreDaemon daemon; - #ifdef WIN32 - return daemon.Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE, - argc, argv); - #else - return daemon.Main(BOX_FILE_BBSTORED_DEFAULT_CONFIG, - argc, argv); - #endif + return daemon.Main(BOX_GET_DEFAULT_BBSTORED_CONFIG_FILE, argc, argv); MAINHELPER_END } |