diff options
Diffstat (limited to 'bin/bbackupd/BackupClientDirectoryRecord.cpp')
| -rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.cpp | 818 |
1 files changed, 617 insertions, 201 deletions
diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp index 84c17dab..90caf2e7 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp @@ -17,21 +17,26 @@ #include <errno.h> #include <string.h> -#include "BackupClientDirectoryRecord.h" -#include "autogen_BackupProtocolClient.h" +#include "autogen_BackupProtocol.h" +#include "autogen_CipherException.h" +#include "autogen_ClientException.h" +#include "Archive.h" #include "BackupClientContext.h" -#include "IOStream.h" -#include "MemBlockStream.h" -#include "CommonException.h" -#include "CollectInBufferStream.h" -#include "BackupStoreFile.h" +#include "BackupClientDirectoryRecord.h" #include "BackupClientInodeToIDMap.h" -#include "FileModificationTime.h" #include "BackupDaemon.h" #include "BackupStoreException.h" -#include "Archive.h" -#include "PathUtils.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileEncodeStream.h" +#include "BufferedStream.h" +#include "CommonException.h" +#include "CollectInBufferStream.h" +#include "FileModificationTime.h" +#include "IOStream.h" #include "Logging.h" +#include "MemBlockStream.h" +#include "PathUtils.h" +#include "RateLimitingStream.h" #include "ReadLoggingStream.h" #include "MemLeakFindOn.h" @@ -51,6 +56,7 @@ BackupClientDirectoryRecord::BackupClientDirectoryRecord(int64_t ObjectID, const mSubDirName(rSubDirName), mInitialSyncDone(false), mSyncDone(false), + mSuppressMultipleLinksWarning(false), mpPendingEntries(0) { ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); @@ -98,6 +104,32 @@ void BackupClientDirectoryRecord::DeleteSubDirectories() mSubDirectories.clear(); } +std::string BackupClientDirectoryRecord::ConvertVssPathToRealPath( + const std::string &rVssPath, + const Location& rBackupLocation) +{ +#ifdef ENABLE_VSS + BOX_TRACE("VSS: ConvertVssPathToRealPath: mIsSnapshotCreated = " << + rBackupLocation.mIsSnapshotCreated); + BOX_TRACE("VSS: ConvertVssPathToRealPath: File/Directory Path = " << + rVssPath.substr(0, rBackupLocation.mSnapshotPath.length())); + BOX_TRACE("VSS: ConvertVssPathToRealPath: Snapshot Path = " << + rBackupLocation.mSnapshotPath); + if (rBackupLocation.mIsSnapshotCreated && + rVssPath.substr(0, rBackupLocation.mSnapshotPath.length()) == + rBackupLocation.mSnapshotPath) + { + std::string convertedPath = rBackupLocation.mPath + + rVssPath.substr(rBackupLocation.mSnapshotPath.length()); + BOX_TRACE("VSS: ConvertVssPathToRealPath: Converted Path = " << + convertedPath); + return convertedPath; + } +#endif + + return rVssPath; +} + // -------------------------------------------------------------------------- // // Function @@ -115,6 +147,7 @@ void BackupClientDirectoryRecord::SyncDirectory( int64_t ContainingDirectoryID, const std::string &rLocalPath, const std::string &rRemotePath, + const Location& rBackupLocation, bool ThisDirHasJustBeenCreated) { BackupClientContext& rContext(rParams.mrContext); @@ -160,10 +193,16 @@ void BackupClientDirectoryRecord::SyncDirectory( // just ignore this error. In a future scan, this // deletion will be noticed, deleted from server, // and this object deleted. - rNotifier.NotifyDirStatFailed(this, rLocalPath, + rNotifier.NotifyDirStatFailed(this, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation), strerror(errno)); return; } + + BOX_TRACE("Stat dir '" << rLocalPath << "' " + "found device/inode " << + dest_st.st_dev << "/" << dest_st.st_ino); + // Store inode number in map so directories are tracked // in case they're renamed { @@ -204,12 +243,12 @@ void BackupClientDirectoryRecord::SyncDirectory( { // Report the error (logs and // eventual email to administrator) - rNotifier.NotifyFileStatFailed(this, rLocalPath, + rNotifier.NotifyFileStatFailed(this, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation), strerror(errno)); // FIXME move to NotifyFileStatFailed() - SetErrorWhenReadingFilesystemObject(rParams, - rLocalPath.c_str()); + SetErrorWhenReadingFilesystemObject(rParams, rLocalPath); // This shouldn't happen, so we'd better not continue THROW_EXCEPTION(CommonException, OSFileError) @@ -221,7 +260,9 @@ void BackupClientDirectoryRecord::SyncDirectory( DIR *dirHandle = 0; try { - rNotifier.NotifyScanDirectory(this, rLocalPath); + std::string nonVssDirPath = ConvertVssPathToRealPath(rLocalPath, + rBackupLocation); + rNotifier.NotifyScanDirectory(this, nonVssDirPath); dirHandle = ::opendir(rLocalPath.c_str()); if(dirHandle == 0) @@ -231,18 +272,20 @@ void BackupClientDirectoryRecord::SyncDirectory( if (errno == EACCES) { rNotifier.NotifyDirListFailed(this, - rLocalPath, "Access denied"); + nonVssDirPath, + "Access denied"); } else { rNotifier.NotifyDirListFailed(this, - rLocalPath, strerror(errno)); + nonVssDirPath, + strerror(errno)); } // Report the error (logs and eventual email // to administrator) SetErrorWhenReadingFilesystemObject(rParams, - rLocalPath.c_str()); + nonVssDirPath); // Ignore this directory for now. return; } @@ -279,6 +322,8 @@ void BackupClientDirectoryRecord::SyncDirectory( // Stat file to get info filename = MakeFullPath(rLocalPath, en->d_name); + std::string realFileName = ConvertVssPathToRealPath(filename, + rBackupLocation); #ifdef WIN32 // Don't stat the file just yet, to ensure @@ -289,9 +334,18 @@ void BackupClientDirectoryRecord::SyncDirectory( // Our emulated readdir() abuses en->d_type, // which would normally contain DT_REG, // DT_DIR, etc, but we only use it here and - // prefer S_IFREG, S_IFDIR... - int type = en->d_type; - #else + // prefer to have the full file attributes. + int type; + if (en->d_type & FILE_ATTRIBUTE_DIRECTORY) + { + type = S_IFDIR; + } + else + { + type = S_IFREG; + } + + #else // !WIN32 if(EMU_LSTAT(filename.c_str(), &file_st) != 0) { if(!(rParams.mrContext.ExcludeDir( @@ -306,15 +360,52 @@ void BackupClientDirectoryRecord::SyncDirectory( // FIXME move to // NotifyFileStatFailed() - SetErrorWhenReadingFilesystemObject( - rParams, filename.c_str()); + SetErrorWhenReadingFilesystemObject(rParams, filename); } // Ignore this entry for now. continue; } - if(file_st.st_dev != dest_st.st_dev) + int type = file_st.st_mode & S_IFMT; + + // ecryptfs reports nlink > 1 for directories + // with contents, but no filesystem supports + // hardlinking directories? so we can ignore + // this if the entry is a directory. + if(file_st.st_nlink != 1 && type == S_IFDIR) + { + BOX_INFO("Ignoring apparent hard link " + "count on directory: " << + filename << ", nlink=" << + file_st.st_nlink); + } + else if(file_st.st_nlink > 1) + { + if(!mSuppressMultipleLinksWarning) + { + BOX_WARNING("File is hard linked, this may " + "cause rename tracking to fail and " + "move files incorrectly in your " + "backup! " << filename << + ", nlink=" << file_st.st_nlink << + " (suppressing further warnings)"); + mSuppressMultipleLinksWarning = true; + } + SetErrorWhenReadingFilesystemObject(rParams, filename); + } + + BOX_TRACE("Stat entry '" << filename << "' " + "found device/inode " << + file_st.st_dev << "/" << + file_st.st_ino); + + /* Workaround for apparent btrfs bug, where + symlinks appear to be on a different filesystem + than their containing directory, thanks to + Toke Hoiland-Jorgensen */ + if(type == S_IFDIR && + file_st.st_dev != dest_st.st_dev) { if(!(rParams.mrContext.ExcludeDir( filename))) @@ -324,8 +415,6 @@ void BackupClientDirectoryRecord::SyncDirectory( } continue; } - - int type = file_st.st_mode & S_IFMT; #endif if(type == S_IFREG || type == S_IFLNK) @@ -333,12 +422,9 @@ void BackupClientDirectoryRecord::SyncDirectory( // File or symbolic link // Exclude it? - if(rParams.mrContext.ExcludeFile(filename)) + if(rParams.mrContext.ExcludeFile(realFileName)) { - rNotifier.NotifyFileExcluded( - this, - filename); - + rNotifier.NotifyFileExcluded(this, realFileName); // Next item! continue; } @@ -351,38 +437,50 @@ void BackupClientDirectoryRecord::SyncDirectory( // Directory // Exclude it? - if(rParams.mrContext.ExcludeDir(filename)) + if(rParams.mrContext.ExcludeDir(realFileName)) { - rNotifier.NotifyDirExcluded( - this, - filename); + rNotifier.NotifyDirExcluded(this, realFileName); // Next item! continue; } + #ifdef WIN32 + // exclude reparse points, as Application Data points to the + // parent directory under Vista and later, and causes an + // infinite loop: + // http://social.msdn.microsoft.com/forums/en-US/windowscompatibility/thread/05d14368-25dd-41c8-bdba-5590bf762a68/ + if (en->d_type & FILE_ATTRIBUTE_REPARSE_POINT) + { + rNotifier.NotifyMountPointSkipped(this, realFileName); + continue; + } + #endif + // Store on list dirs.push_back(std::string(en->d_name)); } - else + else // not a file or directory, what is it? { - if (type == S_IFSOCK || type == S_IFIFO) + if (type == S_IFSOCK +# ifndef WIN32 + || type == S_IFIFO +# endif + ) { // removed notification for these types // see Debian bug 479145, no objections } - else if(rParams.mrContext.ExcludeFile(filename)) + else if(rParams.mrContext.ExcludeFile(realFileName)) { - rNotifier.NotifyFileExcluded( - this, - filename); + rNotifier.NotifyFileExcluded(this, realFileName); } else { - rNotifier.NotifyUnsupportedFileType( - this, filename); - SetErrorWhenReadingFilesystemObject( - rParams, filename.c_str()); + rNotifier.NotifyUnsupportedFileType(this, + realFileName); + SetErrorWhenReadingFilesystemObject(rParams, + realFileName); } continue; @@ -397,13 +495,12 @@ void BackupClientDirectoryRecord::SyncDirectory( if(emu_stat(filename.c_str(), &file_st) != 0) { rNotifier.NotifyFileStatFailed(this, - filename, + ConvertVssPathToRealPath(filename, rBackupLocation), strerror(errno)); // Report the error (logs and // eventual email to administrator) - SetErrorWhenReadingFilesystemObject( - rParams, filename.c_str()); + SetErrorWhenReadingFilesystemObject(rParams, filename); // Ignore this entry for now. continue; @@ -412,7 +509,7 @@ void BackupClientDirectoryRecord::SyncDirectory( if(file_st.st_dev != link_st.st_dev) { rNotifier.NotifyMountPointSkipped(this, - filename); + ConvertVssPathToRealPath(filename, rBackupLocation)); continue; } #endif @@ -432,8 +529,8 @@ void BackupClientDirectoryRecord::SyncDirectory( // Log that this has happened if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) { - rNotifier.NotifyFileModifiedInFuture( - this, filename); + rNotifier.NotifyFileModifiedInFuture(this, + ConvertVssPathToRealPath(filename, rBackupLocation)); rParams.mHaveLoggedWarningAboutFutureFileTimes = true; } } @@ -507,7 +604,7 @@ void BackupClientDirectoryRecord::SyncDirectory( // Do the directory reading bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, - rRemotePath, pdirOnStore, entriesLeftOver, files, dirs); + rRemotePath, rBackupLocation, pdirOnStore, entriesLeftOver, files, dirs); // LAST THING! (think exception safety) // Store the new checksum -- don't fetch things unnecessarily in the future @@ -564,10 +661,11 @@ BackupStoreDirectory *BackupClientDirectoryRecord::FetchDirectoryListing(BackupC BackupProtocolClient &connection(rParams.mrContext.GetConnection()); // Query the directory - std::auto_ptr<BackupProtocolClientSuccess> dirreply(connection.QueryListDirectory( + std::auto_ptr<BackupProtocolSuccess> dirreply(connection.QueryListDirectory( mObjectID, - BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, // both files and directories - BackupProtocolClientListDirectory::Flags_Deleted | BackupProtocolClientListDirectory::Flags_OldVersion, // exclude old/deleted stuff + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, // both files and directories + BackupProtocolListDirectory::Flags_Deleted | + BackupProtocolListDirectory::Flags_OldVersion, // exclude old/deleted stuff true /* want attributes */)); // Retrieve the directory from the stream following @@ -630,11 +728,38 @@ void BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord:: BackupProtocolClient &connection(rParams.mrContext.GetConnection()); // Exception thrown if this doesn't work - MemBlockStream attrStream(attr); + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); connection.QueryChangeDirAttributes(mObjectID, attrModTime, attrStream); } } +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreDirectory::Entry *en, + const std::string& rRemoteDirectoryPath) +{ + BackupStoreFilenameClear fn(en->GetName()); + return DecryptFilename(fn, en->GetObjectID(), rRemoteDirectoryPath); +} + +std::string BackupClientDirectoryRecord::DecryptFilename( + BackupStoreFilenameClear fn, int64_t filenameObjectID, + const std::string& rRemoteDirectoryPath) +{ + std::string filenameClear; + try + { + filenameClear = fn.GetClearFilename(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to decrypt filename for object " << + BOX_FORMAT_OBJECTID(filenameObjectID) << " in " + "directory " << BOX_FORMAT_OBJECTID(mObjectID) << + " (" << rRemoteDirectoryPath << ")"); + throw; + } + return filenameClear; +} // -------------------------------------------------------------------------- // @@ -649,6 +774,7 @@ bool BackupClientDirectoryRecord::UpdateItems( BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rLocalPath, const std::string &rRemotePath, + const Location& rBackupLocation, BackupStoreDirectory *pDirOnStore, std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, std::vector<std::string> &rFiles, @@ -673,7 +799,19 @@ bool BackupClientDirectoryRecord::UpdateItems( BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { - decryptedEntries[BackupStoreFilenameClear(en->GetName()).GetClearFilename()] = en; + std::string filenameClear; + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + decryptedEntries[filenameClear] = en; + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "pretending that the file doesn't " + "exist"); + } } } @@ -686,26 +824,26 @@ bool BackupClientDirectoryRecord::UpdateItems( // Filename of this file std::string filename(MakeFullPath(rLocalPath, *f)); + std::string nonVssFilePath = ConvertVssPathToRealPath(filename, + rBackupLocation); // Get relevant info about file box_time_t modTime = 0; uint64_t attributesHash = 0; int64_t fileSize = 0; InodeRefType inodeNum = 0; - bool hasMultipleHardLinks = true; // BLOCK { // Stat the file EMU_STRUCT_STAT st; if(EMU_LSTAT(filename.c_str(), &st) != 0) { - rNotifier.NotifyFileStatFailed(this, - filename, strerror(errno)); + rNotifier.NotifyFileStatFailed(this, nonVssFilePath, + strerror(errno)); // Report the error (logs and // eventual email to administrator) - SetErrorWhenReadingFilesystemObject(rParams, - filename.c_str()); + SetErrorWhenReadingFilesystemObject(rParams, nonVssFilePath); // Ignore this entry for now. continue; @@ -715,7 +853,6 @@ bool BackupClientDirectoryRecord::UpdateItems( modTime = FileModificationTime(st); fileSize = st.st_size; inodeNum = st.st_ino; - hasMultipleHardLinks = (st.st_nlink > 1); attributesHash = BackupClientFileAttributes::GenerateAttributeHash(st, filename, *f); } @@ -734,7 +871,7 @@ bool BackupClientDirectoryRecord::UpdateItems( } // Check that the entry which might have been found is in fact a file - if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == 0)) + if((en != 0) && !(en->IsFile())) { // Directory exists in the place of this file -- sort it out RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, @@ -760,7 +897,9 @@ bool BackupClientDirectoryRecord::UpdateItems( bool isCurrentVersion = false; box_time_t srvModTime = 0, srvAttributesHash = 0; BackupStoreFilenameClear oldLeafname; - if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname)) + if(rContext.FindFilename(renameObjectID, renameInDirectory, + localPotentialOldName, isDir, isCurrentVersion, + &srvModTime, &srvAttributesHash, &oldLeafname)) { // Only interested if it's a file and the latest version if(!isDir && isCurrentVersion) @@ -780,8 +919,11 @@ bool BackupClientDirectoryRecord::UpdateItems( if(!rContext.StorageLimitExceeded()) { // Rename the existing files (ie include old versions) on the server - connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */, - BackupProtocolClientMoveObject::Flags_MoveAllWithSameName | BackupProtocolClientMoveObject::Flags_AllowMoveOverDeletedObject, + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, storeFilename); // Stop the attempt to delete the file in the original location @@ -790,8 +932,11 @@ bool BackupClientDirectoryRecord::UpdateItems( // Create new entry in the directory for it // -- will be near enough what's actually on the server for the rest to work. - en = pDirOnStore->AddEntry(storeFilename, srvModTime, renameObjectID, 0 /* size in blocks unknown, but not needed */, - BackupStoreDirectory::Entry::Flags_File, srvAttributesHash); + en = pDirOnStore->AddEntry(storeFilename, + srvModTime, renameObjectID, + 0 /* size in blocks unknown, but not needed */, + BackupStoreDirectory::Entry::Flags_File, + srvAttributesHash); // Store the object ID for the inode lookup map later latestObjectID = renameObjectID; @@ -833,6 +978,7 @@ bool BackupClientDirectoryRecord::UpdateItems( // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store bool doUpload = false; + std::string decisionReason = "unknown"; // Only upload a file if the mod time locally is // different to that on the server. @@ -848,16 +994,12 @@ bool BackupClientDirectoryRecord::UpdateItems( if (pDirOnStore != 0 && en == 0) { doUpload = true; - BOX_TRACE("Upload decision: " << - filename << ": will upload " - "(not on server)"); + decisionReason = "not on server"; } else if (modTime >= rParams.mSyncPeriodStart) { doUpload = true; - BOX_TRACE("Upload decision: " << - filename << ": will upload " - "(modified since last sync)"); + decisionReason = "modified since last sync"; } } @@ -874,9 +1016,7 @@ bool BackupClientDirectoryRecord::UpdateItems( > rParams.mMaxUploadWait) { doUpload = true; - BOX_TRACE("Upload decision: " << - filename << ": will upload " - "(continually modified)"); + decisionReason = "continually modified"; } // Then make sure that if files are added with a @@ -892,9 +1032,7 @@ bool BackupClientDirectoryRecord::UpdateItems( en->GetModificationTime() != modTime) { doUpload = true; - BOX_TRACE("Upload decision: " << - filename << ": will upload " - "(mod time changed)"); + decisionReason = "mod time changed"; } // And just to catch really badly off clocks in @@ -905,17 +1043,14 @@ bool BackupClientDirectoryRecord::UpdateItems( rParams.mUploadAfterThisTimeInTheFuture) { doUpload = true; - BOX_TRACE("Upload decision: " << - filename << ": will upload " - "(mod time in the future)"); + decisionReason = "mod time in the future"; } } if (en != 0 && en->GetModificationTime() == modTime) { - BOX_TRACE("Upload decision: " << - filename << ": will not upload " - "(not modified since last upload)"); + doUpload = false; + decisionReason = "not modified since last upload"; } else if (!doUpload) { @@ -924,22 +1059,26 @@ bool BackupClientDirectoryRecord::UpdateItems( box_time_t now = GetCurrentBoxTime(); int age = BoxTimeToSeconds(now - modTime); - BOX_TRACE("Upload decision: " << - filename << ": will not upload " - "(modified too recently: " - "only " << age << " seconds ago)"); + std::ostringstream s; + s << "modified too recently: " + "only " << age << " seconds ago"; + decisionReason = s.str(); } else { - BOX_TRACE("Upload decision: " << - filename << ": will not upload " - "(mod time is " << modTime << + std::ostringstream s; + s << "mod time is " << modTime << " which is outside sync window, " << rParams.mSyncPeriodStart << " to " - << rParams.mSyncPeriodEnd << ")"); + << rParams.mSyncPeriodEnd; + decisionReason = s.str(); } } + BOX_TRACE("Upload decision: " << nonVssFilePath << ": " << + (doUpload ? "will upload" : "will not upload") << + " (" << decisionReason << ")"); + bool fileSynced = true; if (doUpload) @@ -968,7 +1107,9 @@ bool BackupClientDirectoryRecord::UpdateItems( try { latestObjectID = UploadFile(rParams, - filename, storeFilename, + filename, + nonVssFilePath, + storeFilename, fileSize, modTime, attributesHash, noPreviousVersionOnServer); @@ -992,7 +1133,7 @@ bool BackupClientDirectoryRecord::UpdateItems( // retries would probably just cause // more problems. rNotifier.NotifyFileUploadException( - this, filename, e); + this, nonVssFilePath, e); throw; } catch(BoxException &e) @@ -1009,9 +1150,10 @@ bool BackupClientDirectoryRecord::UpdateItems( // code false, to show error in directory allUpdatedSuccessfully = false; // Log it. - SetErrorWhenReadingFilesystemObject(rParams, filename.c_str()); - rNotifier.NotifyFileUploadException( - this, filename, e); + SetErrorWhenReadingFilesystemObject(rParams, + nonVssFilePath); + rNotifier.NotifyFileUploadException(this, + nonVssFilePath, e); } // Update structures if the file was uploaded @@ -1029,8 +1171,7 @@ bool BackupClientDirectoryRecord::UpdateItems( } else { - rNotifier.NotifyFileSkippedServerFull(this, - filename); + rNotifier.NotifyFileSkippedServerFull(this, nonVssFilePath); } } else if(en != 0 && en->GetAttributesHash() != attributesHash) @@ -1050,22 +1191,23 @@ bool BackupClientDirectoryRecord::UpdateItems( { try { - rNotifier.NotifyFileUploadingAttributes( - this, filename); + rNotifier.NotifyFileUploadingAttributes(this, + nonVssFilePath); // Update store BackupClientFileAttributes attr; - attr.ReadAttributes(filename.c_str(), false /* put mod times in the attributes, please */); - MemBlockStream attrStream(attr); + attr.ReadAttributes(filename, + false /* put mod times in the attributes, please */); + std::auto_ptr<IOStream> attrStream( + new MemBlockStream(attr)); connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream); fileSynced = true; } catch (BoxException &e) { - BOX_ERROR("Failed to read or store " - "file attributes for '" << - filename << "', will try " - "again later"); + BOX_ERROR("Failed to read or store file attributes " + "for '" << nonVssFilePath << "', will try again " + "later"); } } } @@ -1109,7 +1251,7 @@ bool BackupClientDirectoryRecord::UpdateItems( { // Use this one BOX_TRACE("Storing uploaded file ID " << - inodeNum << " (" << filename << ") " + inodeNum << " (" << nonVssFilePath << ") " "in ID map as object " << latestObjectID << " with parent " << mObjectID); @@ -1126,7 +1268,12 @@ bool BackupClientDirectoryRecord::UpdateItems( // Found if (dirid != mObjectID) { - BOX_WARNING("Found conflicting parent ID for file ID " << inodeNum << " (" << filename << "): expected " << mObjectID << " but found " << dirid << " (same directory used in two different locations?)"); + BOX_WARNING("Found conflicting parent ID for " + "file ID " << inodeNum << " (" << + nonVssFilePath << "): expected " << + mObjectID << " but found " << dirid << + " (same directory used in two different " + "locations?)"); } ASSERT(dirid == mObjectID); @@ -1136,11 +1283,9 @@ bool BackupClientDirectoryRecord::UpdateItems( // into it. However, in a long running process this may happen occasionally and // not indicate anything wrong. // Run the release version for real life use, where this check is not made. - BOX_TRACE("Storing found file ID " << - inodeNum << " (" << filename << - ") in ID map as object " << - objid << " with parent " << - mObjectID); + BOX_TRACE("Storing found file ID " << inodeNum << + " (" << nonVssFilePath << ") in ID map as " + "object " << objid << " with parent " << mObjectID); idMap.AddToMap(inodeNum, objid, mObjectID /* containing directory */); } @@ -1149,7 +1294,7 @@ bool BackupClientDirectoryRecord::UpdateItems( if (fileSynced) { - rNotifier.NotifyFileSynchronised(this, filename, + rNotifier.NotifyFileSynchronised(this, nonVssFilePath, fileSize); } } @@ -1175,6 +1320,8 @@ bool BackupClientDirectoryRecord::UpdateItems( // Get the local filename std::string dirname(MakeFullPath(rLocalPath, *d)); + std::string nonVssDirPath = ConvertVssPathToRealPath(dirname, + rBackupLocation); // See if it's in the listing (if we have one) BackupStoreFilenameClear storeFilename(*d); @@ -1189,14 +1336,17 @@ bool BackupClientDirectoryRecord::UpdateItems( } // Check that the entry which might have been found is in fact a directory - if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == 0)) + if((en != 0) && !(en->IsDir())) { // Entry exists, but is not a directory. Bad. // Get rid of it. BackupProtocolClient &connection(rContext.GetConnection()); connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); + + std::string filenameClear = DecryptFilename(en, + rRemotePath); rNotifier.NotifyFileDeleted(en->GetObjectID(), - storeFilename.GetClearFilename()); + filenameClear); // Nothing found en = 0; @@ -1266,7 +1416,7 @@ bool BackupClientDirectoryRecord::UpdateItems( BOX_WARNING("Failed to read attributes " "of directory, cannot check " "for rename, assuming new: '" - << dirname << "'"); + << nonVssDirPath << "'"); failedToReadAttributes = true; } @@ -1284,7 +1434,9 @@ bool BackupClientDirectoryRecord::UpdateItems( std::string localPotentialOldName; bool isDir = false; bool isCurrentVersion = false; - if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion)) + if(rContext.FindFilename(renameObjectID, + renameInDirectory, localPotentialOldName, + isDir, isCurrentVersion)) { // Only interested if it's a directory if(isDir && isCurrentVersion) @@ -1309,13 +1461,16 @@ bool BackupClientDirectoryRecord::UpdateItems( // in the else if(...) above will be correct. // Build attribute stream for sending - MemBlockStream attrStream(attr); + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); if(renameDir) { // Rename the existing directory on the server - connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */, - BackupProtocolClientMoveObject::Flags_MoveAllWithSameName | BackupProtocolClientMoveObject::Flags_AllowMoveOverDeletedObject, + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, storeFilename); // Put the latest attributes on it @@ -1332,12 +1487,22 @@ bool BackupClientDirectoryRecord::UpdateItems( else { // Create a new directory - std::auto_ptr<BackupProtocolClientSuccess> dirCreate(connection.QueryCreateDirectory( - mObjectID, attrModTime, storeFilename, attrStream)); + std::auto_ptr<BackupProtocolSuccess> dirCreate( + connection.QueryCreateDirectory( + mObjectID, attrModTime, + storeFilename, attrStream)); subDirObjectID = dirCreate->GetObjectID(); // Flag as having done this for optimisation later haveJustCreatedDirOnServer = true; + + std::string filenameClear = + DecryptFilename(storeFilename, + subDirObjectID, + rRemotePath); + rNotifier.NotifyDirectoryCreated( + subDirObjectID, filenameClear, + nonVssDirPath); } } @@ -1365,8 +1530,8 @@ bool BackupClientDirectoryRecord::UpdateItems( if(psubDirRecord) { // Sync this sub directory too - psubDirRecord->SyncDirectory(rParams, mObjectID, - dirname, rRemotePath + "/" + *d, + psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, + rRemotePath + "/" + *d, rBackupLocation, haveJustCreatedDirOnServer); } @@ -1397,10 +1562,26 @@ bool BackupClientDirectoryRecord::UpdateItems( // If there's an error during the process, it doesn't matter if things // aren't actually deleted, as the whole state will be reset anyway. BackupClientDeleteList &rdel(rContext.GetDeleteList()); + std::string filenameClear; + bool isCorruptFilename = false; + + try + { + filenameClear = DecryptFilename(en, + rRemotePath); + } + catch (CipherException &e) + { + BOX_ERROR("Failed to decrypt a filename, " + "scheduling that file for deletion"); + filenameClear = "<corrupt filename>"; + isCorruptFilename = true; + } - BackupStoreFilenameClear clear(en->GetName()); std::string localName = MakeFullPath(rLocalPath, - clear.GetClearFilename()); + filenameClear); + std::string nonVssLocalName = ConvertVssPathToRealPath(localName, + rBackupLocation); // Delete this entry -- file or directory? if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) @@ -1418,20 +1599,17 @@ bool BackupClientDirectoryRecord::UpdateItems( // If there's a directory record for it in // the sub directory map, delete it now BackupStoreFilenameClear dirname(en->GetName()); - std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(dirname.GetClearFilename())); - if(e != mSubDirectories.end()) + std::map<std::string, BackupClientDirectoryRecord *>::iterator + e(mSubDirectories.find(filenameClear)); + if(e != mSubDirectories.end() && !isCorruptFilename) { // Carefully delete the entry from the map BackupClientDirectoryRecord *rec = e->second; mSubDirectories.erase(e); delete rec; - std::string name = MakeFullPath( - rLocalPath, - dirname.GetClearFilename()); - - BOX_TRACE("Deleted directory record " - "for " << name); + BOX_TRACE("Deleted directory record for " << + nonVssLocalName); } } } @@ -1498,6 +1676,7 @@ void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile( int64_t BackupClientDirectoryRecord::UploadFile( BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rFilename, + const std::string &rNonVssFilePath, const BackupStoreFilename &rStoreFilename, int64_t FileSize, box_time_t ModificationTime, @@ -1512,11 +1691,14 @@ int64_t BackupClientDirectoryRecord::UploadFile( // Info int64_t objID = 0; - bool doNormalUpload = true; + int64_t uploadedSize = -1; // Use a try block to catch store full errors try { + std::auto_ptr<BackupStoreFileEncodeStream> apStreamToUpload; + int64_t diffFromID = 0; + // Might an old version be on the server, and is the file // size over the diffing threshold? if(!NoPreviousVersionOnServer && @@ -1524,14 +1706,14 @@ int64_t BackupClientDirectoryRecord::UploadFile( { // YES -- try to do diff, if possible // First, query the server to see if there's an old version available - std::auto_ptr<BackupProtocolClientSuccess> getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename)); - int64_t diffFromID = getBlockIndex->GetObjectID(); + std::auto_ptr<BackupProtocolSuccess> getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename)); + diffFromID = getBlockIndex->GetObjectID(); if(diffFromID != 0) { // Found an old version - rNotifier.NotifyFileUploadingPatch(this, - rFilename); + rNotifier.NotifyFileUploadingPatch(this, + rNonVssFilePath); // Get the index std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream()); @@ -1543,55 +1725,68 @@ int64_t BackupClientDirectoryRecord::UploadFile( rContext.ManageDiffProcess(); bool isCompletelyDifferent = false; - std::auto_ptr<IOStream> patchStream( - BackupStoreFile::EncodeFileDiff( - rFilename.c_str(), - mObjectID, /* containing directory */ - rStoreFilename, diffFromID, *blockIndexStream, - connection.GetTimeout(), - &rContext, // DiffTimer implementation - 0 /* not interested in the modification time */, - &isCompletelyDifferent)); - - rContext.UnManageDiffProcess(); - // - // Upload the patch to the store - // - std::auto_ptr<BackupProtocolClientSuccess> stored(connection.QueryStoreFile(mObjectID, ModificationTime, - AttributesHash, isCompletelyDifferent?(0):(diffFromID), rStoreFilename, *patchStream)); - - // Get object ID from the result - objID = stored->GetObjectID(); + apStreamToUpload = BackupStoreFile::EncodeFileDiff( + rFilename, + mObjectID, /* containing directory */ + rStoreFilename, diffFromID, *blockIndexStream, + connection.GetTimeout(), + &rContext, // DiffTimer implementation + 0 /* not interested in the modification time */, + &isCompletelyDifferent); - // Don't attempt to upload it again! - doNormalUpload = false; + if(isCompletelyDifferent) + { + diffFromID = 0; + } + + rContext.UnManageDiffProcess(); } } - if(doNormalUpload) + if(!apStreamToUpload.get()) // No patch upload, so do a normal upload { // below threshold or nothing to diff from, so upload whole - rNotifier.NotifyFileUploading(this, rFilename); + rNotifier.NotifyFileUploading(this, rNonVssFilePath); // Prepare to upload, getting a stream which will encode the file as we go along - std::auto_ptr<IOStream> upload( - BackupStoreFile::EncodeFile(rFilename.c_str(), - mObjectID, rStoreFilename, NULL, - &rParams, - &(rParams.mrRunStatusProvider))); - - // Send to store - std::auto_ptr<BackupProtocolClientSuccess> stored( - connection.QueryStoreFile( - mObjectID, ModificationTime, - AttributesHash, - 0 /* no diff from file ID */, - rStoreFilename, *upload)); - - // Get object ID from the result - objID = stored->GetObjectID(); + apStreamToUpload = BackupStoreFile::EncodeFile( + rFilename, mObjectID, /* containing directory */ + rStoreFilename, NULL, &rParams, + &(rParams.mrRunStatusProvider)); + } + + rContext.SetNiceMode(true); + std::auto_ptr<IOStream> apWrappedStream; + + if(rParams.mMaxUploadRate > 0) + { + apWrappedStream.reset(new RateLimitingStream( + *apStreamToUpload, rParams.mMaxUploadRate)); + } + else + { + // Wrap the stream in *something*, so that + // QueryStoreFile() doesn't delete the original + // stream (upload object) and we can retrieve + // the byte counter. + apWrappedStream.reset(new BufferedStream( + *apStreamToUpload)); } + + // Send to store + std::auto_ptr<BackupProtocolSuccess> stored( + connection.QueryStoreFile( + mObjectID, ModificationTime, + AttributesHash, + diffFromID, + rStoreFilename, apWrappedStream)); + + rContext.SetNiceMode(false); + + // Get object ID from the result + objID = stored->GetObjectID(); + uploadedSize = apStreamToUpload->GetTotalBytesSent(); } catch(BoxException &e) { @@ -1605,8 +1800,8 @@ int64_t BackupClientDirectoryRecord::UploadFile( int type, subtype; if(connection.GetLastError(type, subtype)) { - if(type == BackupProtocolClientError::ErrorType - && subtype == BackupProtocolClientError::Err_StorageLimitExceeded) + if(type == BackupProtocolError::ErrorType + && subtype == BackupProtocolError::Err_StorageLimitExceeded) { // The hard limit was exceeded on the server, notify! rParams.mrSysadminNotifier.NotifySysadmin( @@ -1617,7 +1812,7 @@ int64_t BackupClientDirectoryRecord::UploadFile( return 0; } rNotifier.NotifyFileUploadServerError(this, - rFilename, type, subtype); + rNonVssFilePath, type, subtype); } } @@ -1625,7 +1820,8 @@ int64_t BackupClientDirectoryRecord::UploadFile( throw; } - rNotifier.NotifyFileUploaded(this, rFilename, FileSize); + rNotifier.NotifyFileUploaded(this, rNonVssFilePath, FileSize, + uploadedSize); // Return the new object ID of this file return objID; @@ -1641,7 +1837,9 @@ int64_t BackupClientDirectoryRecord::UploadFile( // Created: 29/3/04 // // -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(BackupClientDirectoryRecord::SyncParams &rParams, const char *Filename) +void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string& rFilename) { // Zero hash, so it gets synced properly next time round. ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); @@ -1671,19 +1869,20 @@ BackupClientDirectoryRecord::SyncParams::SyncParams( SysadminNotifier &rSysadminNotifier, ProgressNotifier &rProgressNotifier, BackupClientContext &rContext) - : mSyncPeriodStart(0), - mSyncPeriodEnd(0), - mMaxUploadWait(0), - mMaxFileTimeInFuture(99999999999999999LL), - mFileTrackingSizeThreshold(16*1024), - mDiffingUploadSizeThreshold(16*1024), - mrRunStatusProvider(rRunStatusProvider), - mrSysadminNotifier(rSysadminNotifier), - mrProgressNotifier(rProgressNotifier), - mrContext(rContext), - mReadErrorsOnFilesystemObjects(false), - mUploadAfterThisTimeInTheFuture(99999999999999999LL), - mHaveLoggedWarningAboutFutureFileTimes(false) +: mSyncPeriodStart(0), + mSyncPeriodEnd(0), + mMaxUploadWait(0), + mMaxFileTimeInFuture(99999999999999999LL), + mFileTrackingSizeThreshold(16*1024), + mDiffingUploadSizeThreshold(16*1024), + mrRunStatusProvider(rRunStatusProvider), + mrSysadminNotifier(rSysadminNotifier), + mrProgressNotifier(rProgressNotifier), + mrContext(rContext), + mReadErrorsOnFilesystemObjects(false), + mMaxUploadRate(0), + mUploadAfterThisTimeInTheFuture(99999999999999999LL), + mHaveLoggedWarningAboutFutureFileTimes(false) { } @@ -1874,3 +2073,220 @@ void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const pSubItem->Serialize(rArchive); } } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Location() +// Purpose: Constructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +Location::Location() + : mIDMapIndex(0), + mpExcludeFiles(0), + mpExcludeDirs(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::~Location() +// Purpose: Destructor +// Created: 11/11/03 +// +// -------------------------------------------------------------------------- +Location::~Location() +{ + // Clean up exclude locations + if(mpExcludeDirs != 0) + { + delete mpExcludeDirs; + mpExcludeDirs = 0; + } + if(mpExcludeFiles != 0) + { + delete mpExcludeFiles; + mpExcludeFiles = 0; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, +// using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void Location::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mName); + rArchive.Write(mPath); + rArchive.Write(mIDMapIndex); + + // + // + // + if(mpDirectoryRecord.get() == NULL) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpDirectoryRecord->Serialize(rArchive); + } + + // + // + // + if(!mpExcludeFiles) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpExcludeFiles->Serialize(rArchive); + } + + // + // + // + if(!mpExcludeDirs) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpExcludeDirs->Serialize(rArchive); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Location::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void Location::Deserialize(Archive &rArchive) +{ + // + // + // + mpDirectoryRecord.reset(NULL); + if(mpExcludeFiles) + { + delete mpExcludeFiles; + mpExcludeFiles = NULL; + } + if(mpExcludeDirs) + { + delete mpExcludeDirs; + mpExcludeDirs = NULL; + } + + // + // + // + rArchive.Read(mName); + rArchive.Read(mPath); + rArchive.Read(mIDMapIndex); + + // + // + // + int64_t aMagicMarker = 0; + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, ""); + if(!pSubRecord) + { + throw std::bad_alloc(); + } + + mpDirectoryRecord.reset(pSubRecord); + mpDirectoryRecord->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } + + // + // + // + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mpExcludeFiles = new ExcludeList; + if(!mpExcludeFiles) + { + throw std::bad_alloc(); + } + + mpExcludeFiles->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } + + // + // + // + rArchive.Read(aMagicMarker); + + if(aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mpExcludeDirs = new ExcludeList; + if(!mpExcludeDirs) + { + throw std::bad_alloc(); + } + + mpExcludeDirs->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); + } +} |