diff options
Diffstat (limited to 'bin/bbackupd/BackupClientDirectoryRecord.cpp')
| -rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.cpp | 383 |
1 files changed, 314 insertions, 69 deletions
diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp index cccf2f9b..0d3300cb 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp @@ -1,4 +1,4 @@ -// distribution boxbackup-0.10 (svn version: 494) +// distribution boxbackup-0.11rc1 (svn version: 2023_2024) // // Copyright (c) 2003 - 2006 // Ben Summers and contributors. All rights reserved. @@ -67,6 +67,9 @@ #include "BackupDaemon.h" #include "BackupStoreException.h" #include "Archive.h" +#include "PathUtils.h" +#include "Logging.h" +#include "ReadLoggingStream.h" #include "MemLeakFindOn.h" @@ -175,8 +178,8 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn { // The directory has probably been deleted, so just ignore this error. // In a future scan, this deletion will be noticed, deleted from server, and this object deleted. - TRACE1("Stat failed for '%s' (directory)\n", - rLocalPath.c_str()); + rParams.GetProgressNotifier().NotifyDirStatFailed( + this, rLocalPath, strerror(errno)); return; } // Store inode number in map so directories are tracked in case they're renamed @@ -204,15 +207,48 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn std::vector<std::string> dirs; std::vector<std::string> files; bool downloadDirectoryRecordBecauseOfFutureFiles = false; + + struct stat dir_st; + if(::lstat(rLocalPath.c_str(), &dir_st) != 0) + { + // Report the error (logs and + // eventual email to administrator) + rParams.GetProgressNotifier().NotifyFileStatFailed(this, + rLocalPath, strerror(errno)); + + // FIXME move to NotifyFileStatFailed() + SetErrorWhenReadingFilesystemObject(rParams, + rLocalPath.c_str()); + + // This shouldn't happen, so we'd better not continue + THROW_EXCEPTION(CommonException, OSFileError) + } + // BLOCK { // read the contents... DIR *dirHandle = 0; try { + rParams.GetProgressNotifier().NotifyScanDirectory( + this, rLocalPath); + dirHandle = ::opendir(rLocalPath.c_str()); if(dirHandle == 0) { + // Report the error (logs and + // eventual email to administrator) + if (errno == EACCES) + { + rParams.GetProgressNotifier().NotifyDirListFailed( + this, rLocalPath, "Access denied"); + } + else + { + rParams.GetProgressNotifier().NotifyDirListFailed(this, + rLocalPath, strerror(errno)); + } + // Report the error (logs and eventual email to administrator) SetErrorWhenReadingFilesystemObject(rParams, rLocalPath.c_str()); // Ignore this directory for now. @@ -234,6 +270,8 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn std::string filename; while((en = ::readdir(dirHandle)) != 0) { + rParams.mrContext.DoKeepAlive(); + // Don't need to use LinuxWorkaround_FinishDirentStruct(en, rLocalPath.c_str()); // on Linux, as a stat is performed to get all this info @@ -245,13 +283,28 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn } // Stat file to get info - filename = rLocalPath + DIRECTORY_SEPARATOR + - en->d_name; + filename = MakeFullPath(rLocalPath, en->d_name); + #ifdef WIN32 + // Don't stat the file just yet, to ensure + // that users can exclude unreadable files + // to suppress warnings that they are + // not accessible. + // + // Our emulated readdir() abuses en->d_type, + // which would normally contain DT_REG, + // DT_DIR, etc, but we only use it here and + // prefer S_IFREG, S_IFDIR... + int type = en->d_type; + #else if(::lstat(filename.c_str(), &st) != 0) { // Report the error (logs and // eventual email to administrator) + rParams.GetProgressNotifier().NotifyFileStatFailed(this, + filename, strerror(errno)); + + // FIXME move to NotifyFileStatFailed() SetErrorWhenReadingFilesystemObject( rParams, filename.c_str()); @@ -259,7 +312,17 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn continue; } + if(st.st_dev != dir_st.st_dev) + { + rParams.GetProgressNotifier() + .NotifyMountPointSkipped(this, + filename); + continue; + } + int type = st.st_mode & S_IFMT; + #endif + if(type == S_IFREG || type == S_IFLNK) { // File or symbolic link @@ -267,6 +330,11 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Exclude it? if(rParams.mrContext.ExcludeFile(filename)) { + rParams.GetProgressNotifier() + .NotifyFileExcluded( + this, + filename); + // Next item! continue; } @@ -281,6 +349,11 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Exclude it? if(rParams.mrContext.ExcludeDir(filename)) { + rParams.GetProgressNotifier() + .NotifyDirExcluded( + this, + filename); + // Next item! continue; } @@ -290,11 +363,56 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn } else { + if(rParams.mrContext.ExcludeFile(filename)) + { + rParams.GetProgressNotifier() + .NotifyFileExcluded( + this, + filename); + } + else + { + rParams.GetProgressNotifier() + .NotifyUnsupportedFileType( + this, filename); + SetErrorWhenReadingFilesystemObject( + rParams, filename.c_str()); + } + continue; } // Here if the object is something to back up (file, symlink or dir, not excluded) // So make the information for adding to the checksum + + #ifdef WIN32 + // We didn't stat the file before, + // but now we need the information. + if(::lstat(filename.c_str(), &st) != 0) + { + rParams.GetProgressNotifier() + .NotifyFileStatFailed(this, + filename, + strerror(errno)); + + // Report the error (logs and + // eventual email to administrator) + SetErrorWhenReadingFilesystemObject( + rParams, filename.c_str()); + + // Ignore this entry for now. + continue; + } + + if(st.st_dev != dir_st.st_dev) + { + rParams.GetProgressNotifier() + .NotifyMountPointSkipped(this, + filename); + continue; + } + #endif + checksum_info.mModificationTime = FileModificationTime(st); checksum_info.mAttributeModificationTime = FileAttrModificationTime(st); checksum_info.mSize = st.st_size; @@ -310,8 +428,8 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Log that this has happened if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) { - ::syslog(LOG_ERR, "Some files have modification times excessively in the future. Check clock syncronisation.\n"); - ::syslog(LOG_ERR, "Example file (only one shown) : %s\n", filename.c_str()); + rParams.GetProgressNotifier().NotifyFileModifiedInFuture( + this, filename); rParams.mHaveLoggedWarningAboutFutureFileTimes = true; } } @@ -549,8 +667,11 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP for(std::vector<std::string>::const_iterator f = rFiles.begin(); f != rFiles.end(); ++f) { + // Send keep-alive message if needed + rParams.mrContext.DoKeepAlive(); + // Filename of this file - std::string filename(rLocalPath + DIRECTORY_SEPARATOR + *f); + std::string filename(MakeFullPath(rLocalPath, *f)); // Get relevant info about file box_time_t modTime = 0; @@ -564,7 +685,16 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP struct stat st; if(::lstat(filename.c_str(), &st) != 0) { - THROW_EXCEPTION(CommonException, OSFileError) + rParams.GetProgressNotifier().NotifyFileStatFailed(this, + filename, strerror(errno)); + + // Report the error (logs and + // eventual email to administrator) + SetErrorWhenReadingFilesystemObject(rParams, + filename.c_str()); + + // Ignore this entry for now. + continue; } // Extract required data @@ -683,34 +813,94 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // Need to update? // // Condition for upload: - // modifiction time within sync period + // modification time within sync period // if it's been seen before but not uploaded, is the time from this first sight longer than the MaxUploadWait // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store - if( - ( - // Check the file modified within the acceptable time period we're checking - // If the file isn't on the server, the acceptable time starts at zero. - // Check pDirOnStore and en, because if we didn't download a directory listing, - // pDirOnStore will be zero, but we know it's on the server. - ( ((pDirOnStore != 0 && en == 0) || (modTime >= rParams.mSyncPeriodStart)) && modTime < rParams.mSyncPeriodEnd) - - // However, just in case things are continually modified, we check the first seen time. - // The two compares of syncPeriodEnd and pendingFirstSeenTime are because the values are unsigned. - || (pendingFirstSeenTime != 0 && - (rParams.mSyncPeriodEnd > pendingFirstSeenTime) - && ((rParams.mSyncPeriodEnd - pendingFirstSeenTime) > rParams.mMaxUploadWait)) - - // Then make sure that if files are added with a time less than the sync period start - // (which can easily happen on file server), it gets uploaded. The directory contents checksum - // will pick up the fact it has been added, so the store listing will be available when this happens. - || ((modTime <= rParams.mSyncPeriodStart) && (en != 0) && (en->GetModificationTime() != modTime)) - - // And just to catch really badly off clocks in the future for file server clients, - // just upload the file if it's madly in the future. - || (modTime > rParams.mUploadAfterThisTimeInTheFuture) - ) - // But even then, only upload it if the mod time locally is different to that on the server. - && (en == 0 || en->GetModificationTime() != modTime)) + + bool doUpload = false; + + // Only upload a file if the mod time locally is + // different to that on the server. + + if (en == 0 || en->GetModificationTime() != modTime) + { + // Check the file modified within the acceptable time period we're checking + // If the file isn't on the server, the acceptable time starts at zero. + // Check pDirOnStore and en, because if we didn't download a directory listing, + // pDirOnStore will be zero, but we know it's on the server. + if (modTime < rParams.mSyncPeriodEnd) + { + if (pDirOnStore != 0 && en == 0) + { + doUpload = true; + BOX_TRACE(filename << ": will upload " + "(not on server)"); + } + else if (modTime >= rParams.mSyncPeriodStart) + { + doUpload = true; + BOX_TRACE(filename << ": will upload " + "(modified since last sync)"); + } + } + + // However, just in case things are continually + // modified, we check the first seen time. + // The two compares of syncPeriodEnd and + // pendingFirstSeenTime are because the values + // are unsigned. + + if (!doUpload && + pendingFirstSeenTime != 0 && + rParams.mSyncPeriodEnd > pendingFirstSeenTime && + (rParams.mSyncPeriodEnd - pendingFirstSeenTime) + > rParams.mMaxUploadWait) + { + doUpload = true; + BOX_TRACE(filename << ": will upload " + "(continually modified)"); + } + + // Then make sure that if files are added with a + // time less than the sync period start + // (which can easily happen on file server), it + // gets uploaded. The directory contents checksum + // will pick up the fact it has been added, so the + // store listing will be available when this happens. + + if (!doUpload && + modTime <= rParams.mSyncPeriodStart && + en != 0 && + en->GetModificationTime() != modTime) + { + doUpload = true; + BOX_TRACE(filename << ": will upload " + "(mod time changed)"); + } + + // And just to catch really badly off clocks in + // the future for file server clients, + // just upload the file if it's madly in the future. + + if (!doUpload && modTime > + rParams.mUploadAfterThisTimeInTheFuture) + { + doUpload = true; + BOX_TRACE(filename << ": will upload " + "(mod time in the future)"); + } + } + + if (!doUpload) + { + BOX_TRACE(filename << ": will not upload " + "(no reason to upload, mod time is " + << modTime << " versus sync period " + << rParams.mSyncPeriodStart << " to " + << rParams.mSyncPeriodEnd << ")"); + } + + if (doUpload) { // Make sure we're connected -- must connect here so we know whether // the storage limit has been exceeded, and hence whether or not @@ -735,6 +925,9 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP { // Connection errors should just be passed on to the main handler, retries // would probably just cause more problems. + rParams.GetProgressNotifier() + .NotifyFileUploadException( + this, filename, e); throw; } catch(BoxException &e) @@ -743,8 +936,9 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP allUpdatedSuccessfully = false; // Log it. SetErrorWhenReadingFilesystemObject(rParams, filename.c_str()); - // Log error. - ::syslog(LOG_ERR, "Error code when uploading was (%d/%d), %s", e.GetType(), e.GetSubType(), e.what()); + rParams.GetProgressNotifier() + .NotifyFileUploadException( + this, filename, e); } // Update structures if the file was uploaded successfully. @@ -757,6 +951,11 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP } } } + else + { + rParams.GetProgressNotifier().NotifyFileSkippedServerFull(this, + filename); + } } else if(en != 0 && en->GetAttributesHash() != attributesHash) { @@ -838,6 +1037,9 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP } } } + + rParams.GetProgressNotifier().NotifyFileSynchronised(this, + filename, fileSize); } // Erase contents of files to save space when recursing @@ -855,8 +1057,11 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP for(std::vector<std::string>::const_iterator d = rDirs.begin(); d != rDirs.end(); ++d) { + // Send keep-alive message if needed + rParams.mrContext.DoKeepAlive(); + // Get the local filename - std::string dirname(rLocalPath + DIRECTORY_SEPARATOR + *d); + std::string dirname(MakeFullPath(rLocalPath, *d)); // See if it's in the listing (if we have one) BackupStoreFilenameClear storeFilename(*d); @@ -893,11 +1098,15 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // In the list, just use this pointer psubDirRecord = e->second; } - else if(!rParams.mrContext.StorageLimitExceeded()) // know we've got a connection if we get this far, as dir will have been modified. + else { - // Note: only think about adding directory records if there's space left on the server. - // If there isn't, this step will be repeated when there is some available. - + // Note: if we have exceeded our storage limit, then + // we should not upload any more data, nor create any + // DirectoryRecord representing data that would have + // been uploaded. This step will be repeated when + // there is some space available. + bool doCreateDirectoryRecord = true; + // Need to create the record. But do we need to create the directory on the server? int64_t subDirObjectID = 0; if(en != 0) @@ -905,6 +1114,12 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // No. Exists on the server, and we know about it from the listing. subDirObjectID = en->GetObjectID(); } + else if(rParams.mrContext.StorageLimitExceeded()) + // know we've got a connection if we get this far, + // as dir will have been modified. + { + doCreateDirectoryRecord = false; + } else { // Yes, creation required! @@ -987,20 +1202,23 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP haveJustCreatedDirOnServer = true; } } - - // New an object for this - psubDirRecord = new BackupClientDirectoryRecord(subDirObjectID, *d); - - // Store in list - try - { - mSubDirectories[*d] = psubDirRecord; - } - catch(...) - { - delete psubDirRecord; - psubDirRecord = 0; - throw; + + if (doCreateDirectoryRecord) + { + // New an object for this + psubDirRecord = new BackupClientDirectoryRecord(subDirObjectID, *d); + + // Store in list + try + { + mSubDirectories[*d] = psubDirRecord; + } + catch(...) + { + delete psubDirRecord; + psubDirRecord = 0; + throw; + } } } @@ -1060,7 +1278,13 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP BackupClientDirectoryRecord *rec = e->second; mSubDirectories.erase(e); delete rec; - TRACE2("Deleted directory record for %s/%s\n", rLocalPath.c_str(), dirname.GetClearFilename().c_str()); + + std::string name = MakeFullPath( + rLocalPath, + dirname.GetClearFilename()); + + TRACE1("Deleted directory record for " + "%s\n", name.c_str()); } } } @@ -1132,7 +1356,11 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn if(diffFromID != 0) { - // Found an old version -- get the index + // Found an old version + rParams.GetProgressNotifier().NotifyFileUploadingPatch(this, + rFilename); + + // Get the index std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream()); // @@ -1168,9 +1396,13 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn if(doNormalUpload) { // below threshold or nothing to diff from, so upload whole + rParams.GetProgressNotifier().NotifyFileUploading(this, + rFilename); // Prepare to upload, getting a stream which will encode the file as we go along - std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(rFilename.c_str(), mObjectID, rStoreFilename)); + std::auto_ptr<IOStream> upload( + BackupStoreFile::EncodeFile(rFilename.c_str(), + mObjectID, rStoreFilename)); // Send to store std::auto_ptr<BackupProtocolClientSuccess> stored( @@ -1190,14 +1422,20 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn if(e.GetType() == ConnectionException::ExceptionType && e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) { - // Check and see what error the protocol has -- as it might be an error... + // Check and see what error the protocol has, + // this is more useful to users than the exception. int type, subtype; - if(connection.GetLastError(type, subtype) - && type == BackupProtocolClientError::ErrorType - && subtype == BackupProtocolClientError::Err_StorageLimitExceeded) + if(connection.GetLastError(type, subtype)) { - // The hard limit was exceeded on the server, notify! - rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull); + if(type == BackupProtocolClientError::ErrorType + && subtype == BackupProtocolClientError::Err_StorageLimitExceeded) + { + // The hard limit was exceeded on the server, notify! + rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull); + } + rParams.GetProgressNotifier() + .NotifyFileUploadServerError( + this, rFilename, type, subtype); } } @@ -1205,6 +1443,8 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn throw; } + rParams.GetProgressNotifier().NotifyFileUploaded(this, rFilename, FileSize); + // Return the new object ID of this file return objID; } @@ -1224,8 +1464,11 @@ void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(BackupClie // Zero hash, so it gets synced properly next time round. ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); - // Log the error - ::syslog(LOG_ERR, "Backup object failed, error when reading %s", Filename); + // Log the error - already done by caller + /* + rParams.GetProgressNotifier().NotifyFileReadFailed(this, + Filename, strerror(errno)); + */ // Mark that an error occured in the parameters object rParams.mReadErrorsOnFilesystemObjects = true; @@ -1241,8 +1484,10 @@ void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(BackupClie // Created: 8/3/04 // // -------------------------------------------------------------------------- -BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, BackupClientContext &rContext) - : mSyncPeriodStart(0), +BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, + ProgressNotifier &rProgressNotifier, BackupClientContext &rContext) + : mrProgressNotifier(rProgressNotifier), + mSyncPeriodStart(0), mSyncPeriodEnd(0), mMaxUploadWait(0), mMaxFileTimeInFuture(99999999999999999LL), |