summaryrefslogtreecommitdiff
path: root/bin/bbackupd/BackupClientDirectoryRecord.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'bin/bbackupd/BackupClientDirectoryRecord.cpp')
-rw-r--r--bin/bbackupd/BackupClientDirectoryRecord.cpp576
1 files changed, 392 insertions, 184 deletions
diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp
index 0a0703c2..b8d42d47 100644
--- a/bin/bbackupd/BackupClientDirectoryRecord.cpp
+++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp
@@ -2,7 +2,8 @@
//
// File
// Name: BackupClientDirectoryRecord.cpp
-// Purpose: Implementation of record about directory for backup client
+// Purpose: Implementation of record about directory for
+// backup client
// Created: 2003/10/08
//
// --------------------------------------------------------------------------
@@ -100,16 +101,27 @@ void BackupClientDirectoryRecord::DeleteSubDirectories()
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &, int64_t, const std::string &, bool)
-// Purpose: Syncronise, recusively, a local directory with the server.
+// Name: BackupClientDirectoryRecord::SyncDirectory(i
+// BackupClientDirectoryRecord::SyncParams &,
+// int64_t, const std::string &,
+// const std::string &, bool)
+// Purpose: Recursively synchronise a local directory
+// with the server.
// Created: 2003/10/08
//
// --------------------------------------------------------------------------
-void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &rParams, int64_t ContainingDirectoryID,
- const std::string &rLocalPath, bool ThisDirHasJustBeenCreated)
+void BackupClientDirectoryRecord::SyncDirectory(
+ BackupClientDirectoryRecord::SyncParams &rParams,
+ int64_t ContainingDirectoryID,
+ const std::string &rLocalPath,
+ const std::string &rRemotePath,
+ bool ThisDirHasJustBeenCreated)
{
+ BackupClientContext& rContext(rParams.mrContext);
+ ProgressNotifier& rNotifier(rContext.GetProgressNotifier());
+
// Signal received by daemon?
- if(rParams.mrDaemon.StopRun())
+ if(rParams.mrRunStatusProvider.StopRun())
{
// Yes. Stop now.
THROW_EXCEPTION(BackupStoreException, SignalReceived)
@@ -118,49 +130,66 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
// Start by making some flag changes, marking this sync as not done,
// and on the immediate sub directories.
mSyncDone = false;
- for(std::map<std::string, BackupClientDirectoryRecord *>::iterator i = mSubDirectories.begin();
+ for(std::map<std::string, BackupClientDirectoryRecord *>::iterator
+ i = mSubDirectories.begin();
i != mSubDirectories.end(); ++i)
{
i->second->mSyncDone = false;
}
- // Work out the time in the future after which the file should be uploaded regardless.
- // This is a simple way to avoid having too many problems with file servers when they have
- // clients with badly out of sync clocks.
- rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + rParams.mMaxFileTimeInFuture;
+ // Work out the time in the future after which the file should
+ // be uploaded regardless. This is a simple way to avoid having
+ // too many problems with file servers when they have clients
+ // with badly out of sync clocks.
+ rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() +
+ rParams.mMaxFileTimeInFuture;
- // Build the current state checksum to compare against while getting info from dirs
- // Note checksum is used locally only, so byte order isn't considered.
+ // Build the current state checksum to compare against while
+ // getting info from dirs. Note checksum is used locally only,
+ // so byte order isn't considered.
MD5Digest currentStateChecksum;
+ EMU_STRUCT_STAT dest_st;
// Stat the directory, to get attribute info
+ // If it's a symbolic link, we want the link target here
+ // (as we're about to back up the contents of the directory)
{
- struct stat st;
- if(::stat(rLocalPath.c_str(), &st) != 0)
+ if(EMU_STAT(rLocalPath.c_str(), &dest_st) != 0)
{
- // The directory has probably been deleted, so just ignore this error.
- // In a future scan, this deletion will be noticed, deleted from server, and this object deleted.
- rParams.GetProgressNotifier().NotifyDirStatFailed(
- this, rLocalPath, strerror(errno));
+ // The directory has probably been deleted, so
+ // just ignore this error. In a future scan, this
+ // deletion will be noticed, deleted from server,
+ // and this object deleted.
+ rNotifier.NotifyDirStatFailed(this, rLocalPath,
+ strerror(errno));
return;
}
- // Store inode number in map so directories are tracked in case they're renamed
+ // Store inode number in map so directories are tracked
+ // in case they're renamed
{
- BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap());
- idMap.AddToMap(st.st_ino, mObjectID, ContainingDirectoryID);
+ BackupClientInodeToIDMap &idMap(
+ rParams.mrContext.GetNewIDMap());
+ idMap.AddToMap(dest_st.st_ino, mObjectID,
+ ContainingDirectoryID);
}
// Add attributes to checksum
- currentStateChecksum.Add(&st.st_mode, sizeof(st.st_mode));
- currentStateChecksum.Add(&st.st_uid, sizeof(st.st_uid));
- currentStateChecksum.Add(&st.st_gid, sizeof(st.st_gid));
+ currentStateChecksum.Add(&dest_st.st_mode,
+ sizeof(dest_st.st_mode));
+ currentStateChecksum.Add(&dest_st.st_uid,
+ sizeof(dest_st.st_uid));
+ currentStateChecksum.Add(&dest_st.st_gid,
+ sizeof(dest_st.st_gid));
// Inode to be paranoid about things moving around
- currentStateChecksum.Add(&st.st_ino, sizeof(st.st_ino));
+ currentStateChecksum.Add(&dest_st.st_ino,
+ sizeof(dest_st.st_ino));
#ifdef HAVE_STRUCT_STAT_ST_FLAGS
- currentStateChecksum.Add(&st.st_flags, sizeof(st.st_flags));
+ currentStateChecksum.Add(&dest_st.st_flags,
+ sizeof(dest_st.st_flags));
#endif
StreamableMemBlock xattr;
- BackupClientFileAttributes::FillExtendedAttr(xattr, rLocalPath.c_str());
+ BackupClientFileAttributes::FillExtendedAttr(xattr,
+ rLocalPath.c_str());
currentStateChecksum.Add(xattr.GetBuffer(), xattr.GetSize());
}
@@ -170,13 +199,13 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
std::vector<std::string> files;
bool downloadDirectoryRecordBecauseOfFutureFiles = false;
- struct stat dir_st;
- if(::lstat(rLocalPath.c_str(), &dir_st) != 0)
+ EMU_STRUCT_STAT link_st;
+ if(EMU_LSTAT(rLocalPath.c_str(), &link_st) != 0)
{
// Report the error (logs and
// eventual email to administrator)
- rParams.GetProgressNotifier().NotifyFileStatFailed(this,
- rLocalPath, strerror(errno));
+ rNotifier.NotifyFileStatFailed(this, rLocalPath,
+ strerror(errno));
// FIXME move to NotifyFileStatFailed()
SetErrorWhenReadingFilesystemObject(rParams,
@@ -192,8 +221,7 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
DIR *dirHandle = 0;
try
{
- rParams.GetProgressNotifier().NotifyScanDirectory(
- this, rLocalPath);
+ rNotifier.NotifyScanDirectory(this, rLocalPath);
dirHandle = ::opendir(rLocalPath.c_str());
if(dirHandle == 0)
@@ -202,17 +230,19 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
// eventual email to administrator)
if (errno == EACCES)
{
- rParams.GetProgressNotifier().NotifyDirListFailed(
- this, rLocalPath, "Access denied");
+ rNotifier.NotifyDirListFailed(this,
+ rLocalPath, "Access denied");
}
else
{
- rParams.GetProgressNotifier().NotifyDirListFailed(this,
+ rNotifier.NotifyDirListFailed(this,
rLocalPath, strerror(errno));
}
- // Report the error (logs and eventual email to administrator)
- SetErrorWhenReadingFilesystemObject(rParams, rLocalPath.c_str());
+ // Report the error (logs and eventual email
+ // to administrator)
+ SetErrorWhenReadingFilesystemObject(rParams,
+ rLocalPath.c_str());
// Ignore this directory for now.
return;
}
@@ -228,14 +258,17 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
::memset(&checksum_info, 0, sizeof(checksum_info));
struct dirent *en = 0;
- struct stat st;
+ EMU_STRUCT_STAT file_st;
std::string filename;
while((en = ::readdir(dirHandle)) != 0)
{
rParams.mrContext.DoKeepAlive();
- // Don't need to use LinuxWorkaround_FinishDirentStruct(en, rLocalPath.c_str());
- // on Linux, as a stat is performed to get all this info
+ // Don't need to use
+ // LinuxWorkaround_FinishDirentStruct(en,
+ // rLocalPath.c_str());
+ // on Linux, as a stat is performed to
+ // get all this info
if(en->d_name[0] == '.' &&
(en->d_name[1] == '\0' || (en->d_name[1] == '.' && en->d_name[2] == '\0')))
@@ -259,11 +292,11 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
// prefer S_IFREG, S_IFDIR...
int type = en->d_type;
#else
- if(::lstat(filename.c_str(), &st) != 0)
+ if(EMU_LSTAT(filename.c_str(), &file_st) != 0)
{
// Report the error (logs and
// eventual email to administrator)
- rParams.GetProgressNotifier().NotifyFileStatFailed(this,
+ rNotifier.NotifyFileStatFailed(this,
filename, strerror(errno));
// FIXME move to NotifyFileStatFailed()
@@ -274,19 +307,18 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
continue;
}
- if(st.st_dev != dir_st.st_dev)
+ if(file_st.st_dev != dest_st.st_dev)
{
if(!(rParams.mrContext.ExcludeDir(
filename)))
{
- rParams.GetProgressNotifier()
- .NotifyMountPointSkipped(
- this, filename);
+ rNotifier.NotifyMountPointSkipped(
+ this, filename);
}
continue;
}
- int type = st.st_mode & S_IFMT;
+ int type = file_st.st_mode & S_IFMT;
#endif
if(type == S_IFREG || type == S_IFLNK)
@@ -296,8 +328,7 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
// Exclude it?
if(rParams.mrContext.ExcludeFile(filename))
{
- rParams.GetProgressNotifier()
- .NotifyFileExcluded(
+ rNotifier.NotifyFileExcluded(
this,
filename);
@@ -315,8 +346,7 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
// Exclude it?
if(rParams.mrContext.ExcludeDir(filename))
{
- rParams.GetProgressNotifier()
- .NotifyDirExcluded(
+ rNotifier.NotifyDirExcluded(
this,
filename);
@@ -327,19 +357,22 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
// Store on list
dirs.push_back(std::string(en->d_name));
}
+ else if (type == S_IFSOCK || type == S_IFIFO)
+ {
+ // removed notification for these types
+ // see Debian bug 479145, no objections
+ }
else
{
if(rParams.mrContext.ExcludeFile(filename))
{
- rParams.GetProgressNotifier()
- .NotifyFileExcluded(
+ rNotifier.NotifyFileExcluded(
this,
filename);
}
else
{
- rParams.GetProgressNotifier()
- .NotifyUnsupportedFileType(
+ rNotifier.NotifyUnsupportedFileType(
this, filename);
SetErrorWhenReadingFilesystemObject(
rParams, filename.c_str());
@@ -354,10 +387,9 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
#ifdef WIN32
// We didn't stat the file before,
// but now we need the information.
- if(::lstat(filename.c_str(), &st) != 0)
+ if(emu_stat(filename.c_str(), &file_st) != 0)
{
- rParams.GetProgressNotifier()
- .NotifyFileStatFailed(this,
+ rNotifier.NotifyFileStatFailed(this,
filename,
strerror(errno));
@@ -370,18 +402,17 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
continue;
}
- if(st.st_dev != dir_st.st_dev)
+ if(file_st.st_dev != link_st.st_dev)
{
- rParams.GetProgressNotifier()
- .NotifyMountPointSkipped(this,
+ rNotifier.NotifyMountPointSkipped(this,
filename);
continue;
}
#endif
- checksum_info.mModificationTime = FileModificationTime(st);
- checksum_info.mAttributeModificationTime = FileAttrModificationTime(st);
- checksum_info.mSize = st.st_size;
+ checksum_info.mModificationTime = FileModificationTime(file_st);
+ checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st);
+ checksum_info.mSize = file_st.st_size;
currentStateChecksum.Add(&checksum_info, sizeof(checksum_info));
currentStateChecksum.Add(en->d_name, strlen(en->d_name));
@@ -394,7 +425,7 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
// Log that this has happened
if(!rParams.mHaveLoggedWarningAboutFutureFileTimes)
{
- rParams.GetProgressNotifier().NotifyFileModifiedInFuture(
+ rNotifier.NotifyFileModifiedInFuture(
this, filename);
rParams.mHaveLoggedWarningAboutFutureFileTimes = true;
}
@@ -468,7 +499,8 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn
}
// Do the directory reading
- bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, pdirOnStore, entriesLeftOver, files, dirs);
+ bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath,
+ rRemotePath, pdirOnStore, entriesLeftOver, files, dirs);
// LAST THING! (think exception safety)
// Store the new checksum -- don't fetch things unnecessarily in the future
@@ -604,11 +636,18 @@ void BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord::
// Created: 2003/10/09
//
// --------------------------------------------------------------------------
-bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &rParams,
- const std::string &rLocalPath, BackupStoreDirectory *pDirOnStore,
+bool BackupClientDirectoryRecord::UpdateItems(
+ BackupClientDirectoryRecord::SyncParams &rParams,
+ const std::string &rLocalPath,
+ const std::string &rRemotePath,
+ BackupStoreDirectory *pDirOnStore,
std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver,
- std::vector<std::string> &rFiles, const std::vector<std::string> &rDirs)
+ std::vector<std::string> &rFiles,
+ const std::vector<std::string> &rDirs)
{
+ BackupClientContext& rContext(rParams.mrContext);
+ ProgressNotifier& rNotifier(rContext.GetProgressNotifier());
+
bool allUpdatedSuccessfully = true;
// Decrypt all the directory entries.
@@ -634,7 +673,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
f != rFiles.end(); ++f)
{
// Send keep-alive message if needed
- rParams.mrContext.DoKeepAlive();
+ rContext.DoKeepAlive();
// Filename of this file
std::string filename(MakeFullPath(rLocalPath, *f));
@@ -648,10 +687,10 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
// BLOCK
{
// Stat the file
- struct stat st;
- if(::lstat(filename.c_str(), &st) != 0)
+ EMU_STRUCT_STAT st;
+ if(EMU_LSTAT(filename.c_str(), &st) != 0)
{
- rParams.GetProgressNotifier().NotifyFileStatFailed(this,
+ rNotifier.NotifyFileStatFailed(this,
filename, strerror(errno));
// Report the error (logs and
@@ -689,7 +728,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == 0))
{
// Directory exists in the place of this file -- sort it out
- RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, en->GetObjectID(), *f);
+ RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore,
+ en, *f);
en = 0;
}
@@ -701,7 +741,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
// 2) It's not in the store
// Do we know about the inode number?
- const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap());
+ const BackupClientInodeToIDMap &idMap(rContext.GetCurrentIDMap());
int64_t renameObjectID = 0, renameInDirectory = 0;
if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory))
{
@@ -711,24 +751,24 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
bool isCurrentVersion = false;
box_time_t srvModTime = 0, srvAttributesHash = 0;
BackupStoreFilenameClear oldLeafname;
- if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname))
+ if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname))
{
// Only interested if it's a file and the latest version
if(!isDir && isCurrentVersion)
{
// Check that the object we found in the ID map doesn't exist on disc
- struct stat st;
- if(::stat(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT)
+ EMU_STRUCT_STAT st;
+ if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT)
{
// Doesn't exist locally, but does exist on the server.
// Therefore we can safely rename it to this new file.
// Get the connection to the server
- BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+ BackupProtocolClient &connection(rContext.GetConnection());
// Only do this step if there is room on the server.
// This step will be repeated later when there is space available
- if(!rParams.mrContext.StorageLimitExceeded())
+ if(!rContext.StorageLimitExceeded())
{
// Rename the existing files (ie include old versions) on the server
connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */,
@@ -736,7 +776,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
storeFilename);
// Stop the attempt to delete the file in the original location
- BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList());
+ BackupClientDeleteList &rdelList(rContext.GetDeleteList());
rdelList.StopFileDeletion(renameInDirectory, oldLeafname);
// Create new entry in the directory for it
@@ -799,13 +839,15 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
if (pDirOnStore != 0 && en == 0)
{
doUpload = true;
- BOX_TRACE(filename << ": will upload "
+ BOX_TRACE("Upload decision: " <<
+ filename << ": will upload "
"(not on server)");
}
else if (modTime >= rParams.mSyncPeriodStart)
{
doUpload = true;
- BOX_TRACE(filename << ": will upload "
+ BOX_TRACE("Upload decision: " <<
+ filename << ": will upload "
"(modified since last sync)");
}
}
@@ -823,7 +865,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
> rParams.mMaxUploadWait)
{
doUpload = true;
- BOX_TRACE(filename << ": will upload "
+ BOX_TRACE("Upload decision: " <<
+ filename << ": will upload "
"(continually modified)");
}
@@ -840,7 +883,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
en->GetModificationTime() != modTime)
{
doUpload = true;
- BOX_TRACE(filename << ": will upload "
+ BOX_TRACE("Upload decision: " <<
+ filename << ": will upload "
"(mod time changed)");
}
@@ -852,64 +896,121 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
rParams.mUploadAfterThisTimeInTheFuture)
{
doUpload = true;
- BOX_TRACE(filename << ": will upload "
+ BOX_TRACE("Upload decision: " <<
+ filename << ": will upload "
"(mod time in the future)");
}
}
-
- if (!doUpload)
+
+ if (en != 0 && en->GetModificationTime() == modTime)
{
- BOX_TRACE(filename << ": will not upload "
- "(no reason to upload, mod time is "
- << modTime << " versus sync period "
- << rParams.mSyncPeriodStart << " to "
- << rParams.mSyncPeriodEnd << ")");
+ BOX_TRACE("Upload decision: " <<
+ filename << ": will not upload "
+ "(not modified since last upload)");
+ }
+ else if (!doUpload)
+ {
+ if (modTime > rParams.mSyncPeriodEnd)
+ {
+ box_time_t now = GetCurrentBoxTime();
+ int age = BoxTimeToSeconds(now -
+ modTime);
+ BOX_TRACE("Upload decision: " <<
+ filename << ": will not upload "
+ "(modified too recently: "
+ "only " << age << "seconds ago)");
+ }
+ else
+ {
+ BOX_TRACE("Upload decision: " <<
+ filename << ": will not upload "
+ "(mod time is " << modTime <<
+ " which is outside sync window, "
+ << rParams.mSyncPeriodStart << " to "
+ << rParams.mSyncPeriodEnd << ")");
+ }
}
+ bool fileSynced = true;
+
if (doUpload)
{
+ // Upload needed, don't mark sync success until
+ // we've actually done it
+ fileSynced = false;
+
// Make sure we're connected -- must connect here so we know whether
// the storage limit has been exceeded, and hence whether or not
// to actually upload the file.
- rParams.mrContext.GetConnection();
+ rContext.GetConnection();
// Only do this step if there is room on the server.
// This step will be repeated later when there is space available
- if(!rParams.mrContext.StorageLimitExceeded())
+ if(!rContext.StorageLimitExceeded())
{
- // Upload the file to the server, recording the object ID it returns
- bool noPreviousVersionOnServer = ((pDirOnStore != 0) && (en == 0));
+ // Upload the file to the server, recording the
+ // object ID it returns
+ bool noPreviousVersionOnServer =
+ ((pDirOnStore != 0) && (en == 0));
- // Surround this in a try/catch block, to catch errrors, but still continue
+ // Surround this in a try/catch block, to
+ // catch errors, but still continue
bool uploadSuccess = false;
try
{
- latestObjectID = UploadFile(rParams, filename, storeFilename, fileSize, modTime, attributesHash, noPreviousVersionOnServer);
- uploadSuccess = true;
+ latestObjectID = UploadFile(rParams,
+ filename, storeFilename,
+ fileSize, modTime,
+ attributesHash,
+ noPreviousVersionOnServer);
+
+ if (latestObjectID == 0)
+ {
+ // storage limit exceeded
+ rParams.mrContext.SetStorageLimitExceeded();
+ uploadSuccess = false;
+ allUpdatedSuccessfully = false;
+ }
+ else
+ {
+ uploadSuccess = true;
+ }
}
catch(ConnectionException &e)
{
- // Connection errors should just be passed on to the main handler, retries
- // would probably just cause more problems.
- rParams.GetProgressNotifier()
- .NotifyFileUploadException(
- this, filename, e);
+ // Connection errors should just be
+ // passed on to the main handler,
+ // retries would probably just cause
+ // more problems.
+ rNotifier.NotifyFileUploadException(
+ this, filename, e);
throw;
}
catch(BoxException &e)
{
- // an error occured -- make return code false, to show error in directory
+ if (e.GetType() == BackupStoreException::ExceptionType &&
+ e.GetSubType() == BackupStoreException::SignalReceived)
+ {
+ // abort requested, pass the
+ // exception on up.
+ throw;
+ }
+
+ // an error occured -- make return
+ // code false, to show error in directory
allUpdatedSuccessfully = false;
// Log it.
SetErrorWhenReadingFilesystemObject(rParams, filename.c_str());
- rParams.GetProgressNotifier()
- .NotifyFileUploadException(
- this, filename, e);
+ rNotifier.NotifyFileUploadException(
+ this, filename, e);
}
- // Update structures if the file was uploaded successfully.
+ // Update structures if the file was uploaded
+ // successfully.
if(uploadSuccess)
{
+ fileSynced = true;
+
// delete from pending entries
if(pendingFirstSeenTime != 0 && mpPendingEntries != 0)
{
@@ -919,28 +1020,41 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
}
else
{
- rParams.GetProgressNotifier().NotifyFileSkippedServerFull(this,
+ rNotifier.NotifyFileSkippedServerFull(this,
filename);
}
}
else if(en != 0 && en->GetAttributesHash() != attributesHash)
{
// Attributes have probably changed, upload them again.
- // If the attributes have changed enough, the directory hash will have changed too,
- // and so the dir will have been downloaded, and the entry will be available.
+ // If the attributes have changed enough, the directory
+ // hash will have changed too, and so the dir will have
+ // been downloaded, and the entry will be available.
// Get connection
- BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+ BackupProtocolClient &connection(rContext.GetConnection());
// Only do this step if there is room on the server.
- // This step will be repeated later when there is space available
- if(!rParams.mrContext.StorageLimitExceeded())
+ // This step will be repeated later when there is
+ // space available
+ if(!rContext.StorageLimitExceeded())
{
- // Update store
- BackupClientFileAttributes attr;
- attr.ReadAttributes(filename.c_str(), false /* put mod times in the attributes, please */);
- MemBlockStream attrStream(attr);
- connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream);
+ try
+ {
+ // Update store
+ BackupClientFileAttributes attr;
+ attr.ReadAttributes(filename.c_str(), false /* put mod times in the attributes, please */);
+ MemBlockStream attrStream(attr);
+ connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream);
+ fileSynced = true;
+ }
+ catch (BoxException &e)
+ {
+ BOX_ERROR("Failed to read or store "
+ "file attributes for '" <<
+ filename << "', will try "
+ "again later");
+ }
}
}
@@ -976,36 +1090,56 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
if(fileSize >= rParams.mFileTrackingSizeThreshold)
{
// Get the map
- BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap());
+ BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap());
// Need to get an ID from somewhere...
if(latestObjectID != 0)
{
// Use this one
+ BOX_TRACE("Storing uploaded file ID " <<
+ inodeNum << " (" << filename << ") "
+ "in ID map as object " <<
+ latestObjectID << " with parent " <<
+ mObjectID);
idMap.AddToMap(inodeNum, latestObjectID, mObjectID /* containing directory */);
}
else
{
// Don't know it -- haven't sent anything to the store, and didn't get a listing.
// Look it up in the current map, and if it's there, use that.
- const BackupClientInodeToIDMap &currentIDMap(rParams.mrContext.GetCurrentIDMap());
+ const BackupClientInodeToIDMap &currentIDMap(rContext.GetCurrentIDMap());
int64_t objid = 0, dirid = 0;
if(currentIDMap.Lookup(inodeNum, objid, dirid))
{
// Found
+ if (dirid != mObjectID)
+ {
+ BOX_WARNING("Found conflicting parent ID for file ID " << inodeNum << " (" << filename << "): expected " << mObjectID << " but found " << dirid << " (same directory used in two different locations?)");
+ }
+
ASSERT(dirid == mObjectID);
+
// NOTE: If the above assert fails, an inode number has been reused by the OS,
// or there is a problem somewhere. If this happened on a short test run, look
// into it. However, in a long running process this may happen occasionally and
- // not indiciate anything wrong.
+ // not indicate anything wrong.
// Run the release version for real life use, where this check is not made.
- idMap.AddToMap(inodeNum, objid, mObjectID /* containing directory */);
+ BOX_TRACE("Storing found file ID " <<
+ inodeNum << " (" << filename <<
+ ") in ID map as object " <<
+ objid << " with parent " <<
+ mObjectID);
+ idMap.AddToMap(inodeNum, objid,
+ mObjectID /* containing directory */);
}
}
}
- rParams.GetProgressNotifier().NotifyFileSynchronised(this,
- filename, fileSize);
+ if (fileSynced)
+ {
+ rNotifier.NotifyFileSynchronised(this, filename,
+ fileSize);
+ }
}
// Erase contents of files to save space when recursing
@@ -1014,7 +1148,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
// Delete the pending entries, if the map is entry
if(mpPendingEntries != 0 && mpPendingEntries->size() == 0)
{
- TRACE1("Deleting mpPendingEntries from dir ID %lld\n", mObjectID);
+ BOX_TRACE("Deleting mpPendingEntries from dir ID " <<
+ BOX_FORMAT_OBJECTID(mObjectID));
delete mpPendingEntries;
mpPendingEntries = 0;
}
@@ -1024,7 +1159,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
d != rDirs.end(); ++d)
{
// Send keep-alive message if needed
- rParams.mrContext.DoKeepAlive();
+ rContext.DoKeepAlive();
// Get the local filename
std::string dirname(MakeFullPath(rLocalPath, *d));
@@ -1044,21 +1179,27 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
// Check that the entry which might have been found is in fact a directory
if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == 0))
{
- // Entry exists, but is not a directory. Bad. Get rid of it.
- BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+ // Entry exists, but is not a directory. Bad.
+ // Get rid of it.
+ BackupProtocolClient &connection(rContext.GetConnection());
connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename);
+ rNotifier.NotifyFileDeleted(en->GetObjectID(),
+ storeFilename.GetClearFilename());
// Nothing found
en = 0;
}
- // Flag for having created directory, so can optimise the recusive call not to
- // read it again, because we know it's empty.
+ // Flag for having created directory, so can optimise the
+ // recursive call not to read it again, because we know
+ // it's empty.
bool haveJustCreatedDirOnServer = false;
// Next, see if it's in the list of sub directories
BackupClientDirectoryRecord *psubDirRecord = 0;
- std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(*d));
+ std::map<std::string, BackupClientDirectoryRecord *>::iterator
+ e(mSubDirectories.find(*d));
+
if(e != mSubDirectories.end())
{
// In the list, just use this pointer
@@ -1080,7 +1221,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
// No. Exists on the server, and we know about it from the listing.
subDirObjectID = en->GetObjectID();
}
- else if(rParams.mrContext.StorageLimitExceeded())
+ else if(rContext.StorageLimitExceeded())
// know we've got a connection if we get this far,
// as dir will have been modified.
{
@@ -1098,29 +1239,47 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
box_time_t attrModTime = 0;
InodeRefType inodeNum = 0;
BackupClientFileAttributes attr;
- attr.ReadAttributes(dirname.c_str(), true /* directories have zero mod times */,
- 0 /* not interested in mod time */, &attrModTime, 0 /* not file size */,
- &inodeNum);
+ bool failedToReadAttributes = false;
+
+ try
+ {
+ attr.ReadAttributes(dirname.c_str(),
+ true /* directories have zero mod times */,
+ 0 /* not interested in mod time */,
+ &attrModTime, 0 /* not file size */,
+ &inodeNum);
+ }
+ catch (BoxException &e)
+ {
+ BOX_WARNING("Failed to read attributes "
+ "of directory, cannot check "
+ "for rename, assuming new: '"
+ << dirname << "'");
+ failedToReadAttributes = true;
+ }
// Check to see if the directory been renamed
// First, do we have a record in the ID map?
int64_t renameObjectID = 0, renameInDirectory = 0;
bool renameDir = false;
- const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap());
- if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory))
+ const BackupClientInodeToIDMap &idMap(
+ rContext.GetCurrentIDMap());
+
+ if(!failedToReadAttributes && idMap.Lookup(inodeNum,
+ renameObjectID, renameInDirectory))
{
// Look up on the server to get the name, to build the local filename
std::string localPotentialOldName;
bool isDir = false;
bool isCurrentVersion = false;
- if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion))
+ if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion))
{
// Only interested if it's a directory
if(isDir && isCurrentVersion)
{
// Check that the object doesn't exist already
- struct stat st;
- if(::stat(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT)
+ EMU_STRUCT_STAT st;
+ if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT)
{
// Doesn't exist locally, but does exist on the server.
// Therefore we can safely rename it.
@@ -1131,7 +1290,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
}
// Get connection
- BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+ BackupProtocolClient &connection(rContext.GetConnection());
// Don't do a check for storage limit exceeded here, because if we get to this
// stage, a connection will have been opened, and the status known, so the check
@@ -1151,7 +1310,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream);
// Stop it being deleted later
- BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList());
+ BackupClientDeleteList &rdelList(
+ rContext.GetDeleteList());
rdelList.StopDirectoryDeletion(renameObjectID);
// This is the ID for the renamed directory
@@ -1188,12 +1348,14 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
}
}
- ASSERT(psubDirRecord != 0 || rParams.mrContext.StorageLimitExceeded());
+ ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded());
if(psubDirRecord)
{
// Sync this sub directory too
- psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, haveJustCreatedDirOnServer);
+ psubDirRecord->SyncDirectory(rParams, mObjectID,
+ dirname, rRemotePath + "/" + *d,
+ haveJustCreatedDirOnServer);
}
// Zero pointer in rEntriesLeftOver, if we have a pointer to zero
@@ -1222,20 +1384,27 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
// to a list, which is actually deleted at the very end of the session.
// If there's an error during the process, it doesn't matter if things
// aren't actually deleted, as the whole state will be reset anyway.
- BackupClientDeleteList &rdel(rParams.mrContext.GetDeleteList());
+ BackupClientDeleteList &rdel(rContext.GetDeleteList());
+
+ BackupStoreFilenameClear clear(en->GetName());
+ std::string localName = MakeFullPath(rLocalPath,
+ clear.GetClearFilename());
// Delete this entry -- file or directory?
if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0)
{
// Set a pending deletion for the file
- rdel.AddFileDelete(mObjectID, en->GetName());
+ rdel.AddFileDelete(mObjectID, en->GetName(),
+ localName);
}
else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0)
{
// Set as a pending deletion for the directory
- rdel.AddDirectoryDelete(en->GetObjectID());
+ rdel.AddDirectoryDelete(en->GetObjectID(),
+ localName);
- // If there's a directory record for it in the sub directory map, delete it now
+ // If there's a directory record for it in
+ // the sub directory map, delete it now
BackupStoreFilenameClear dirname(en->GetName());
std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(dirname.GetClearFilename()));
if(e != mSubDirectories.end())
@@ -1249,8 +1418,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
rLocalPath,
dirname.GetClearFilename());
- TRACE1("Deleted directory record for "
- "%s\n", name.c_str());
+ BOX_TRACE("Deleted directory record "
+ "for " << name);
}
}
}
@@ -1270,14 +1439,24 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP
// Created: 9/7/04
//
// --------------------------------------------------------------------------
-void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, int64_t ObjectID, const std::string &rFilename)
+void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(
+ SyncParams &rParams,
+ BackupStoreDirectory* pDirOnStore,
+ BackupStoreDirectory::Entry* pEntry,
+ const std::string &rFilename)
{
// First, delete the directory
BackupProtocolClient &connection(rParams.mrContext.GetConnection());
- connection.QueryDeleteDirectory(ObjectID);
+ connection.QueryDeleteDirectory(pEntry->GetObjectID());
+
+ BackupStoreFilenameClear clear(pEntry->GetName());
+ rParams.mrContext.GetProgressNotifier().NotifyDirectoryDeleted(
+ pEntry->GetObjectID(), clear.GetClearFilename());
// Then, delete any directory record
- std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(rFilename));
+ std::map<std::string, BackupClientDirectoryRecord *>::iterator
+ e(mSubDirectories.find(rFilename));
+
if(e != mSubDirectories.end())
{
// A record exists for this, remove it
@@ -1294,16 +1473,30 @@ void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &rPara
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &, const std::string &, const BackupStoreFilename &, int64_t, box_time_t, box_time_t, bool)
-// Purpose: Private. Upload a file to the server -- may send a patch instead of the whole thing
+// Name: BackupClientDirectoryRecord::UploadFile(
+// BackupClientDirectoryRecord::SyncParams &,
+// const std::string &,
+// const BackupStoreFilename &,
+// int64_t, box_time_t, box_time_t, bool)
+// Purpose: Private. Upload a file to the server. May send
+// a patch instead of the whole thing
// Created: 20/1/04
//
// --------------------------------------------------------------------------
-int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rFilename, const BackupStoreFilename &rStoreFilename,
- int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer)
+int64_t BackupClientDirectoryRecord::UploadFile(
+ BackupClientDirectoryRecord::SyncParams &rParams,
+ const std::string &rFilename,
+ const BackupStoreFilename &rStoreFilename,
+ int64_t FileSize,
+ box_time_t ModificationTime,
+ box_time_t AttributesHash,
+ bool NoPreviousVersionOnServer)
{
+ BackupClientContext& rContext(rParams.mrContext);
+ ProgressNotifier& rNotifier(rContext.GetProgressNotifier());
+
// Get the connection
- BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+ BackupProtocolClient &connection(rContext.GetConnection());
// Info
int64_t objID = 0;
@@ -1312,8 +1505,10 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn
// Use a try block to catch store full errors
try
{
- // Might an old version be on the server, and is the file size over the diffing threshold?
- if(!NoPreviousVersionOnServer && FileSize >= rParams.mDiffingUploadSizeThreshold)
+ // Might an old version be on the server, and is the file
+ // size over the diffing threshold?
+ if(!NoPreviousVersionOnServer &&
+ FileSize >= rParams.mDiffingUploadSizeThreshold)
{
// YES -- try to do diff, if possible
// First, query the server to see if there's an old version available
@@ -1323,7 +1518,7 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn
if(diffFromID != 0)
{
// Found an old version
- rParams.GetProgressNotifier().NotifyFileUploadingPatch(this,
+ rNotifier.NotifyFileUploadingPatch(this,
rFilename);
// Get the index
@@ -1333,7 +1528,7 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn
// Diff the file
//
- rParams.mrContext.ManageDiffProcess();
+ rContext.ManageDiffProcess();
bool isCompletelyDifferent = false;
std::auto_ptr<IOStream> patchStream(
@@ -1342,11 +1537,11 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn
mObjectID, /* containing directory */
rStoreFilename, diffFromID, *blockIndexStream,
connection.GetTimeout(),
- &rParams.mrContext, // DiffTimer implementation
+ &rContext, // DiffTimer implementation
0 /* not interested in the modification time */,
&isCompletelyDifferent));
- rParams.mrContext.UnManageDiffProcess();
+ rContext.UnManageDiffProcess();
//
// Upload the patch to the store
@@ -1354,6 +1549,9 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn
std::auto_ptr<BackupProtocolClientSuccess> stored(connection.QueryStoreFile(mObjectID, ModificationTime,
AttributesHash, isCompletelyDifferent?(0):(diffFromID), rStoreFilename, *patchStream));
+ // Get object ID from the result
+ objID = stored->GetObjectID();
+
// Don't attempt to upload it again!
doNormalUpload = false;
}
@@ -1362,13 +1560,14 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn
if(doNormalUpload)
{
// below threshold or nothing to diff from, so upload whole
- rParams.GetProgressNotifier().NotifyFileUploading(this,
- rFilename);
+ rNotifier.NotifyFileUploading(this, rFilename);
// Prepare to upload, getting a stream which will encode the file as we go along
std::auto_ptr<IOStream> upload(
BackupStoreFile::EncodeFile(rFilename.c_str(),
- mObjectID, rStoreFilename));
+ mObjectID, rStoreFilename, NULL,
+ &rParams,
+ &(rParams.mrRunStatusProvider)));
// Send to store
std::auto_ptr<BackupProtocolClientSuccess> stored(
@@ -1384,9 +1583,10 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn
}
catch(BoxException &e)
{
- rParams.mrContext.UnManageDiffProcess();
+ rContext.UnManageDiffProcess();
- if(e.GetType() == ConnectionException::ExceptionType && e.GetSubType() == ConnectionException::Protocol_UnexpectedReply)
+ if(e.GetType() == ConnectionException::ExceptionType &&
+ e.GetSubType() == ConnectionException::Protocol_UnexpectedReply)
{
// Check and see what error the protocol has,
// this is more useful to users than the exception.
@@ -1397,11 +1597,15 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn
&& subtype == BackupProtocolClientError::Err_StorageLimitExceeded)
{
// The hard limit was exceeded on the server, notify!
- rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull);
+ rParams.mrSysadminNotifier.NotifySysadmin(
+ SysadminNotifier::StoreFull);
+ // return an error code instead of
+ // throwing an exception that we
+ // can't debug.
+ return 0;
}
- rParams.GetProgressNotifier()
- .NotifyFileUploadServerError(
- this, rFilename, type, subtype);
+ rNotifier.NotifyFileUploadServerError(this,
+ rFilename, type, subtype);
}
}
@@ -1409,7 +1613,7 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn
throw;
}
- rParams.GetProgressNotifier().NotifyFileUploaded(this, rFilename, FileSize);
+ rNotifier.NotifyFileUploaded(this, rFilename, FileSize);
// Return the new object ID of this file
return objID;
@@ -1450,16 +1654,20 @@ void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(BackupClie
// Created: 8/3/04
//
// --------------------------------------------------------------------------
-BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon,
- ProgressNotifier &rProgressNotifier, BackupClientContext &rContext)
- : mrProgressNotifier(rProgressNotifier),
- mSyncPeriodStart(0),
+BackupClientDirectoryRecord::SyncParams::SyncParams(
+ RunStatusProvider &rRunStatusProvider,
+ SysadminNotifier &rSysadminNotifier,
+ ProgressNotifier &rProgressNotifier,
+ BackupClientContext &rContext)
+ : mSyncPeriodStart(0),
mSyncPeriodEnd(0),
mMaxUploadWait(0),
mMaxFileTimeInFuture(99999999999999999LL),
mFileTrackingSizeThreshold(16*1024),
mDiffingUploadSizeThreshold(16*1024),
- mrDaemon(rDaemon),
+ mrRunStatusProvider(rRunStatusProvider),
+ mrSysadminNotifier(rSysadminNotifier),
+ mrProgressNotifier(rProgressNotifier),
mrContext(rContext),
mReadErrorsOnFilesystemObjects(false),
mUploadAfterThisTimeInTheFuture(99999999999999999LL),
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-18 05:24:05 +0000