// -------------------------------------------------------------------------- // // File // Name: BackupClientDirectoryRecord.cpp // Purpose: Implementation of record about directory for backup client // Created: 2003/10/08 // // -------------------------------------------------------------------------- #include "Box.h" #include #include #include #include "BackupClientDirectoryRecord.h" #include "autogen_BackupProtocolClient.h" #include "BackupClientContext.h" #include "IOStream.h" #include "MemBlockStream.h" #include "CommonException.h" #include "CollectInBufferStream.h" #include "BackupStoreFile.h" #include "BackupClientInodeToIDMap.h" #include "FileModificationTime.h" #include "BackupDaemon.h" #include "BackupStoreException.h" #ifdef PLATFORM_LINUX #include "LinuxWorkaround.h" #endif #include "MemLeakFindOn.h" typedef std::map DecryptedEntriesMap_t; // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::BackupClientDirectoryRecord() // Purpose: Constructor // Created: 2003/10/08 // // -------------------------------------------------------------------------- BackupClientDirectoryRecord::BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName) : mObjectID(ObjectID), mSubDirName(rSubDirName), mInitialSyncDone(false), mSyncDone(false), mpPendingEntries(0) { ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::~BackupClientDirectoryRecord() // Purpose: Destructor // Created: 2003/10/08 // // -------------------------------------------------------------------------- BackupClientDirectoryRecord::~BackupClientDirectoryRecord() { // Make deletion recursive DeleteSubDirectories(); // Delete maps if(mpPendingEntries != 0) { delete mpPendingEntries; mpPendingEntries = 0; } } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::DeleteSubDirectories(); // Purpose: Delete all sub directory entries // Created: 2003/10/09 // // -------------------------------------------------------------------------- void BackupClientDirectoryRecord::DeleteSubDirectories() { // Delete all pointers for(std::map::iterator i = mSubDirectories.begin(); i != mSubDirectories.end(); ++i) { delete i->second; } // Empty list mSubDirectories.clear(); } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &, int64_t, const std::string &, bool) // Purpose: Syncronise, recusively, a local directory with the server. // Created: 2003/10/08 // // -------------------------------------------------------------------------- void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &rParams, int64_t ContainingDirectoryID, const std::string &rLocalPath, bool ThisDirHasJustBeenCreated) { // Signal received by daemon? if(rParams.mrDaemon.StopRun()) { // Yes. Stop now. THROW_EXCEPTION(BackupStoreException, SignalReceived) } // Start by making some flag changes, marking this sync as not done, // and on the immediate sub directories. mSyncDone = false; for(std::map::iterator i = mSubDirectories.begin(); i != mSubDirectories.end(); ++i) { i->second->mSyncDone = false; } // Work out the time in the future after which the file should be uploaded regardless. // This is a simple way to avoid having too many problems with file servers when they have // clients with badly out of sync clocks. rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + rParams.mMaxFileTimeInFuture; // Build the current state checksum to compare against while getting info from dirs // Note checksum is used locally only, so byte order isn't considered. MD5Digest currentStateChecksum; // Stat the directory, to get attribute info { struct stat st; if(::stat(rLocalPath.c_str(), &st) != 0) { // The directory has probably been deleted, so just ignore this error. // In a future scan, this deletion will be noticed, deleted from server, and this object deleted. TRACE1("Stat failed for '%s' (directory)\n", rLocalPath.c_str()); return; } // Store inode number in map so directories are tracked in case they're renamed { BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap()); idMap.AddToMap(st.st_ino, mObjectID, ContainingDirectoryID); } // Add attributes to checksum currentStateChecksum.Add(&st.st_mode, sizeof(st.st_mode)); currentStateChecksum.Add(&st.st_uid, sizeof(st.st_uid)); currentStateChecksum.Add(&st.st_gid, sizeof(st.st_gid)); // Inode to be paranoid about things moving around currentStateChecksum.Add(&st.st_ino, sizeof(st.st_ino)); #ifndef PLATFORM_stat_NO_st_flags currentStateChecksum.Add(&st.st_flags, sizeof(st.st_flags)); #endif // n PLATFORM_stat_NO_st_flags StreamableMemBlock xattr; BackupClientFileAttributes::FillExtendedAttr(xattr, rLocalPath.c_str()); currentStateChecksum.Add(xattr.GetBuffer(), xattr.GetSize()); } // Read directory entries, building arrays of names // First, need to read the contents of the directory. std::vector dirs; std::vector files; bool downloadDirectoryRecordBecauseOfFutureFiles = false; // BLOCK { // read the contents... DIR *dirHandle = 0; try { dirHandle = ::opendir(rLocalPath.c_str()); if(dirHandle == 0) { // Report the error (logs and eventual email to administrator) SetErrorWhenReadingFilesystemObject(rParams, rLocalPath.c_str()); // Ignore this directory for now. return; } // Basic structure for checksum info struct { box_time_t mModificationTime; box_time_t mAttributeModificationTime; int64_t mSize; // And then the name follows } checksum_info; // Be paranoid about structure packing ::memset(&checksum_info, 0, sizeof(checksum_info)); struct dirent *en = 0; struct stat st; std::string filename; while((en = ::readdir(dirHandle)) != 0) { // Don't need to use LinuxWorkaround_FinishDirentStruct(en, rLocalPath.c_str()); // on Linux, as a stat is performed to get all this info if(en->d_name[0] == '.' && (en->d_name[1] == '\0' || (en->d_name[1] == '.' && en->d_name[2] == '\0'))) { // ignore, it's . or .. continue; } // Stat file to get info filename = rLocalPath + DIRECTORY_SEPARATOR + en->d_name; if(::lstat(filename.c_str(), &st) != 0) { TRACE1("Stat failed for '%s' (contents)\n", filename.c_str()); THROW_EXCEPTION(CommonException, OSFileError) } int type = st.st_mode & S_IFMT; if(type == S_IFREG || type == S_IFLNK) { // File or symbolic link // Exclude it? if(rParams.mrContext.ExcludeFile(filename)) { // Next item! continue; } // Store on list files.push_back(std::string(en->d_name)); } else if(type == S_IFDIR) { // Directory // Exclude it? if(rParams.mrContext.ExcludeDir(filename)) { // Next item! continue; } // Store on list dirs.push_back(std::string(en->d_name)); } else { continue; } // Here if the object is something to back up (file, symlink or dir, not excluded) // So make the information for adding to the checksum checksum_info.mModificationTime = FileModificationTime(st); checksum_info.mAttributeModificationTime = FileAttrModificationTime(st); checksum_info.mSize = st.st_size; currentStateChecksum.Add(&checksum_info, sizeof(checksum_info)); currentStateChecksum.Add(en->d_name, strlen(en->d_name)); // If the file has been modified madly into the future, download the // directory record anyway to ensure that it doesn't get uploaded // every single time the disc is scanned. if(checksum_info.mModificationTime > rParams.mUploadAfterThisTimeInTheFuture) { downloadDirectoryRecordBecauseOfFutureFiles = true; // Log that this has happened if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) { ::syslog(LOG_ERR, "Some files have modification times excessively in the future. Check clock syncronisation.\n"); ::syslog(LOG_ERR, "Example file (only one shown) : %s\n", filename.c_str()); rParams.mHaveLoggedWarningAboutFutureFileTimes = true; } } } if(::closedir(dirHandle) != 0) { THROW_EXCEPTION(CommonException, OSFileError) } dirHandle = 0; } catch(...) { if(dirHandle != 0) { ::closedir(dirHandle); } throw; } } // Finish off the checksum, and compare with the one currently stored bool checksumDifferent = true; currentStateChecksum.Finish(); if(mInitialSyncDone && currentStateChecksum.DigestMatches(mStateChecksum)) { // The checksum is the same, and there was one to compare with checksumDifferent = false; } // Pointer to potentially downloaded store directory info BackupStoreDirectory *pdirOnStore = 0; try { // Want to get the directory listing? if(ThisDirHasJustBeenCreated) { // Avoid sending another command to the server when we know it's empty pdirOnStore = new BackupStoreDirectory(mObjectID, ContainingDirectoryID); } else { // Consider asking the store for it if(!mInitialSyncDone || checksumDifferent || downloadDirectoryRecordBecauseOfFutureFiles) { pdirOnStore = FetchDirectoryListing(rParams); } } // Make sure the attributes are up to date -- if there's space on the server // and this directory has not just been created (because it's attributes will be correct in this case) // and the checksum is different, implying they *MIGHT* be different. if((!ThisDirHasJustBeenCreated) && checksumDifferent && (!rParams.mrContext.StorageLimitExceeded())) { UpdateAttributes(rParams, pdirOnStore, rLocalPath); } // Create the list of pointers to directory entries std::vector entriesLeftOver; if(pdirOnStore) { entriesLeftOver.resize(pdirOnStore->GetNumberOfEntries(), 0); BackupStoreDirectory::Iterator i(*pdirOnStore); // Copy in pointers to all the entries for(unsigned int l = 0; l < pdirOnStore->GetNumberOfEntries(); ++l) { entriesLeftOver[l] = i.Next(); } } // Do the directory reading bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, pdirOnStore, entriesLeftOver, files, dirs); // LAST THING! (think exception safety) // Store the new checksum -- don't fetch things unnecessarily in the future // But... only if 1) the storage limit isn't exceeded -- make sure things are done again if // the directory is modified later // and 2) All the objects within the directory were stored successfully. if(!rParams.mrContext.StorageLimitExceeded() && updateCompleteSuccess) { currentStateChecksum.CopyDigestTo(mStateChecksum); } } catch(...) { // Bad things have happened -- clean up if(pdirOnStore != 0) { delete pdirOnStore; pdirOnStore = 0; } // Set things so that we get a full go at stuff later ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); throw; } // Clean up directory on store if(pdirOnStore != 0) { delete pdirOnStore; pdirOnStore = 0; } // Flag things as having happened. mInitialSyncDone = true; mSyncDone = true; } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::FetchDirectoryListing(BackupClientDirectoryRecord::SyncParams &) // Purpose: Fetch the directory listing of this directory from the store. // Created: 2003/10/09 // // -------------------------------------------------------------------------- BackupStoreDirectory *BackupClientDirectoryRecord::FetchDirectoryListing(BackupClientDirectoryRecord::SyncParams &rParams) { BackupStoreDirectory *pdir = 0; try { // Get connection to store BackupProtocolClient &connection(rParams.mrContext.GetConnection()); // Query the directory std::auto_ptr dirreply(connection.QueryListDirectory( mObjectID, BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, // both files and directories BackupProtocolClientListDirectory::Flags_Deleted | BackupProtocolClientListDirectory::Flags_OldVersion, // exclude old/deleted stuff true /* want attributes */)); // Retrieve the directory from the stream following pdir = new BackupStoreDirectory; ASSERT(pdir != 0); std::auto_ptr dirstream(connection.ReceiveStream()); pdir->ReadFromStream(*dirstream, connection.GetTimeout()); } catch(...) { delete pdir; pdir = 0; throw; } return pdir; } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord::SyncParams &, const std::string &) // Purpose: Sets the attributes of the directory on the store, if necessary // Created: 2003/10/09 // // -------------------------------------------------------------------------- void BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord::SyncParams &rParams, BackupStoreDirectory *pDirOnStore, const std::string &rLocalPath) { // Get attributes for the directory BackupClientFileAttributes attr; box_time_t attrModTime = 0; attr.ReadAttributes(rLocalPath.c_str(), true /* directories have zero mod times */, 0 /* no modification time */, &attrModTime); // Assume attributes need updating, unless proved otherwise bool updateAttr = true; // Got a listing to compare with? ASSERT(pDirOnStore == 0 || (pDirOnStore != 0 && pDirOnStore->HasAttributes())); if(pDirOnStore != 0 && pDirOnStore->HasAttributes()) { const StreamableMemBlock &storeAttrEnc(pDirOnStore->GetAttributes()); // Explict decryption BackupClientFileAttributes storeAttr(storeAttrEnc); // Compare the attributes if(attr.Compare(storeAttr, true, true /* ignore both modification times */)) { // No update necessary updateAttr = false; } } // Update them? if(updateAttr) { // Get connection to store BackupProtocolClient &connection(rParams.mrContext.GetConnection()); // Exception thrown if this doesn't work MemBlockStream attrStream(attr); connection.QueryChangeDirAttributes(mObjectID, attrModTime, attrStream); } } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &, const std::string &, BackupStoreDirectory *, std::vector &) // Purpose: Update the items stored on the server. The rFiles vector will be erased after it's used to save space. // Returns true if all items were updated successfully. (If not, the failures will have been logged). // Created: 2003/10/09 // // -------------------------------------------------------------------------- bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rLocalPath, BackupStoreDirectory *pDirOnStore, std::vector &rEntriesLeftOver, std::vector &rFiles, const std::vector &rDirs) { bool allUpdatedSuccessfully = true; // Decrypt all the directory entries. // It would be nice to be able to just compare the encrypted versions, however this doesn't work // in practise because there can be multiple encodings of the same filename using different // methods (although each method will result in the same string for the same filename.) This // happens when the server fixes a broken store, and gives plain text generated filenames. // So if we didn't do things like this, then you wouldn't be able to recover from bad things // happening with the server. DecryptedEntriesMap_t decryptedEntries; if(pDirOnStore != 0) { BackupStoreDirectory::Iterator i(*pDirOnStore); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { decryptedEntries[BackupStoreFilenameClear(en->GetName()).GetClearFilename()] = en; } } // Do files for(std::vector::const_iterator f = rFiles.begin(); f != rFiles.end(); ++f) { // Filename of this file std::string filename(rLocalPath + DIRECTORY_SEPARATOR + *f); // Get relevant info about file box_time_t modTime = 0; uint64_t attributesHash = 0; int64_t fileSize = 0; ino_t inodeNum = 0; bool hasMultipleHardLinks = true; // BLOCK { // Stat the file struct stat st; if(::lstat(filename.c_str(), &st) != 0) { THROW_EXCEPTION(CommonException, OSFileError) } // Extract required data modTime = FileModificationTime(st); fileSize = st.st_size; inodeNum = st.st_ino; hasMultipleHardLinks = (st.st_nlink > 1); attributesHash = BackupClientFileAttributes::GenerateAttributeHash(st, filename, *f); } // See if it's in the listing (if we have one) BackupStoreFilenameClear storeFilename(*f); BackupStoreDirectory::Entry *en = 0; int64_t latestObjectID = 0; if(pDirOnStore != 0) { DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*f)); if(i != decryptedEntries.end()) { en = i->second; latestObjectID = en->GetObjectID(); } } // Check that the entry which might have been found is in fact a file if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == 0)) { // Directory exists in the place of this file -- sort it out RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, en->GetObjectID(), *f); en = 0; } // Check for renaming? if(pDirOnStore != 0 && en == 0) { // We now know... // 1) File has just been added // 2) It's not in the store // Do we know about the inode number? const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap()); int64_t renameObjectID = 0, renameInDirectory = 0; if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) { // Look up on the server to get the name, to build the local filename std::string localPotentialOldName; bool isDir = false; bool isCurrentVersion = false; box_time_t srvModTime = 0, srvAttributesHash = 0; BackupStoreFilenameClear oldLeafname; if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname)) { // Only interested if it's a file and the latest version if(!isDir && isCurrentVersion) { // Check that the object we found in the ID map doesn't exist on disc struct stat st; if(::stat(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) { // Doesn't exist locally, but does exist on the server. // Therefore we can safely rename it to this new file. // Get the connection to the server BackupProtocolClient &connection(rParams.mrContext.GetConnection()); // Only do this step if there is room on the server. // This step will be repeated later when there is space available if(!rParams.mrContext.StorageLimitExceeded()) { // Rename the existing files (ie include old versions) on the server connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */, BackupProtocolClientMoveObject::Flags_MoveAllWithSameName | BackupProtocolClientMoveObject::Flags_AllowMoveOverDeletedObject, storeFilename); // Stop the attempt to delete the file in the original location BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList()); rdelList.StopFileDeletion(renameInDirectory, oldLeafname); // Create new entry in the directory for it // -- will be near enough what's actually on the server for the rest to work. en = pDirOnStore->AddEntry(storeFilename, srvModTime, renameObjectID, 0 /* size in blocks unknown, but not needed */, BackupStoreDirectory::Entry::Flags_File, srvAttributesHash); // Store the object ID for the inode lookup map later latestObjectID = renameObjectID; } } } } } } // Is it in the mPendingEntries list? box_time_t pendingFirstSeenTime = 0; // ie not seen if(mpPendingEntries != 0) { std::map::const_iterator i(mpPendingEntries->find(*f)); if(i != mpPendingEntries->end()) { // found it -- set flag pendingFirstSeenTime = i->second; } } // If pDirOnStore == 0, then this must have been after an initial sync: ASSERT(pDirOnStore != 0 || mInitialSyncDone); // So, if pDirOnStore == 0, then we know that everything before syncPeriodStart // is either on the server, or in the toupload list. If the directory had changed, // we'd have got a directory listing. // // At this point, if (pDirOnStore == 0 && en == 0), we can assume it's on the server with a // mod time < syncPeriodStart, or didn't exist before that time. // // But if en != 0, then we need to compare modification times to avoid uploading it again. // Need to update? // // Condition for upload: // modifiction time within sync period // if it's been seen before but not uploaded, is the time from this first sight longer than the MaxUploadWait // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store if( ( // Check the file modified within the acceptable time period we're checking // If the file isn't on the server, the acceptable time starts at zero. // Check pDirOnStore and en, because if we didn't download a directory listing, // pDirOnStore will be zero, but we know it's on the server. ( ((pDirOnStore != 0 && en == 0) || (modTime >= rParams.mSyncPeriodStart)) && modTime < rParams.mSyncPeriodEnd) // However, just in case things are continually modified, we check the first seen time. // The two compares of syncPeriodEnd and pendingFirstSeenTime are because the values are unsigned. || (pendingFirstSeenTime != 0 && (rParams.mSyncPeriodEnd > pendingFirstSeenTime) && ((rParams.mSyncPeriodEnd - pendingFirstSeenTime) > rParams.mMaxUploadWait)) // Then make sure that if files are added with a time less than the sync period start // (which can easily happen on file server), it gets uploaded. The directory contents checksum // will pick up the fact it has been added, so the store listing will be available when this happens. || ((modTime <= rParams.mSyncPeriodStart) && (en != 0) && (en->GetModificationTime() != modTime)) // And just to catch really badly off clocks in the future for file server clients, // just upload the file if it's madly in the future. || (modTime > rParams.mUploadAfterThisTimeInTheFuture) ) // But even then, only upload it if the mod time locally is different to that on the server. && (en == 0 || en->GetModificationTime() != modTime)) { // Make sure we're connected -- must connect here so we know whether // the storage limit has been exceeded, and hence whether or not // to actually upload the file. rParams.mrContext.GetConnection(); // Only do this step if there is room on the server. // This step will be repeated later when there is space available if(!rParams.mrContext.StorageLimitExceeded()) { // Upload the file to the server, recording the object ID it returns bool noPreviousVersionOnServer = ((pDirOnStore != 0) && (en == 0)); // Surround this in a try/catch block, to catch errrors, but still continue bool uploadSuccess = false; try { latestObjectID = UploadFile(rParams, filename, storeFilename, fileSize, modTime, attributesHash, noPreviousVersionOnServer); uploadSuccess = true; } catch(ConnectionException &e) { // Connection errors should just be passed on to the main handler, retries // would probably just cause more problems. throw; } catch(BoxException &e) { // an error occured -- make return code false, to show error in directory allUpdatedSuccessfully = false; // Log it. SetErrorWhenReadingFilesystemObject(rParams, filename.c_str()); // Log error. ::syslog(LOG_ERR, "Error code when uploading was (%d/%d), %s", e.GetType(), e.GetSubType(), e.what()); } // Update structures if the file was uploaded successfully. if(uploadSuccess) { // delete from pending entries if(pendingFirstSeenTime != 0 && mpPendingEntries != 0) { mpPendingEntries->erase(*f); } } } } else if(en != 0 && en->GetAttributesHash() != attributesHash) { // Attributes have probably changed, upload them again. // If the attributes have changed enough, the directory hash will have changed too, // and so the dir will have been downloaded, and the entry will be available. // Get connection BackupProtocolClient &connection(rParams.mrContext.GetConnection()); // Only do this step if there is room on the server. // This step will be repeated later when there is space available if(!rParams.mrContext.StorageLimitExceeded()) { // Update store BackupClientFileAttributes attr; attr.ReadAttributes(filename.c_str(), false /* put mod times in the attributes, please */); MemBlockStream attrStream(attr); connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream); } } if(modTime >= rParams.mSyncPeriodEnd) { // Allocate? if(mpPendingEntries == 0) { mpPendingEntries = new std::map; } // Adding to mPendingEntries list if(pendingFirstSeenTime == 0) { // Haven't seen this before -- add to list! (*mpPendingEntries)[*f] = modTime; } } // Zero pointer in rEntriesLeftOver, if we have a pointer to zero if(en != 0) { for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) { if(rEntriesLeftOver[l] == en) { rEntriesLeftOver[l] = 0; break; } } } // Does this file need an entry in the ID map? if(fileSize >= rParams.mFileTrackingSizeThreshold) { // Get the map BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap()); // Need to get an ID from somewhere... if(latestObjectID != 0) { // Use this one idMap.AddToMap(inodeNum, latestObjectID, mObjectID /* containing directory */); } else { // Don't know it -- haven't sent anything to the store, and didn't get a listing. // Look it up in the current map, and if it's there, use that. const BackupClientInodeToIDMap ¤tIDMap(rParams.mrContext.GetCurrentIDMap()); int64_t objid = 0, dirid = 0; if(currentIDMap.Lookup(inodeNum, objid, dirid)) { // Found ASSERT(dirid == mObjectID); // NOTE: If the above assert fails, an inode number has been reused by the OS, // or there is a problem somewhere. If this happened on a short test run, look // into it. However, in a long running process this may happen occasionally and // not indiciate anything wrong. // Run the release version for real life use, where this check is not made. idMap.AddToMap(inodeNum, objid, mObjectID /* containing directory */); } } } } // Erase contents of files to save space when recursing rFiles.clear(); // Delete the pending entries, if the map is entry if(mpPendingEntries != 0 && mpPendingEntries->size() == 0) { TRACE1("Deleting mpPendingEntries from dir ID %lld\n", mObjectID); delete mpPendingEntries; mpPendingEntries = 0; } // Do directories for(std::vector::const_iterator d = rDirs.begin(); d != rDirs.end(); ++d) { // Get the local filename std::string dirname(rLocalPath + DIRECTORY_SEPARATOR + *d); // See if it's in the listing (if we have one) BackupStoreFilenameClear storeFilename(*d); BackupStoreDirectory::Entry *en = 0; if(pDirOnStore != 0) { DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*d)); if(i != decryptedEntries.end()) { en = i->second; } } // Check that the entry which might have been found is in fact a directory if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == 0)) { // Entry exists, but is not a directory. Bad. Get rid of it. BackupProtocolClient &connection(rParams.mrContext.GetConnection()); connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); // Nothing found en = 0; } // Flag for having created directory, so can optimise the recusive call not to // read it again, because we know it's empty. bool haveJustCreatedDirOnServer = false; // Next, see if it's in the list of sub directories BackupClientDirectoryRecord *psubDirRecord = 0; std::map::iterator e(mSubDirectories.find(*d)); if(e != mSubDirectories.end()) { // In the list, just use this pointer psubDirRecord = e->second; } else if(!rParams.mrContext.StorageLimitExceeded()) // know we've got a connection if we get this far, as dir will have been modified. { // Note: only think about adding directory records if there's space left on the server. // If there isn't, this step will be repeated when there is some available. // Need to create the record. But do we need to create the directory on the server? int64_t subDirObjectID = 0; if(en != 0) { // No. Exists on the server, and we know about it from the listing. subDirObjectID = en->GetObjectID(); } else { // Yes, creation required! // It is known that the it doesn't exist: // if pDirOnStore == 0, then the directory has had an initial sync, and hasn't been modified. // so it has definately been created already. // if en == 0 but pDirOnStore != 0, well... obviously it doesn't exist. // Get attributes box_time_t attrModTime = 0; ino_t inodeNum = 0; BackupClientFileAttributes attr; attr.ReadAttributes(dirname.c_str(), true /* directories have zero mod times */, 0 /* not interested in mod time */, &attrModTime, 0 /* not file size */, &inodeNum); // Check to see if the directory been renamed // First, do we have a record in the ID map? int64_t renameObjectID = 0, renameInDirectory = 0; bool renameDir = false; const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap()); if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) { // Look up on the server to get the name, to build the local filename std::string localPotentialOldName; bool isDir = false; bool isCurrentVersion = false; if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion)) { // Only interested if it's a directory if(isDir && isCurrentVersion) { // Check that the object doesn't exist already struct stat st; if(::stat(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) { // Doesn't exist locally, but does exist on the server. // Therefore we can safely rename it. renameDir = true; } } } } // Get connection BackupProtocolClient &connection(rParams.mrContext.GetConnection()); // Don't do a check for storage limit exceeded here, because if we get to this // stage, a connection will have been opened, and the status known, so the check // in the else if(...) above will be correct. // Build attribute stream for sending MemBlockStream attrStream(attr); if(renameDir) { // Rename the existing directory on the server connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */, BackupProtocolClientMoveObject::Flags_MoveAllWithSameName | BackupProtocolClientMoveObject::Flags_AllowMoveOverDeletedObject, storeFilename); // Put the latest attributes on it connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream); // Stop it being deleted later BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList()); rdelList.StopDirectoryDeletion(renameObjectID); // This is the ID for the renamed directory subDirObjectID = renameObjectID; } else { // Create a new directory std::auto_ptr dirCreate(connection.QueryCreateDirectory( mObjectID, attrModTime, storeFilename, attrStream)); subDirObjectID = dirCreate->GetObjectID(); // Flag as having done this for optimisation later haveJustCreatedDirOnServer = true; } } // New an object for this psubDirRecord = new BackupClientDirectoryRecord(subDirObjectID, *d); // Store in list try { mSubDirectories[*d] = psubDirRecord; } catch(...) { delete psubDirRecord; psubDirRecord = 0; throw; } } ASSERT(psubDirRecord != 0 || rParams.mrContext.StorageLimitExceeded()); if(psubDirRecord) { // Sync this sub directory too psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, haveJustCreatedDirOnServer); } // Zero pointer in rEntriesLeftOver, if we have a pointer to zero if(en != 0) { for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) { if(rEntriesLeftOver[l] == en) { rEntriesLeftOver[l] = 0; break; } } } } // Delete everything which is on the store, but not on disc for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) { if(rEntriesLeftOver[l] != 0) { BackupStoreDirectory::Entry *en = rEntriesLeftOver[l]; // These entries can't be deleted immediately, as it would prevent // renaming and moving of objects working properly. So we add them // to a list, which is actually deleted at the very end of the session. // If there's an error during the process, it doesn't matter if things // aren't actually deleted, as the whole state will be reset anyway. BackupClientDeleteList &rdel(rParams.mrContext.GetDeleteList()); // Delete this entry -- file or directory? if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) { // Set a pending deletion for the file rdel.AddFileDelete(mObjectID, en->GetName()); } else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) { // Set as a pending deletion for the directory rdel.AddDirectoryDelete(en->GetObjectID()); // If there's a directory record for it in the sub directory map, delete it now BackupStoreFilenameClear dirname(en->GetName()); std::map::iterator e(mSubDirectories.find(dirname.GetClearFilename())); if(e != mSubDirectories.end()) { // Carefully delete the entry from the map BackupClientDirectoryRecord *rec = e->second; mSubDirectories.erase(e); delete rec; TRACE2("Deleted directory record for %s/%s\n", rLocalPath.c_str(), dirname.GetClearFilename().c_str()); } } } } // Return success flag (will be false if some files failed) return allUpdatedSuccessfully; } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &, BackupStoreDirectory *, int64_t, const std::string &) // Purpose: Called to resolve difficulties when a directory is found on the // store where a file is to be uploaded. // Created: 9/7/04 // // -------------------------------------------------------------------------- void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, int64_t ObjectID, const std::string &rFilename) { // First, delete the directory BackupProtocolClient &connection(rParams.mrContext.GetConnection()); connection.QueryDeleteDirectory(ObjectID); // Then, delete any directory record std::map::iterator e(mSubDirectories.find(rFilename)); if(e != mSubDirectories.end()) { // A record exists for this, remove it BackupClientDirectoryRecord *psubDirRecord = e->second; mSubDirectories.erase(e); // And delete the object delete psubDirRecord; } } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &, const std::string &, const BackupStoreFilename &, int64_t, box_time_t, box_time_t, bool) // Purpose: Private. Upload a file to the server -- may send a patch instead of the whole thing // Created: 20/1/04 // // -------------------------------------------------------------------------- int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rFilename, const BackupStoreFilename &rStoreFilename, int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer) { // Get the connection BackupProtocolClient &connection(rParams.mrContext.GetConnection()); // Info int64_t objID = 0; bool doNormalUpload = true; // Use a try block to catch store full errors try { // Might an old version be on the server, and is the file size over the diffing threshold? if(!NoPreviousVersionOnServer && FileSize >= rParams.mDiffingUploadSizeThreshold) { // YES -- try to do diff, if possible // First, query the server to see if there's an old version available std::auto_ptr getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename)); int64_t diffFromID = getBlockIndex->GetObjectID(); if(diffFromID != 0) { // Found an old version -- get the index std::auto_ptr blockIndexStream(connection.ReceiveStream()); // Diff the file bool isCompletelyDifferent = false; std::auto_ptr patchStream(BackupStoreFile::EncodeFileDiff(rFilename.c_str(), mObjectID, /* containing directory */ rStoreFilename, diffFromID, *blockIndexStream, connection.GetTimeout(), 0 /* not interested in the modification time */, &isCompletelyDifferent)); // Upload the patch to the store std::auto_ptr stored(connection.QueryStoreFile(mObjectID, ModificationTime, AttributesHash, isCompletelyDifferent?(0):(diffFromID), rStoreFilename, *patchStream)); // Don't attempt to upload it again! doNormalUpload = false; } } if(doNormalUpload) { // below threshold or nothing to diff from, so upload whole // Prepare to upload, getting a stream which will encode the file as we go along std::auto_ptr upload(BackupStoreFile::EncodeFile(rFilename.c_str(), mObjectID, rStoreFilename)); // Send to store std::auto_ptr stored(connection.QueryStoreFile(mObjectID, ModificationTime, AttributesHash, 0 /* no diff from file ID */, rStoreFilename, *upload)); // Get object ID from the result objID = stored->GetObjectID(); } } catch(BoxException &e) { if(e.GetType() == ConnectionException::ExceptionType && e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) { // Check and see what error the protocol has -- as it might be an error... int type, subtype; if(connection.GetLastError(type, subtype) && type == BackupProtocolClientError::ErrorType && subtype == BackupProtocolClientError::Err_StorageLimitExceeded) { // The hard limit was exceeded on the server, notify! rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull); } } // Send the error on it's way throw; } // Return the new object ID of this file return objID; } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(SyncParams &, const char *) // Purpose: Sets the error state when there were problems reading an object // from the filesystem. // Created: 29/3/04 // // -------------------------------------------------------------------------- void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(BackupClientDirectoryRecord::SyncParams &rParams, const char *Filename) { // Zero hash, so it gets synced properly next time round. ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); // Log the error ::syslog(LOG_ERR, "Backup object failed, error when reading %s", Filename); // Mark that an error occured in the parameters object rParams.mReadErrorsOnFilesystemObjects = true; } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::SyncParams::SyncParams(BackupClientContext &) // Purpose: Constructor // Created: 8/3/04 // // -------------------------------------------------------------------------- BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, BackupClientContext &rContext) : mSyncPeriodStart(0), mSyncPeriodEnd(0), mMaxUploadWait(0), mMaxFileTimeInFuture(99999999999999999LL), mFileTrackingSizeThreshold(16*1024), mDiffingUploadSizeThreshold(16*1024), mrDaemon(rDaemon), mrContext(rContext), mReadErrorsOnFilesystemObjects(false), mUploadAfterThisTimeInTheFuture(99999999999999999LL), mHaveLoggedWarningAboutFutureFileTimes(false) { } // -------------------------------------------------------------------------- // // Function // Name: BackupClientDirectoryRecord::SyncParams::~SyncParams() // Purpose: Destructor // Created: 8/3/04 // // -------------------------------------------------------------------------- BackupClientDirectoryRecord::SyncParams::~SyncParams() { }