diff options
Diffstat (limited to 'lib/backupstore/HousekeepStoreAccount.cpp')
| -rw-r--r-- | lib/backupstore/HousekeepStoreAccount.cpp | 620 |
1 files changed, 331 insertions, 289 deletions
diff --git a/lib/backupstore/HousekeepStoreAccount.cpp b/lib/backupstore/HousekeepStoreAccount.cpp index 75feda7f..d5acf62c 100644 --- a/lib/backupstore/HousekeepStoreAccount.cpp +++ b/lib/backupstore/HousekeepStoreAccount.cpp @@ -2,7 +2,7 @@ // // File // Name: HousekeepStoreAccount.cpp -// Purpose: +// Purpose: // Created: 11/12/03 // // -------------------------------------------------------------------------- @@ -51,6 +51,7 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, mDeletionSizeTarget(0), mPotentialDeletionsTotalSize(0), mMaxSizeInPotentialDeletions(0), + mErrorCount(0), mBlocksUsed(0), mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), @@ -61,8 +62,6 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, mBlocksInDirectoriesDelta(0), mFilesDeleted(0), mEmptyDirectoriesDeleted(0), - mSuppressRefCountChangeWarnings(false), - mRefCountsAdjusted(0), mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY) { std::ostringstream tag; @@ -80,6 +79,20 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, // -------------------------------------------------------------------------- HousekeepStoreAccount::~HousekeepStoreAccount() { + if(mapNewRefs.get()) + { + // Discard() can throw exception, but destructors aren't supposed to do that, so + // just catch and log them. + try + { + mapNewRefs->Discard(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to destroy housekeeper: discarding the refcount " + "database threw an exception: " << e.what()); + } + } } // -------------------------------------------------------------------------- @@ -105,7 +118,7 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) { if(KeepTryingForever) { - BOX_WARNING("Failed to lock account for housekeeping, " + BOX_INFO("Failed to lock account for housekeeping, " "still trying..."); while(!writeLock.TryAndGetLock(writeLockFilename, 0600 /* restrictive file permissions */)) @@ -143,204 +156,108 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) mDeletionSizeTarget = 0; } - // initialise the refcount database - mNewRefCounts.clear(); - // try to pre-allocate as much memory as we need - mNewRefCounts.reserve(info->GetLastObjectIDUsed()); - // initialise the refcount of the root entry - mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0); - mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1; + BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); + mapNewRefs = BackupStoreRefCountDatabase::Create(account); // Scan the directory for potential things to delete // This will also remove eligible items marked with RemoveASAP - bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID); + bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, + *info); - // If scan directory stopped for some reason, probably parent - // instructed to terminate, stop now. if(!continueHousekeeping) { - // If any files were marked "delete now", then update - // the size of the store. - if(mBlocksUsedDelta != 0 || - mBlocksInOldFilesDelta != 0 || - mBlocksInDeletedFilesDelta != 0) - { - info->ChangeBlocksUsed(mBlocksUsedDelta); - info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); - info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); - - // Save the store info back - info->ReportChangesTo(*pOldInfo); - info->Save(); - } - - return false; + // The scan was incomplete, so the new block counts are + // incorrect, we can't rely on them. It's better to discard + // the new info and adjust the old one instead. + info = pOldInfo; + + // We're about to reset counters and exit, so report what + // happened now. + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " removed " << + (0 - mBlocksUsedDelta) << " blocks (" << + mFilesDeleted << " files, " << + mEmptyDirectoriesDeleted << " dirs) and the directory " + "scan was interrupted"); } - // Log any difference in opinion between the values recorded in - // the store info, and the values just calculated for space usage. - // BLOCK - { - int64_t used = info->GetBlocksUsed(); - int64_t usedOld = info->GetBlocksInOldFiles(); - int64_t usedDeleted = info->GetBlocksInDeletedFiles(); - int64_t usedDirectories = info->GetBlocksInDirectories(); - - // If the counts were wrong, taking into account RemoveASAP - // items deleted, log a message - if((used + mBlocksUsedDelta) != mBlocksUsed - || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles - || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles - || usedDirectories != mBlocksInDirectories) - { - // Log this - BOX_ERROR("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " found " - "and fixed wrong block counts: " - "used (" << - (used + mBlocksUsedDelta) << "," << - mBlocksUsed << "), old (" << - (usedOld + mBlocksInOldFilesDelta) << "," << - mBlocksInOldFiles << "), deleted (" << - (usedDeleted + mBlocksInDeletedFilesDelta) << - "," << mBlocksInDeletedFiles << "), dirs (" << - usedDirectories << "," << mBlocksInDirectories - << ")"); - } - - // If the current values don't match, store them - if(used != mBlocksUsed - || usedOld != mBlocksInOldFiles - || usedDeleted != mBlocksInDeletedFiles - || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta)) - { - // Set corrected values in store info - info->CorrectAllUsedValues(mBlocksUsed, - mBlocksInOldFiles, mBlocksInDeletedFiles, - mBlocksInDirectories + mBlocksInDirectoriesDelta); - - info->ReportChangesTo(*pOldInfo); - info->Save(); - } - } - - // Reset the delta counts for files, as they will include - // RemoveASAP flagged files deleted during the initial scan. + // If housekeeping made any changes, such as deleting RemoveASAP files, + // the differences in block counts will be recorded in the deltas. + info->ChangeBlocksUsed(mBlocksUsedDelta); + info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); + info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); - // keep for reporting + // Reset the delta counts for files, as they will include + // RemoveASAP flagged files deleted during the initial scan. + // keep removeASAPBlocksUsedDelta for reporting int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta; - mBlocksUsedDelta = 0; mBlocksInOldFilesDelta = 0; mBlocksInDeletedFilesDelta = 0; - - // Go and delete items from the accounts - bool deleteInterrupted = DeleteFiles(); - - // If that wasn't interrupted, remove any empty directories which - // are also marked as deleted in their containing directory - if(!deleteInterrupted) - { - deleteInterrupted = DeleteEmptyDirectories(); - } - - // Log deletion if anything was deleted - if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) - { - BOX_INFO("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " " - "removed " << - (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << - " blocks (" << mFilesDeleted << " files, " << - mEmptyDirectoriesDeleted << " dirs)" << - (deleteInterrupted?" and was interrupted":"")); - } + // If scan directory stopped for some reason, probably parent + // instructed to terminate, stop now. + // // We can only update the refcount database if we successfully // finished our scan of all directories, otherwise we don't actually // know which of the new counts are valid and which aren't - // (we might not have seen second references to some objects, etc.) + // (we might not have seen second references to some objects, etc.). - BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); - std::auto_ptr<BackupStoreRefCountDatabase> apReferences; + if(!continueHousekeeping) + { + mapNewRefs->Discard(); + info->Save(); + return false; + } + + // Report any UNexpected changes, and consider them to be errors. + // Do this before applying the expected changes below. + mErrorCount += info->ReportChangesTo(*pOldInfo); + info->Save(); + + // Try to load the old reference count database and check whether + // any counts have changed. We want to compare the mapNewRefs to + // apOldRefs before we delete any files, because that will also change + // the reference count in a way that's not an error. - // try to load the reference count database try { - apReferences = BackupStoreRefCountDatabase::Load(account, - false); + std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs = + BackupStoreRefCountDatabase::Load(account, false); + mErrorCount += mapNewRefs->ReportChangesTo(*apOldRefs); } catch(BoxException &e) { - BOX_WARNING("Reference count database is missing or corrupted " - "during housekeeping, creating a new one."); - mSuppressRefCountChangeWarnings = true; - BackupStoreRefCountDatabase::CreateForRegeneration(account); - apReferences = BackupStoreRefCountDatabase::Load(account, - false); + BOX_WARNING("Reference count database was missing or " + "corrupted during housekeeping, cannot check it for " + "errors."); + mErrorCount++; } - int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed(); + // Go and delete items from the accounts + bool deleteInterrupted = DeleteFiles(*info); - for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID; - ObjectID < mNewRefCounts.size(); ObjectID++) + // If that wasn't interrupted, remove any empty directories which + // are also marked as deleted in their containing directory + if(!deleteInterrupted) { - if (ObjectID > LastUsedObjectIdOnDisk) - { - if (!mSuppressRefCountChangeWarnings) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " not found in database, added" - " with " << mNewRefCounts[ObjectID] << - " references"); - } - apReferences->SetRefCount(ObjectID, - mNewRefCounts[ObjectID]); - mRefCountsAdjusted++; - LastUsedObjectIdOnDisk = ObjectID; - continue; - } - - BackupStoreRefCountDatabase::refcount_t OldRefCount = - apReferences->GetRefCount(ObjectID); - - if (OldRefCount != mNewRefCounts[ObjectID]) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " changed from " << OldRefCount << - " to " << mNewRefCounts[ObjectID]); - apReferences->SetRefCount(ObjectID, - mNewRefCounts[ObjectID]); - mRefCountsAdjusted++; - } + deleteInterrupted = DeleteEmptyDirectories(*info); } - // zero excess references in the database - for (int64_t ObjectID = mNewRefCounts.size(); - ObjectID <= LastUsedObjectIdOnDisk; ObjectID++) + // Log deletion if anything was deleted + if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) { - BackupStoreRefCountDatabase::refcount_t OldRefCount = - apReferences->GetRefCount(ObjectID); - BackupStoreRefCountDatabase::refcount_t NewRefCount = 0; - - if (OldRefCount != NewRefCount) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " changed from " << OldRefCount << - " to " << NewRefCount << " (not found)"); - apReferences->SetRefCount(ObjectID, NewRefCount); - mRefCountsAdjusted++; - } + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " " + "removed " << + (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << + " blocks (" << mFilesDeleted << " files, " << + mEmptyDirectoriesDeleted << " dirs)" << + (deleteInterrupted?" and was interrupted":"")); } - // force file to be saved and closed before releasing the lock below - apReferences.reset(); - - // Make sure the delta's won't cause problems if the counts are - // really wrong, and it wasn't fixed because the store was + // Make sure the delta's won't cause problems if the counts are + // really wrong, and it wasn't fixed because the store was // updated during the scan. if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) { @@ -348,28 +265,31 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) } if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles())) { - mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); + mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); } if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles())) { - mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); + mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); } if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories())) { mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories()); } - + // Update the usage counts in the store info->ChangeBlocksUsed(mBlocksUsedDelta); info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta); - + // Save the store info back - info->ReportChangesTo(*pOldInfo); info->Save(); - - // Explicity release the lock (would happen automatically on + + // force file to be saved and closed before releasing the lock below + mapNewRefs->Commit(); + mapNewRefs.reset(); + + // Explicity release the lock (would happen automatically on // going out of scope, included for code clarity) writeLock.ReleaseLock(); @@ -405,7 +325,8 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF // Created: 11/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) +bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID, + BackupStoreInfo& rBackupStoreInfo) { #ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) @@ -430,18 +351,19 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Open it. std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, objectFilename)); - + // Add the size of the directory on disc to the size being calculated int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); mBlocksInDirectories += originalDirSizeInBlocks; mBlocksUsed += originalDirSizeInBlocks; - + // Read the directory in BackupStoreDirectory dir; BufferedStream buf(*dirStream); dir.ReadFromStream(buf, IOStream::TimeOutInfinite); + dir.SetUserInfo1_SizeInBlocks(originalDirSizeInBlocks); dirStream->Close(); - + // Is it empty? if(dir.GetNumberOfEntries() == 0) { @@ -459,11 +381,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) while((en = i.Next()) != 0) { // This directory references this object - if (mNewRefCounts.size() <= en->GetObjectID()) - { - mNewRefCounts.resize(en->GetObjectID() + 1, 0); - } - mNewRefCounts[en->GetObjectID()]++; + mapNewRefs->AddReference(en->GetObjectID()); } } @@ -482,14 +400,15 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { int16_t enFlags = en->GetFlags(); if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0 - && (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) + && (en->IsDeleted() || en->IsOld())) { // Delete this immediately. - DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks); - + DeleteFile(ObjectID, en->GetObjectID(), dir, + objectFilename, rBackupStoreInfo); + // flag as having done something deletedSomething = true; - + // Must start the loop from the beginning again, as iterator is now // probably invalid. break; @@ -497,7 +416,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } } while(deletedSomething); } - + // BLOCK { // Add files to the list of potential deletions @@ -517,9 +436,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) int16_t enFlags = en->GetFlags(); int64_t enSizeInBlocks = en->GetSizeInBlocks(); mBlocksUsed += enSizeInBlocks; - if(enFlags & BackupStoreDirectory::Entry::Flags_OldVersion) mBlocksInOldFiles += enSizeInBlocks; - if(enFlags & BackupStoreDirectory::Entry::Flags_Deleted) mBlocksInDeletedFiles += enSizeInBlocks; - + if(en->IsOld()) mBlocksInOldFiles += enSizeInBlocks; + if(en->IsDeleted()) mBlocksInDeletedFiles += enSizeInBlocks; + // Work out ages of this version from the last mark int32_t enVersionAge = 0; std::map<version_t, int32_t>::iterator enVersionAgeI( @@ -536,9 +455,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) markVersionAges[version_t(en->GetName().GetEncodedFilename(), en->GetMarkNumber())] = enVersionAge; } // enVersionAge is now the age of this version. - + // Potentially add it to the list if it's deleted, if it's an old version or deleted - if((enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) + if(en->IsOld() || en->IsDeleted()) { // Is deleted / old version. DelEn d; @@ -547,17 +466,15 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) d.mSizeInBlocks = en->GetSizeInBlocks(); d.mMarkNumber = en->GetMarkNumber(); d.mVersionAgeWithinMark = enVersionAge; - d.mIsFlagDeleted = (enFlags & - BackupStoreDirectory::Entry::Flags_Deleted) - ? true : false; - + d.mIsFlagDeleted = en->IsDeleted(); + // Add it to the list mPotentialDeletions.insert(d); - + // Update various counts mPotentialDeletionsTotalSize += d.mSizeInBlocks; if(d.mSizeInBlocks > mMaxSizeInPotentialDeletions) mMaxSizeInPotentialDeletions = d.mSizeInBlocks; - + // Too much in the list of potential deletions? // (check against the deletion target + the max size in deletions, so that we never delete things // and take the total size below the deletion size target) @@ -565,7 +482,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { int64_t sizeToRemove = mPotentialDeletionsTotalSize - (mDeletionSizeTarget + mMaxSizeInPotentialDeletions); bool recalcMaxSize = false; - + while(sizeToRemove > 0) { // Make iterator for the last element, while checking that there's something there in the first place. @@ -577,7 +494,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } // Make this into an iterator pointing to the last element in the set --i; - + // Delete this one? if(sizeToRemove > i->mSizeInBlocks) { @@ -595,7 +512,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) break; } } - + if(recalcMaxSize) { // Because an object which was the maximum size recorded was deleted from the set @@ -615,23 +532,22 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } } + // Recurse into subdirectories { - // Recurse into subdirectories BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0) { - // Next level - ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir); - - if(!ScanDirectory(en->GetObjectID())) + ASSERT(en->IsDir()); + + if(!ScanDirectory(en->GetObjectID(), rBackupStoreInfo)) { // Halting operation return false; } } } - + return true; } @@ -648,11 +564,11 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &x, const HousekeepStoreAccount::DelEn &y) { // STL spec says this: - // A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. + // A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. // The sort order here is intended to preserve the entries of most value, that is, the newest objects // which are on a mark boundary. - + // Reverse order age, so oldest goes first if(x.mVersionAgeWithinMark > y.mVersionAgeWithinMark) { @@ -687,7 +603,7 @@ bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount // Created: 15/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::DeleteFiles() +bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo) { // Only delete files if the deletion target is greater than zero // (otherwise we delete one file each time round, which gradually deletes the old versions) @@ -718,21 +634,33 @@ bool HousekeepStoreAccount::DeleteFiles() // Get the filename std::string dirFilename; BackupStoreDirectory dir; - int64_t dirSizeInBlocksOrig = 0; { MakeObjectFilename(i->mInDirectory, dirFilename); std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename)); - dirSizeInBlocksOrig = dirStream->GetDiscUsageInBlocks(); dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + dir.SetUserInfo1_SizeInBlocks(dirStream->GetDiscUsageInBlocks()); } - + // Delete the file - DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig); - BOX_INFO("Housekeeping removed " << - (i->mIsFlagDeleted ? "deleted" : "old") << - " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << - " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); - + BackupStoreRefCountDatabase::refcount_t refs = + DeleteFile(i->mInDirectory, i->mObjectID, dir, + dirFilename, rBackupStoreInfo); + if(refs == 0) + { + BOX_INFO("Housekeeping removed " << + (i->mIsFlagDeleted ? "deleted" : "old") << + " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << + " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); + } + else + { + BOX_TRACE("Housekeeping preserved " << + (i->mIsFlagDeleted ? "deleted" : "old") << + " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << + " in dir " << BOX_FORMAT_OBJECTID(i->mInDirectory) << + " with " << refs << " references"); + } + // Stop if the deletion target has been matched or exceeded // (checking here rather than at the beginning will tend to reduce the // space to slightly less than the soft limit, which will allow the backup @@ -754,11 +682,17 @@ bool HousekeepStoreAccount::DeleteFiles() // BackupStoreDirectory &, const std::string &, int64_t) // Purpose: Delete a file. Takes the directory already loaded // in and the filename, for efficiency in both the -// usage scenarios. +// usage scenarios. Returns the number of references +// remaining. If it's zero, the file was removed from +// disk as unused. // Created: 15/7/04 // // -------------------------------------------------------------------------- -void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks) + +BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( + int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, + const std::string &rDirectoryFilename, + BackupStoreInfo& rBackupStoreInfo) { // Find the entry inside the directory bool wasDeleted = false; @@ -768,6 +702,9 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba std::auto_ptr<RaidFileWrite> padjustedEntry; // BLOCK { + BackupStoreRefCountDatabase::refcount_t refs = + mapNewRefs->GetRefCount(ObjectID); + BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID); if(pentry == 0) { @@ -775,26 +712,47 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba BOX_FORMAT_ACCOUNT(mAccountID) << " " "found error: object " << BOX_FORMAT_OBJECTID(ObjectID) << " " - "not found in dir " << + "not found in dir " << BOX_FORMAT_OBJECTID(InDirectory) << ", " "indicates logic error/corruption? Run " "bbstoreaccounts check <accid> fix"); - return; + mErrorCount++; + return refs; } - + // Record the flags it's got set - wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0); - wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0); + wasDeleted = pentry->IsDeleted(); + wasOldVersion = pentry->IsOld(); // Check this should be deleted if(!wasDeleted && !wasOldVersion) { - // Things changed size we were last around - return; + // Things changed since we were last around + return refs; } - + // Record size deletedFileSizeInBlocks = pentry->GetSizeInBlocks(); - + + if(refs > 1) + { + // Not safe to merge patches if someone else has a + // reference to this object, so just remove the + // directory entry and return. + rDirectory.DeleteEntry(ObjectID); + if(wasDeleted) + { + rBackupStoreInfo.AdjustNumDeletedFiles(-1); + } + + if(wasOldVersion) + { + rBackupStoreInfo.AdjustNumOldFiles(-1); + } + + mapNewRefs->RemoveReference(ObjectID); + return refs - 1; + } + // If the entry is involved in a chain of patches, it needs to be handled // a bit more carefully. if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0) @@ -826,7 +784,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba else { // This entry is in the middle of a chain, and two patches need combining. - + // First, adjust the directory entries BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer()); if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID @@ -838,7 +796,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba pnewer->SetDependsOlder(pentry->GetDependsOlder()); polder->SetDependsNewer(pentry->GetDependsNewer()); } - + // COMMON CODE to both cases // Generate the filename of the older version @@ -853,7 +811,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename)); // And open a write file to overwrite the other directory entry padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, - objFilenameOlder, mNewRefCounts[ObjectID])); + objFilenameOlder, mapNewRefs->GetRefCount(ObjectID))); padjustedEntry->Open(true /* allow overwriting */); if(pentry->GetDependsNewer() == 0) @@ -867,84 +825,86 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry); } // The file will be committed later when the directory is safely commited. - + // Work out the adjusted size int64_t newSize = padjustedEntry->GetDiscUsageInBlocks(); int64_t sizeDelta = newSize - polder->GetSizeInBlocks(); mBlocksUsedDelta += sizeDelta; - if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0) + if(polder->IsDeleted()) { mBlocksInDeletedFilesDelta += sizeDelta; } - if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0) + if(polder->IsOld()) { mBlocksInOldFilesDelta += sizeDelta; } polder->SetSizeInBlocks(newSize); } - + // pentry no longer valid } - + // Delete it from the directory rDirectory.DeleteEntry(ObjectID); - + // Save directory back to disc // BLOCK - int64_t dirRevisedSize = 0; { RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename, - mNewRefCounts[InDirectory]); + mapNewRefs->GetRefCount(InDirectory)); writeDir.Open(true /* allow overwriting */); rDirectory.WriteToStream(writeDir); - // get the disc usage (must do this before commiting it) - dirRevisedSize = writeDir.GetDiscUsageInBlocks(); + // Get the disc usage (must do this before commiting it) + int64_t new_size = writeDir.GetDiscUsageInBlocks(); // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - // adjust usage counts for this directory - if(dirRevisedSize > 0) - { - int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks; - mBlocksUsedDelta += adjust; - mBlocksInDirectoriesDelta += adjust; - } + // Adjust block counts if the directory itself changed in size + int64_t original_size = rDirectory.GetUserInfo1_SizeInBlocks(); + int64_t adjust = new_size - original_size; + mBlocksUsedDelta += adjust; + mBlocksInDirectoriesDelta += adjust; + + UpdateDirectorySize(rDirectory, new_size); } // Commit any new adjusted entry if(padjustedEntry.get() != 0) { padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - padjustedEntry.reset(); // delete it now + padjustedEntry.reset(); // delete it now } - // Drop reference count by one. If it reaches zero, delete the file. - if(--mNewRefCounts[ObjectID] == 0) + // Drop reference count by one. Must now be zero, to delete the file. + bool remaining_refs = mapNewRefs->RemoveReference(ObjectID); + ASSERT(!remaining_refs); + + // Delete from disc + BOX_TRACE("Removing unreferenced object " << + BOX_FORMAT_OBJECTID(ObjectID)); + std::string objFilename; + MakeObjectFilename(ObjectID, objFilename); + RaidFileWrite del(mStoreDiscSet, objFilename, mapNewRefs->GetRefCount(ObjectID)); + del.Delete(); + + // Adjust counts for the file + ++mFilesDeleted; + mBlocksUsedDelta -= deletedFileSizeInBlocks; + + if(wasDeleted) { - // Delete from disc - BOX_TRACE("Removing unreferenced object " << - BOX_FORMAT_OBJECTID(ObjectID)); - std::string objFilename; - MakeObjectFilename(ObjectID, objFilename); - RaidFileWrite del(mStoreDiscSet, objFilename, - mNewRefCounts[ObjectID]); - del.Delete(); + mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks; + rBackupStoreInfo.AdjustNumDeletedFiles(-1); } - else + + if(wasOldVersion) { - BOX_TRACE("Preserving object " << - BOX_FORMAT_OBJECTID(ObjectID) << " with " << - mNewRefCounts[ObjectID] << " references"); + mBlocksInOldFilesDelta -= deletedFileSizeInBlocks; + rBackupStoreInfo.AdjustNumOldFiles(-1); } - // Adjust counts for the file - ++mFilesDeleted; - mBlocksUsedDelta -= deletedFileSizeInBlocks; - if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks; - if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks; - // Delete the directory? // Do this if... dir has zero entries, and is marked as deleted in it's containing directory if(rDirectory.GetNumberOfEntries() == 0) @@ -952,8 +912,81 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba // Candidate for deletion mEmptyDirectories.push_back(InDirectory); } + + return 0; } +// -------------------------------------------------------------------------- +// +// Function +// Name: HousekeepStoreAccount::UpdateDirectorySize( +// BackupStoreDirectory& rDirectory, +// IOStream::pos_type new_size_in_blocks) +// Purpose: Update the directory size, modifying the parent +// directory's entry for this directory if necessary. +// Created: 05/03/14 +// +// -------------------------------------------------------------------------- + +void HousekeepStoreAccount::UpdateDirectorySize( + BackupStoreDirectory& rDirectory, + IOStream::pos_type new_size_in_blocks) +{ +#ifndef BOX_RELEASE_BUILD + { + std::string dirFilename; + MakeObjectFilename(rDirectory.GetObjectID(), dirFilename); + std::auto_ptr<RaidFileRead> dirStream( + RaidFileRead::Open(mStoreDiscSet, dirFilename)); + ASSERT(new_size_in_blocks == dirStream->GetDiscUsageInBlocks()); + } +#endif + + IOStream::pos_type old_size_in_blocks = + rDirectory.GetUserInfo1_SizeInBlocks(); + + if(new_size_in_blocks == old_size_in_blocks) + { + return; + } + + rDirectory.SetUserInfo1_SizeInBlocks(new_size_in_blocks); + + if (rDirectory.GetObjectID() == BACKUPSTORE_ROOT_DIRECTORY_ID) + { + return; + } + + std::string parentFilename; + MakeObjectFilename(rDirectory.GetContainerID(), parentFilename); + std::auto_ptr<RaidFileRead> parentStream( + RaidFileRead::Open(mStoreDiscSet, parentFilename)); + BackupStoreDirectory parent(*parentStream); + parentStream.reset(); + + BackupStoreDirectory::Entry* en = + parent.FindEntryByID(rDirectory.GetObjectID()); + ASSERT(en); + + if (en->GetSizeInBlocks() != old_size_in_blocks) + { + BOX_WARNING("Directory " << + BOX_FORMAT_OBJECTID(rDirectory.GetObjectID()) << + " entry in directory " << + BOX_FORMAT_OBJECTID(rDirectory.GetContainerID()) << + " had incorrect size " << en->GetSizeInBlocks() << + ", should have been " << old_size_in_blocks); + mErrorCount++; + } + + en->SetSizeInBlocks(new_size_in_blocks); + + RaidFileWrite writeDir(mStoreDiscSet, parentFilename, + mapNewRefs->GetRefCount(rDirectory.GetContainerID())); + writeDir.Open(true /* allow overwriting */); + parent.WriteToStream(writeDir); + writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); +} // -------------------------------------------------------------------------- // @@ -964,7 +997,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba // Created: 15/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::DeleteEmptyDirectories() +bool HousekeepStoreAccount::DeleteEmptyDirectories(BackupStoreInfo& rBackupStoreInfo) { while(mEmptyDirectories.size() > 0) { @@ -992,7 +1025,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() continue; } - DeleteEmptyDirectory(*i, toExamine); + DeleteEmptyDirectory(*i, toExamine, rBackupStoreInfo); } // Remove contents of empty directories @@ -1000,13 +1033,13 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() // Swap in new, so it's examined next time round mEmptyDirectories.swap(toExamine); } - + // Not interrupted return false; } void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, - std::vector<int64_t>& rToExamine) + std::vector<int64_t>& rToExamine, BackupStoreInfo& rBackupStoreInfo) { // Load up the directory to potentially delete std::string dirFilename; @@ -1046,16 +1079,18 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, std::auto_ptr<RaidFileRead> containingDirStream( RaidFileRead::Open(mStoreDiscSet, containingDirFilename)); - containingDirSizeInBlocksOrig = + containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks(); containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite); + containingDir.SetUserInfo1_SizeInBlocks(containingDirSizeInBlocksOrig); } // Find the entry - BackupStoreDirectory::Entry *pdirentry = + BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID()); - if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)) + // TODO FIXME invert test and reduce indentation + if((pdirentry != 0) && pdirentry->IsDeleted()) { // Should be deleted containingDir.DeleteEntry(dir.GetObjectID()); @@ -1068,7 +1103,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Write revised parent directory RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename, - mNewRefCounts[containingDir.GetObjectID()]); + mapNewRefs->GetRefCount(containingDir.GetObjectID())); writeDir.Open(true /* allow overwriting */); containingDir.WriteToStream(writeDir); @@ -1077,6 +1112,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + UpdateDirectorySize(containingDir, dirSize); // adjust usage counts for this directory if(dirSize > 0) @@ -1086,17 +1122,22 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, mBlocksInDirectoriesDelta += adjust; } - - if (--mNewRefCounts[dir.GetObjectID()] == 0) + if (mapNewRefs->RemoveReference(dir.GetObjectID())) { - // Delete the directory itself - RaidFileWrite del(mStoreDiscSet, dirFilename, - mNewRefCounts[dir.GetObjectID()]); - del.Delete(); + // Still referenced + BOX_TRACE("Housekeeping spared empty deleted dir " << + BOX_FORMAT_OBJECTID(dirId) << " due to " << + mapNewRefs->GetRefCount(dir.GetObjectID()) << + "remaining references"); + return; } - BOX_INFO("Housekeeping removed empty deleted dir " << + // Delete the directory itself + BOX_INFO("Housekeeping removing empty deleted dir " << BOX_FORMAT_OBJECTID(dirId)); + RaidFileWrite del(mStoreDiscSet, dirFilename, + mapNewRefs->GetRefCount(dir.GetObjectID())); + del.Delete(); // And adjust usage counts for the directory that's // just been deleted @@ -1105,6 +1146,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Update count ++mEmptyDirectoriesDeleted; + rBackupStoreInfo.AdjustNumDirectories(-1); } } |