summaryrefslogtreecommitdiff
path: root/lib/backupstore/HousekeepStoreAccount.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/backupstore/HousekeepStoreAccount.cpp')
-rw-r--r--lib/backupstore/HousekeepStoreAccount.cpp321
1 files changed, 127 insertions, 194 deletions
diff --git a/lib/backupstore/HousekeepStoreAccount.cpp b/lib/backupstore/HousekeepStoreAccount.cpp
index 75feda7f..41f43150 100644
--- a/lib/backupstore/HousekeepStoreAccount.cpp
+++ b/lib/backupstore/HousekeepStoreAccount.cpp
@@ -61,8 +61,6 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID,
mBlocksInDirectoriesDelta(0),
mFilesDeleted(0),
mEmptyDirectoriesDeleted(0),
- mSuppressRefCountChangeWarnings(false),
- mRefCountsAdjusted(0),
mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
{
std::ostringstream tag;
@@ -105,7 +103,7 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
{
if(KeepTryingForever)
{
- BOX_WARNING("Failed to lock account for housekeeping, "
+ BOX_INFO("Failed to lock account for housekeeping, "
"still trying...");
while(!writeLock.TryAndGetLock(writeLockFilename,
0600 /* restrictive file permissions */))
@@ -143,99 +141,105 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
mDeletionSizeTarget = 0;
}
- // initialise the refcount database
- mNewRefCounts.clear();
- // try to pre-allocate as much memory as we need
- mNewRefCounts.reserve(info->GetLastObjectIDUsed());
- // initialise the refcount of the root entry
- mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0);
- mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1;
+ BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
+ mapNewRefs = BackupStoreRefCountDatabase::Create(account);
// Scan the directory for potential things to delete
// This will also remove eligible items marked with RemoveASAP
- bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
+ bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID,
+ *info);
+
+ if(!continueHousekeeping)
+ {
+ // The scan was incomplete, so the new block counts are
+ // incorrect, we can't rely on them. It's better to discard
+ // the new info and adjust the old one instead.
+ info = pOldInfo;
+
+ // We're about to reset counters and exit, so report what
+ // happened now.
+ BOX_INFO("Housekeeping on account " <<
+ BOX_FORMAT_ACCOUNT(mAccountID) << " removed " <<
+ (0 - mBlocksUsedDelta) << " blocks (" <<
+ mFilesDeleted << " files, " <<
+ mEmptyDirectoriesDeleted << " dirs) and the directory "
+ "scan was interrupted");
+ }
+
+ // If housekeeping made any changes, such as deleting RemoveASAP files,
+ // the differences in block counts will be recorded in the deltas.
+ info->ChangeBlocksUsed(mBlocksUsedDelta);
+ info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+ info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+
+ // Reset the delta counts for files, as they will include
+ // RemoveASAP flagged files deleted during the initial scan.
+ // keep removeASAPBlocksUsedDelta for reporting
+ int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
+ mBlocksUsedDelta = 0;
+ mBlocksInOldFilesDelta = 0;
+ mBlocksInDeletedFilesDelta = 0;
// If scan directory stopped for some reason, probably parent
// instructed to terminate, stop now.
+ //
+ // We can only update the refcount database if we successfully
+ // finished our scan of all directories, otherwise we don't actually
+ // know which of the new counts are valid and which aren't
+ // (we might not have seen second references to some objects, etc.).
+
if(!continueHousekeeping)
{
- // If any files were marked "delete now", then update
- // the size of the store.
- if(mBlocksUsedDelta != 0 ||
- mBlocksInOldFilesDelta != 0 ||
- mBlocksInDeletedFilesDelta != 0)
- {
- info->ChangeBlocksUsed(mBlocksUsedDelta);
- info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
- info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
-
- // Save the store info back
- info->ReportChangesTo(*pOldInfo);
- info->Save();
- }
-
+ mapNewRefs->Discard();
+ info->Save();
return false;
}
- // Log any difference in opinion between the values recorded in
- // the store info, and the values just calculated for space usage.
- // BLOCK
+ // Report any UNexpected changes, and consider them to be errors.
+ // Do this before applying the expected changes below.
+ mErrorCount += info->ReportChangesTo(*pOldInfo);
+ info->Save();
+
+ // We want to compare the mapNewRefs to apOldRefs before we delete any
+ // files, because that will also change the reference count in a way that's not an error.
+
+ // try to load the reference count database
+ try
{
- int64_t used = info->GetBlocksUsed();
- int64_t usedOld = info->GetBlocksInOldFiles();
- int64_t usedDeleted = info->GetBlocksInDeletedFiles();
- int64_t usedDirectories = info->GetBlocksInDirectories();
-
- // If the counts were wrong, taking into account RemoveASAP
- // items deleted, log a message
- if((used + mBlocksUsedDelta) != mBlocksUsed
- || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
- || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles
- || usedDirectories != mBlocksInDirectories)
+ std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs =
+ BackupStoreRefCountDatabase::Load(account, false);
+
+ int64_t LastUsedObjectIdOnDisk = mapNewRefs->GetLastObjectIDUsed();
+
+ if (apOldRefs->GetLastObjectIDUsed() > LastUsedObjectIdOnDisk)
{
- // Log this
- BOX_ERROR("Housekeeping on account " <<
- BOX_FORMAT_ACCOUNT(mAccountID) << " found "
- "and fixed wrong block counts: "
- "used (" <<
- (used + mBlocksUsedDelta) << "," <<
- mBlocksUsed << "), old (" <<
- (usedOld + mBlocksInOldFilesDelta) << "," <<
- mBlocksInOldFiles << "), deleted (" <<
- (usedDeleted + mBlocksInDeletedFilesDelta) <<
- "," << mBlocksInDeletedFiles << "), dirs (" <<
- usedDirectories << "," << mBlocksInDirectories
- << ")");
+ LastUsedObjectIdOnDisk = apOldRefs->GetLastObjectIDUsed();
}
-
- // If the current values don't match, store them
- if(used != mBlocksUsed
- || usedOld != mBlocksInOldFiles
- || usedDeleted != mBlocksInDeletedFiles
- || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
- {
- // Set corrected values in store info
- info->CorrectAllUsedValues(mBlocksUsed,
- mBlocksInOldFiles, mBlocksInDeletedFiles,
- mBlocksInDirectories + mBlocksInDirectoriesDelta);
-
- info->ReportChangesTo(*pOldInfo);
- info->Save();
+
+ for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
+ ObjectID < LastUsedObjectIdOnDisk; ObjectID++)
+ {
+ if (apOldRefs->GetRefCount(ObjectID) !=
+ mapNewRefs->GetRefCount(ObjectID))
+ {
+ BOX_WARNING("Reference count of object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " changed from " <<
+ apOldRefs->GetRefCount(ObjectID) <<
+ " to " << mapNewRefs->GetRefCount(ObjectID));
+ mErrorCount++;
+ }
}
}
-
- // Reset the delta counts for files, as they will include
- // RemoveASAP flagged files deleted during the initial scan.
-
- // keep for reporting
- int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
+ catch(BoxException &e)
+ {
+ BOX_WARNING("Reference count database was missing or "
+ "corrupted during housekeeping, cannot check for "
+ "errors.");
+ }
- mBlocksUsedDelta = 0;
- mBlocksInOldFilesDelta = 0;
- mBlocksInDeletedFilesDelta = 0;
-
// Go and delete items from the accounts
- bool deleteInterrupted = DeleteFiles();
+ bool deleteInterrupted = DeleteFiles(*info);
// If that wasn't interrupted, remove any empty directories which
// are also marked as deleted in their containing directory
@@ -256,89 +260,6 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
(deleteInterrupted?" and was interrupted":""));
}
- // We can only update the refcount database if we successfully
- // finished our scan of all directories, otherwise we don't actually
- // know which of the new counts are valid and which aren't
- // (we might not have seen second references to some objects, etc.)
-
- BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
- std::auto_ptr<BackupStoreRefCountDatabase> apReferences;
-
- // try to load the reference count database
- try
- {
- apReferences = BackupStoreRefCountDatabase::Load(account,
- false);
- }
- catch(BoxException &e)
- {
- BOX_WARNING("Reference count database is missing or corrupted "
- "during housekeeping, creating a new one.");
- mSuppressRefCountChangeWarnings = true;
- BackupStoreRefCountDatabase::CreateForRegeneration(account);
- apReferences = BackupStoreRefCountDatabase::Load(account,
- false);
- }
-
- int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed();
-
- for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
- ObjectID < mNewRefCounts.size(); ObjectID++)
- {
- if (ObjectID > LastUsedObjectIdOnDisk)
- {
- if (!mSuppressRefCountChangeWarnings)
- {
- BOX_WARNING("Reference count of object " <<
- BOX_FORMAT_OBJECTID(ObjectID) <<
- " not found in database, added"
- " with " << mNewRefCounts[ObjectID] <<
- " references");
- }
- apReferences->SetRefCount(ObjectID,
- mNewRefCounts[ObjectID]);
- mRefCountsAdjusted++;
- LastUsedObjectIdOnDisk = ObjectID;
- continue;
- }
-
- BackupStoreRefCountDatabase::refcount_t OldRefCount =
- apReferences->GetRefCount(ObjectID);
-
- if (OldRefCount != mNewRefCounts[ObjectID])
- {
- BOX_WARNING("Reference count of object " <<
- BOX_FORMAT_OBJECTID(ObjectID) <<
- " changed from " << OldRefCount <<
- " to " << mNewRefCounts[ObjectID]);
- apReferences->SetRefCount(ObjectID,
- mNewRefCounts[ObjectID]);
- mRefCountsAdjusted++;
- }
- }
-
- // zero excess references in the database
- for (int64_t ObjectID = mNewRefCounts.size();
- ObjectID <= LastUsedObjectIdOnDisk; ObjectID++)
- {
- BackupStoreRefCountDatabase::refcount_t OldRefCount =
- apReferences->GetRefCount(ObjectID);
- BackupStoreRefCountDatabase::refcount_t NewRefCount = 0;
-
- if (OldRefCount != NewRefCount)
- {
- BOX_WARNING("Reference count of object " <<
- BOX_FORMAT_OBJECTID(ObjectID) <<
- " changed from " << OldRefCount <<
- " to " << NewRefCount << " (not found)");
- apReferences->SetRefCount(ObjectID, NewRefCount);
- mRefCountsAdjusted++;
- }
- }
-
- // force file to be saved and closed before releasing the lock below
- apReferences.reset();
-
// Make sure the delta's won't cause problems if the counts are
// really wrong, and it wasn't fixed because the store was
// updated during the scan.
@@ -364,11 +285,14 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta);
-
+
// Save the store info back
- info->ReportChangesTo(*pOldInfo);
info->Save();
+ // force file to be saved and closed before releasing the lock below
+ mapNewRefs->Commit();
+ mapNewRefs.reset();
+
// Explicity release the lock (would happen automatically on
// going out of scope, included for code clarity)
writeLock.ReleaseLock();
@@ -405,7 +329,8 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF
// Created: 11/12/03
//
// --------------------------------------------------------------------------
-bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
+bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID,
+ BackupStoreInfo& rBackupStoreInfo)
{
#ifndef WIN32
if((--mCountUntilNextInterprocessMsgCheck) <= 0)
@@ -459,11 +384,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
while((en = i.Next()) != 0)
{
// This directory references this object
- if (mNewRefCounts.size() <= en->GetObjectID())
- {
- mNewRefCounts.resize(en->GetObjectID() + 1, 0);
- }
- mNewRefCounts[en->GetObjectID()]++;
+ mapNewRefs->AddReference(en->GetObjectID());
}
}
@@ -482,10 +403,11 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
{
int16_t enFlags = en->GetFlags();
if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0
- && (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+ && (en->IsDeleted() || en->IsOld()))
{
// Delete this immediately.
- DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks);
+ DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename,
+ originalDirSizeInBlocks, rBackupStoreInfo);
// flag as having done something
deletedSomething = true;
@@ -624,7 +546,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
// Next level
ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir);
- if(!ScanDirectory(en->GetObjectID()))
+ if(!ScanDirectory(en->GetObjectID(), rBackupStoreInfo))
{
// Halting operation
return false;
@@ -687,7 +609,7 @@ bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount
// Created: 15/12/03
//
// --------------------------------------------------------------------------
-bool HousekeepStoreAccount::DeleteFiles()
+bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo)
{
// Only delete files if the deletion target is greater than zero
// (otherwise we delete one file each time round, which gradually deletes the old versions)
@@ -727,12 +649,13 @@ bool HousekeepStoreAccount::DeleteFiles()
}
// Delete the file
- DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig);
+ DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename,
+ dirSizeInBlocksOrig, rBackupStoreInfo);
BOX_INFO("Housekeeping removed " <<
(i->mIsFlagDeleted ? "deleted" : "old") <<
" file " << BOX_FORMAT_OBJECTID(i->mObjectID) <<
" from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory));
-
+
// Stop if the deletion target has been matched or exceeded
// (checking here rather than at the beginning will tend to reduce the
// space to slightly less than the soft limit, which will allow the backup
@@ -758,7 +681,9 @@ bool HousekeepStoreAccount::DeleteFiles()
// Created: 15/7/04
//
// --------------------------------------------------------------------------
-void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks)
+void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID,
+ BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename,
+ int64_t OriginalDirSizeInBlocks, BackupStoreInfo& rBackupStoreInfo)
{
// Find the entry inside the directory
bool wasDeleted = false;
@@ -783,8 +708,8 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
}
// Record the flags it's got set
- wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0);
- wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0);
+ wasDeleted = pentry->IsDeleted();
+ wasOldVersion = pentry->IsOld();
// Check this should be deleted
if(!wasDeleted && !wasOldVersion)
{
@@ -853,7 +778,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename));
// And open a write file to overwrite the other directory entry
padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet,
- objFilenameOlder, mNewRefCounts[ObjectID]));
+ objFilenameOlder, mapNewRefs->GetRefCount(ObjectID)));
padjustedEntry->Open(true /* allow overwriting */);
if(pentry->GetDependsNewer() == 0)
@@ -872,11 +797,11 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
int64_t newSize = padjustedEntry->GetDiscUsageInBlocks();
int64_t sizeDelta = newSize - polder->GetSizeInBlocks();
mBlocksUsedDelta += sizeDelta;
- if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)
+ if(polder->IsDeleted())
{
mBlocksInDeletedFilesDelta += sizeDelta;
}
- if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0)
+ if(polder->IsOld())
{
mBlocksInOldFilesDelta += sizeDelta;
}
@@ -894,17 +819,17 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
int64_t dirRevisedSize = 0;
{
RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename,
- mNewRefCounts[InDirectory]);
+ mapNewRefs->GetRefCount(InDirectory));
writeDir.Open(true /* allow overwriting */);
rDirectory.WriteToStream(writeDir);
- // get the disc usage (must do this before commiting it)
+ // Get the disc usage (must do this before commiting it)
dirRevisedSize = writeDir.GetDiscUsageInBlocks();
// Commit directory
writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
- // adjust usage counts for this directory
+ // Adjust block counts if the directory itself changed in size
if(dirRevisedSize > 0)
{
int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks;
@@ -921,29 +846,38 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
}
// Drop reference count by one. If it reaches zero, delete the file.
- if(--mNewRefCounts[ObjectID] == 0)
+ if(!mapNewRefs->RemoveReference(ObjectID))
{
// Delete from disc
BOX_TRACE("Removing unreferenced object " <<
BOX_FORMAT_OBJECTID(ObjectID));
std::string objFilename;
MakeObjectFilename(ObjectID, objFilename);
- RaidFileWrite del(mStoreDiscSet, objFilename,
- mNewRefCounts[ObjectID]);
+ RaidFileWrite del(mStoreDiscSet, objFilename, 0);
del.Delete();
}
else
{
BOX_TRACE("Preserving object " <<
BOX_FORMAT_OBJECTID(ObjectID) << " with " <<
- mNewRefCounts[ObjectID] << " references");
+ mapNewRefs->GetRefCount(ObjectID) << " references");
}
// Adjust counts for the file
++mFilesDeleted;
mBlocksUsedDelta -= deletedFileSizeInBlocks;
- if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
- if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
+
+ if(wasDeleted)
+ {
+ mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
+ rBackupStoreInfo.AdjustNumDeletedFiles(-1);
+ }
+
+ if(wasOldVersion)
+ {
+ mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
+ rBackupStoreInfo.AdjustNumOldFiles(-1);
+ }
// Delete the directory?
// Do this if... dir has zero entries, and is marked as deleted in it's containing directory
@@ -1068,7 +1002,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
// Write revised parent directory
RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename,
- mNewRefCounts[containingDir.GetObjectID()]);
+ mapNewRefs->GetRefCount(containingDir.GetObjectID()));
writeDir.Open(true /* allow overwriting */);
containingDir.WriteToStream(writeDir);
@@ -1087,11 +1021,10 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
}
- if (--mNewRefCounts[dir.GetObjectID()] == 0)
+ if (mapNewRefs->RemoveReference(dir.GetObjectID()))
{
// Delete the directory itself
- RaidFileWrite del(mStoreDiscSet, dirFilename,
- mNewRefCounts[dir.GetObjectID()]);
+ RaidFileWrite del(mStoreDiscSet, dirFilename, 0);
del.Delete();
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-08 06:44:05 +0000