summaryrefslogtreecommitdiff
path: root/lib/backupstore
diff options
context:
space:
mode:
authorChris Wilson <chris+github@qwirx.com>2012-06-29 22:15:36 +0000
committerChris Wilson <chris+github@qwirx.com>2012-06-29 22:15:36 +0000
commita9e867a2392349a76b3dbac30b3b85b22b478579 (patch)
treefb915aa3b3e403ceade8b1b851647c6c56fd2867 /lib/backupstore
parent9b89929355f6d1522e66a2919af0fb07dc28c508 (diff)
Add housekeep command to bbstoreaccounts to run housekeeping right now.
Diffstat (limited to 'lib/backupstore')
-rw-r--r--lib/backupstore/HousekeepStoreAccount.cpp1104
-rw-r--r--lib/backupstore/HousekeepStoreAccount.h111
2 files changed, 1215 insertions, 0 deletions
diff --git a/lib/backupstore/HousekeepStoreAccount.cpp b/lib/backupstore/HousekeepStoreAccount.cpp
new file mode 100644
index 00000000..311ae433
--- /dev/null
+++ b/lib/backupstore/HousekeepStoreAccount.cpp
@@ -0,0 +1,1104 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: HousekeepStoreAccount.cpp
+// Purpose:
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include <map>
+
+#include "autogen_BackupStoreException.h"
+#include "BackupConstants.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreRefCountDatabase.h"
+#include "BufferedStream.h"
+#include "HousekeepStoreAccount.h"
+#include "NamedLock.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "StoreStructure.h"
+
+#include "MemLeakFindOn.h"
+
+// check every 32 directories scanned/files deleted
+#define POLL_INTERPROCESS_MSG_CHECK_FREQUENCY 32
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: HousekeepStoreAccount::HousekeepStoreAccount(int, const std::string &, int, BackupStoreDaemon &)
+// Purpose: Constructor
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+HousekeepStoreAccount::HousekeepStoreAccount(int AccountID,
+ const std::string &rStoreRoot, int StoreDiscSet,
+ HousekeepingCallback* pHousekeepingCallback)
+ : mAccountID(AccountID),
+ mStoreRoot(rStoreRoot),
+ mStoreDiscSet(StoreDiscSet),
+ mpHousekeepingCallback(pHousekeepingCallback),
+ mDeletionSizeTarget(0),
+ mPotentialDeletionsTotalSize(0),
+ mMaxSizeInPotentialDeletions(0),
+ mBlocksUsed(0),
+ mBlocksInOldFiles(0),
+ mBlocksInDeletedFiles(0),
+ mBlocksInDirectories(0),
+ mBlocksUsedDelta(0),
+ mBlocksInOldFilesDelta(0),
+ mBlocksInDeletedFilesDelta(0),
+ mBlocksInDirectoriesDelta(0),
+ mFilesDeleted(0),
+ mEmptyDirectoriesDeleted(0),
+ mSuppressRefCountChangeWarnings(false),
+ mRefCountsAdjusted(0),
+ mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: HousekeepStoreAccount::~HousekeepStoreAccount()
+// Purpose: Destructor
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+HousekeepStoreAccount::~HousekeepStoreAccount()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: HousekeepStoreAccount::DoHousekeeping()
+// Purpose: Perform the housekeeping
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
+{
+ BOX_TRACE("Starting housekeeping on account " <<
+ BOX_FORMAT_ACCOUNT(mAccountID));
+
+ // Attempt to lock the account
+ std::string writeLockFilename;
+ StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet,
+ writeLockFilename);
+ NamedLock writeLock;
+ if(!writeLock.TryAndGetLock(writeLockFilename.c_str(),
+ 0600 /* restrictive file permissions */))
+ {
+ if(KeepTryingForever)
+ {
+ BOX_WARNING("Failed to lock account for housekeeping, "
+ "still trying...");
+ while(!writeLock.TryAndGetLock(writeLockFilename,
+ 0600 /* restrictive file permissions */))
+ {
+ sleep(1);
+ }
+ }
+ else
+ {
+ // Couldn't lock the account -- just stop now
+ return false;
+ }
+ }
+
+ // Load the store info to find necessary info for the housekeeping
+ std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(mAccountID,
+ mStoreRoot, mStoreDiscSet, false /* Read/Write */));
+ std::auto_ptr<BackupStoreInfo> pOldInfo(
+ BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet,
+ true /* Read Only */));
+
+ // Tag log output to identify account
+ std::ostringstream tag;
+ tag << "hk/" << BOX_FORMAT_ACCOUNT(mAccountID) << "/" <<
+ info->GetAccountName();
+ Logging::Tagger tagWithClientID(tag.str());
+
+ // Calculate how much should be deleted
+ mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit();
+ if(mDeletionSizeTarget < 0)
+ {
+ mDeletionSizeTarget = 0;
+ }
+
+ // initialise the refcount database
+ mNewRefCounts.clear();
+ // try to pre-allocate as much memory as we need
+ mNewRefCounts.reserve(info->GetLastObjectIDUsed());
+ // initialise the refcount of the root entry
+ mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0);
+ mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1;
+
+ // Scan the directory for potential things to delete
+ // This will also remove eligible items marked with RemoveASAP
+ bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
+
+ // If scan directory stopped for some reason, probably parent
+ // instructed to terminate, stop now.
+ if(!continueHousekeeping)
+ {
+ // If any files were marked "delete now", then update
+ // the size of the store.
+ if(mBlocksUsedDelta != 0 ||
+ mBlocksInOldFilesDelta != 0 ||
+ mBlocksInDeletedFilesDelta != 0)
+ {
+ info->ChangeBlocksUsed(mBlocksUsedDelta);
+ info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+ info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+
+ // Save the store info back
+ info->ReportChangesTo(*pOldInfo);
+ info->Save();
+ }
+
+ return false;
+ }
+
+ // Log any difference in opinion between the values recorded in
+ // the store info, and the values just calculated for space usage.
+ // BLOCK
+ {
+ int64_t used = info->GetBlocksUsed();
+ int64_t usedOld = info->GetBlocksInOldFiles();
+ int64_t usedDeleted = info->GetBlocksInDeletedFiles();
+ int64_t usedDirectories = info->GetBlocksInDirectories();
+
+ // If the counts were wrong, taking into account RemoveASAP
+ // items deleted, log a message
+ if((used + mBlocksUsedDelta) != mBlocksUsed
+ || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
+ || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles
+ || usedDirectories != mBlocksInDirectories)
+ {
+ // Log this
+ BOX_ERROR("Housekeeping on account " <<
+ BOX_FORMAT_ACCOUNT(mAccountID) << " found "
+ "and fixed wrong block counts: "
+ "used (" <<
+ (used + mBlocksUsedDelta) << "," <<
+ mBlocksUsed << "), old (" <<
+ (usedOld + mBlocksInOldFilesDelta) << "," <<
+ mBlocksInOldFiles << "), deleted (" <<
+ (usedDeleted + mBlocksInDeletedFilesDelta) <<
+ "," << mBlocksInDeletedFiles << "), dirs (" <<
+ usedDirectories << "," << mBlocksInDirectories
+ << ")");
+ }
+
+ // If the current values don't match, store them
+ if(used != mBlocksUsed
+ || usedOld != mBlocksInOldFiles
+ || usedDeleted != mBlocksInDeletedFiles
+ || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
+ {
+ // Set corrected values in store info
+ info->CorrectAllUsedValues(mBlocksUsed,
+ mBlocksInOldFiles, mBlocksInDeletedFiles,
+ mBlocksInDirectories + mBlocksInDirectoriesDelta);
+
+ info->ReportChangesTo(*pOldInfo);
+ info->Save();
+ }
+ }
+
+ // Reset the delta counts for files, as they will include
+ // RemoveASAP flagged files deleted during the initial scan.
+
+ // keep for reporting
+ int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
+
+ mBlocksUsedDelta = 0;
+ mBlocksInOldFilesDelta = 0;
+ mBlocksInDeletedFilesDelta = 0;
+
+ // Go and delete items from the accounts
+ bool deleteInterrupted = DeleteFiles();
+
+ // If that wasn't interrupted, remove any empty directories which
+ // are also marked as deleted in their containing directory
+ if(!deleteInterrupted)
+ {
+ deleteInterrupted = DeleteEmptyDirectories();
+ }
+
+ // Log deletion if anything was deleted
+ if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0)
+ {
+ BOX_INFO("Housekeeping on account " <<
+ BOX_FORMAT_ACCOUNT(mAccountID) << " "
+ "removed " <<
+ (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) <<
+ " blocks (" << mFilesDeleted << " files, " <<
+ mEmptyDirectoriesDeleted << " dirs)" <<
+ (deleteInterrupted?" and was interrupted":""));
+ }
+
+ // We can only update the refcount database if we successfully
+ // finished our scan of all directories, otherwise we don't actually
+ // know which of the new counts are valid and which aren't
+ // (we might not have seen second references to some objects, etc.)
+
+ BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
+ std::auto_ptr<BackupStoreRefCountDatabase> apReferences;
+
+ // try to load the reference count database
+ try
+ {
+ apReferences = BackupStoreRefCountDatabase::Load(account,
+ false);
+ }
+ catch(BoxException &e)
+ {
+ BOX_WARNING("Reference count database is missing or corrupted "
+ "during housekeeping, creating a new one.");
+ mSuppressRefCountChangeWarnings = true;
+ BackupStoreRefCountDatabase::CreateForRegeneration(account);
+ apReferences = BackupStoreRefCountDatabase::Load(account,
+ false);
+ }
+
+ int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed();
+
+ for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
+ ObjectID < mNewRefCounts.size(); ObjectID++)
+ {
+ if (ObjectID > LastUsedObjectIdOnDisk)
+ {
+ if (!mSuppressRefCountChangeWarnings)
+ {
+ BOX_WARNING("Reference count of object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " not found in database, added"
+ " with " << mNewRefCounts[ObjectID] <<
+ " references");
+ }
+ apReferences->SetRefCount(ObjectID,
+ mNewRefCounts[ObjectID]);
+ mRefCountsAdjusted++;
+ LastUsedObjectIdOnDisk = ObjectID;
+ continue;
+ }
+
+ BackupStoreRefCountDatabase::refcount_t OldRefCount =
+ apReferences->GetRefCount(ObjectID);
+
+ if (OldRefCount != mNewRefCounts[ObjectID])
+ {
+ BOX_WARNING("Reference count of object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " changed from " << OldRefCount <<
+ " to " << mNewRefCounts[ObjectID]);
+ apReferences->SetRefCount(ObjectID,
+ mNewRefCounts[ObjectID]);
+ mRefCountsAdjusted++;
+ }
+ }
+
+ // zero excess references in the database
+ for (int64_t ObjectID = mNewRefCounts.size();
+ ObjectID <= LastUsedObjectIdOnDisk; ObjectID++)
+ {
+ BackupStoreRefCountDatabase::refcount_t OldRefCount =
+ apReferences->GetRefCount(ObjectID);
+ BackupStoreRefCountDatabase::refcount_t NewRefCount = 0;
+
+ if (OldRefCount != NewRefCount)
+ {
+ BOX_WARNING("Reference count of object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " changed from " << OldRefCount <<
+ " to " << NewRefCount << " (not found)");
+ apReferences->SetRefCount(ObjectID, NewRefCount);
+ mRefCountsAdjusted++;
+ }
+ }
+
+ // force file to be saved and closed before releasing the lock below
+ apReferences.reset();
+
+ // Make sure the delta's won't cause problems if the counts are
+ // really wrong, and it wasn't fixed because the store was
+ // updated during the scan.
+ if(mBlocksUsedDelta < (0 - info->GetBlocksUsed()))
+ {
+ mBlocksUsedDelta = (0 - info->GetBlocksUsed());
+ }
+ if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles()))
+ {
+ mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles());
+ }
+ if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles()))
+ {
+ mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles());
+ }
+ if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories()))
+ {
+ mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories());
+ }
+
+ // Update the usage counts in the store
+ info->ChangeBlocksUsed(mBlocksUsedDelta);
+ info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+ info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+ info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta);
+
+ // Save the store info back
+ info->ReportChangesTo(*pOldInfo);
+ info->Save();
+
+ // Explicity release the lock (would happen automatically on
+ // going out of scope, included for code clarity)
+ writeLock.ReleaseLock();
+
+ BOX_TRACE("Finished housekeeping on account " <<
+ BOX_FORMAT_ACCOUNT(mAccountID));
+ return true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: HousekeepStoreAccount::MakeObjectFilename(int64_t, std::string &)
+// Purpose: Generate and place the filename for a given object ID
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut)
+{
+ // Delegate to utility function
+ StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rFilenameOut, false /* don't bother ensuring the directory exists */);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: HousekeepStoreAccount::ScanDirectory(int64_t)
+// Purpose: Private. Scan a directory for potentially deleteable
+// items, and add them to the list. Returns true if the
+// scan should continue.
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
+{
+#ifndef WIN32
+ if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+ {
+ mCountUntilNextInterprocessMsgCheck =
+ POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+
+ // Check for having to stop
+ // Include account ID here as the specified account is locked
+ if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))
+ {
+ // Need to abort now
+ return false;
+ }
+ }
+#endif
+
+ // Get the filename
+ std::string objectFilename;
+ MakeObjectFilename(ObjectID, objectFilename);
+
+ // Open it.
+ std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet,
+ objectFilename));
+
+ // Add the size of the directory on disc to the size being calculated
+ int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
+ mBlocksInDirectories += originalDirSizeInBlocks;
+ mBlocksUsed += originalDirSizeInBlocks;
+
+ // Read the directory in
+ BackupStoreDirectory dir;
+ BufferedStream buf(*dirStream);
+ dir.ReadFromStream(buf, IOStream::TimeOutInfinite);
+ dirStream->Close();
+
+ // Is it empty?
+ if(dir.GetNumberOfEntries() == 0)
+ {
+ // Add it to the list of directories to potentially delete
+ mEmptyDirectories.push_back(dir.GetObjectID());
+ }
+
+ // Calculate reference counts first, before we start requesting
+ // files to be deleted.
+ // BLOCK
+ {
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+
+ while((en = i.Next()) != 0)
+ {
+ // This directory references this object
+ if (mNewRefCounts.size() <= en->GetObjectID())
+ {
+ mNewRefCounts.resize(en->GetObjectID() + 1, 0);
+ }
+ mNewRefCounts[en->GetObjectID()]++;
+ }
+ }
+
+ // BLOCK
+ {
+ // Remove any files which are marked for removal as soon
+ // as they become old or deleted.
+ bool deletedSomething = false;
+ do
+ {
+ // Iterate through the directory
+ deletedSomething = false;
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+ {
+ int16_t enFlags = en->GetFlags();
+ if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0
+ && (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+ {
+ // Delete this immediately.
+ DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks);
+
+ // flag as having done something
+ deletedSomething = true;
+
+ // Must start the loop from the beginning again, as iterator is now
+ // probably invalid.
+ break;
+ }
+ }
+ } while(deletedSomething);
+ }
+
+ // BLOCK
+ {
+ // Add files to the list of potential deletions
+
+ // map to count the distance from the mark
+ typedef std::pair<std::string, int32_t> version_t;
+ std::map<version_t, int32_t> markVersionAges;
+ // map of pair (filename, mark number) -> version age
+
+ // NOTE: use a reverse iterator to allow the distance from mark stuff to work
+ BackupStoreDirectory::ReverseIterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+ {
+ // Update recalculated usage sizes
+ int16_t enFlags = en->GetFlags();
+ int64_t enSizeInBlocks = en->GetSizeInBlocks();
+ mBlocksUsed += enSizeInBlocks;
+ if(enFlags & BackupStoreDirectory::Entry::Flags_OldVersion) mBlocksInOldFiles += enSizeInBlocks;
+ if(enFlags & BackupStoreDirectory::Entry::Flags_Deleted) mBlocksInDeletedFiles += enSizeInBlocks;
+
+ // Work out ages of this version from the last mark
+ int32_t enVersionAge = 0;
+ std::map<version_t, int32_t>::iterator enVersionAgeI(
+ markVersionAges.find(
+ version_t(en->GetName().GetEncodedFilename(),
+ en->GetMarkNumber())));
+ if(enVersionAgeI != markVersionAges.end())
+ {
+ enVersionAge = enVersionAgeI->second + 1;
+ enVersionAgeI->second = enVersionAge;
+ }
+ else
+ {
+ markVersionAges[version_t(en->GetName().GetEncodedFilename(), en->GetMarkNumber())] = enVersionAge;
+ }
+ // enVersionAge is now the age of this version.
+
+ // Potentially add it to the list if it's deleted, if it's an old version or deleted
+ if((enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+ {
+ // Is deleted / old version.
+ DelEn d;
+ d.mObjectID = en->GetObjectID();
+ d.mInDirectory = ObjectID;
+ d.mSizeInBlocks = en->GetSizeInBlocks();
+ d.mMarkNumber = en->GetMarkNumber();
+ d.mVersionAgeWithinMark = enVersionAge;
+ d.mIsFlagDeleted = (enFlags &
+ BackupStoreDirectory::Entry::Flags_Deleted)
+ ? true : false;
+
+ // Add it to the list
+ mPotentialDeletions.insert(d);
+
+ // Update various counts
+ mPotentialDeletionsTotalSize += d.mSizeInBlocks;
+ if(d.mSizeInBlocks > mMaxSizeInPotentialDeletions) mMaxSizeInPotentialDeletions = d.mSizeInBlocks;
+
+ // Too much in the list of potential deletions?
+ // (check against the deletion target + the max size in deletions, so that we never delete things
+ // and take the total size below the deletion size target)
+ if(mPotentialDeletionsTotalSize > (mDeletionSizeTarget + mMaxSizeInPotentialDeletions))
+ {
+ int64_t sizeToRemove = mPotentialDeletionsTotalSize - (mDeletionSizeTarget + mMaxSizeInPotentialDeletions);
+ bool recalcMaxSize = false;
+
+ while(sizeToRemove > 0)
+ {
+ // Make iterator for the last element, while checking that there's something there in the first place.
+ std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.end());
+ if(i != mPotentialDeletions.begin())
+ {
+ // Nothing left in set
+ break;
+ }
+ // Make this into an iterator pointing to the last element in the set
+ --i;
+
+ // Delete this one?
+ if(sizeToRemove > i->mSizeInBlocks)
+ {
+ sizeToRemove -= i->mSizeInBlocks;
+ if(i->mSizeInBlocks >= mMaxSizeInPotentialDeletions)
+ {
+ // Will need to recalculate the maximum size now, because we've just deleted that element
+ recalcMaxSize = true;
+ }
+ mPotentialDeletions.erase(i);
+ }
+ else
+ {
+ // Over the size to remove, so stop now
+ break;
+ }
+ }
+
+ if(recalcMaxSize)
+ {
+ // Because an object which was the maximum size recorded was deleted from the set
+ // it's necessary to recalculate this maximum.
+ mMaxSizeInPotentialDeletions = 0;
+ std::set<DelEn, DelEnCompare>::const_iterator i(mPotentialDeletions.begin());
+ for(; i != mPotentialDeletions.end(); ++i)
+ {
+ if(i->mSizeInBlocks > mMaxSizeInPotentialDeletions)
+ {
+ mMaxSizeInPotentialDeletions = i->mSizeInBlocks;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ {
+ // Recurse into subdirectories
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
+ {
+ // Next level
+ ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir);
+
+ if(!ScanDirectory(en->GetObjectID()))
+ {
+ // Halting operation
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &, const HousekeepStoreAccount::DelEnd &)
+// Purpose: Comparison function for set
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &x, const HousekeepStoreAccount::DelEn &y)
+{
+ // STL spec says this:
+ // A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second.
+
+ // The sort order here is intended to preserve the entries of most value, that is, the newest objects
+ // which are on a mark boundary.
+
+ // Reverse order age, so oldest goes first
+ if(x.mVersionAgeWithinMark > y.mVersionAgeWithinMark)
+ {
+ return true;
+ }
+ else if(x.mVersionAgeWithinMark < y.mVersionAgeWithinMark)
+ {
+ return false;
+ }
+
+ // but mark number in ascending order, so that the oldest marks are deleted first
+ if(x.mMarkNumber < y.mMarkNumber)
+ {
+ return true;
+ }
+ else if(x.mMarkNumber > y.mMarkNumber)
+ {
+ return false;
+ }
+
+ // Just compare object ID now to put the oldest objects first
+ return x.mObjectID < y.mObjectID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: HousekeepStoreAccount::DeleteFiles()
+// Purpose: Delete the files targeted for deletion, returning
+// true if the operation was interrupted
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DeleteFiles()
+{
+ // Only delete files if the deletion target is greater than zero
+ // (otherwise we delete one file each time round, which gradually deletes the old versions)
+ if(mDeletionSizeTarget <= 0)
+ {
+ // Not interrupted
+ return false;
+ }
+
+ // Iterate through the set of potential deletions, until enough has been deleted.
+ // (there is likely to be more in the set than should be actually deleted).
+ for(std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i)
+ {
+#ifndef WIN32
+ if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+ {
+ mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+ // Check for having to stop
+ if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked
+ {
+ // Need to abort now
+ return true;
+ }
+ }
+#endif
+
+ // Load up the directory it's in
+ // Get the filename
+ std::string dirFilename;
+ BackupStoreDirectory dir;
+ int64_t dirSizeInBlocksOrig = 0;
+ {
+ MakeObjectFilename(i->mInDirectory, dirFilename);
+ std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename));
+ dirSizeInBlocksOrig = dirStream->GetDiscUsageInBlocks();
+ dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+ }
+
+ // Delete the file
+ DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig);
+ BOX_INFO("Housekeeping removed " <<
+ (i->mIsFlagDeleted ? "deleted" : "old") <<
+ " file " << BOX_FORMAT_OBJECTID(i->mObjectID) <<
+ " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory));
+
+ // Stop if the deletion target has been matched or exceeded
+ // (checking here rather than at the beginning will tend to reduce the
+ // space to slightly less than the soft limit, which will allow the backup
+ // client to start uploading files again)
+ if((0 - mBlocksUsedDelta) >= mDeletionSizeTarget)
+ {
+ break;
+ }
+ }
+
+ return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: HousekeepStoreAccount::DeleteFile(int64_t, int64_t,
+// BackupStoreDirectory &, const std::string &, int64_t)
+// Purpose: Delete a file. Takes the directory already loaded
+// in and the filename, for efficiency in both the
+// usage scenarios.
+// Created: 15/7/04
+//
+// --------------------------------------------------------------------------
+void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks)
+{
+ // Find the entry inside the directory
+ bool wasDeleted = false;
+ bool wasOldVersion = false;
+ int64_t deletedFileSizeInBlocks = 0;
+ // A pointer to an object which requires committing if the directory save goes OK
+ std::auto_ptr<RaidFileWrite> padjustedEntry;
+ // BLOCK
+ {
+ BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID);
+ if(pentry == 0)
+ {
+ BOX_ERROR("Housekeeping on account " <<
+ BOX_FORMAT_ACCOUNT(mAccountID) << " "
+ "found error: object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) << " "
+ "not found in dir " <<
+ BOX_FORMAT_OBJECTID(InDirectory) << ", "
+ "indicates logic error/corruption? Run "
+ "bbstoreaccounts check <accid> fix");
+ return;
+ }
+
+ // Record the flags it's got set
+ wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0);
+ wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0);
+ // Check this should be deleted
+ if(!wasDeleted && !wasOldVersion)
+ {
+ // Things changed size we were last around
+ return;
+ }
+
+ // Record size
+ deletedFileSizeInBlocks = pentry->GetSizeInBlocks();
+
+ // If the entry is involved in a chain of patches, it needs to be handled
+ // a bit more carefully.
+ if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0)
+ {
+ // This entry is a patch from a newer entry. Just need to update the info on that entry.
+ BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
+ if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID)
+ {
+ THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+ }
+ // Change the info in the newer entry so that this no longer points to this entry
+ pnewer->SetDependsOlder(0);
+ }
+ else if(pentry->GetDependsOlder() != 0)
+ {
+ BackupStoreDirectory::Entry *polder = rDirectory.FindEntryByID(pentry->GetDependsOlder());
+ if(pentry->GetDependsNewer() == 0)
+ {
+ // There exists an older version which depends on this one. Need to combine the two over that one.
+
+ // Adjust the other entry in the directory
+ if(polder == 0 || polder->GetDependsNewer() != ObjectID)
+ {
+ THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+ }
+ // Change the info in the older entry so that this no longer points to this entry
+ polder->SetDependsNewer(0);
+ }
+ else
+ {
+ // This entry is in the middle of a chain, and two patches need combining.
+
+ // First, adjust the directory entries
+ BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
+ if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID
+ || polder == 0 || polder->GetDependsNewer() != ObjectID)
+ {
+ THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+ }
+ // Remove the middle entry from the linked list by simply using the values from this entry
+ pnewer->SetDependsOlder(pentry->GetDependsOlder());
+ polder->SetDependsNewer(pentry->GetDependsNewer());
+ }
+
+ // COMMON CODE to both cases
+
+ // Generate the filename of the older version
+ std::string objFilenameOlder;
+ MakeObjectFilename(pentry->GetDependsOlder(), objFilenameOlder);
+ // Open it twice (it's the diff)
+ std::auto_ptr<RaidFileRead> pdiff(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
+ std::auto_ptr<RaidFileRead> pdiff2(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
+ // Open this file
+ std::string objFilename;
+ MakeObjectFilename(ObjectID, objFilename);
+ std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename));
+ // And open a write file to overwrite the other directory entry
+ padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet,
+ objFilenameOlder, mNewRefCounts[ObjectID]));
+ padjustedEntry->Open(true /* allow overwriting */);
+
+ if(pentry->GetDependsNewer() == 0)
+ {
+ // There exists an older version which depends on this one. Need to combine the two over that one.
+ BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, *padjustedEntry);
+ }
+ else
+ {
+ // This entry is in the middle of a chain, and two patches need combining.
+ BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry);
+ }
+ // The file will be committed later when the directory is safely commited.
+
+ // Work out the adjusted size
+ int64_t newSize = padjustedEntry->GetDiscUsageInBlocks();
+ int64_t sizeDelta = newSize - polder->GetSizeInBlocks();
+ mBlocksUsedDelta += sizeDelta;
+ if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)
+ {
+ mBlocksInDeletedFilesDelta += sizeDelta;
+ }
+ if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0)
+ {
+ mBlocksInOldFilesDelta += sizeDelta;
+ }
+ polder->SetSizeInBlocks(newSize);
+ }
+
+ // pentry no longer valid
+ }
+
+ // Delete it from the directory
+ rDirectory.DeleteEntry(ObjectID);
+
+ // Save directory back to disc
+ // BLOCK
+ int64_t dirRevisedSize = 0;
+ {
+ RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename,
+ mNewRefCounts[InDirectory]);
+ writeDir.Open(true /* allow overwriting */);
+ rDirectory.WriteToStream(writeDir);
+
+ // get the disc usage (must do this before commiting it)
+ dirRevisedSize = writeDir.GetDiscUsageInBlocks();
+
+ // Commit directory
+ writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+ // adjust usage counts for this directory
+ if(dirRevisedSize > 0)
+ {
+ int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks;
+ mBlocksUsedDelta += adjust;
+ mBlocksInDirectoriesDelta += adjust;
+ }
+ }
+
+ // Commit any new adjusted entry
+ if(padjustedEntry.get() != 0)
+ {
+ padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+ padjustedEntry.reset(); // delete it now
+ }
+
+ // Drop reference count by one. If it reaches zero, delete the file.
+ if(--mNewRefCounts[ObjectID] == 0)
+ {
+ // Delete from disc
+ BOX_TRACE("Removing unreferenced object " <<
+ BOX_FORMAT_OBJECTID(ObjectID));
+ std::string objFilename;
+ MakeObjectFilename(ObjectID, objFilename);
+ RaidFileWrite del(mStoreDiscSet, objFilename,
+ mNewRefCounts[ObjectID]);
+ del.Delete();
+ }
+ else
+ {
+ BOX_TRACE("Preserving object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) << " with " <<
+ mNewRefCounts[ObjectID] << " references");
+ }
+
+ // Adjust counts for the file
+ ++mFilesDeleted;
+ mBlocksUsedDelta -= deletedFileSizeInBlocks;
+ if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
+ if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
+
+ // Delete the directory?
+ // Do this if... dir has zero entries, and is marked as deleted in it's containing directory
+ if(rDirectory.GetNumberOfEntries() == 0)
+ {
+ // Candidate for deletion
+ mEmptyDirectories.push_back(InDirectory);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: HousekeepStoreAccount::DeleteEmptyDirectories()
+// Purpose: Remove any empty directories which are also marked as deleted in their containing directory,
+// returning true if the opertaion was interrupted
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DeleteEmptyDirectories()
+{
+ while(mEmptyDirectories.size() > 0)
+ {
+ std::vector<int64_t> toExamine;
+
+ // Go through list
+ for(std::vector<int64_t>::const_iterator i(mEmptyDirectories.begin()); i != mEmptyDirectories.end(); ++i)
+ {
+#ifndef WIN32
+ if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+ {
+ mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+ // Check for having to stop
+ if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked
+ {
+ // Need to abort now
+ return true;
+ }
+ }
+#endif
+
+ // Do not delete the root directory
+ if(*i == BACKUPSTORE_ROOT_DIRECTORY_ID)
+ {
+ continue;
+ }
+
+ DeleteEmptyDirectory(*i, toExamine);
+ }
+
+ // Remove contents of empty directories
+ mEmptyDirectories.clear();
+ // Swap in new, so it's examined next time round
+ mEmptyDirectories.swap(toExamine);
+ }
+
+ // Not interrupted
+ return false;
+}
+
+void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
+ std::vector<int64_t>& rToExamine)
+{
+ // Load up the directory to potentially delete
+ std::string dirFilename;
+ BackupStoreDirectory dir;
+ int64_t dirSizeInBlocks = 0;
+
+ // BLOCK
+ {
+ MakeObjectFilename(dirId, dirFilename);
+ // Check it actually exists (just in case it gets
+ // added twice to the list)
+ if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename))
+ {
+ // doesn't exist, next!
+ return;
+ }
+ // load
+ std::auto_ptr<RaidFileRead> dirStream(
+ RaidFileRead::Open(mStoreDiscSet, dirFilename));
+ dirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
+ dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+ }
+
+ // Make sure this directory is actually empty
+ if(dir.GetNumberOfEntries() != 0)
+ {
+ // Not actually empty, try next one
+ return;
+ }
+
+ // Candidate for deletion... open containing directory
+ std::string containingDirFilename;
+ BackupStoreDirectory containingDir;
+ int64_t containingDirSizeInBlocksOrig = 0;
+ {
+ MakeObjectFilename(dir.GetContainerID(), containingDirFilename);
+ std::auto_ptr<RaidFileRead> containingDirStream(
+ RaidFileRead::Open(mStoreDiscSet,
+ containingDirFilename));
+ containingDirSizeInBlocksOrig =
+ containingDirStream->GetDiscUsageInBlocks();
+ containingDir.ReadFromStream(*containingDirStream,
+ IOStream::TimeOutInfinite);
+ }
+
+ // Find the entry
+ BackupStoreDirectory::Entry *pdirentry =
+ containingDir.FindEntryByID(dir.GetObjectID());
+ if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0))
+ {
+ // Should be deleted
+ containingDir.DeleteEntry(dir.GetObjectID());
+
+ // Is the containing dir now a candidate for deletion?
+ if(containingDir.GetNumberOfEntries() == 0)
+ {
+ rToExamine.push_back(containingDir.GetObjectID());
+ }
+
+ // Write revised parent directory
+ RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename,
+ mNewRefCounts[containingDir.GetObjectID()]);
+ writeDir.Open(true /* allow overwriting */);
+ containingDir.WriteToStream(writeDir);
+
+ // get the disc usage (must do this before commiting it)
+ int64_t dirSize = writeDir.GetDiscUsageInBlocks();
+
+ // Commit directory
+ writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+ // adjust usage counts for this directory
+ if(dirSize > 0)
+ {
+ int64_t adjust = dirSize - containingDirSizeInBlocksOrig;
+ mBlocksUsedDelta += adjust;
+ mBlocksInDirectoriesDelta += adjust;
+ }
+
+
+ if (--mNewRefCounts[dir.GetObjectID()] == 0)
+ {
+ // Delete the directory itself
+ RaidFileWrite del(mStoreDiscSet, dirFilename,
+ mNewRefCounts[dir.GetObjectID()]);
+ del.Delete();
+ }
+
+ BOX_INFO("Housekeeping removed empty deleted dir " <<
+ BOX_FORMAT_OBJECTID(dirId));
+
+ // And adjust usage counts for the directory that's
+ // just been deleted
+ mBlocksUsedDelta -= dirSizeInBlocks;
+ mBlocksInDirectoriesDelta -= dirSizeInBlocks;
+
+ // Update count
+ ++mEmptyDirectoriesDeleted;
+ }
+}
+
diff --git a/lib/backupstore/HousekeepStoreAccount.h b/lib/backupstore/HousekeepStoreAccount.h
new file mode 100644
index 00000000..61cff868
--- /dev/null
+++ b/lib/backupstore/HousekeepStoreAccount.h
@@ -0,0 +1,111 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: HousekeepStoreAccount.h
+// Purpose: Action class to perform housekeeping on a store account
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef HOUSEKEEPSTOREACCOUNT__H
+#define HOUSEKEEPSTOREACCOUNT__H
+
+#include <string>
+#include <set>
+#include <vector>
+
+class BackupStoreDirectory;
+
+class HousekeepingCallback
+{
+ public:
+ virtual ~HousekeepingCallback() {}
+ virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0) = 0;
+};
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: HousekeepStoreAccount
+// Purpose: Action class to perform housekeeping on a store account
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+class HousekeepStoreAccount
+{
+public:
+ HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot,
+ int StoreDiscSet, HousekeepingCallback* pHousekeepingCallback);
+ ~HousekeepStoreAccount();
+
+ bool DoHousekeeping(bool KeepTryingForever = false);
+ int GetRefCountsAdjusted() { return mRefCountsAdjusted; }
+
+private:
+ // utility functions
+ void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut);
+
+ bool ScanDirectory(int64_t ObjectID);
+ bool DeleteFiles();
+ bool DeleteEmptyDirectories();
+ void DeleteEmptyDirectory(int64_t dirId,
+ std::vector<int64_t>& rToExamine);
+ void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks);
+
+private:
+ typedef struct
+ {
+ int64_t mObjectID;
+ int64_t mInDirectory;
+ int64_t mSizeInBlocks;
+ int32_t mMarkNumber;
+ int32_t mVersionAgeWithinMark; // 0 == current, 1 latest old version, etc
+ bool mIsFlagDeleted; // false for files flagged "Old"
+ } DelEn;
+
+ struct DelEnCompare
+ {
+ bool operator()(const DelEn &x, const DelEn &y);
+ };
+
+ int mAccountID;
+ std::string mStoreRoot;
+ int mStoreDiscSet;
+ HousekeepingCallback* mpHousekeepingCallback;
+
+ int64_t mDeletionSizeTarget;
+
+ std::set<DelEn, DelEnCompare> mPotentialDeletions;
+ int64_t mPotentialDeletionsTotalSize;
+ int64_t mMaxSizeInPotentialDeletions;
+
+ // List of directories which are empty, and might be good for deleting
+ std::vector<int64_t> mEmptyDirectories;
+
+ // The re-calculated blocks used stats
+ int64_t mBlocksUsed;
+ int64_t mBlocksInOldFiles;
+ int64_t mBlocksInDeletedFiles;
+ int64_t mBlocksInDirectories;
+
+ // Deltas from deletion
+ int64_t mBlocksUsedDelta;
+ int64_t mBlocksInOldFilesDelta;
+ int64_t mBlocksInDeletedFilesDelta;
+ int64_t mBlocksInDirectoriesDelta;
+
+ // Deletion count
+ int64_t mFilesDeleted;
+ int64_t mEmptyDirectoriesDeleted;
+
+ // New reference count list
+ std::vector<uint32_t> mNewRefCounts;
+ bool mSuppressRefCountChangeWarnings;
+ int mRefCountsAdjusted;
+
+ // Poll frequency
+ int mCountUntilNextInterprocessMsgCheck;
+};
+
+#endif // HOUSEKEEPSTOREACCOUNT__H
+