Create new refcount database atomically during account check.

Use a temporary refcount db for check instead of an in-memory vector. This
avoid the memory usage problems created by using the vector on large accounts,
but may require us to improve the efficiency of the refcount database itself
to avoid large numbers of small I/O operations. That is very doable now that
we're using a class for it.

Fix some inconsistencies and mistakes in handling reference counts and
info counters during account checks (more to come).

6 files changed, 284 insertions, 256 deletions

diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp
index d9efcd5d..6a4d846c 100644
--- a/lib/backupstore/BackupStoreAccounts.cpp
+++ b/lib/backupstore/BackupStoreAccounts.cpp
@@ -120,10 +120,7 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit,
 		info->Save();
 
 		// Create the refcount database
-		BackupStoreRefCountDatabase::CreateNew(Entry);
-		std::auto_ptr<BackupStoreRefCountDatabase> refcount(
-			BackupStoreRefCountDatabase::Load(Entry, false));
-		refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID);
+		BackupStoreRefCountDatabase::Create(Entry)->Commit();
 	}
 
 	// As the original user...
diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp
index 5e3e0b54..97380037 100644
--- a/lib/backupstore/BackupStoreCheck.cpp
+++ b/lib/backupstore/BackupStoreCheck.cpp
@@ -405,7 +405,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
 			fileOK = false;
 		}
 		// info and refcount databases are OK in the root directory
-		else if(*i == "info" || *i == "refcount.db")
+		else if(*i == "info" || *i == "refcount.db" || *i == "refcount.rdb")
 		{
 			fileOK = true;
 		}
diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp
index 26f9acca..86f75a55 100644
--- a/lib/backupstore/BackupStoreRefCountDatabase.cpp
+++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp
@@ -9,6 +9,8 @@
 
 #include "Box.h"
 
+#include <stdio.h>
+
 #include <algorithm>
 
 #include "BackupStoreRefCountDatabase.h"
@@ -34,12 +36,82 @@
 //
 // --------------------------------------------------------------------------
 BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const
-	BackupStoreAccountDatabase::Entry& rAccount)
+	BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly,
+	bool Temporary, std::auto_ptr<FileStream> apDatabaseFile)
 : mAccount(rAccount),
-  mFilename(GetFilename(rAccount)),
-  mReadOnly(true),
-  mIsModified(false)
+  mFilename(GetFilename(rAccount, Temporary)),
+  mReadOnly(ReadOnly),
+  mIsModified(false),
+  mIsTemporaryFile(Temporary),
+  mapDatabaseFile(apDatabaseFile)
 {
+	ASSERT(!(ReadOnly && Temporary)); // being both doesn't make sense
+}
+
+void BackupStoreRefCountDatabase::Commit()
+{
+	if (!mIsTemporaryFile)
+	{
+		THROW_EXCEPTION_MESSAGE(CommonException, Internal,
+			"Cannot commit a permanent reference count database");
+	}
+
+	if (!mapDatabaseFile.get())
+	{
+		THROW_EXCEPTION_MESSAGE(CommonException, Internal,
+			"Reference count database is already closed");
+	}
+
+	mapDatabaseFile->Close();
+	mapDatabaseFile.reset();
+
+	std::string Final_Filename = GetFilename(mAccount, false);
+
+	#ifdef WIN32
+	if(FileExists(Final_Filename) && unlink(Final_Filename.c_str()) != 0)
+	{
+		THROW_SYS_FILE_ERROR("Failed to delete old permanent refcount "
+			"database file", mFilename, CommonException, OSFileError);
+	}
+	#endif
+
+	if(rename(mFilename.c_str(), Final_Filename.c_str()) != 0)
+	{
+		THROW_SYS_ERROR("Failed to rename temporary refcount database "
+			"file from " << mFilename << " to " <<
+			Final_Filename, CommonException, OSFileError);
+	}
+
+	mFilename = Final_Filename;
+	mIsModified = false;
+	mIsTemporaryFile = false;
+}
+
+void BackupStoreRefCountDatabase::Discard()
+{
+	if (!mIsTemporaryFile)
+	{
+		THROW_EXCEPTION_MESSAGE(CommonException, Internal,
+			"Cannot discard a permanent reference count database");
+	}
+
+	if (!mapDatabaseFile.get())
+	{
+		THROW_EXCEPTION_MESSAGE(CommonException, Internal,
+			"Reference count database is already closed");
+	}
+
+	mapDatabaseFile->Close();
+	mapDatabaseFile.reset();
+
+	if(unlink(mFilename.c_str()) != 0)
+	{
+		THROW_SYS_FILE_ERROR("Failed to delete temporary refcount database "
+			"file ", mFilename, CommonException, OSFileError);
+	}
+
+	mIsModified = false;
+	mIsTemporaryFile = false;
 }
 
 // --------------------------------------------------------------------------
@@ -52,16 +124,27 @@ BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const
 // --------------------------------------------------------------------------
 BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase()
 {
+	if (mIsTemporaryFile)
+	{
+		THROW_EXCEPTION_MESSAGE(CommonException, Internal,
+			"BackupStoreRefCountDatabase destroyed without "
+			"explicit commit or discard");
+		Discard();
+	}
 }
 
 std::string BackupStoreRefCountDatabase::GetFilename(const
-	BackupStoreAccountDatabase::Entry& rAccount)
+	BackupStoreAccountDatabase::Entry& rAccount, bool Temporary)
 {
 	std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount);
 	ASSERT(RootDir[RootDir.size() - 1] == '/' ||
 		RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR);
 
-	std::string fn(RootDir + REFCOUNT_FILENAME ".db");
+	std::string fn(RootDir + REFCOUNT_FILENAME ".rdb");
+	if(Temporary)
+	{
+		fn += "X";
+	}
 	RaidFileController &rcontroller(RaidFileController::GetController());
 	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet()));
 	return RaidFileUtil::MakeWriteFileName(rdiscSet, fn);
@@ -77,35 +160,48 @@ std::string BackupStoreRefCountDatabase::GetFilename(const
 //		Created: 2003/08/28
 //
 // --------------------------------------------------------------------------
-void BackupStoreRefCountDatabase::Create(const
-	BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite)
+std::auto_ptr<BackupStoreRefCountDatabase>
+	BackupStoreRefCountDatabase::Create
+	(const BackupStoreAccountDatabase::Entry& rAccount)
 {
 	// Initial header
 	refcount_StreamFormat hdr;
 	hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE);
 	hdr.mAccountID = htonl(rAccount.GetID());
 	
-	// Generate the filename
-	std::string Filename = GetFilename(rAccount);
+	std::string Filename = GetFilename(rAccount, true); // temporary
 
 	// Open the file for writing
-	if (FileExists(Filename) && !AllowOverwrite)
-	{
-		THROW_FILE_ERROR("Failed to overwrite refcount database: "
-			"not allowed here", Filename, RaidFileException,
-			CannotOverwriteExistingFile);
-	}
-
-	int flags = O_CREAT | O_BINARY | O_RDWR;
-	if (!AllowOverwrite)
+	if (FileExists(Filename))
 	{
-		flags |= O_EXCL;
+		BOX_WARNING(BOX_FILE_MESSAGE(Filename, "Overwriting existing "
+			"temporary reference count database"));
+		if (unlink(Filename.c_str()) != 0)
+		{
+			THROW_SYS_FILE_ERROR("Failed to delete old temporary "
+				"reference count database file", Filename,
+				CommonException, OSFileError);
+		}
 	}
 
+	int flags = O_CREAT | O_BINARY | O_RDWR | O_EXCL;
 	std::auto_ptr<FileStream> DatabaseFile(new FileStream(Filename, flags));
 	
 	// Write header
 	DatabaseFile->Write(&hdr, sizeof(hdr));
+
+	// Make new object
+	std::auto_ptr<BackupStoreRefCountDatabase> refcount(
+		new BackupStoreRefCountDatabase(rAccount, false, true,
+			DatabaseFile));
+	
+	// The root directory must always have one reference for a database
+	// to be valid, so set that now on the new database. This will leave
+	// mIsModified set to true.
+	refcount->SetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1);
+
+	// return it to caller
+	return refcount;
 }
 
 // --------------------------------------------------------------------------
@@ -122,12 +218,13 @@ void BackupStoreRefCountDatabase::Create(const
 std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load(
 	const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly)
 {
-	// Generate the filename
-	std::string filename = GetFilename(rAccount);
+	// Generate the filename. Cannot open a temporary database, so it must
+	// be a permanent one.
+	std::string Filename = GetFilename(rAccount, false);
 	int flags = ReadOnly ? O_RDONLY : O_RDWR;
 
 	// Open the file for read/write
-	std::auto_ptr<FileStream> dbfile(new FileStream(filename,
+	std::auto_ptr<FileStream> dbfile(new FileStream(Filename,
 		flags | O_BINARY));
 	
 	// Read in a header
@@ -135,7 +232,7 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load(
 	if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
 	{
 		THROW_FILE_ERROR("Failed to read refcount database: "
-			"short read", filename, BackupStoreException,
+			"short read", Filename, BackupStoreException,
 			CouldNotLoadStoreInfo);
 	}
 	
@@ -144,16 +241,14 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load(
 		(int32_t)ntohl(hdr.mAccountID) != rAccount.GetID())
 	{
 		THROW_FILE_ERROR("Failed to read refcount database: "
-			"bad magic number", filename, BackupStoreException,
+			"bad magic number", Filename, BackupStoreException,
 			BadStoreInfoOnLoad);
 	}
 	
 	// Make new object
-	std::auto_ptr<BackupStoreRefCountDatabase> refcount(new BackupStoreRefCountDatabase(rAccount));
-	
-	// Put in basic location info
-	refcount->mReadOnly = ReadOnly;
-	refcount->mapDatabaseFile = dbfile;
+	std::auto_ptr<BackupStoreRefCountDatabase> refcount(
+		new BackupStoreRefCountDatabase(rAccount, ReadOnly, false,
+			dbfile));
 	
 	// return it to caller
 	return refcount;
@@ -229,6 +324,7 @@ void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID,
 	mapDatabaseFile->Seek(offset, SEEK_SET);
 	refcount_t RefCountNetOrder = htonl(NewRefCount);
 	mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder));
+	mIsModified = true;
 }
 
 bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID)
diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h
index 93c79afb..f0c1222f 100644
--- a/lib/backupstore/BackupStoreRefCountDatabase.h
+++ b/lib/backupstore/BackupStoreRefCountDatabase.h
@@ -15,6 +15,7 @@
 #include <vector>
 
 #include "BackupStoreAccountDatabase.h"
+#include "BackupStoreConstants.h"
 #include "FileStream.h"
 
 class BackupStoreCheck;
@@ -59,17 +60,17 @@ public:
 private:
 	// Creation through static functions only
 	BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry&
-		rAccount);
+		rAccount, bool ReadOnly, bool Temporary,
+		std::auto_ptr<FileStream> apDatabaseFile);
 	// No copying allowed
 	BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &);
 	
 public:
-	// Create a new database for a new account. This method will refuse
-	// to overwrite any existing file.
-	static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount)
-	{
-		Create(rAccount, false);
-	}
+	// Create a new database for a new account.
+	static std::auto_ptr<BackupStoreRefCountDatabase> Create
+		(const BackupStoreAccountDatabase::Entry& rAccount);
+	void Commit();
+	void Discard();
 	
 	// Load it from the store
 	static std::auto_ptr<BackupStoreRefCountDatabase> Load(const
@@ -87,20 +88,9 @@ public:
 	bool RemoveReference(int64_t ObjectID);
 
 private:
-	// Create a new database for an existing account. Used during
-	// account checking if opening the old database throws an exception.
-	// This method will overwrite any existing file.
-	static void CreateForRegeneration(const
-		BackupStoreAccountDatabase::Entry& rAccount)
-	{
-		Create(rAccount, true);
-	}
-
-	static void Create(const BackupStoreAccountDatabase::Entry& rAccount,
-		bool AllowOverwrite);
-
 	static std::string GetFilename(const BackupStoreAccountDatabase::Entry&
-		rAccount);
+		rAccount, bool Temporary);
+
 	IOStream::pos_type GetSize() const
 	{
 		return mapDatabaseFile->GetPosition() +
@@ -122,7 +112,13 @@ private:
 	std::string mFilename;
 	bool mReadOnly;
 	bool mIsModified;
+	bool mIsTemporaryFile;
 	std::auto_ptr<FileStream> mapDatabaseFile;
+
+	bool NeedsCommitOrDiscard()
+	{
+		return mapDatabaseFile.get() && mIsModified && mIsTemporaryFile;
+	}
 };
 
 #endif // BACKUPSTOREREFCOUNTDATABASE__H
diff --git a/lib/backupstore/HousekeepStoreAccount.cpp b/lib/backupstore/HousekeepStoreAccount.cpp
index 75feda7f..41f43150 100644
--- a/lib/backupstore/HousekeepStoreAccount.cpp
+++ b/lib/backupstore/HousekeepStoreAccount.cpp
@@ -61,8 +61,6 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID,
 	  mBlocksInDirectoriesDelta(0),
 	  mFilesDeleted(0),
 	  mEmptyDirectoriesDeleted(0),
-	  mSuppressRefCountChangeWarnings(false),
-	  mRefCountsAdjusted(0),
 	  mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
 {
 	std::ostringstream tag;
@@ -105,7 +103,7 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
 	{
 		if(KeepTryingForever)
 		{
-			BOX_WARNING("Failed to lock account for housekeeping, "
+			BOX_INFO("Failed to lock account for housekeeping, "
 				"still trying...");
 			while(!writeLock.TryAndGetLock(writeLockFilename,
 				0600 /* restrictive file permissions */))
@@ -143,99 +141,105 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
 		mDeletionSizeTarget = 0;
 	}
 
-	// initialise the refcount database
-	mNewRefCounts.clear();
-	// try to pre-allocate as much memory as we need
-	mNewRefCounts.reserve(info->GetLastObjectIDUsed());
-	// initialise the refcount of the root entry
-	mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0);
-	mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1;
+	BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
+	mapNewRefs = BackupStoreRefCountDatabase::Create(account);
 
 	// Scan the directory for potential things to delete
 	// This will also remove eligible items marked with RemoveASAP
-	bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
+	bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID,
+		*info);
+
+	if(!continueHousekeeping)
+	{
+		// The scan was incomplete, so the new block counts are
+		// incorrect, we can't rely on them. It's better to discard
+		// the new info and adjust the old one instead.
+		info = pOldInfo;
+
+		// We're about to reset counters and exit, so report what
+		// happened now.
+		BOX_INFO("Housekeeping on account " << 
+			BOX_FORMAT_ACCOUNT(mAccountID) << " removed " <<
+			(0 - mBlocksUsedDelta) << " blocks (" << 
+			mFilesDeleted << " files, " <<
+			mEmptyDirectoriesDeleted << " dirs) and the directory "
+			"scan was interrupted");
+	}
+
+	// If housekeeping made any changes, such as deleting RemoveASAP files,
+	// the differences in block counts will be recorded in the deltas.
+	info->ChangeBlocksUsed(mBlocksUsedDelta);
+	info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+	info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+
+	// Reset the delta counts for files, as they will include 
+	// RemoveASAP flagged files deleted during the initial scan.
+	// keep removeASAPBlocksUsedDelta for reporting
+	int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
+	mBlocksUsedDelta = 0;
+	mBlocksInOldFilesDelta = 0;
+	mBlocksInDeletedFilesDelta = 0;
 
 	// If scan directory stopped for some reason, probably parent
 	// instructed to terminate, stop now.
+	//
+	// We can only update the refcount database if we successfully
+	// finished our scan of all directories, otherwise we don't actually
+	// know which of the new counts are valid and which aren't
+	// (we might not have seen second references to some objects, etc.).
+
 	if(!continueHousekeeping)
 	{
-		// If any files were marked "delete now", then update
-		// the size of the store.
-		if(mBlocksUsedDelta != 0 || 
-			mBlocksInOldFilesDelta != 0 ||
-			mBlocksInDeletedFilesDelta != 0)
-		{
-			info->ChangeBlocksUsed(mBlocksUsedDelta);
-			info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
-			info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
-
-			// Save the store info back
-			info->ReportChangesTo(*pOldInfo);
-			info->Save();
-		}
-	
+		mapNewRefs->Discard();
+		info->Save();
 		return false;
 	}
 
-	// Log any difference in opinion between the values recorded in
-	// the store info, and the values just calculated for space usage.
-	// BLOCK
+	// Report any UNexpected changes, and consider them to be errors.
+	// Do this before applying the expected changes below.
+	mErrorCount += info->ReportChangesTo(*pOldInfo);
+	info->Save();
+
+	// We want to compare the mapNewRefs to apOldRefs before we delete any
+	// files, because that will also change the reference count in a way that's not an error.
+
+	// try to load the reference count database
+	try
 	{
-		int64_t used = info->GetBlocksUsed();
-		int64_t usedOld = info->GetBlocksInOldFiles();
-		int64_t usedDeleted = info->GetBlocksInDeletedFiles();
-		int64_t usedDirectories = info->GetBlocksInDirectories();
-
-		// If the counts were wrong, taking into account RemoveASAP 
-		// items deleted, log a message
-		if((used + mBlocksUsedDelta) != mBlocksUsed 
-			|| (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
-			|| (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles 
-			|| usedDirectories != mBlocksInDirectories)
+		std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs =
+			BackupStoreRefCountDatabase::Load(account, false);
+
+		int64_t LastUsedObjectIdOnDisk = mapNewRefs->GetLastObjectIDUsed();
+
+		if (apOldRefs->GetLastObjectIDUsed() > LastUsedObjectIdOnDisk)
 		{
-			// Log this
-			BOX_ERROR("Housekeeping on account " << 
-				BOX_FORMAT_ACCOUNT(mAccountID) << " found "
-				"and fixed wrong block counts: "
-				"used (" <<
-				(used + mBlocksUsedDelta) << "," <<
-				mBlocksUsed << "), old (" <<
-				(usedOld + mBlocksInOldFilesDelta) << "," <<
-				mBlocksInOldFiles << "), deleted (" <<
-				(usedDeleted + mBlocksInDeletedFilesDelta) <<
-				"," << mBlocksInDeletedFiles << "), dirs (" <<
-				usedDirectories << "," << mBlocksInDirectories
-				<< ")");
+			LastUsedObjectIdOnDisk = apOldRefs->GetLastObjectIDUsed();
 		}
-		
-		// If the current values don't match, store them
-		if(used != mBlocksUsed 
-			|| usedOld != mBlocksInOldFiles
-			|| usedDeleted != mBlocksInDeletedFiles 
-			|| usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
-		{	
-			// Set corrected values in store info
-			info->CorrectAllUsedValues(mBlocksUsed,
-				mBlocksInOldFiles, mBlocksInDeletedFiles,
-				mBlocksInDirectories + mBlocksInDirectoriesDelta);
-
-			info->ReportChangesTo(*pOldInfo);
-			info->Save();
+
+		for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
+			ObjectID < LastUsedObjectIdOnDisk; ObjectID++)
+		{
+			if (apOldRefs->GetRefCount(ObjectID) !=
+				mapNewRefs->GetRefCount(ObjectID))
+			{
+				BOX_WARNING("Reference count of object " <<
+					BOX_FORMAT_OBJECTID(ObjectID) <<
+					" changed from " <<
+					apOldRefs->GetRefCount(ObjectID) <<
+					" to " << mapNewRefs->GetRefCount(ObjectID));
+				mErrorCount++;
+			}
 		}
 	}
-	
-	// Reset the delta counts for files, as they will include 
-	// RemoveASAP flagged files deleted during the initial scan.
-
-	// keep for reporting
-	int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
+	catch(BoxException &e)
+	{
+		BOX_WARNING("Reference count database was missing or "
+			"corrupted during housekeeping, cannot check for "
+			"errors.");
+	}
 
-	mBlocksUsedDelta = 0;
-	mBlocksInOldFilesDelta = 0;
-	mBlocksInDeletedFilesDelta = 0;
-	
 	// Go and delete items from the accounts
-	bool deleteInterrupted = DeleteFiles();
+	bool deleteInterrupted = DeleteFiles(*info);
 	
 	// If that wasn't interrupted, remove any empty directories which 
 	// are also marked as deleted in their containing directory
@@ -256,89 +260,6 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
 			(deleteInterrupted?" and was interrupted":""));
 	}
 
-	// We can only update the refcount database if we successfully
-	// finished our scan of all directories, otherwise we don't actually
-	// know which of the new counts are valid and which aren't
-	// (we might not have seen second references to some objects, etc.)
-
-	BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
-	std::auto_ptr<BackupStoreRefCountDatabase> apReferences;
-
-	// try to load the reference count database
-	try
-	{
-		apReferences = BackupStoreRefCountDatabase::Load(account,
-			false);
-	}
-	catch(BoxException &e)
-	{
-		BOX_WARNING("Reference count database is missing or corrupted "
-			"during housekeeping, creating a new one.");
-		mSuppressRefCountChangeWarnings = true;
-		BackupStoreRefCountDatabase::CreateForRegeneration(account);
-		apReferences = BackupStoreRefCountDatabase::Load(account,
-			false);
-	}
-
-	int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed();
-
-	for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
-		ObjectID < mNewRefCounts.size(); ObjectID++)
-	{
-		if (ObjectID > LastUsedObjectIdOnDisk)
-		{
-			if (!mSuppressRefCountChangeWarnings)
-			{
-				BOX_WARNING("Reference count of object " <<
-					BOX_FORMAT_OBJECTID(ObjectID) <<
-					" not found in database, added"
-					" with " << mNewRefCounts[ObjectID] <<
-					" references");
-			}
-			apReferences->SetRefCount(ObjectID,
-				mNewRefCounts[ObjectID]);
-			mRefCountsAdjusted++;
-			LastUsedObjectIdOnDisk = ObjectID;
-			continue;
-		}
-
-		BackupStoreRefCountDatabase::refcount_t OldRefCount =
-			apReferences->GetRefCount(ObjectID);
-
-		if (OldRefCount != mNewRefCounts[ObjectID])
-		{
-			BOX_WARNING("Reference count of object " <<
-				BOX_FORMAT_OBJECTID(ObjectID) <<
-				" changed from " << OldRefCount <<
-				" to " << mNewRefCounts[ObjectID]);
-			apReferences->SetRefCount(ObjectID,
-				mNewRefCounts[ObjectID]);
-			mRefCountsAdjusted++;
-		}
-	}
-
-	// zero excess references in the database
-	for (int64_t ObjectID = mNewRefCounts.size(); 
-		ObjectID <= LastUsedObjectIdOnDisk; ObjectID++)
-	{
-		BackupStoreRefCountDatabase::refcount_t OldRefCount =
-			apReferences->GetRefCount(ObjectID);
-		BackupStoreRefCountDatabase::refcount_t NewRefCount = 0;
-
-		if (OldRefCount != NewRefCount)
-		{
-			BOX_WARNING("Reference count of object " <<
-				BOX_FORMAT_OBJECTID(ObjectID) <<
-				" changed from " << OldRefCount <<
-				" to " << NewRefCount << " (not found)");
-			apReferences->SetRefCount(ObjectID, NewRefCount);
-			mRefCountsAdjusted++;
-		}
-	}
-
-	// force file to be saved and closed before releasing the lock below
-	apReferences.reset();
-	
 	// Make sure the delta's won't cause problems if the counts are 
 	// really wrong, and it wasn't fixed because the store was 
 	// updated during the scan.
@@ -364,11 +285,14 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
 	info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
 	info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
 	info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta);
-	
+
 	// Save the store info back
-	info->ReportChangesTo(*pOldInfo);
 	info->Save();
 	
+	// force file to be saved and closed before releasing the lock below
+	mapNewRefs->Commit();
+	mapNewRefs.reset();
+
 	// Explicity release the lock (would happen automatically on 
 	// going out of scope, included for code clarity)
 	writeLock.ReleaseLock();
@@ -405,7 +329,8 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF
 //		Created: 11/12/03
 //
 // --------------------------------------------------------------------------
-bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
+bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID,
+	BackupStoreInfo& rBackupStoreInfo)
 {
 #ifndef WIN32
 	if((--mCountUntilNextInterprocessMsgCheck) <= 0)
@@ -459,11 +384,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
 		while((en = i.Next()) != 0)
 		{
 			// This directory references this object
-			if (mNewRefCounts.size() <= en->GetObjectID())
-			{
-				mNewRefCounts.resize(en->GetObjectID() + 1, 0);
-			}
-			mNewRefCounts[en->GetObjectID()]++;
+			mapNewRefs->AddReference(en->GetObjectID());
 		}
 	}
 
@@ -482,10 +403,11 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
 			{
 				int16_t enFlags = en->GetFlags();
 				if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0
-					&& (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+					&& (en->IsDeleted() || en->IsOld()))
 				{
 					// Delete this immediately.
-					DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks);
+					DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename,
+						originalDirSizeInBlocks, rBackupStoreInfo);
 					
 					// flag as having done something
 					deletedSomething = true;
@@ -624,7 +546,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
 			// Next level
 			ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir);
 			
-			if(!ScanDirectory(en->GetObjectID()))
+			if(!ScanDirectory(en->GetObjectID(), rBackupStoreInfo))
 			{
 				// Halting operation
 				return false;
@@ -687,7 +609,7 @@ bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount
 //		Created: 15/12/03
 //
 // --------------------------------------------------------------------------
-bool HousekeepStoreAccount::DeleteFiles()
+bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo)
 {
 	// Only delete files if the deletion target is greater than zero
 	// (otherwise we delete one file each time round, which gradually deletes the old versions)
@@ -727,12 +649,13 @@ bool HousekeepStoreAccount::DeleteFiles()
 		}
 		
 		// Delete the file
-		DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig);
+		DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename,
+			dirSizeInBlocksOrig, rBackupStoreInfo);
 		BOX_INFO("Housekeeping removed " <<
 			(i->mIsFlagDeleted ? "deleted" : "old") <<
 			" file " << BOX_FORMAT_OBJECTID(i->mObjectID) <<
 			" from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory));
-		
+
 		// Stop if the deletion target has been matched or exceeded
 		// (checking here rather than at the beginning will tend to reduce the
 		// space to slightly less than the soft limit, which will allow the backup
@@ -758,7 +681,9 @@ bool HousekeepStoreAccount::DeleteFiles()
 //		Created: 15/7/04
 //
 // --------------------------------------------------------------------------
-void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks)
+void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID,
+	BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename,
+	int64_t OriginalDirSizeInBlocks, BackupStoreInfo& rBackupStoreInfo)
 {
 	// Find the entry inside the directory
 	bool wasDeleted = false;
@@ -783,8 +708,8 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
 		}
 		
 		// Record the flags it's got set
-		wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0);
-		wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0);
+		wasDeleted = pentry->IsDeleted();
+		wasOldVersion = pentry->IsOld();
 		// Check this should be deleted
 		if(!wasDeleted && !wasOldVersion)
 		{
@@ -853,7 +778,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
 			std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename));
 			// And open a write file to overwrite the other directory entry
 			padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet,
-				objFilenameOlder, mNewRefCounts[ObjectID]));
+				objFilenameOlder, mapNewRefs->GetRefCount(ObjectID)));
 			padjustedEntry->Open(true /* allow overwriting */);
 
 			if(pentry->GetDependsNewer() == 0)
@@ -872,11 +797,11 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
 			int64_t newSize = padjustedEntry->GetDiscUsageInBlocks();
 			int64_t sizeDelta = newSize - polder->GetSizeInBlocks();
 			mBlocksUsedDelta += sizeDelta;
-			if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)
+			if(polder->IsDeleted())
 			{
 				mBlocksInDeletedFilesDelta += sizeDelta;
 			}
-			if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0)
+			if(polder->IsOld())
 			{
 				mBlocksInOldFilesDelta += sizeDelta;
 			}
@@ -894,17 +819,17 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
 	int64_t dirRevisedSize = 0;
 	{
 		RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename,
-			mNewRefCounts[InDirectory]);
+			mapNewRefs->GetRefCount(InDirectory));
 		writeDir.Open(true /* allow overwriting */);
 		rDirectory.WriteToStream(writeDir);
 
-		// get the disc usage (must do this before commiting it)
+		// Get the disc usage (must do this before commiting it)
 		dirRevisedSize = writeDir.GetDiscUsageInBlocks();
 
 		// Commit directory
 		writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
 
-		// adjust usage counts for this directory
+		// Adjust block counts if the directory itself changed in size
 		if(dirRevisedSize > 0)
 		{
 			int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks;
@@ -921,29 +846,38 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
 	}
 
 	// Drop reference count by one. If it reaches zero, delete the file.
-	if(--mNewRefCounts[ObjectID] == 0)
+	if(!mapNewRefs->RemoveReference(ObjectID))
 	{
 		// Delete from disc
 		BOX_TRACE("Removing unreferenced object " <<
 			BOX_FORMAT_OBJECTID(ObjectID));
 		std::string objFilename;
 		MakeObjectFilename(ObjectID, objFilename);
-		RaidFileWrite del(mStoreDiscSet, objFilename,
-			mNewRefCounts[ObjectID]);
+		RaidFileWrite del(mStoreDiscSet, objFilename, 0);
 		del.Delete();
 	}
 	else
 	{
 		BOX_TRACE("Preserving object " <<
 			BOX_FORMAT_OBJECTID(ObjectID) << " with " <<
-			mNewRefCounts[ObjectID] << " references");
+			mapNewRefs->GetRefCount(ObjectID) << " references");
 	}
 
 	// Adjust counts for the file
 	++mFilesDeleted;
 	mBlocksUsedDelta -= deletedFileSizeInBlocks;
-	if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
-	if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
+
+	if(wasDeleted)
+	{
+		mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
+		rBackupStoreInfo.AdjustNumDeletedFiles(-1);
+	}
+
+	if(wasOldVersion)
+	{
+		mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
+		rBackupStoreInfo.AdjustNumOldFiles(-1);
+	}
 	
 	// Delete the directory?
 	// Do this if... dir has zero entries, and is marked as deleted in it's containing directory
@@ -1068,7 +1002,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
 
 		// Write revised parent directory
 		RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename,
-			mNewRefCounts[containingDir.GetObjectID()]);
+			mapNewRefs->GetRefCount(containingDir.GetObjectID()));
 		writeDir.Open(true /* allow overwriting */);
 		containingDir.WriteToStream(writeDir);
 
@@ -1087,11 +1021,10 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
 		}
 
 		
-		if (--mNewRefCounts[dir.GetObjectID()] == 0)
+		if (mapNewRefs->RemoveReference(dir.GetObjectID()))
 		{
 			// Delete the directory itself
-			RaidFileWrite del(mStoreDiscSet, dirFilename,
-				mNewRefCounts[dir.GetObjectID()]);
+			RaidFileWrite del(mStoreDiscSet, dirFilename, 0);
 			del.Delete();
 		}
 
diff --git a/lib/backupstore/HousekeepStoreAccount.h b/lib/backupstore/HousekeepStoreAccount.h
index cfa05a2e..a63c5536 100644
--- a/lib/backupstore/HousekeepStoreAccount.h
+++ b/lib/backupstore/HousekeepStoreAccount.h
@@ -15,6 +15,7 @@
 #include <vector>
 
 class BackupStoreDirectory;
+class BackupStoreRefCountDatabase;
 
 class HousekeepingCallback
 {
@@ -39,18 +40,22 @@ public:
 	~HousekeepStoreAccount();
 	
 	bool DoHousekeeping(bool KeepTryingForever = false);
-	int GetRefCountsAdjusted() { return mRefCountsAdjusted; }
+	int GetErrorCount() { return mErrorCount; }
 	
 private:
 	// utility functions
 	void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut);
 
-	bool ScanDirectory(int64_t ObjectID);
-	bool DeleteFiles();
+	bool ScanDirectory(int64_t ObjectID, BackupStoreInfo& rBackupStoreInfo);
+	bool DeleteFiles(BackupStoreInfo& rBackupStoreInfo);
 	bool DeleteEmptyDirectories();
 	void DeleteEmptyDirectory(int64_t dirId,
 		std::vector<int64_t>& rToExamine);
-	void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks);
+	void DeleteFile(int64_t InDirectory, int64_t ObjectID,
+		BackupStoreDirectory &rDirectory,
+		const std::string &rDirectoryFilename,
+		int64_t OriginalDirSizeInBlocks,
+		BackupStoreInfo& rBackupStoreInfo);
 
 private:
 	typedef struct
@@ -81,6 +86,9 @@ private:
 	
 	// List of directories which are empty, and might be good for deleting
 	std::vector<int64_t> mEmptyDirectories;
+
+	// Count of errors found and fixed
+	int64_t mErrorCount;
 	
 	// The re-calculated blocks used stats
 	int64_t mBlocksUsed;
@@ -99,9 +107,7 @@ private:
 	int64_t mEmptyDirectoriesDeleted;
 
 	// New reference count list
-	std::vector<uint32_t> mNewRefCounts;
-	bool mSuppressRefCountChangeWarnings;
-	int mRefCountsAdjusted;
+	std::auto_ptr<BackupStoreRefCountDatabase> mapNewRefs;
 	
 	// Poll frequency
 	int mCountUntilNextInterprocessMsgCheck;