summaryrefslogtreecommitdiff
path: root/lib/backupstore
diff options
context:
space:
mode:
Diffstat (limited to 'lib/backupstore')
-rw-r--r--lib/backupstore/BackupStoreAccounts.cpp5
-rw-r--r--lib/backupstore/BackupStoreCheck.cpp2
-rw-r--r--lib/backupstore/BackupStoreRefCountDatabase.cpp156
-rw-r--r--lib/backupstore/BackupStoreRefCountDatabase.h36
-rw-r--r--lib/backupstore/HousekeepStoreAccount.cpp321
-rw-r--r--lib/backupstore/HousekeepStoreAccount.h20
6 files changed, 284 insertions, 256 deletions
diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp
index d9efcd5d..6a4d846c 100644
--- a/lib/backupstore/BackupStoreAccounts.cpp
+++ b/lib/backupstore/BackupStoreAccounts.cpp
@@ -120,10 +120,7 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit,
info->Save();
// Create the refcount database
- BackupStoreRefCountDatabase::CreateNew(Entry);
- std::auto_ptr<BackupStoreRefCountDatabase> refcount(
- BackupStoreRefCountDatabase::Load(Entry, false));
- refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID);
+ BackupStoreRefCountDatabase::Create(Entry)->Commit();
}
// As the original user...
diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp
index 5e3e0b54..97380037 100644
--- a/lib/backupstore/BackupStoreCheck.cpp
+++ b/lib/backupstore/BackupStoreCheck.cpp
@@ -405,7 +405,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
fileOK = false;
}
// info and refcount databases are OK in the root directory
- else if(*i == "info" || *i == "refcount.db")
+ else if(*i == "info" || *i == "refcount.db" || *i == "refcount.rdb")
{
fileOK = true;
}
diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp
index 26f9acca..86f75a55 100644
--- a/lib/backupstore/BackupStoreRefCountDatabase.cpp
+++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp
@@ -9,6 +9,8 @@
#include "Box.h"
+#include <stdio.h>
+
#include <algorithm>
#include "BackupStoreRefCountDatabase.h"
@@ -34,12 +36,82 @@
//
// --------------------------------------------------------------------------
BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const
- BackupStoreAccountDatabase::Entry& rAccount)
+ BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly,
+ bool Temporary, std::auto_ptr<FileStream> apDatabaseFile)
: mAccount(rAccount),
- mFilename(GetFilename(rAccount)),
- mReadOnly(true),
- mIsModified(false)
+ mFilename(GetFilename(rAccount, Temporary)),
+ mReadOnly(ReadOnly),
+ mIsModified(false),
+ mIsTemporaryFile(Temporary),
+ mapDatabaseFile(apDatabaseFile)
{
+ ASSERT(!(ReadOnly && Temporary)); // being both doesn't make sense
+}
+
+void BackupStoreRefCountDatabase::Commit()
+{
+ if (!mIsTemporaryFile)
+ {
+ THROW_EXCEPTION_MESSAGE(CommonException, Internal,
+ "Cannot commit a permanent reference count database");
+ }
+
+ if (!mapDatabaseFile.get())
+ {
+ THROW_EXCEPTION_MESSAGE(CommonException, Internal,
+ "Reference count database is already closed");
+ }
+
+ mapDatabaseFile->Close();
+ mapDatabaseFile.reset();
+
+ std::string Final_Filename = GetFilename(mAccount, false);
+
+ #ifdef WIN32
+ if(FileExists(Final_Filename) && unlink(Final_Filename.c_str()) != 0)
+ {
+ THROW_SYS_FILE_ERROR("Failed to delete old permanent refcount "
+ "database file", mFilename, CommonException, OSFileError);
+ }
+ #endif
+
+ if(rename(mFilename.c_str(), Final_Filename.c_str()) != 0)
+ {
+ THROW_SYS_ERROR("Failed to rename temporary refcount database "
+ "file from " << mFilename << " to " <<
+ Final_Filename, CommonException, OSFileError);
+ }
+
+ mFilename = Final_Filename;
+ mIsModified = false;
+ mIsTemporaryFile = false;
+}
+
+void BackupStoreRefCountDatabase::Discard()
+{
+ if (!mIsTemporaryFile)
+ {
+ THROW_EXCEPTION_MESSAGE(CommonException, Internal,
+ "Cannot discard a permanent reference count database");
+ }
+
+ if (!mapDatabaseFile.get())
+ {
+ THROW_EXCEPTION_MESSAGE(CommonException, Internal,
+ "Reference count database is already closed");
+ }
+
+ mapDatabaseFile->Close();
+ mapDatabaseFile.reset();
+
+ if(unlink(mFilename.c_str()) != 0)
+ {
+ THROW_SYS_FILE_ERROR("Failed to delete temporary refcount database "
+ "file ", mFilename, CommonException, OSFileError);
+ }
+
+ mIsModified = false;
+ mIsTemporaryFile = false;
}
// --------------------------------------------------------------------------
@@ -52,16 +124,27 @@ BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const
// --------------------------------------------------------------------------
BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase()
{
+ if (mIsTemporaryFile)
+ {
+ THROW_EXCEPTION_MESSAGE(CommonException, Internal,
+ "BackupStoreRefCountDatabase destroyed without "
+ "explicit commit or discard");
+ Discard();
+ }
}
std::string BackupStoreRefCountDatabase::GetFilename(const
- BackupStoreAccountDatabase::Entry& rAccount)
+ BackupStoreAccountDatabase::Entry& rAccount, bool Temporary)
{
std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount);
ASSERT(RootDir[RootDir.size() - 1] == '/' ||
RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR);
- std::string fn(RootDir + REFCOUNT_FILENAME ".db");
+ std::string fn(RootDir + REFCOUNT_FILENAME ".rdb");
+ if(Temporary)
+ {
+ fn += "X";
+ }
RaidFileController &rcontroller(RaidFileController::GetController());
RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet()));
return RaidFileUtil::MakeWriteFileName(rdiscSet, fn);
@@ -77,35 +160,48 @@ std::string BackupStoreRefCountDatabase::GetFilename(const
// Created: 2003/08/28
//
// --------------------------------------------------------------------------
-void BackupStoreRefCountDatabase::Create(const
- BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite)
+std::auto_ptr<BackupStoreRefCountDatabase>
+ BackupStoreRefCountDatabase::Create
+ (const BackupStoreAccountDatabase::Entry& rAccount)
{
// Initial header
refcount_StreamFormat hdr;
hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE);
hdr.mAccountID = htonl(rAccount.GetID());
- // Generate the filename
- std::string Filename = GetFilename(rAccount);
+ std::string Filename = GetFilename(rAccount, true); // temporary
// Open the file for writing
- if (FileExists(Filename) && !AllowOverwrite)
- {
- THROW_FILE_ERROR("Failed to overwrite refcount database: "
- "not allowed here", Filename, RaidFileException,
- CannotOverwriteExistingFile);
- }
-
- int flags = O_CREAT | O_BINARY | O_RDWR;
- if (!AllowOverwrite)
+ if (FileExists(Filename))
{
- flags |= O_EXCL;
+ BOX_WARNING(BOX_FILE_MESSAGE(Filename, "Overwriting existing "
+ "temporary reference count database"));
+ if (unlink(Filename.c_str()) != 0)
+ {
+ THROW_SYS_FILE_ERROR("Failed to delete old temporary "
+ "reference count database file", Filename,
+ CommonException, OSFileError);
+ }
}
+ int flags = O_CREAT | O_BINARY | O_RDWR | O_EXCL;
std::auto_ptr<FileStream> DatabaseFile(new FileStream(Filename, flags));
// Write header
DatabaseFile->Write(&hdr, sizeof(hdr));
+
+ // Make new object
+ std::auto_ptr<BackupStoreRefCountDatabase> refcount(
+ new BackupStoreRefCountDatabase(rAccount, false, true,
+ DatabaseFile));
+
+ // The root directory must always have one reference for a database
+ // to be valid, so set that now on the new database. This will leave
+ // mIsModified set to true.
+ refcount->SetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1);
+
+ // return it to caller
+ return refcount;
}
// --------------------------------------------------------------------------
@@ -122,12 +218,13 @@ void BackupStoreRefCountDatabase::Create(const
std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load(
const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly)
{
- // Generate the filename
- std::string filename = GetFilename(rAccount);
+ // Generate the filename. Cannot open a temporary database, so it must
+ // be a permanent one.
+ std::string Filename = GetFilename(rAccount, false);
int flags = ReadOnly ? O_RDONLY : O_RDWR;
// Open the file for read/write
- std::auto_ptr<FileStream> dbfile(new FileStream(filename,
+ std::auto_ptr<FileStream> dbfile(new FileStream(Filename,
flags | O_BINARY));
// Read in a header
@@ -135,7 +232,7 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load(
if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
{
THROW_FILE_ERROR("Failed to read refcount database: "
- "short read", filename, BackupStoreException,
+ "short read", Filename, BackupStoreException,
CouldNotLoadStoreInfo);
}
@@ -144,16 +241,14 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load(
(int32_t)ntohl(hdr.mAccountID) != rAccount.GetID())
{
THROW_FILE_ERROR("Failed to read refcount database: "
- "bad magic number", filename, BackupStoreException,
+ "bad magic number", Filename, BackupStoreException,
BadStoreInfoOnLoad);
}
// Make new object
- std::auto_ptr<BackupStoreRefCountDatabase> refcount(new BackupStoreRefCountDatabase(rAccount));
-
- // Put in basic location info
- refcount->mReadOnly = ReadOnly;
- refcount->mapDatabaseFile = dbfile;
+ std::auto_ptr<BackupStoreRefCountDatabase> refcount(
+ new BackupStoreRefCountDatabase(rAccount, ReadOnly, false,
+ dbfile));
// return it to caller
return refcount;
@@ -229,6 +324,7 @@ void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID,
mapDatabaseFile->Seek(offset, SEEK_SET);
refcount_t RefCountNetOrder = htonl(NewRefCount);
mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder));
+ mIsModified = true;
}
bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID)
diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h
index 93c79afb..f0c1222f 100644
--- a/lib/backupstore/BackupStoreRefCountDatabase.h
+++ b/lib/backupstore/BackupStoreRefCountDatabase.h
@@ -15,6 +15,7 @@
#include <vector>
#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreConstants.h"
#include "FileStream.h"
class BackupStoreCheck;
@@ -59,17 +60,17 @@ public:
private:
// Creation through static functions only
BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry&
- rAccount);
+ rAccount, bool ReadOnly, bool Temporary,
+ std::auto_ptr<FileStream> apDatabaseFile);
// No copying allowed
BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &);
public:
- // Create a new database for a new account. This method will refuse
- // to overwrite any existing file.
- static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount)
- {
- Create(rAccount, false);
- }
+ // Create a new database for a new account.
+ static std::auto_ptr<BackupStoreRefCountDatabase> Create
+ (const BackupStoreAccountDatabase::Entry& rAccount);
+ void Commit();
+ void Discard();
// Load it from the store
static std::auto_ptr<BackupStoreRefCountDatabase> Load(const
@@ -87,20 +88,9 @@ public:
bool RemoveReference(int64_t ObjectID);
private:
- // Create a new database for an existing account. Used during
- // account checking if opening the old database throws an exception.
- // This method will overwrite any existing file.
- static void CreateForRegeneration(const
- BackupStoreAccountDatabase::Entry& rAccount)
- {
- Create(rAccount, true);
- }
-
- static void Create(const BackupStoreAccountDatabase::Entry& rAccount,
- bool AllowOverwrite);
-
static std::string GetFilename(const BackupStoreAccountDatabase::Entry&
- rAccount);
+ rAccount, bool Temporary);
+
IOStream::pos_type GetSize() const
{
return mapDatabaseFile->GetPosition() +
@@ -122,7 +112,13 @@ private:
std::string mFilename;
bool mReadOnly;
bool mIsModified;
+ bool mIsTemporaryFile;
std::auto_ptr<FileStream> mapDatabaseFile;
+
+ bool NeedsCommitOrDiscard()
+ {
+ return mapDatabaseFile.get() && mIsModified && mIsTemporaryFile;
+ }
};
#endif // BACKUPSTOREREFCOUNTDATABASE__H
diff --git a/lib/backupstore/HousekeepStoreAccount.cpp b/lib/backupstore/HousekeepStoreAccount.cpp
index 75feda7f..41f43150 100644
--- a/lib/backupstore/HousekeepStoreAccount.cpp
+++ b/lib/backupstore/HousekeepStoreAccount.cpp
@@ -61,8 +61,6 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID,
mBlocksInDirectoriesDelta(0),
mFilesDeleted(0),
mEmptyDirectoriesDeleted(0),
- mSuppressRefCountChangeWarnings(false),
- mRefCountsAdjusted(0),
mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
{
std::ostringstream tag;
@@ -105,7 +103,7 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
{
if(KeepTryingForever)
{
- BOX_WARNING("Failed to lock account for housekeeping, "
+ BOX_INFO("Failed to lock account for housekeeping, "
"still trying...");
while(!writeLock.TryAndGetLock(writeLockFilename,
0600 /* restrictive file permissions */))
@@ -143,99 +141,105 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
mDeletionSizeTarget = 0;
}
- // initialise the refcount database
- mNewRefCounts.clear();
- // try to pre-allocate as much memory as we need
- mNewRefCounts.reserve(info->GetLastObjectIDUsed());
- // initialise the refcount of the root entry
- mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0);
- mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1;
+ BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
+ mapNewRefs = BackupStoreRefCountDatabase::Create(account);
// Scan the directory for potential things to delete
// This will also remove eligible items marked with RemoveASAP
- bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
+ bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID,
+ *info);
+
+ if(!continueHousekeeping)
+ {
+ // The scan was incomplete, so the new block counts are
+ // incorrect, we can't rely on them. It's better to discard
+ // the new info and adjust the old one instead.
+ info = pOldInfo;
+
+ // We're about to reset counters and exit, so report what
+ // happened now.
+ BOX_INFO("Housekeeping on account " <<
+ BOX_FORMAT_ACCOUNT(mAccountID) << " removed " <<
+ (0 - mBlocksUsedDelta) << " blocks (" <<
+ mFilesDeleted << " files, " <<
+ mEmptyDirectoriesDeleted << " dirs) and the directory "
+ "scan was interrupted");
+ }
+
+ // If housekeeping made any changes, such as deleting RemoveASAP files,
+ // the differences in block counts will be recorded in the deltas.
+ info->ChangeBlocksUsed(mBlocksUsedDelta);
+ info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+ info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+
+ // Reset the delta counts for files, as they will include
+ // RemoveASAP flagged files deleted during the initial scan.
+ // keep removeASAPBlocksUsedDelta for reporting
+ int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
+ mBlocksUsedDelta = 0;
+ mBlocksInOldFilesDelta = 0;
+ mBlocksInDeletedFilesDelta = 0;
// If scan directory stopped for some reason, probably parent
// instructed to terminate, stop now.
+ //
+ // We can only update the refcount database if we successfully
+ // finished our scan of all directories, otherwise we don't actually
+ // know which of the new counts are valid and which aren't
+ // (we might not have seen second references to some objects, etc.).
+
if(!continueHousekeeping)
{
- // If any files were marked "delete now", then update
- // the size of the store.
- if(mBlocksUsedDelta != 0 ||
- mBlocksInOldFilesDelta != 0 ||
- mBlocksInDeletedFilesDelta != 0)
- {
- info->ChangeBlocksUsed(mBlocksUsedDelta);
- info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
- info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
-
- // Save the store info back
- info->ReportChangesTo(*pOldInfo);
- info->Save();
- }
-
+ mapNewRefs->Discard();
+ info->Save();
return false;
}
- // Log any difference in opinion between the values recorded in
- // the store info, and the values just calculated for space usage.
- // BLOCK
+ // Report any UNexpected changes, and consider them to be errors.
+ // Do this before applying the expected changes below.
+ mErrorCount += info->ReportChangesTo(*pOldInfo);
+ info->Save();
+
+ // We want to compare the mapNewRefs to apOldRefs before we delete any
+ // files, because that will also change the reference count in a way that's not an error.
+
+ // try to load the reference count database
+ try
{
- int64_t used = info->GetBlocksUsed();
- int64_t usedOld = info->GetBlocksInOldFiles();
- int64_t usedDeleted = info->GetBlocksInDeletedFiles();
- int64_t usedDirectories = info->GetBlocksInDirectories();
-
- // If the counts were wrong, taking into account RemoveASAP
- // items deleted, log a message
- if((used + mBlocksUsedDelta) != mBlocksUsed
- || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
- || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles
- || usedDirectories != mBlocksInDirectories)
+ std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs =
+ BackupStoreRefCountDatabase::Load(account, false);
+
+ int64_t LastUsedObjectIdOnDisk = mapNewRefs->GetLastObjectIDUsed();
+
+ if (apOldRefs->GetLastObjectIDUsed() > LastUsedObjectIdOnDisk)
{
- // Log this
- BOX_ERROR("Housekeeping on account " <<
- BOX_FORMAT_ACCOUNT(mAccountID) << " found "
- "and fixed wrong block counts: "
- "used (" <<
- (used + mBlocksUsedDelta) << "," <<
- mBlocksUsed << "), old (" <<
- (usedOld + mBlocksInOldFilesDelta) << "," <<
- mBlocksInOldFiles << "), deleted (" <<
- (usedDeleted + mBlocksInDeletedFilesDelta) <<
- "," << mBlocksInDeletedFiles << "), dirs (" <<
- usedDirectories << "," << mBlocksInDirectories
- << ")");
+ LastUsedObjectIdOnDisk = apOldRefs->GetLastObjectIDUsed();
}
-
- // If the current values don't match, store them
- if(used != mBlocksUsed
- || usedOld != mBlocksInOldFiles
- || usedDeleted != mBlocksInDeletedFiles
- || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
- {
- // Set corrected values in store info
- info->CorrectAllUsedValues(mBlocksUsed,
- mBlocksInOldFiles, mBlocksInDeletedFiles,
- mBlocksInDirectories + mBlocksInDirectoriesDelta);
-
- info->ReportChangesTo(*pOldInfo);
- info->Save();
+
+ for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
+ ObjectID < LastUsedObjectIdOnDisk; ObjectID++)
+ {
+ if (apOldRefs->GetRefCount(ObjectID) !=
+ mapNewRefs->GetRefCount(ObjectID))
+ {
+ BOX_WARNING("Reference count of object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " changed from " <<
+ apOldRefs->GetRefCount(ObjectID) <<
+ " to " << mapNewRefs->GetRefCount(ObjectID));
+ mErrorCount++;
+ }
}
}
-
- // Reset the delta counts for files, as they will include
- // RemoveASAP flagged files deleted during the initial scan.
-
- // keep for reporting
- int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;
+ catch(BoxException &e)
+ {
+ BOX_WARNING("Reference count database was missing or "
+ "corrupted during housekeeping, cannot check for "
+ "errors.");
+ }
- mBlocksUsedDelta = 0;
- mBlocksInOldFilesDelta = 0;
- mBlocksInDeletedFilesDelta = 0;
-
// Go and delete items from the accounts
- bool deleteInterrupted = DeleteFiles();
+ bool deleteInterrupted = DeleteFiles(*info);
// If that wasn't interrupted, remove any empty directories which
// are also marked as deleted in their containing directory
@@ -256,89 +260,6 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
(deleteInterrupted?" and was interrupted":""));
}
- // We can only update the refcount database if we successfully
- // finished our scan of all directories, otherwise we don't actually
- // know which of the new counts are valid and which aren't
- // (we might not have seen second references to some objects, etc.)
-
- BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
- std::auto_ptr<BackupStoreRefCountDatabase> apReferences;
-
- // try to load the reference count database
- try
- {
- apReferences = BackupStoreRefCountDatabase::Load(account,
- false);
- }
- catch(BoxException &e)
- {
- BOX_WARNING("Reference count database is missing or corrupted "
- "during housekeeping, creating a new one.");
- mSuppressRefCountChangeWarnings = true;
- BackupStoreRefCountDatabase::CreateForRegeneration(account);
- apReferences = BackupStoreRefCountDatabase::Load(account,
- false);
- }
-
- int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed();
-
- for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
- ObjectID < mNewRefCounts.size(); ObjectID++)
- {
- if (ObjectID > LastUsedObjectIdOnDisk)
- {
- if (!mSuppressRefCountChangeWarnings)
- {
- BOX_WARNING("Reference count of object " <<
- BOX_FORMAT_OBJECTID(ObjectID) <<
- " not found in database, added"
- " with " << mNewRefCounts[ObjectID] <<
- " references");
- }
- apReferences->SetRefCount(ObjectID,
- mNewRefCounts[ObjectID]);
- mRefCountsAdjusted++;
- LastUsedObjectIdOnDisk = ObjectID;
- continue;
- }
-
- BackupStoreRefCountDatabase::refcount_t OldRefCount =
- apReferences->GetRefCount(ObjectID);
-
- if (OldRefCount != mNewRefCounts[ObjectID])
- {
- BOX_WARNING("Reference count of object " <<
- BOX_FORMAT_OBJECTID(ObjectID) <<
- " changed from " << OldRefCount <<
- " to " << mNewRefCounts[ObjectID]);
- apReferences->SetRefCount(ObjectID,
- mNewRefCounts[ObjectID]);
- mRefCountsAdjusted++;
- }
- }
-
- // zero excess references in the database
- for (int64_t ObjectID = mNewRefCounts.size();
- ObjectID <= LastUsedObjectIdOnDisk; ObjectID++)
- {
- BackupStoreRefCountDatabase::refcount_t OldRefCount =
- apReferences->GetRefCount(ObjectID);
- BackupStoreRefCountDatabase::refcount_t NewRefCount = 0;
-
- if (OldRefCount != NewRefCount)
- {
- BOX_WARNING("Reference count of object " <<
- BOX_FORMAT_OBJECTID(ObjectID) <<
- " changed from " << OldRefCount <<
- " to " << NewRefCount << " (not found)");
- apReferences->SetRefCount(ObjectID, NewRefCount);
- mRefCountsAdjusted++;
- }
- }
-
- // force file to be saved and closed before releasing the lock below
- apReferences.reset();
-
// Make sure the delta's won't cause problems if the counts are
// really wrong, and it wasn't fixed because the store was
// updated during the scan.
@@ -364,11 +285,14 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta);
-
+
// Save the store info back
- info->ReportChangesTo(*pOldInfo);
info->Save();
+ // force file to be saved and closed before releasing the lock below
+ mapNewRefs->Commit();
+ mapNewRefs.reset();
+
// Explicity release the lock (would happen automatically on
// going out of scope, included for code clarity)
writeLock.ReleaseLock();
@@ -405,7 +329,8 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF
// Created: 11/12/03
//
// --------------------------------------------------------------------------
-bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
+bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID,
+ BackupStoreInfo& rBackupStoreInfo)
{
#ifndef WIN32
if((--mCountUntilNextInterprocessMsgCheck) <= 0)
@@ -459,11 +384,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
while((en = i.Next()) != 0)
{
// This directory references this object
- if (mNewRefCounts.size() <= en->GetObjectID())
- {
- mNewRefCounts.resize(en->GetObjectID() + 1, 0);
- }
- mNewRefCounts[en->GetObjectID()]++;
+ mapNewRefs->AddReference(en->GetObjectID());
}
}
@@ -482,10 +403,11 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
{
int16_t enFlags = en->GetFlags();
if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0
- && (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+ && (en->IsDeleted() || en->IsOld()))
{
// Delete this immediately.
- DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks);
+ DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename,
+ originalDirSizeInBlocks, rBackupStoreInfo);
// flag as having done something
deletedSomething = true;
@@ -624,7 +546,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
// Next level
ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir);
- if(!ScanDirectory(en->GetObjectID()))
+ if(!ScanDirectory(en->GetObjectID(), rBackupStoreInfo))
{
// Halting operation
return false;
@@ -687,7 +609,7 @@ bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount
// Created: 15/12/03
//
// --------------------------------------------------------------------------
-bool HousekeepStoreAccount::DeleteFiles()
+bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo)
{
// Only delete files if the deletion target is greater than zero
// (otherwise we delete one file each time round, which gradually deletes the old versions)
@@ -727,12 +649,13 @@ bool HousekeepStoreAccount::DeleteFiles()
}
// Delete the file
- DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig);
+ DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename,
+ dirSizeInBlocksOrig, rBackupStoreInfo);
BOX_INFO("Housekeeping removed " <<
(i->mIsFlagDeleted ? "deleted" : "old") <<
" file " << BOX_FORMAT_OBJECTID(i->mObjectID) <<
" from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory));
-
+
// Stop if the deletion target has been matched or exceeded
// (checking here rather than at the beginning will tend to reduce the
// space to slightly less than the soft limit, which will allow the backup
@@ -758,7 +681,9 @@ bool HousekeepStoreAccount::DeleteFiles()
// Created: 15/7/04
//
// --------------------------------------------------------------------------
-void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks)
+void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID,
+ BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename,
+ int64_t OriginalDirSizeInBlocks, BackupStoreInfo& rBackupStoreInfo)
{
// Find the entry inside the directory
bool wasDeleted = false;
@@ -783,8 +708,8 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
}
// Record the flags it's got set
- wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0);
- wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0);
+ wasDeleted = pentry->IsDeleted();
+ wasOldVersion = pentry->IsOld();
// Check this should be deleted
if(!wasDeleted && !wasOldVersion)
{
@@ -853,7 +778,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename));
// And open a write file to overwrite the other directory entry
padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet,
- objFilenameOlder, mNewRefCounts[ObjectID]));
+ objFilenameOlder, mapNewRefs->GetRefCount(ObjectID)));
padjustedEntry->Open(true /* allow overwriting */);
if(pentry->GetDependsNewer() == 0)
@@ -872,11 +797,11 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
int64_t newSize = padjustedEntry->GetDiscUsageInBlocks();
int64_t sizeDelta = newSize - polder->GetSizeInBlocks();
mBlocksUsedDelta += sizeDelta;
- if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)
+ if(polder->IsDeleted())
{
mBlocksInDeletedFilesDelta += sizeDelta;
}
- if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0)
+ if(polder->IsOld())
{
mBlocksInOldFilesDelta += sizeDelta;
}
@@ -894,17 +819,17 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
int64_t dirRevisedSize = 0;
{
RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename,
- mNewRefCounts[InDirectory]);
+ mapNewRefs->GetRefCount(InDirectory));
writeDir.Open(true /* allow overwriting */);
rDirectory.WriteToStream(writeDir);
- // get the disc usage (must do this before commiting it)
+ // Get the disc usage (must do this before commiting it)
dirRevisedSize = writeDir.GetDiscUsageInBlocks();
// Commit directory
writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
- // adjust usage counts for this directory
+ // Adjust block counts if the directory itself changed in size
if(dirRevisedSize > 0)
{
int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks;
@@ -921,29 +846,38 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
}
// Drop reference count by one. If it reaches zero, delete the file.
- if(--mNewRefCounts[ObjectID] == 0)
+ if(!mapNewRefs->RemoveReference(ObjectID))
{
// Delete from disc
BOX_TRACE("Removing unreferenced object " <<
BOX_FORMAT_OBJECTID(ObjectID));
std::string objFilename;
MakeObjectFilename(ObjectID, objFilename);
- RaidFileWrite del(mStoreDiscSet, objFilename,
- mNewRefCounts[ObjectID]);
+ RaidFileWrite del(mStoreDiscSet, objFilename, 0);
del.Delete();
}
else
{
BOX_TRACE("Preserving object " <<
BOX_FORMAT_OBJECTID(ObjectID) << " with " <<
- mNewRefCounts[ObjectID] << " references");
+ mapNewRefs->GetRefCount(ObjectID) << " references");
}
// Adjust counts for the file
++mFilesDeleted;
mBlocksUsedDelta -= deletedFileSizeInBlocks;
- if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
- if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
+
+ if(wasDeleted)
+ {
+ mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
+ rBackupStoreInfo.AdjustNumDeletedFiles(-1);
+ }
+
+ if(wasOldVersion)
+ {
+ mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
+ rBackupStoreInfo.AdjustNumOldFiles(-1);
+ }
// Delete the directory?
// Do this if... dir has zero entries, and is marked as deleted in it's containing directory
@@ -1068,7 +1002,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
// Write revised parent directory
RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename,
- mNewRefCounts[containingDir.GetObjectID()]);
+ mapNewRefs->GetRefCount(containingDir.GetObjectID()));
writeDir.Open(true /* allow overwriting */);
containingDir.WriteToStream(writeDir);
@@ -1087,11 +1021,10 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId,
}
- if (--mNewRefCounts[dir.GetObjectID()] == 0)
+ if (mapNewRefs->RemoveReference(dir.GetObjectID()))
{
// Delete the directory itself
- RaidFileWrite del(mStoreDiscSet, dirFilename,
- mNewRefCounts[dir.GetObjectID()]);
+ RaidFileWrite del(mStoreDiscSet, dirFilename, 0);
del.Delete();
}
diff --git a/lib/backupstore/HousekeepStoreAccount.h b/lib/backupstore/HousekeepStoreAccount.h
index cfa05a2e..a63c5536 100644
--- a/lib/backupstore/HousekeepStoreAccount.h
+++ b/lib/backupstore/HousekeepStoreAccount.h
@@ -15,6 +15,7 @@
#include <vector>
class BackupStoreDirectory;
+class BackupStoreRefCountDatabase;
class HousekeepingCallback
{
@@ -39,18 +40,22 @@ public:
~HousekeepStoreAccount();
bool DoHousekeeping(bool KeepTryingForever = false);
- int GetRefCountsAdjusted() { return mRefCountsAdjusted; }
+ int GetErrorCount() { return mErrorCount; }
private:
// utility functions
void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut);
- bool ScanDirectory(int64_t ObjectID);
- bool DeleteFiles();
+ bool ScanDirectory(int64_t ObjectID, BackupStoreInfo& rBackupStoreInfo);
+ bool DeleteFiles(BackupStoreInfo& rBackupStoreInfo);
bool DeleteEmptyDirectories();
void DeleteEmptyDirectory(int64_t dirId,
std::vector<int64_t>& rToExamine);
- void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks);
+ void DeleteFile(int64_t InDirectory, int64_t ObjectID,
+ BackupStoreDirectory &rDirectory,
+ const std::string &rDirectoryFilename,
+ int64_t OriginalDirSizeInBlocks,
+ BackupStoreInfo& rBackupStoreInfo);
private:
typedef struct
@@ -81,6 +86,9 @@ private:
// List of directories which are empty, and might be good for deleting
std::vector<int64_t> mEmptyDirectories;
+
+ // Count of errors found and fixed
+ int64_t mErrorCount;
// The re-calculated blocks used stats
int64_t mBlocksUsed;
@@ -99,9 +107,7 @@ private:
int64_t mEmptyDirectoriesDeleted;
// New reference count list
- std::vector<uint32_t> mNewRefCounts;
- bool mSuppressRefCountChangeWarnings;
- int mRefCountsAdjusted;
+ std::auto_ptr<BackupStoreRefCountDatabase> mapNewRefs;
// Poll frequency
int mCountUntilNextInterprocessMsgCheck;
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-19 14:32:36 +0000