summaryrefslogtreecommitdiff
path: root/bin/bbstored
diff options
context:
space:
mode:
Diffstat (limited to 'bin/bbstored')
-rw-r--r--bin/bbstored/BBStoreDHousekeeping.cpp3
-rw-r--r--bin/bbstored/BackupStoreContext.cpp55
-rw-r--r--bin/bbstored/BackupStoreContext.h5
-rw-r--r--bin/bbstored/BackupStoreDaemon.h11
-rw-r--r--bin/bbstored/HousekeepStoreAccount.cpp151
-rw-r--r--bin/bbstored/HousekeepStoreAccount.h17
-rw-r--r--bin/bbstored/Makefile.extra2
7 files changed, 210 insertions, 34 deletions
diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp
index 1c1767ca..7f799008 100644
--- a/bin/bbstored/BBStoreDHousekeeping.cpp
+++ b/bin/bbstored/BBStoreDHousekeeping.cpp
@@ -108,7 +108,8 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded()
mpAccounts->GetAccountRoot(*i, rootDir, discSet);
// Do housekeeping on this account
- HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this);
+ HousekeepStoreAccount housekeeping(*i, rootDir,
+ discSet, this);
housekeeping.DoHousekeeping();
}
}
diff --git a/bin/bbstored/BackupStoreContext.cpp b/bin/bbstored/BackupStoreContext.cpp
index 990be05d..2e915f57 100644
--- a/bin/bbstored/BackupStoreContext.cpp
+++ b/bin/bbstored/BackupStoreContext.cpp
@@ -191,6 +191,23 @@ void BackupStoreContext::LoadStoreInfo()
// Keep the pointer to it
mpStoreInfo = i;
+
+ BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet);
+
+ // try to load the reference count database
+ try
+ {
+ mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+ }
+ catch(BoxException &e)
+ {
+ BOX_WARNING("Reference count database is missing or corrupted, "
+ "creating a new one, expect housekeeping to find and "
+ "fix problems with reference counts later.");
+
+ BackupStoreRefCountDatabase::CreateForRegeneration(account);
+ mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+ }
}
@@ -395,15 +412,18 @@ int64_t BackupStoreContext::AllocateObjectID()
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupStoreContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
-// Purpose: Add a file to the store, from a given stream, into a specified directory.
-// Returns object ID of the new file.
+// Name: BackupStoreContext::AddFile(IOStream &, int64_t,
+// int64_t, int64_t, const BackupStoreFilename &, bool)
+// Purpose: Add a file to the store, from a given stream, into
+// a specified directory. Returns object ID of the new
+// file.
// Created: 2003/09/03
//
// --------------------------------------------------------------------------
-int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime,
- int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
- bool MarkFileWithSameNameAsOldVersions)
+int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory,
+ int64_t ModificationTime, int64_t AttributesHash,
+ int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
+ bool MarkFileWithSameNameAsOldVersions)
{
if(mpStoreInfo.get() == 0)
{
@@ -670,6 +690,9 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_
mpStoreInfo->ChangeBlocksUsed(blocksUsed);
mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles);
+ // Increment reference count on the new directory to one
+ mapRefCount->AddReference(id);
+
// Save the store info -- can cope if this exceptions because infomation
// will be rebuilt by housekeeping, and ID allocation can recover.
SaveStoreInfo();
@@ -768,8 +791,9 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &)
-// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found.
+// Name: BackupStoreContext::UndeleteFile(int64_t, int64_t)
+// Purpose: Undeletes a file, if it exists, returning true if
+// the file existed.
// Created: 2003/10/21
//
// --------------------------------------------------------------------------
@@ -933,8 +957,10 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupStoreContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &)
-// Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely.
+// Name: BackupStoreContext::AddDirectory(int64_t,
+// const BackupStoreFilename &, bool &)
+// Purpose: Creates a directory (or just returns the ID of an
+// existing one). rAlreadyExists set appropraitely.
// Created: 2003/09/04
//
// --------------------------------------------------------------------------
@@ -974,7 +1000,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF
// Allocate the next ID
int64_t id = AllocateObjectID();
- // Create a blank directory with the given attributes on disc
+ // Create an empty directory with the given attributes on disc
std::string fn;
MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
{
@@ -998,11 +1024,14 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF
// Not added to cache, so don't set the size in the directory
}
- // Then add it into the directory
+ // Then add it into the parent directory
try
{
dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */);
SaveDirectory(dir, InDirectory);
+
+ // Increment reference count on the new directory to one
+ mapRefCount->AddReference(id);
}
catch(...)
{
@@ -1016,7 +1045,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF
// Don't worry about the incremented number in the store info
throw;
}
-
+
// Save the store info (may be postponed)
SaveStoreInfo();
diff --git a/bin/bbstored/BackupStoreContext.h b/bin/bbstored/BackupStoreContext.h
index 4cfdb601..6053e4b8 100644
--- a/bin/bbstored/BackupStoreContext.h
+++ b/bin/bbstored/BackupStoreContext.h
@@ -14,6 +14,7 @@
#include <map>
#include <memory>
+#include "BackupStoreRefCountDatabase.h"
#include "NamedLock.h"
#include "ProtocolObject.h"
#include "Utils.h"
@@ -137,7 +138,6 @@ private:
void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete);
int64_t AllocateObjectID();
-private:
int32_t mClientID;
HousekeepingInterface &mrDaemon;
int mProtocolPhase;
@@ -150,6 +150,9 @@ private:
// Store info
std::auto_ptr<BackupStoreInfo> mpStoreInfo;
+
+ // Refcount database
+ std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount;
// Directory cache
std::map<int64_t, BackupStoreDirectory*> mDirectoryCache;
diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h
index a5d216f4..49af5b81 100644
--- a/bin/bbstored/BackupStoreDaemon.h
+++ b/bin/bbstored/BackupStoreDaemon.h
@@ -14,11 +14,11 @@
#include "BoxPortsAndFiles.h"
#include "BackupConstants.h"
#include "BackupStoreContext.h"
+#include "HousekeepStoreAccount.h"
#include "IOStreamGetLine.h"
class BackupStoreAccounts;
class BackupStoreAccountDatabase;
-class HousekeepStoreAccount;
// --------------------------------------------------------------------------
//
@@ -29,10 +29,8 @@ class HousekeepStoreAccount;
//
// --------------------------------------------------------------------------
class BackupStoreDaemon : public ServerTLS<BOX_PORT_BBSTORED>,
- HousekeepingInterface
+ HousekeepingInterface, HousekeepingCallback
{
- friend class HousekeepStoreAccount;
-
public:
BackupStoreDaemon();
~BackupStoreDaemon();
@@ -64,10 +62,13 @@ protected:
// Housekeeping functions
void HousekeepingProcess();
- bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0);
void LogConnectionStats(const char *commonName, const SocketStreamTLS &s);
+public:
+ // HousekeepingInterface implementation
+ virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0);
+
private:
BackupStoreAccountDatabase *mpAccountDatabase;
BackupStoreAccounts *mpAccounts;
diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp
index dbb9b544..0ccbcf23 100644
--- a/bin/bbstored/HousekeepStoreAccount.cpp
+++ b/bin/bbstored/HousekeepStoreAccount.cpp
@@ -39,11 +39,13 @@
// Created: 11/12/03
//
// --------------------------------------------------------------------------
-HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon)
+HousekeepStoreAccount::HousekeepStoreAccount(int AccountID,
+ const std::string &rStoreRoot, int StoreDiscSet,
+ HousekeepingCallback* pHousekeepingCallback)
: mAccountID(AccountID),
mStoreRoot(rStoreRoot),
mStoreDiscSet(StoreDiscSet),
- mrDaemon(rDaemon),
+ mpHousekeepingCallback(pHousekeepingCallback),
mDeletionSizeTarget(0),
mPotentialDeletionsTotalSize(0),
mMaxSizeInPotentialDeletions(0),
@@ -57,6 +59,7 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, const std::string &r
mBlocksInDirectoriesDelta(0),
mFilesDeleted(0),
mEmptyDirectoriesDeleted(0),
+ mSuppressRefCountChangeWarnings(false),
mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
{
}
@@ -81,7 +84,7 @@ HousekeepStoreAccount::~HousekeepStoreAccount()
// Created: 11/12/03
//
// --------------------------------------------------------------------------
-void HousekeepStoreAccount::DoHousekeeping()
+void HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
{
// Attempt to lock the account
std::string writeLockFilename;
@@ -91,8 +94,21 @@ void HousekeepStoreAccount::DoHousekeeping()
if(!writeLock.TryAndGetLock(writeLockFilename.c_str(),
0600 /* restrictive file permissions */))
{
- // Couldn't lock the account -- just stop now
- return;
+ if(KeepTryingForever)
+ {
+ BOX_WARNING("Failed to lock account for housekeeping, "
+ "still trying...");
+ while(!writeLock.TryAndGetLock(writeLockFilename,
+ 0600 /* restrictive file permissions */))
+ {
+ sleep(1);
+ }
+ }
+ else
+ {
+ // Couldn't lock the account -- just stop now
+ return;
+ }
}
// Load the store info to find necessary info for the housekeeping
@@ -106,6 +122,14 @@ void HousekeepStoreAccount::DoHousekeeping()
mDeletionSizeTarget = 0;
}
+ // initialise the refcount database
+ mNewRefCounts.clear();
+ // try to pre-allocate as much memory as we need
+ mNewRefCounts.reserve(info->GetLastObjectIDUsed());
+ // initialise the refcount of the root entry
+ mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0);
+ mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1;
+
// Scan the directory for potential things to delete
// This will also remove eligible items marked with RemoveASAP
bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
@@ -207,6 +231,86 @@ void HousekeepStoreAccount::DoHousekeeping()
mEmptyDirectoriesDeleted << " dirs)" <<
(deleteInterrupted?" and was interrupted":""));
}
+
+ // We can only update the refcount database if we successfully
+ // finished our scan of all directories, otherwise we don't actually
+ // know which of the new counts are valid and which aren't
+ // (we might not have seen second references to some objects, etc.)
+
+ BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
+ std::auto_ptr<BackupStoreRefCountDatabase> apReferences;
+
+ // try to load the reference count database
+ try
+ {
+ apReferences = BackupStoreRefCountDatabase::Load(account,
+ false);
+ }
+ catch(BoxException &e)
+ {
+ BOX_WARNING("Reference count database is missing or corrupted "
+ "during housekeeping, creating a new one.");
+ mSuppressRefCountChangeWarnings = true;
+ BackupStoreRefCountDatabase::CreateForRegeneration(account);
+ apReferences = BackupStoreRefCountDatabase::Load(account,
+ false);
+ }
+
+ int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed();
+
+ for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
+ ObjectID < mNewRefCounts.size(); ObjectID++)
+ {
+ if (ObjectID > LastUsedObjectIdOnDisk)
+ {
+ if (!mSuppressRefCountChangeWarnings)
+ {
+ BOX_WARNING("Reference count of object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " not found in database, added"
+ " with " << mNewRefCounts[ObjectID] <<
+ " references");
+ }
+ apReferences->SetRefCount(ObjectID,
+ mNewRefCounts[ObjectID]);
+ LastUsedObjectIdOnDisk = ObjectID;
+ continue;
+ }
+
+ BackupStoreRefCountDatabase::refcount_t OldRefCount =
+ apReferences->GetRefCount(ObjectID);
+
+ if (OldRefCount != mNewRefCounts[ObjectID])
+ {
+ BOX_WARNING("Reference count of object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " changed from " << OldRefCount <<
+ " to " << mNewRefCounts[ObjectID]);
+ apReferences->SetRefCount(ObjectID,
+ mNewRefCounts[ObjectID]);
+ }
+ }
+
+ // zero excess references in the database
+ for (int64_t ObjectID = mNewRefCounts.size();
+ ObjectID <= LastUsedObjectIdOnDisk; ObjectID++)
+ {
+ BackupStoreRefCountDatabase::refcount_t OldRefCount =
+ apReferences->GetRefCount(ObjectID);
+ BackupStoreRefCountDatabase::refcount_t NewRefCount = 0;
+
+ if (OldRefCount != NewRefCount)
+ {
+ BOX_WARNING("Reference count of object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " changed from " << OldRefCount <<
+ " to " << NewRefCount << " (not found)");
+ apReferences->SetRefCount(ObjectID, NewRefCount);
+ }
+ }
+
+ // force file to be saved and closed before releasing the lock below
+ apReferences.reset();
// Make sure the delta's won't cause problems if the counts are
// really wrong, and it wasn't fixed because the store was
@@ -279,7 +383,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
// Check for having to stop
// Include account ID here as the specified account is locked
- if(mrDaemon.CheckForInterProcessMsg(mAccountID))
+ if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))
{
// Need to abort now
return false;
@@ -312,7 +416,25 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
// Add it to the list of directories to potentially delete
mEmptyDirectories.push_back(dir.GetObjectID());
}
-
+
+ // Calculate reference counts first, before we start requesting
+ // files to be deleted.
+ // BLOCK
+ {
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+
+ while((en = i.Next()) != 0)
+ {
+ // This directory references this object
+ if (mNewRefCounts.size() <= en->GetObjectID())
+ {
+ mNewRefCounts.resize(en->GetObjectID() + 1, 0);
+ }
+ mNewRefCounts[en->GetObjectID()]++;
+ }
+ }
+
// BLOCK
{
// Remove any files which are marked for removal as soon
@@ -551,7 +673,7 @@ bool HousekeepStoreAccount::DeleteFiles()
{
mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
// Check for having to stop
- if(mrDaemon.CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked
+ if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked
{
// Need to abort now
return true;
@@ -761,13 +883,22 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
padjustedEntry.reset(); // delete it now
}
- // Delete from disc
+ // Drop reference count by one. If it reaches zero, delete the file.
+ if(--mNewRefCounts[ObjectID] == 0)
{
+ BOX_TRACE("Removing unreferenced object " <<
+ BOX_FORMAT_OBJECTID(ObjectID));
std::string objFilename;
MakeObjectFilename(ObjectID, objFilename);
RaidFileWrite del(mStoreDiscSet, objFilename);
del.Delete();
}
+ else
+ {
+ BOX_TRACE("Preserving object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) << " with " <<
+ mNewRefCounts[ObjectID] << " references");
+ }
// Adjust counts for the file
++mFilesDeleted;
@@ -808,7 +939,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories()
{
mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
// Check for having to stop
- if(mrDaemon.CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked
+ if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked
{
// Need to abort now
return true;
diff --git a/bin/bbstored/HousekeepStoreAccount.h b/bin/bbstored/HousekeepStoreAccount.h
index 5c2a9885..1dd6d79c 100644
--- a/bin/bbstored/HousekeepStoreAccount.h
+++ b/bin/bbstored/HousekeepStoreAccount.h
@@ -17,6 +17,12 @@
class BackupStoreDaemon;
class BackupStoreDirectory;
+class HousekeepingCallback
+{
+ public:
+ virtual ~HousekeepingCallback() {}
+ virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0) = 0;
+};
// --------------------------------------------------------------------------
//
@@ -29,10 +35,11 @@ class BackupStoreDirectory;
class HousekeepStoreAccount
{
public:
- HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon);
+ HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot,
+ int StoreDiscSet, HousekeepingCallback* pHousekeepingCallback);
~HousekeepStoreAccount();
- void DoHousekeeping();
+ void DoHousekeeping(bool KeepTryingForever = false);
private:
@@ -65,7 +72,7 @@ private:
int mAccountID;
std::string mStoreRoot;
int mStoreDiscSet;
- BackupStoreDaemon &mrDaemon;
+ HousekeepingCallback* mpHousekeepingCallback;
int64_t mDeletionSizeTarget;
@@ -91,6 +98,10 @@ private:
// Deletion count
int64_t mFilesDeleted;
int64_t mEmptyDirectoriesDeleted;
+
+ // New reference count list
+ std::vector<uint32_t> mNewRefCounts;
+ bool mSuppressRefCountChangeWarnings;
// Poll frequency
int mCountUntilNextInterprocessMsgCheck;
diff --git a/bin/bbstored/Makefile.extra b/bin/bbstored/Makefile.extra
index 94bc3fb9..6562647d 100644
--- a/bin/bbstored/Makefile.extra
+++ b/bin/bbstored/Makefile.extra
@@ -5,5 +5,5 @@ GEN_CMD_SRV = $(MAKEPROTOCOL) Server backupprotocol.txt
# AUTOGEN SEEDING
autogen_BackupProtocolServer.cpp autogen_BackupProtocolServer.h: $(MAKEPROTOCOL) backupprotocol.txt
- $(PERL) $(GEN_CMD_SRV)
+ $(_PERL) $(GEN_CMD_SRV)
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-28 00:16:52 +0000