summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Wilson <chris+github@qwirx.com>2009-06-27 11:38:52 +0000
committerChris Wilson <chris+github@qwirx.com>2009-06-27 11:38:52 +0000
commitcac51d0017b98ea98a16cdd2100e4fe20d3a74ac (patch)
treeb33106585470584d080aa6f9b614871656ed9e63
parentc6bab91bf0940f900a1831d4246f52d5768dd495 (diff)
Add code for BackupStoreRefCountDatabase.
Create a refcount database when creating a new account. Add an easier way to get the account root directory.
-rw-r--r--lib/backupstore/BackupStoreAccounts.cpp19
-rw-r--r--lib/backupstore/BackupStoreAccounts.h9
-rw-r--r--lib/backupstore/BackupStoreRefCountDatabase.cpp319
-rw-r--r--lib/backupstore/BackupStoreRefCountDatabase.h125
4 files changed, 464 insertions, 8 deletions
diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp
index eb10b385..5c7e4d38 100644
--- a/lib/backupstore/BackupStoreAccounts.cpp
+++ b/lib/backupstore/BackupStoreAccounts.cpp
@@ -14,6 +14,7 @@
#include "BoxPortsAndFiles.h"
#include "BackupStoreAccounts.h"
#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreRefCountDatabase.h"
#include "RaidFileWrite.h"
#include "BackupStoreInfo.h"
#include "BackupStoreDirectory.h"
@@ -61,6 +62,10 @@ BackupStoreAccounts::~BackupStoreAccounts()
// --------------------------------------------------------------------------
void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername)
{
+ // Create the entry in the database
+ BackupStoreAccountDatabase::Entry Entry(mrDatabase.AddEntry(ID,
+ DiscSet));
+
{
// Become the user specified in the config file?
std::auto_ptr<UnixUser> user;
@@ -100,15 +105,17 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit,
// Save it back
info->Save();
+
+ // Create the refcount database
+ BackupStoreRefCountDatabase::CreateNew(Entry);
+ std::auto_ptr<BackupStoreRefCountDatabase> refcount(
+ BackupStoreRefCountDatabase::Load(Entry, false));
+ refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID);
}
// As the original user...
-
- // Create the entry in the database
- mrDatabase.AddEntry(ID, DiscSet);
-
// Write the database back
- mrDatabase.Write();
+ mrDatabase.Write();
}
@@ -138,7 +145,7 @@ void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, i
// Created: 2003/08/21
//
// --------------------------------------------------------------------------
-std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const
+std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet)
{
char accid[64]; // big enough!
::sprintf(accid, "%08x" DIRECTORY_SEPARATOR, ID);
diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h
index 0c3dd103..224d7353 100644
--- a/lib/backupstore/BackupStoreAccounts.h
+++ b/lib/backupstore/BackupStoreAccounts.h
@@ -12,7 +12,7 @@
#include <string>
-class BackupStoreAccountDatabase;
+#include "BackupStoreAccountDatabase.h"
// --------------------------------------------------------------------------
//
@@ -35,9 +35,14 @@ public:
bool AccountExists(int32_t ID);
void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const;
+ static std::string GetAccountRoot(const
+ BackupStoreAccountDatabase::Entry &rEntry)
+ {
+ return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet());
+ }
private:
- std::string MakeAccountRootDir(int32_t ID, int DiscSet) const;
+ static std::string MakeAccountRootDir(int32_t ID, int DiscSet);
private:
BackupStoreAccountDatabase &mrDatabase;
diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp
new file mode 100644
index 00000000..92ba947d
--- /dev/null
+++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp
@@ -0,0 +1,319 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreRefCountDatabase.cpp
+// Purpose: Backup store object reference count database storage
+// Created: 2009/06/01
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <algorithm>
+
+#include "BackupStoreRefCountDatabase.h"
+#include "BackupStoreException.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreAccounts.h"
+#include "RaidFileController.h"
+#include "RaidFileUtil.h"
+#include "RaidFileException.h"
+
+#include "MemLeakFindOn.h"
+
+#define REFCOUNT_MAGIC_VALUE 0x52656643 // RefC
+#define REFCOUNT_FILENAME "refcount"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreRefCountDatabase::BackupStoreRefCountDatabase()
+// Purpose: Default constructor
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const
+ BackupStoreAccountDatabase::Entry& rAccount)
+: mAccount(rAccount),
+ mFilename(GetFilename(rAccount)),
+ mReadOnly(true),
+ mIsModified(false)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase
+// Purpose: Destructor
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase()
+{
+}
+
+std::string BackupStoreRefCountDatabase::GetFilename(const
+ BackupStoreAccountDatabase::Entry& rAccount)
+{
+ std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount);
+ ASSERT(RootDir[RootDir.size() - 1] == '/' ||
+ RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR);
+
+ std::string fn(RootDir + "refcount.db");
+ RaidFileController &rcontroller(RaidFileController::GetController());
+ RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet()));
+ return RaidFileUtil::MakeWriteFileName(rdiscSet, fn);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreRefCountDatabase::Create(int32_t,
+// const std::string &, int, bool)
+// Purpose: Create a new database, overwriting an existing
+// one only if AllowOverwrite is true.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreRefCountDatabase::Create(const
+ BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite)
+{
+ // Initial header
+ refcount_StreamFormat hdr;
+ hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE);
+ hdr.mAccountID = htonl(rAccount.GetID());
+
+ // Generate the filename
+ std::string Filename = GetFilename(rAccount);
+
+ // Open the file for writing
+ if (FileExists(Filename) && !AllowOverwrite)
+ {
+ BOX_ERROR("Attempted to overwrite refcount database file: " <<
+ Filename);
+ THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile);
+ }
+
+ int flags = O_CREAT | O_BINARY | O_RDWR;
+ if (!AllowOverwrite)
+ {
+ flags |= O_EXCL;
+ }
+
+ std::auto_ptr<FileStream> DatabaseFile(new FileStream(Filename, flags));
+
+ // Write header
+ DatabaseFile->Write(&hdr, sizeof(hdr));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreRefCountDatabase::Load(int32_t AccountID,
+// BackupStoreAccountDatabase& rAccountDatabase,
+// bool ReadOnly);
+// Purpose: Loads the info from disc, given the root
+// information. Can be marked as read only.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load(
+ const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly)
+{
+ // Generate the filename
+ std::string filename = GetFilename(rAccount);
+ int flags = ReadOnly ? O_RDONLY : O_RDWR;
+
+ // Open the file for read/write
+ std::auto_ptr<FileStream> dbfile(new FileStream(filename,
+ flags | O_BINARY));
+
+ // Read in a header
+ refcount_StreamFormat hdr;
+ if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+ }
+
+ // Check it
+ if(ntohl(hdr.mMagicValue) != REFCOUNT_MAGIC_VALUE ||
+ (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID())
+ {
+ THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+ }
+
+ // Make new object
+ std::auto_ptr<BackupStoreRefCountDatabase> refcount(new BackupStoreRefCountDatabase(rAccount));
+
+ // Put in basic location info
+ refcount->mReadOnly = ReadOnly;
+ refcount->mapDatabaseFile = dbfile;
+
+ // return it to caller
+ return refcount;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreRefCountDatabase::Save()
+// Purpose: Save modified info back to disc
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+/*
+void BackupStoreRefCountDatabase::Save()
+{
+ // Make sure we're initialised (although should never come to this)
+ if(mFilename.empty() || mAccount.GetID() == 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Can we do this?
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ // Then... open a write file
+ RaidFileWrite rf(mAccount.GetDiscSet(), mFilename);
+ rf.Open(true); // allow overwriting
+
+ // Make header
+ info_StreamFormat hdr;
+ hdr.mMagicValue = htonl(INFO_MAGIC_VALUE);
+ hdr.mAccountID = htonl(mAccountID);
+ hdr.mClientStoreMarker = box_hton64(mClientStoreMarker);
+ hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed);
+ hdr.mBlocksUsed = box_hton64(mBlocksUsed);
+ hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles);
+ hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles);
+ hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories);
+ hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit);
+ hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit);
+ hdr.mCurrentMarkNumber = 0;
+ hdr.mOptionsPresent = 0;
+ hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size());
+
+ // Write header
+ rf.Write(&hdr, sizeof(hdr));
+
+ // Write the deleted object list
+ if(mDeletedDirectories.size() > 0)
+ {
+ int64_t objs[NUM_DELETED_DIRS_BLOCK];
+
+ int tosave = mDeletedDirectories.size();
+ std::vector<int64_t>::iterator i(mDeletedDirectories.begin());
+ while(tosave > 0)
+ {
+ // How many in this one?
+ int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave));
+
+ // Add them
+ for(int t = 0; t < b; ++t)
+ {
+ ASSERT(i != mDeletedDirectories.end());
+ objs[t] = box_hton64((*i));
+ i++;
+ }
+
+ // Write
+ rf.Write(objs, b * sizeof(int64_t));
+
+ // Number saved
+ tosave -= b;
+ }
+ }
+
+ // Commit it to disc, converting it to RAID now
+ rf.Commit(true);
+
+ // Mark is as not modified
+ mIsModified = false;
+}
+*/
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreRefCountDatabase::GetRefCount(int64_t
+// ObjectID)
+// Purpose: Get the number of references to the specified object
+// out of the database
+// Created: 2009/06/01
+//
+// --------------------------------------------------------------------------
+int32_t BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const
+{
+ IOStream::pos_type offset = GetOffset(ObjectID);
+
+ if (GetSize() < offset + GetEntrySize())
+ {
+ BOX_ERROR("attempted read of unknown refcount for object " <<
+ BOX_FORMAT_OBJECTID(ObjectID));
+ THROW_EXCEPTION(BackupStoreException,
+ UnknownObjectRefCountRequested);
+ }
+
+ mapDatabaseFile->Seek(offset, SEEK_SET);
+
+ refcount_t refcount;
+ if (mapDatabaseFile->Read(&refcount, sizeof(refcount)) !=
+ sizeof(refcount))
+ {
+ BOX_LOG_SYS_ERROR("short read on refcount database: " <<
+ mFilename);
+ THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo);
+ }
+
+ return ntohl(refcount);
+}
+
+int64_t BackupStoreRefCountDatabase::GetLastObjectIDUsed() const
+{
+ return (GetSize() - sizeof(refcount_StreamFormat)) /
+ sizeof(refcount_t);
+}
+
+void BackupStoreRefCountDatabase::AddReference(int64_t ObjectID)
+{
+ refcount_t refcount;
+
+ if (ObjectID > GetLastObjectIDUsed())
+ {
+ // new object, assume no previous references
+ refcount = 0;
+ }
+ else
+ {
+ // read previous value from database
+ refcount = GetRefCount(ObjectID);
+ }
+
+ refcount++;
+
+ SetRefCount(ObjectID, refcount);
+}
+
+void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID,
+ refcount_t NewRefCount)
+{
+ IOStream::pos_type offset = GetOffset(ObjectID);
+ mapDatabaseFile->Seek(offset, SEEK_SET);
+ refcount_t RefCountNetOrder = htonl(NewRefCount);
+ mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder));
+}
+
+bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID)
+{
+ refcount_t refcount = GetRefCount(ObjectID); // must exist in database
+ ASSERT(refcount > 0);
+ refcount--;
+ SetRefCount(ObjectID, refcount);
+ return (refcount > 0);
+}
+
diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h
new file mode 100644
index 00000000..0a65b6a0
--- /dev/null
+++ b/lib/backupstore/BackupStoreRefCountDatabase.h
@@ -0,0 +1,125 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreRefCountDatabase.h
+// Purpose: Main backup store information storage
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREREFCOUNTDATABASE__H
+#define BACKUPSTOREREFCOUNTDATABASE__H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "BackupStoreAccountDatabase.h"
+#include "FileStream.h"
+
+class BackupStoreCheck;
+class BackupStoreContext;
+
+// set packing to one byte
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+ uint32_t mMagicValue; // also the version number
+ uint32_t mAccountID;
+} refcount_StreamFormat;
+
+// Use default packing
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreRefCountDatabase
+// Purpose: Backup store reference count database storage
+// Created: 2009/06/01
+//
+// --------------------------------------------------------------------------
+class BackupStoreRefCountDatabase
+{
+ friend class BackupStoreCheck;
+ friend class BackupStoreContext;
+public:
+ ~BackupStoreRefCountDatabase();
+private:
+ // Creation through static functions only
+ BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry&
+ rAccount);
+ // No copying allowed
+ BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &);
+
+public:
+ // Create a new database for a new account. This method will refuse
+ // to overwrite any existing file.
+ static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount)
+ {
+ Create(rAccount, false);
+ }
+
+ // Load it from the store
+ static std::auto_ptr<BackupStoreRefCountDatabase> Load(const
+ BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly);
+
+ // Data access functions
+ int32_t GetRefCount(int64_t ObjectID) const;
+ int64_t GetLastObjectIDUsed() const;
+
+ // Data modification functions
+ void AddReference(int64_t ObjectID);
+ bool RemoveReference(int64_t ObjectID);
+
+private:
+ // Create a new database for an existing account. Used during
+ // account checking if opening the old database throws an exception.
+ // This method will overwrite any existing file.
+ static void CreateForRegeneration(const
+ BackupStoreAccountDatabase::Entry& rAccount)
+ {
+ Create(rAccount, true);
+ }
+
+ static void Create(const BackupStoreAccountDatabase::Entry& rAccount,
+ bool AllowOverwrite);
+
+ typedef int32_t refcount_t;
+
+ static std::string GetFilename(const BackupStoreAccountDatabase::Entry&
+ rAccount);
+ IOStream::pos_type GetSize() const
+ {
+ return mapDatabaseFile->GetPosition() +
+ mapDatabaseFile->BytesLeftToRead();
+ }
+ IOStream::pos_type GetEntrySize() const
+ {
+ return sizeof(refcount_t);
+ }
+ IOStream::pos_type GetOffset(int64_t ObjectID) const
+ {
+ return ((ObjectID - 1) * GetEntrySize()) +
+ sizeof(refcount_StreamFormat);
+ }
+ void SetRefCount(int64_t ObjectID, refcount_t NewRefCount);
+
+ // Location information
+ BackupStoreAccountDatabase::Entry mAccount;
+ std::string mFilename;
+ bool mReadOnly;
+ bool mIsModified;
+ std::auto_ptr<FileStream> mapDatabaseFile;
+};
+
+#endif // BACKUPSTOREREFCOUNTDATABASE__H