From cac51d0017b98ea98a16cdd2100e4fe20d3a74ac Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sat, 27 Jun 2009 11:38:52 +0000 Subject: Add code for BackupStoreRefCountDatabase. Create a refcount database when creating a new account. Add an easier way to get the account root directory. --- lib/backupstore/BackupStoreAccounts.cpp | 19 +- lib/backupstore/BackupStoreAccounts.h | 9 +- lib/backupstore/BackupStoreRefCountDatabase.cpp | 319 ++++++++++++++++++++++++ lib/backupstore/BackupStoreRefCountDatabase.h | 125 ++++++++++ 4 files changed, 464 insertions(+), 8 deletions(-) create mode 100644 lib/backupstore/BackupStoreRefCountDatabase.cpp create mode 100644 lib/backupstore/BackupStoreRefCountDatabase.h diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp index eb10b385..5c7e4d38 100644 --- a/lib/backupstore/BackupStoreAccounts.cpp +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -14,6 +14,7 @@ #include "BoxPortsAndFiles.h" #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" +#include "BackupStoreRefCountDatabase.h" #include "RaidFileWrite.h" #include "BackupStoreInfo.h" #include "BackupStoreDirectory.h" @@ -61,6 +62,10 @@ BackupStoreAccounts::~BackupStoreAccounts() // -------------------------------------------------------------------------- void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername) { + // Create the entry in the database + BackupStoreAccountDatabase::Entry Entry(mrDatabase.AddEntry(ID, + DiscSet)); + { // Become the user specified in the config file? std::auto_ptr user; @@ -100,15 +105,17 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, // Save it back info->Save(); + + // Create the refcount database + BackupStoreRefCountDatabase::CreateNew(Entry); + std::auto_ptr refcount( + BackupStoreRefCountDatabase::Load(Entry, false)); + refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID); } // As the original user... - - // Create the entry in the database - mrDatabase.AddEntry(ID, DiscSet); - // Write the database back - mrDatabase.Write(); + mrDatabase.Write(); } @@ -138,7 +145,7 @@ void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, i // Created: 2003/08/21 // // -------------------------------------------------------------------------- -std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const +std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) { char accid[64]; // big enough! ::sprintf(accid, "%08x" DIRECTORY_SEPARATOR, ID); diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h index 0c3dd103..224d7353 100644 --- a/lib/backupstore/BackupStoreAccounts.h +++ b/lib/backupstore/BackupStoreAccounts.h @@ -12,7 +12,7 @@ #include -class BackupStoreAccountDatabase; +#include "BackupStoreAccountDatabase.h" // -------------------------------------------------------------------------- // @@ -35,9 +35,14 @@ public: bool AccountExists(int32_t ID); void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const; + static std::string GetAccountRoot(const + BackupStoreAccountDatabase::Entry &rEntry) + { + return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet()); + } private: - std::string MakeAccountRootDir(int32_t ID, int DiscSet) const; + static std::string MakeAccountRootDir(int32_t ID, int DiscSet); private: BackupStoreAccountDatabase &mrDatabase; diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp new file mode 100644 index 00000000..92ba947d --- /dev/null +++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp @@ -0,0 +1,319 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreRefCountDatabase.cpp +// Purpose: Backup store object reference count database storage +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#include "BackupStoreRefCountDatabase.h" +#include "BackupStoreException.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreAccounts.h" +#include "RaidFileController.h" +#include "RaidFileUtil.h" +#include "RaidFileException.h" + +#include "MemLeakFindOn.h" + +#define REFCOUNT_MAGIC_VALUE 0x52656643 // RefC +#define REFCOUNT_FILENAME "refcount" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::BackupStoreRefCountDatabase() +// Purpose: Default constructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const + BackupStoreAccountDatabase::Entry& rAccount) +: mAccount(rAccount), + mFilename(GetFilename(rAccount)), + mReadOnly(true), + mIsModified(false) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase +// Purpose: Destructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase() +{ +} + +std::string BackupStoreRefCountDatabase::GetFilename(const + BackupStoreAccountDatabase::Entry& rAccount) +{ + std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount); + ASSERT(RootDir[RootDir.size() - 1] == '/' || + RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); + + std::string fn(RootDir + "refcount.db"); + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet())); + return RaidFileUtil::MakeWriteFileName(rdiscSet, fn); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Create(int32_t, +// const std::string &, int, bool) +// Purpose: Create a new database, overwriting an existing +// one only if AllowOverwrite is true. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreRefCountDatabase::Create(const + BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite) +{ + // Initial header + refcount_StreamFormat hdr; + hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE); + hdr.mAccountID = htonl(rAccount.GetID()); + + // Generate the filename + std::string Filename = GetFilename(rAccount); + + // Open the file for writing + if (FileExists(Filename) && !AllowOverwrite) + { + BOX_ERROR("Attempted to overwrite refcount database file: " << + Filename); + THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile); + } + + int flags = O_CREAT | O_BINARY | O_RDWR; + if (!AllowOverwrite) + { + flags |= O_EXCL; + } + + std::auto_ptr DatabaseFile(new FileStream(Filename, flags)); + + // Write header + DatabaseFile->Write(&hdr, sizeof(hdr)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Load(int32_t AccountID, +// BackupStoreAccountDatabase& rAccountDatabase, +// bool ReadOnly); +// Purpose: Loads the info from disc, given the root +// information. Can be marked as read only. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupStoreRefCountDatabase::Load( + const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly) +{ + // Generate the filename + std::string filename = GetFilename(rAccount); + int flags = ReadOnly ? O_RDONLY : O_RDWR; + + // Open the file for read/write + std::auto_ptr dbfile(new FileStream(filename, + flags | O_BINARY)); + + // Read in a header + refcount_StreamFormat hdr; + if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Check it + if(ntohl(hdr.mMagicValue) != REFCOUNT_MAGIC_VALUE || + (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID()) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + + // Make new object + std::auto_ptr refcount(new BackupStoreRefCountDatabase(rAccount)); + + // Put in basic location info + refcount->mReadOnly = ReadOnly; + refcount->mapDatabaseFile = dbfile; + + // return it to caller + return refcount; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Save() +// Purpose: Save modified info back to disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +/* +void BackupStoreRefCountDatabase::Save() +{ + // Make sure we're initialised (although should never come to this) + if(mFilename.empty() || mAccount.GetID() == 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Can we do this? + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Then... open a write file + RaidFileWrite rf(mAccount.GetDiscSet(), mFilename); + rf.Open(true); // allow overwriting + + // Make header + info_StreamFormat hdr; + hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); + hdr.mAccountID = htonl(mAccountID); + hdr.mClientStoreMarker = box_hton64(mClientStoreMarker); + hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed); + hdr.mBlocksUsed = box_hton64(mBlocksUsed); + hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles); + hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles); + hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories); + hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit); + hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit); + hdr.mCurrentMarkNumber = 0; + hdr.mOptionsPresent = 0; + hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size()); + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Write the deleted object list + if(mDeletedDirectories.size() > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int tosave = mDeletedDirectories.size(); + std::vector::iterator i(mDeletedDirectories.begin()); + while(tosave > 0) + { + // How many in this one? + int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); + + // Add them + for(int t = 0; t < b; ++t) + { + ASSERT(i != mDeletedDirectories.end()); + objs[t] = box_hton64((*i)); + i++; + } + + // Write + rf.Write(objs, b * sizeof(int64_t)); + + // Number saved + tosave -= b; + } + } + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Mark is as not modified + mIsModified = false; +} +*/ + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::GetRefCount(int64_t +// ObjectID) +// Purpose: Get the number of references to the specified object +// out of the database +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- +int32_t BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const +{ + IOStream::pos_type offset = GetOffset(ObjectID); + + if (GetSize() < offset + GetEntrySize()) + { + BOX_ERROR("attempted read of unknown refcount for object " << + BOX_FORMAT_OBJECTID(ObjectID)); + THROW_EXCEPTION(BackupStoreException, + UnknownObjectRefCountRequested); + } + + mapDatabaseFile->Seek(offset, SEEK_SET); + + refcount_t refcount; + if (mapDatabaseFile->Read(&refcount, sizeof(refcount)) != + sizeof(refcount)) + { + BOX_LOG_SYS_ERROR("short read on refcount database: " << + mFilename); + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo); + } + + return ntohl(refcount); +} + +int64_t BackupStoreRefCountDatabase::GetLastObjectIDUsed() const +{ + return (GetSize() - sizeof(refcount_StreamFormat)) / + sizeof(refcount_t); +} + +void BackupStoreRefCountDatabase::AddReference(int64_t ObjectID) +{ + refcount_t refcount; + + if (ObjectID > GetLastObjectIDUsed()) + { + // new object, assume no previous references + refcount = 0; + } + else + { + // read previous value from database + refcount = GetRefCount(ObjectID); + } + + refcount++; + + SetRefCount(ObjectID, refcount); +} + +void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID, + refcount_t NewRefCount) +{ + IOStream::pos_type offset = GetOffset(ObjectID); + mapDatabaseFile->Seek(offset, SEEK_SET); + refcount_t RefCountNetOrder = htonl(NewRefCount); + mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder)); +} + +bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) +{ + refcount_t refcount = GetRefCount(ObjectID); // must exist in database + ASSERT(refcount > 0); + refcount--; + SetRefCount(ObjectID, refcount); + return (refcount > 0); +} + diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h new file mode 100644 index 00000000..0a65b6a0 --- /dev/null +++ b/lib/backupstore/BackupStoreRefCountDatabase.h @@ -0,0 +1,125 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreRefCountDatabase.h +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREREFCOUNTDATABASE__H +#define BACKUPSTOREREFCOUNTDATABASE__H + +#include +#include +#include + +#include "BackupStoreAccountDatabase.h" +#include "FileStream.h" + +class BackupStoreCheck; +class BackupStoreContext; + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + uint32_t mMagicValue; // also the version number + uint32_t mAccountID; +} refcount_StreamFormat; + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreRefCountDatabase +// Purpose: Backup store reference count database storage +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- +class BackupStoreRefCountDatabase +{ + friend class BackupStoreCheck; + friend class BackupStoreContext; +public: + ~BackupStoreRefCountDatabase(); +private: + // Creation through static functions only + BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry& + rAccount); + // No copying allowed + BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &); + +public: + // Create a new database for a new account. This method will refuse + // to overwrite any existing file. + static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount) + { + Create(rAccount, false); + } + + // Load it from the store + static std::auto_ptr Load(const + BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); + + // Data access functions + int32_t GetRefCount(int64_t ObjectID) const; + int64_t GetLastObjectIDUsed() const; + + // Data modification functions + void AddReference(int64_t ObjectID); + bool RemoveReference(int64_t ObjectID); + +private: + // Create a new database for an existing account. Used during + // account checking if opening the old database throws an exception. + // This method will overwrite any existing file. + static void CreateForRegeneration(const + BackupStoreAccountDatabase::Entry& rAccount) + { + Create(rAccount, true); + } + + static void Create(const BackupStoreAccountDatabase::Entry& rAccount, + bool AllowOverwrite); + + typedef int32_t refcount_t; + + static std::string GetFilename(const BackupStoreAccountDatabase::Entry& + rAccount); + IOStream::pos_type GetSize() const + { + return mapDatabaseFile->GetPosition() + + mapDatabaseFile->BytesLeftToRead(); + } + IOStream::pos_type GetEntrySize() const + { + return sizeof(refcount_t); + } + IOStream::pos_type GetOffset(int64_t ObjectID) const + { + return ((ObjectID - 1) * GetEntrySize()) + + sizeof(refcount_StreamFormat); + } + void SetRefCount(int64_t ObjectID, refcount_t NewRefCount); + + // Location information + BackupStoreAccountDatabase::Entry mAccount; + std::string mFilename; + bool mReadOnly; + bool mIsModified; + std::auto_ptr mapDatabaseFile; +}; + +#endif // BACKUPSTOREREFCOUNTDATABASE__H -- cgit v1.2.3