Add code for BackupStoreRefCountDatabase.

Create a refcount database when creating a new account.

Add an easier way to get the account root directory.

4 files changed, 464 insertions, 8 deletions

diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp
index eb10b385..5c7e4d38 100644
--- a/lib/backupstore/BackupStoreAccounts.cpp
+++ b/lib/backupstore/BackupStoreAccounts.cpp
@@ -14,6 +14,7 @@
 #include "BoxPortsAndFiles.h"
 #include "BackupStoreAccounts.h"
 #include "BackupStoreAccountDatabase.h"
+#include "BackupStoreRefCountDatabase.h"
 #include "RaidFileWrite.h"
 #include "BackupStoreInfo.h"
 #include "BackupStoreDirectory.h"
@@ -61,6 +62,10 @@ BackupStoreAccounts::~BackupStoreAccounts()
 // --------------------------------------------------------------------------
 void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername)
 {
+	// Create the entry in the database
+	BackupStoreAccountDatabase::Entry Entry(mrDatabase.AddEntry(ID,
+		DiscSet));
+	
 	{
 		// Become the user specified in the config file?
 		std::auto_ptr<UnixUser> user;
@@ -100,15 +105,17 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit,
 		
 		// Save it back
 		info->Save();
+
+		// Create the refcount database
+		BackupStoreRefCountDatabase::CreateNew(Entry);
+		std::auto_ptr<BackupStoreRefCountDatabase> refcount(
+			BackupStoreRefCountDatabase::Load(Entry, false));
+		refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID);
 	}
 
 	// As the original user...
-
-	// Create the entry in the database
-	mrDatabase.AddEntry(ID, DiscSet);
-	
 	// Write the database back
-	mrDatabase.Write();	
+	mrDatabase.Write();
 }
 
 
@@ -138,7 +145,7 @@ void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, i
 //		Created: 2003/08/21
 //
 // --------------------------------------------------------------------------
-std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const
+std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet)
 {
 	char accid[64];	// big enough!
 	::sprintf(accid, "%08x" DIRECTORY_SEPARATOR, ID);
diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h
index 0c3dd103..224d7353 100644
--- a/lib/backupstore/BackupStoreAccounts.h
+++ b/lib/backupstore/BackupStoreAccounts.h
@@ -12,7 +12,7 @@
 
 #include <string>
 
-class BackupStoreAccountDatabase;
+#include "BackupStoreAccountDatabase.h"
 
 // --------------------------------------------------------------------------
 //
@@ -35,9 +35,14 @@ public:
 
 	bool AccountExists(int32_t ID);
 	void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const;
+	static std::string GetAccountRoot(const
+		BackupStoreAccountDatabase::Entry &rEntry)
+	{
+		return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet());
+	}
 
 private:
-	std::string MakeAccountRootDir(int32_t ID, int DiscSet) const;
+	static std::string MakeAccountRootDir(int32_t ID, int DiscSet);
 
 private:
 	BackupStoreAccountDatabase &mrDatabase;
diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp
new file mode 100644
index 00000000..92ba947d
--- /dev/null
+++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp
@@ -0,0 +1,319 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreRefCountDatabase.cpp
+//		Purpose: Backup store object reference count database storage
+//		Created: 2009/06/01
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <algorithm>
+
+#include "BackupStoreRefCountDatabase.h"
+#include "BackupStoreException.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreAccounts.h"
+#include "RaidFileController.h"
+#include "RaidFileUtil.h"
+#include "RaidFileException.h"
+
+#include "MemLeakFindOn.h"
+
+#define REFCOUNT_MAGIC_VALUE	0x52656643 // RefC
+#define REFCOUNT_FILENAME	"refcount"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::BackupStoreRefCountDatabase()
+//		Purpose: Default constructor
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const
+	BackupStoreAccountDatabase::Entry& rAccount)
+: mAccount(rAccount),
+  mFilename(GetFilename(rAccount)),
+  mReadOnly(true),
+  mIsModified(false)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase
+//		Purpose: Destructor
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase()
+{
+}
+
+std::string BackupStoreRefCountDatabase::GetFilename(const
+	BackupStoreAccountDatabase::Entry& rAccount)
+{
+	std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount);
+	ASSERT(RootDir[RootDir.size() - 1] == '/' ||
+		RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR);
+
+	std::string fn(RootDir + "refcount.db");
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet()));
+	return RaidFileUtil::MakeWriteFileName(rdiscSet, fn);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::Create(int32_t,
+//			 const std::string &, int, bool)
+//		Purpose: Create a new database, overwriting an existing
+//			 one only if AllowOverwrite is true.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreRefCountDatabase::Create(const
+	BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite)
+{
+	// Initial header
+	refcount_StreamFormat hdr;
+	hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE);
+	hdr.mAccountID = htonl(rAccount.GetID());
+	
+	// Generate the filename
+	std::string Filename = GetFilename(rAccount);
+
+	// Open the file for writing
+	if (FileExists(Filename) && !AllowOverwrite)
+	{
+		BOX_ERROR("Attempted to overwrite refcount database file: " <<
+			Filename);
+		THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile);
+	}
+
+	int flags = O_CREAT | O_BINARY | O_RDWR;
+	if (!AllowOverwrite)
+	{
+		flags |= O_EXCL;
+	}
+
+	std::auto_ptr<FileStream> DatabaseFile(new FileStream(Filename, flags));
+	
+	// Write header
+	DatabaseFile->Write(&hdr, sizeof(hdr));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::Load(int32_t AccountID,
+//			 BackupStoreAccountDatabase& rAccountDatabase,
+//			 bool ReadOnly);
+//		Purpose: Loads the info from disc, given the root
+//			 information. Can be marked as read only.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load(
+	const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly)
+{
+	// Generate the filename
+	std::string filename = GetFilename(rAccount);
+	int flags = ReadOnly ? O_RDONLY : O_RDWR;
+
+	// Open the file for read/write
+	std::auto_ptr<FileStream> dbfile(new FileStream(filename,
+		flags | O_BINARY));
+	
+	// Read in a header
+	refcount_StreamFormat hdr;
+	if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+	}
+	
+	// Check it
+	if(ntohl(hdr.mMagicValue) != REFCOUNT_MAGIC_VALUE ||
+		(int32_t)ntohl(hdr.mAccountID) != rAccount.GetID())
+	{
+		THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+	}
+	
+	// Make new object
+	std::auto_ptr<BackupStoreRefCountDatabase> refcount(new BackupStoreRefCountDatabase(rAccount));
+	
+	// Put in basic location info
+	refcount->mReadOnly = ReadOnly;
+	refcount->mapDatabaseFile = dbfile;
+	
+	// return it to caller
+	return refcount;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::Save()
+//		Purpose: Save modified info back to disc
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+/*
+void BackupStoreRefCountDatabase::Save()
+{
+	// Make sure we're initialised (although should never come to this)
+	if(mFilename.empty() || mAccount.GetID() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, Internal)
+	}
+
+	// Can we do this?
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	
+	// Then... open a write file
+	RaidFileWrite rf(mAccount.GetDiscSet(), mFilename);
+	rf.Open(true);		// allow overwriting
+	
+	// Make header
+	info_StreamFormat hdr;
+	hdr.mMagicValue 			= htonl(INFO_MAGIC_VALUE);
+	hdr.mAccountID 				= htonl(mAccountID);
+	hdr.mClientStoreMarker			= box_hton64(mClientStoreMarker);
+	hdr.mLastObjectIDUsed			= box_hton64(mLastObjectIDUsed);
+	hdr.mBlocksUsed 				= box_hton64(mBlocksUsed);
+	hdr.mBlocksInOldFiles 			= box_hton64(mBlocksInOldFiles);
+	hdr.mBlocksInDeletedFiles 		= box_hton64(mBlocksInDeletedFiles);
+	hdr.mBlocksInDirectories		= box_hton64(mBlocksInDirectories);
+	hdr.mBlocksSoftLimit			= box_hton64(mBlocksSoftLimit);
+	hdr.mBlocksHardLimit			= box_hton64(mBlocksHardLimit);
+	hdr.mCurrentMarkNumber			= 0;
+	hdr.mOptionsPresent				= 0;
+	hdr.mNumberDeletedDirectories	= box_hton64(mDeletedDirectories.size());
+	
+	// Write header
+	rf.Write(&hdr, sizeof(hdr));
+	
+	// Write the deleted object list
+	if(mDeletedDirectories.size() > 0)
+	{
+		int64_t objs[NUM_DELETED_DIRS_BLOCK];
+		
+		int tosave = mDeletedDirectories.size();
+		std::vector<int64_t>::iterator i(mDeletedDirectories.begin());
+		while(tosave > 0)
+		{
+			// How many in this one?
+			int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave));
+			
+			// Add them
+			for(int t = 0; t < b; ++t)
+			{
+				ASSERT(i != mDeletedDirectories.end());
+				objs[t] = box_hton64((*i));
+				i++;
+			}
+
+			// Write			
+			rf.Write(objs, b * sizeof(int64_t));
+			
+			// Number saved
+			tosave -= b;
+		}
+	}
+
+	// Commit it to disc, converting it to RAID now
+	rf.Commit(true);
+	
+	// Mark is as not modified
+	mIsModified = false;
+}
+*/
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreRefCountDatabase::GetRefCount(int64_t
+//			 ObjectID)
+//		Purpose: Get the number of references to the specified object
+//			 out of the database
+//		Created: 2009/06/01
+//
+// --------------------------------------------------------------------------
+int32_t BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const
+{
+	IOStream::pos_type offset = GetOffset(ObjectID);
+
+	if (GetSize() < offset + GetEntrySize())
+	{
+		BOX_ERROR("attempted read of unknown refcount for object " <<
+			BOX_FORMAT_OBJECTID(ObjectID));
+		THROW_EXCEPTION(BackupStoreException,
+			UnknownObjectRefCountRequested);
+	}
+
+	mapDatabaseFile->Seek(offset, SEEK_SET);
+
+	refcount_t refcount;
+	if (mapDatabaseFile->Read(&refcount, sizeof(refcount)) !=
+		sizeof(refcount))
+	{
+		BOX_LOG_SYS_ERROR("short read on refcount database: " <<
+			mFilename);
+		THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo);
+	}
+
+	return ntohl(refcount);
+}
+
+int64_t BackupStoreRefCountDatabase::GetLastObjectIDUsed() const
+{
+	return (GetSize() - sizeof(refcount_StreamFormat)) /
+		sizeof(refcount_t);
+}
+
+void BackupStoreRefCountDatabase::AddReference(int64_t ObjectID)
+{
+	refcount_t refcount;
+
+	if (ObjectID > GetLastObjectIDUsed())
+	{
+		// new object, assume no previous references
+		refcount = 0;
+	}
+	else
+	{
+		// read previous value from database
+		refcount = GetRefCount(ObjectID);
+	}
+
+	refcount++;
+
+	SetRefCount(ObjectID, refcount);
+}
+
+void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID,
+	refcount_t NewRefCount)
+{
+	IOStream::pos_type offset = GetOffset(ObjectID);
+	mapDatabaseFile->Seek(offset, SEEK_SET);
+	refcount_t RefCountNetOrder = htonl(NewRefCount);
+	mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder));
+}
+
+bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID)
+{
+	refcount_t refcount = GetRefCount(ObjectID); // must exist in database
+	ASSERT(refcount > 0);
+	refcount--;
+	SetRefCount(ObjectID, refcount);
+	return (refcount > 0);
+}
+
diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h
new file mode 100644
index 00000000..0a65b6a0
--- /dev/null
+++ b/lib/backupstore/BackupStoreRefCountDatabase.h
@@ -0,0 +1,125 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreRefCountDatabase.h
+//		Purpose: Main backup store information storage
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREREFCOUNTDATABASE__H
+#define BACKUPSTOREREFCOUNTDATABASE__H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "BackupStoreAccountDatabase.h"
+#include "FileStream.h"
+
+class BackupStoreCheck;
+class BackupStoreContext;
+
+// set packing to one byte
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+	uint32_t mMagicValue;	// also the version number
+	uint32_t mAccountID;
+} refcount_StreamFormat;
+
+// Use default packing
+#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreRefCountDatabase
+//		Purpose: Backup store reference count database storage
+//		Created: 2009/06/01
+//
+// --------------------------------------------------------------------------
+class BackupStoreRefCountDatabase
+{
+	friend class BackupStoreCheck;
+	friend class BackupStoreContext;
+public:
+	~BackupStoreRefCountDatabase();
+private:
+	// Creation through static functions only
+	BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry&
+		rAccount);
+	// No copying allowed
+	BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &);
+	
+public:
+	// Create a new database for a new account. This method will refuse
+	// to overwrite any existing file.
+	static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount)
+	{
+		Create(rAccount, false);
+	}
+	
+	// Load it from the store
+	static std::auto_ptr<BackupStoreRefCountDatabase> Load(const
+		BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly);
+	
+	// Data access functions
+	int32_t GetRefCount(int64_t ObjectID) const;
+	int64_t GetLastObjectIDUsed() const;
+	
+	// Data modification functions
+	void AddReference(int64_t ObjectID);
+	bool RemoveReference(int64_t ObjectID);
+
+private:
+	// Create a new database for an existing account. Used during
+	// account checking if opening the old database throws an exception.
+	// This method will overwrite any existing file.
+	static void CreateForRegeneration(const
+		BackupStoreAccountDatabase::Entry& rAccount)
+	{
+		Create(rAccount, true);
+	}
+
+	static void Create(const BackupStoreAccountDatabase::Entry& rAccount,
+		bool AllowOverwrite);
+
+	typedef int32_t refcount_t;
+
+	static std::string GetFilename(const BackupStoreAccountDatabase::Entry&
+		rAccount);
+	IOStream::pos_type GetSize() const
+	{
+		return mapDatabaseFile->GetPosition() +
+			mapDatabaseFile->BytesLeftToRead();
+	}
+	IOStream::pos_type GetEntrySize() const
+	{
+		return sizeof(refcount_t);
+	}
+	IOStream::pos_type GetOffset(int64_t ObjectID) const
+	{
+		return ((ObjectID - 1) * GetEntrySize()) +
+			sizeof(refcount_StreamFormat);
+	}
+	void SetRefCount(int64_t ObjectID, refcount_t NewRefCount);
+	
+	// Location information
+	BackupStoreAccountDatabase::Entry mAccount;
+	std::string mFilename;
+	bool mReadOnly;
+	bool mIsModified;
+	std::auto_ptr<FileStream> mapDatabaseFile;
+};
+
+#endif // BACKUPSTOREREFCOUNTDATABASE__H