Box Backup 0.09 with a few tweeks

14 files changed, 4976 insertions, 0 deletions

diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp
new file mode 100755
index 00000000..2eb19089
--- /dev/null
+++ b/bin/bbstored/BBStoreDHousekeeping.cpp
@@ -0,0 +1,175 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BBStoreDHousekeeping.cpp
+//		Purpose: Implementation of housekeeping functions for bbstored
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <syslog.h>
+
+#include "BackupStoreDaemon.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreAccounts.h"
+#include "HousekeepStoreAccount.h"
+#include "BoxTime.h"
+#include "Configuration.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::HousekeepingProcess()
+//		Purpose: Do housekeeping
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::HousekeepingProcess()
+{
+	// Get the time between housekeeping runs
+	const Configuration &rconfig(GetConfiguration());
+	int64_t housekeepingInterval = SecondsToBoxTime((uint32_t)rconfig.GetKeyValueInt("TimeBetweenHousekeeping"));
+	
+	int64_t lastHousekeepingRun = 0;
+
+	while(!StopRun())
+	{
+		// Time now
+		int64_t timeNow = GetCurrentBoxTime();
+		// Do housekeeping if the time interval has elapsed since the last check
+		if((timeNow - lastHousekeepingRun) >= housekeepingInterval)
+		{
+			// Store the time
+			lastHousekeepingRun = timeNow;
+			::syslog(LOG_INFO, "Starting housekeeping");
+
+			// Get the list of accounts
+			std::vector<int32_t> accounts;
+			if(mpAccountDatabase)
+			{
+				mpAccountDatabase->GetAllAccountIDs(accounts);
+			}
+			
+			SetProcessTitle("housekeeping, active");
+			
+			// Check them all
+			for(std::vector<int32_t>::const_iterator i = accounts.begin(); i != accounts.end(); ++i)
+			{
+				try
+				{
+					if(mpAccounts)
+					{
+						// Get the account root
+						std::string rootDir;
+						int discSet = 0;
+						mpAccounts->GetAccountRoot(*i, rootDir, discSet);
+						
+						// Do housekeeping on this account
+						HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this);
+						housekeeping.DoHousekeeping();
+					}
+				}
+				catch(BoxException &e)
+				{
+					::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account",
+						*i, e.what(), e.GetType(), e.GetSubType());
+				}
+				catch(std::exception &e)
+				{
+					::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account",
+						*i, e.what());
+				}
+				catch(...)
+				{
+					::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account",
+						*i);
+				}
+				
+				// Check to see if there's any message pending
+				CheckForInterProcessMsg(0 /* no account */);
+		
+				// Stop early?
+				if(StopRun())
+				{
+					break;
+				}
+			}
+			
+			::syslog(LOG_INFO, "Finished housekeeping");
+		}
+
+		// Placed here for accuracy, if StopRun() is true, for example.
+		SetProcessTitle("housekeeping, idle");
+		
+		// Calculate how long should wait before doing the next housekeeping run
+		timeNow = GetCurrentBoxTime();
+		int64_t secondsToGo = BoxTimeToSeconds((lastHousekeepingRun + housekeepingInterval) - timeNow);
+		if(secondsToGo < 1) secondsToGo = 1;
+		if(secondsToGo > 60) secondsToGo = 60;
+		int32_t millisecondsToGo = ((int)secondsToGo) * 1000;
+	
+		// Check to see if there's any message pending
+		CheckForInterProcessMsg(0 /* no account */, millisecondsToGo);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::CheckForInterProcessMsg(int, int)
+//		Purpose: Process a message, returning true if the housekeeping process
+//				 should abort for the specified account.
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitTime)
+{
+	// First, check to see if it's EOF -- this means something has gone wrong, and the housekeeping should terminate.
+	if(mInterProcessComms.IsEOF())
+	{
+		SetTerminateWanted();
+		return true;
+	}
+
+	// Get a line, and process the message
+	std::string line;
+	if(mInterProcessComms.GetLine(line, false /* no pre-processing */, MaximumWaitTime))
+	{
+		TRACE1("housekeeping received command '%s' over interprocess comms\n", line.c_str());
+	
+		int account = 0;
+	
+		if(line == "h")
+		{
+			// HUP signal received by main process
+			SetReloadConfigWanted();
+			return true;
+		}
+		else if(line == "t")
+		{
+			// Terminate signal received by main process
+			SetTerminateWanted();
+			return true;
+		}
+		else if(sscanf(line.c_str(), "r%x", &account) == 1)
+		{
+			// Main process is trying to lock an account -- are we processing it?
+			if(account == AccountNum)
+			{
+				// Yes! -- need to stop now so when it retries to get the lock, it will succeed
+				::syslog(LOG_INFO, "Housekeeping giving way to connection for account 0x%08x", AccountNum);
+				return true;
+			}
+		}
+	}
+	
+	return false;
+}
+
+
diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp
new file mode 100755
index 00000000..1a021a58
--- /dev/null
+++ b/bin/bbstored/BackupCommands.cpp
@@ -0,0 +1,861 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupCommands.cpp
+//		Purpose: Implement commands for the Backup store protocol
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <syslog.h>
+
+#include "autogen_BackupProtocolServer.h"
+#include "BackupConstants.h"
+#include "BackupContext.h"
+#include "CollectInBufferStream.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreException.h"
+#include "BackupStoreFile.h"
+#include "StreamableMemBlock.h"
+#include "BackupStoreConstants.h"
+#include "RaidFileController.h"
+#include "BackupStoreInfo.h"
+#include "RaidFileController.h"
+#include "FileStream.h"
+
+#include "MemLeakFindOn.h"
+
+#define CHECK_PHASE(phase)																						\
+	if(rContext.GetPhase() != BackupContext::phase)																\
+	{																											\
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(										\
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_NotInRightProtocolPhase));		\
+	}
+
+#define CHECK_WRITEABLE_SESSION																					\
+	if(rContext.SessionIsReadOnly())																			\
+	{																											\
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(										\
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_SessionReadOnly));				\
+	}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerVersion::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Return the current version, or an error if the requested version isn't allowed
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerVersion::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Version)
+
+	// Correct version?
+	if(mVersion != BACKUP_STORE_SERVER_VERSION)
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_WrongVersion));
+	}
+
+	// Mark the next phase
+	rContext.SetPhase(BackupContext::Phase_Login);
+
+	// Return our version
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerVersion(BACKUP_STORE_SERVER_VERSION));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerLogin::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Return the current version, or an error if the requested version isn't allowed
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerLogin::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Login)
+
+	// Check given client ID against the ID in the certificate certificate
+	// and that the client actually has an account on this machine
+	if(mClientID != rContext.GetClientID() || !rContext.GetClientHasAccount())
+	{
+		::syslog(LOG_INFO, "Failed login: Client ID presented was %08X", mClientID);
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_BadLogin));
+	}
+
+	// If we need to write, check that nothing else has got a write lock
+	if((mFlags & Flags_ReadOnly) != Flags_ReadOnly)
+	{
+		// See if the context will get the lock
+		if(!rContext.AttemptToGetWriteLock())
+		{
+			::syslog(LOG_INFO, "Failed to get write lock (for Client ID %08X)", mClientID);
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_CannotLockStoreForWriting));			
+		}
+		
+		// Debug: check we got the lock
+		ASSERT(!rContext.SessionIsReadOnly());
+	}
+	
+	// Load the store info
+	rContext.LoadStoreInfo();
+
+	// Get the last client store marker
+	int64_t clientStoreMarker = rContext.GetClientStoreMarker();
+
+	// Mark the next phase
+	rContext.SetPhase(BackupContext::Phase_Commands);
+	
+	// Log login
+	::syslog(LOG_INFO, "Login: Client ID %08X, %s", mClientID, ((mFlags & Flags_ReadOnly) != Flags_ReadOnly)?"Read/Write":"Read-only");
+
+	// Get the usage info for reporting to the client
+	int64_t blocksUsed = 0, blocksSoftLimit = 0, blocksHardLimit = 0;
+	rContext.GetStoreDiscUsageInfo(blocksUsed, blocksSoftLimit, blocksHardLimit);
+
+	// Return success
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerLoginConfirmed(clientStoreMarker, blocksUsed, blocksSoftLimit, blocksHardLimit));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerFinished::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Marks end of conversation (Protocol framework handles this)
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	::syslog(LOG_INFO, "Session finished");
+
+	// Let the context know about it
+	rContext.ReceivedFinishCommand();
+
+	// can be called in any phase
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerFinished);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerListDirectory::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Command to list a directory
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerListDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Ask the context for a directory
+	const BackupStoreDirectory &rdir(rContext.GetDirectory(mObjectID));
+	
+	// Store the listing to a stream
+	std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream);
+	rdir.WriteToStream(*stream, mFlagsMustBeSet, mFlagsNotToBeSet, mSendAttributes,
+		false /* never send dependency info to the client */);
+	stream->SetForReading();
+	
+	// Get the protocol to send the stream
+	rProtocol.SendStreamAfterCommand(stream.release());
+
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerStoreFile::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Command to store a file on the server
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerStoreFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+	
+	// Check that the diff from file actually exists, if it's specified
+	if(mDiffFromFileID != 0)
+	{
+		if(!rContext.ObjectExists(mDiffFromFileID, BackupContext::ObjectExists_File))
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DiffFromFileDoesNotExist));	
+		}
+	}
+	
+	// A stream follows, which contains the file
+	std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream());
+	
+	// Ask the context to store it
+	int64_t id = 0;
+	try
+	{
+		id = rContext.AddFile(*dirstream, mDirectoryObjectID, mModificationTime, mAttributesHash, mDiffFromFileID,
+			 mFilename, true /* mark files with same name as old versions */);
+	}
+	catch(BackupStoreException &e)
+	{
+		if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify)
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_FileDoesNotVerify));			
+		}
+		else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit)
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_StorageLimitExceeded));			
+		}
+		else
+		{
+			throw;
+		}
+	}
+	
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(id));
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetObject::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Command to get an arbitary object from the server
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetObject::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Check the object exists
+	if(!rContext.ObjectExists(mObjectID))
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(NoObject));
+	}
+
+	// Open the object
+	std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID));
+
+	// Stream it to the peer
+	rProtocol.SendStreamAfterCommand(object.release());
+
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetFile::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Command to get an file object from the server -- may have to do a bit of 
+//				 work to get the object.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Check the objects exist
+	if(!rContext.ObjectExists(mObjectID)
+		|| !rContext.ObjectExists(mInDirectory))
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DoesNotExist));			
+	}
+
+	// Get the directory it's in
+	const BackupStoreDirectory &rdir(rContext.GetDirectory(mInDirectory));
+
+	// Find the object within the directory
+	BackupStoreDirectory::Entry *pfileEntry = rdir.FindEntryByID(mObjectID);
+	if(pfileEntry == 0)
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DoesNotExistInDirectory));			
+	}
+
+	// The result
+	std::auto_ptr<IOStream> stream;
+
+	// Does this depend on anything?
+	if(pfileEntry->GetDependsNewer() != 0)
+	{
+		// File exists, but is a patch from a new version. Generate the older version.
+		std::vector<int64_t> patchChain;
+		int64_t id = mObjectID;
+		BackupStoreDirectory::Entry *en = 0;
+		do
+		{
+			patchChain.push_back(id);
+			en = rdir.FindEntryByID(id);
+			if(en == 0)
+			{
+				::syslog(LOG_ERR, "Object %llx in dir %llx for account %x references object %llx which does not exist in dir",
+					mObjectID, mInDirectory, rContext.GetClientID(), id);
+				return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+					BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_PatchConsistencyError));			
+			}
+			id = en->GetDependsNewer();
+		} while(en != 0 && id != 0);
+		
+		// OK! The last entry in the chain is the full file, the others are patches back from it.
+		// Open the last one, which is the current from file
+		std::auto_ptr<IOStream> from(rContext.OpenObject(patchChain[patchChain.size() - 1]));
+		
+		// Then, for each patch in the chain, do a combine
+		for(int p = ((int)patchChain.size()) - 2; p >= 0; --p)
+		{
+			// ID of patch
+			int64_t patchID = patchChain[p];
+			
+			// Open it a couple of times
+			std::auto_ptr<IOStream> diff(rContext.OpenObject(patchID));
+			std::auto_ptr<IOStream> diff2(rContext.OpenObject(patchID));
+			
+			// Choose a temporary filename for the result of the combination
+			std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp",
+				p + 16 /* rotate which disc it's on */));
+			
+			// Open the temporary file
+			std::auto_ptr<IOStream> combined;
+			try
+			{
+				{
+					// Write nastily to allow this to work with gcc 2.x
+					std::auto_ptr<IOStream> t(new FileStream(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL));
+					combined = t;
+				}
+				// Unlink immediately as it's a temporary file
+				if(::unlink(tempFn.c_str()) != 0)
+				{
+					THROW_EXCEPTION(CommonException, OSFileError);
+				}
+			}
+			catch(...)
+			{
+				// Make sure it goes
+				::unlink(tempFn.c_str());
+				throw;
+			}
+			
+			// Do the combining
+			BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined);
+			
+			// Move to the beginning of the combined file
+			combined->Seek(0, IOStream::SeekType_Absolute);
+			
+			// Then shuffle round for the next go
+			from = combined;
+		}
+		
+		// Now, from contains a nice file to send to the client. Reorder it
+		{
+			// Write nastily to allow this to work with gcc 2.x
+			std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */));
+			stream = t;
+		}
+		
+		// Release from file to avoid double deletion
+		from.release();
+	}
+	else
+	{
+		// Simple case: file already exists on disc ready to go
+	
+		// Open the object
+		std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID));
+		
+		// Verify it
+		if(!BackupStoreFile::VerifyEncodedFileFormat(*object))
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_FileDoesNotVerify));			
+		}
+		
+		// Reset stream -- seek to beginning
+		object->Seek(0, IOStream::SeekType_Absolute);
+		
+		// Reorder the stream/file into stream order
+		{
+			// Write nastily to allow this to work with gcc 2.x
+			std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(object.get(), true /* take ownership */));
+			stream = t;
+		}
+
+		// Object will be deleted when the stream is deleted, so can release the object auto_ptr here to
+		// avoid premature deletiong
+		object.release();
+	}
+
+	// Stream the reordered stream to the peer
+	rProtocol.SendStreamAfterCommand(stream.get());
+	
+	// Don't delete the stream here
+	stream.release();
+
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerCreateDirectory::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Create directory command
+//		Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerCreateDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+	
+	// Get the stream containing the attributes
+	std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
+	// Collect the attributes -- do this now so no matter what the outcome, 
+	// the data has been absorbed.
+	StreamableMemBlock attr;
+	attr.Set(*attrstream, rProtocol.GetTimeout());
+	
+	// Check to see if the hard limit has been exceeded
+	if(rContext.HardLimitExceeded())
+	{
+		// Won't allow creation if the limit has been exceeded
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_StorageLimitExceeded));			
+	}
+
+	bool alreadyExists = false;
+	int64_t id = rContext.AddDirectory(mContainingDirectoryID, mDirectoryName, attr, mAttributesModTime, alreadyExists);
+	
+	if(alreadyExists)
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DirectoryAlreadyExists));			
+	}
+
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(id));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerChangeDirAttributes::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Change attributes on directory
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerChangeDirAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Get the stream containing the attributes
+	std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
+	// Collect the attributes -- do this now so no matter what the outcome, 
+	// the data has been absorbed.
+	StreamableMemBlock attr;
+	attr.Set(*attrstream, rProtocol.GetTimeout());
+
+	// Get the context to do it's magic
+	rContext.ChangeDirAttributes(mObjectID, attr, mAttributesModTime);
+
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerSetReplacementFileAttributes::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Change attributes on directory
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerSetReplacementFileAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Get the stream containing the attributes
+	std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
+	// Collect the attributes -- do this now so no matter what the outcome, 
+	// the data has been absorbed.
+	StreamableMemBlock attr;
+	attr.Set(*attrstream, rProtocol.GetTimeout());
+
+	// Get the context to do it's magic
+	int64_t objectID = 0;
+	if(!rContext.ChangeFileAttributes(mFilename, mInDirectory, attr, mAttributesHash, objectID))
+	{
+		// Didn't exist
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DoesNotExist));			
+	}
+
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Delete a file
+//		Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Context handles this
+	int64_t objectID = 0;
+	rContext.DeleteFile(mFilename, mInDirectory, objectID);
+
+	// return the object ID or zero for not found
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Delete a directory
+//		Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Check it's not asking for the root directory to be deleted
+	if(mObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_CannotDeleteRoot));			
+	}
+
+	// Context handles this
+	rContext.DeleteDirectory(mObjectID);
+
+	// return the object ID
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Undelete a directory
+//		Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Check it's not asking for the root directory to be deleted
+	if(mObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_CannotDeleteRoot));			
+	}
+
+	// Context handles this
+	rContext.DeleteDirectory(mObjectID, true /* undelete */);
+
+	// return the object ID
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Command to set the client's store marker
+//		Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Set the marker
+	rContext.SetClientStoreMarker(mClientStoreMarker);
+
+	// return store marker set
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mClientStoreMarker));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Command to move an object from one directory to another
+//		Created: 2003/11/12
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+	
+	// Let context do this, but modify error reporting on exceptions...
+	try
+	{
+		rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory,
+			mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName,
+			(mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject);
+	}
+	catch(BackupStoreException &e)
+	{
+		if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory)
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DoesNotExist));			
+		}
+		else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory)
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_TargetNameExists));			
+		}
+		else
+		{
+			throw;
+		}
+	}
+
+	// Return the object ID
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Command to find the name of an object
+//		Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	
+	// Create a stream for the list of filenames
+	std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream);
+
+	// Object and directory IDs
+	int64_t objectID = mObjectID;
+	int64_t dirID = mContainingDirectoryID;
+	
+	// Data to return in the reply
+	int32_t numNameElements = 0;
+	int16_t objectFlags = 0;
+	int64_t modTime = 0;
+	uint64_t attrModHash = 0;
+	bool haveModTimes = false;
+	
+	do
+	{
+		// Check the directory really exists
+		if(!rContext.ObjectExists(dirID, BackupContext::ObjectExists_Directory))
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0));
+		}
+
+		// Load up the directory
+		const BackupStoreDirectory &rdir(rContext.GetDirectory(dirID));
+
+		// Find the element in this directory and store it's name
+		if(objectID != ObjectID_DirectoryOnly)
+		{
+			const BackupStoreDirectory::Entry *en = rdir.FindEntryByID(objectID);
+
+			// If this can't be found, then there is a problem... tell the caller it can't be found
+			if(en == 0)
+			{
+				// Abort!
+				return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0));
+			}
+			
+			// Store flags?
+			if(objectFlags == 0)
+			{
+				objectFlags = en->GetFlags();
+			}
+			
+			// Store modification times?
+			if(!haveModTimes)
+			{
+				modTime = en->GetModificationTime();
+				attrModHash = en->GetAttributesHash();
+				haveModTimes = true;
+			}
+			
+			// Store the name in the stream
+			en->GetName().WriteToStream(*stream);
+			
+			// Count of name elements
+			++numNameElements;
+		}
+		
+		// Setup for next time round
+		objectID = dirID;
+		dirID = rdir.GetContainerID();
+
+	} while(objectID != 0 && objectID != BACKUPSTORE_ROOT_DIRECTORY_ID);
+
+	// Stream to send?
+	if(numNameElements > 0)
+	{
+		// Get the stream ready to go
+		stream->SetForReading();	
+		// Tell the protocol to send the stream
+		rProtocol.SendStreamAfterCommand(stream.release());
+	}
+
+	// Make reply
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(numNameElements, modTime, attrModHash, objectFlags));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Get the block index from a file, by ID
+//		Created: 19/1/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Open the file
+	std::auto_ptr<IOStream> stream(rContext.OpenObject(mObjectID));
+	
+	// Move the file pointer to the block index
+	BackupStoreFile::MoveStreamPositionToBlockIndex(*stream);
+	
+	// Return the stream to the client
+	rProtocol.SendStreamAfterCommand(stream.release());
+
+	// Return the object ID
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Get the block index from a file, by name within a directory
+//		Created: 19/1/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Get the directory
+	const BackupStoreDirectory &dir(rContext.GetDirectory(mInDirectory));
+	
+	// Find the latest object ID within it which has the same name
+	int64_t objectID = 0;
+	BackupStoreDirectory::Iterator i(dir);
+	BackupStoreDirectory::Entry *en = 0;
+	while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+	{
+		if(en->GetName() == mFilename)
+		{
+			// Store the ID, if it's a newer ID than the last one
+			if(en->GetObjectID() > objectID)
+			{
+				objectID = en->GetObjectID();
+			}
+		}
+	}
+	
+	// Found anything?
+	if(objectID == 0)
+	{
+		// No... return a zero object ID
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(0));
+	}
+
+	// Open the file
+	std::auto_ptr<IOStream> stream(rContext.OpenObject(objectID));
+	
+	// Move the file pointer to the block index
+	BackupStoreFile::MoveStreamPositionToBlockIndex(*stream);
+	
+	// Return the stream to the client
+	rProtocol.SendStreamAfterCommand(stream.release());
+
+	// Return the object ID
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Return the amount of disc space used
+//		Created: 19/4/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Get store info from context
+	const BackupStoreInfo &rinfo(rContext.GetBackupStoreInfo());
+	
+	// Find block size
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(rinfo.GetDiscSetNumber()));
+	
+	// Return info
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerAccountUsage(
+		rinfo.GetBlocksUsed(),
+		rinfo.GetBlocksInOldFiles(),
+		rinfo.GetBlocksInDeletedFiles(),
+		rinfo.GetBlocksInDirectories(),
+		rinfo.GetBlocksSoftLimit(),
+		rinfo.GetBlocksHardLimit(),
+		rdiscSet.GetBlockSize()
+	));
+}
+
diff --git a/bin/bbstored/BackupConstants.h b/bin/bbstored/BackupConstants.h
new file mode 100755
index 00000000..515b3bcd
--- /dev/null
+++ b/bin/bbstored/BackupConstants.h
@@ -0,0 +1,23 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupConstants.h
+//		Purpose: Constants for the backup server and client
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCONSTANTS__H
+#define BACKUPCONSTANTS__H
+
+#define BACKUP_STORE_DEFAULT_ACCOUNT_DATABASE_FILE	"/etc/box/backupstoreaccounts"
+
+// 15 minutes to timeout (milliseconds)
+#define	BACKUP_STORE_TIMEOUT			(15*60*1000)
+
+// Should the store daemon convert files to Raid immediately?
+#define	BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY	true
+
+#endif // BACKUPCONSTANTS__H
+
+
diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp
new file mode 100755
index 00000000..c796c13a
--- /dev/null
+++ b/bin/bbstored/BackupContext.cpp
@@ -0,0 +1,1650 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupContext.cpp
+//		Purpose: Context for backup store server
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "BackupContext.h"
+#include "RaidFileWrite.h"
+#include "RaidFileRead.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreException.h"
+#include "BackupStoreInfo.h"
+#include "BackupConstants.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreObjectMagic.h"
+#include "StoreStructure.h"
+#include "BackupStoreDaemon.h"
+#include "RaidFileController.h"
+#include "FileStream.h"
+
+#include "MemLeakFindOn.h"
+
+
+// Maximum number of directories to keep in the cache
+// When the cache is bigger than this, everything gets
+// deleted.
+#ifdef NDEBUG
+	#define	MAX_CACHE_SIZE	32
+#else
+	#define	MAX_CACHE_SIZE	2
+#endif
+
+// Allow the housekeeping process 4 seconds to release an account
+#define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT	4
+
+// Maximum amount of store info updates before it's actually saved to disc.
+#define STORE_INFO_SAVE_DELAY	96
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::BackupContext()
+//		Purpose: Constructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupContext::BackupContext(int32_t ClientID, BackupStoreDaemon &rDaemon)
+	: mClientID(ClientID),
+	  mrDaemon(rDaemon),
+	  mProtocolPhase(Phase_START),
+	  mClientHasAccount(false),
+	  mStoreDiscSet(-1),
+	  mReadOnly(true),
+	  mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::~BackupContext()
+//		Purpose: Destructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupContext::~BackupContext()
+{
+	// Delete the objects in the cache
+	for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i)
+	{
+		delete (i->second);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::CleanUp()
+//		Purpose: Clean up after a connection
+//		Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+void BackupContext::CleanUp()
+{
+	// Make sure the store info is saved, if it has been loaded, isn't read only and has been modified
+	if(mpStoreInfo.get() && !(mpStoreInfo->IsReadOnly()) && mpStoreInfo->IsModified())
+	{
+		mpStoreInfo->Save();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::ReceivedFinishCommand()
+//		Purpose: Called when the finish command is received by the protocol
+//		Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+void BackupContext::ReceivedFinishCommand()
+{
+	if(!mReadOnly && mpStoreInfo.get())
+	{
+		// Save the store info, not delayed
+		SaveStoreInfo(false);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::AttemptToGetWriteLock()
+//		Purpose: Attempt to get a write lock for the store, and if so, unset the read only flags
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+bool BackupContext::AttemptToGetWriteLock()
+{
+	// Make the filename of the write lock file
+	std::string writeLockFile;
+	StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile);
+
+	// Request the lock
+	bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */);
+	
+	if(!gotLock)
+	{
+		// The housekeeping process might have the thing open -- ask it to stop
+		char msg[256];
+		int msgLen = sprintf(msg, "r%x\n", mClientID);
+		// Send message
+		mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen);
+		
+		// Then try again a few times
+		int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT;
+		do
+		{
+			::sleep(1 /* second */);
+			--tries;
+			gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */);
+			
+		} while(!gotLock && tries > 0);
+	}
+	
+	if(gotLock)
+	{
+		// Got the lock, mark as not read only
+		mReadOnly = false;
+	}
+	
+	return gotLock;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::LoadStoreInfo()
+//		Purpose: Load the store info from disc
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+void BackupContext::LoadStoreInfo()
+{
+	if(mpStoreInfo.get() != 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded)
+	}
+	
+	// Load it up!
+	std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly));
+	
+	// Check it
+	if(i->GetAccountID() != mClientID)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount)
+	}
+	
+	// Keep the pointer to it
+	mpStoreInfo = i;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::SaveStoreInfo(bool)
+//		Purpose: Potentially delayed saving of the store info
+//		Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+void BackupContext::SaveStoreInfo(bool AllowDelay)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+
+	// Can delay saving it a little while?
+	if(AllowDelay)
+	{
+		--mSaveStoreInfoDelay;
+		if(mSaveStoreInfoDelay > 0)
+		{
+			return;
+		}
+	}
+
+	// Want to save now	
+	mpStoreInfo->Save();
+
+	// Set count for next delay
+	mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::MakeObjectFilename(int64_t, std::string &, bool)
+//		Purpose: Create the filename of an object in the store, optionally creating the 
+//				 containing directory if it doesn't already exist.
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+void BackupContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists)
+{
+	// Delegate to utility function
+	StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::GetDirectoryInternal(int64_t)
+//		Purpose: Return a reference to a directory. Valid only until the 
+//				 next time a function which affects directories is called.
+//				 Mainly this funciton, and creation of files.
+//				 Private version of this, which returns non-const directories.
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory &BackupContext::GetDirectoryInternal(int64_t ObjectID)
+{
+	// Get the filename
+	std::string filename;
+	MakeObjectFilename(ObjectID, filename);
+	
+	// Already in cache?
+	std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID));
+	if(item != mDirectoryCache.end())
+	{
+		// Check the revision ID of the file -- does it need refreshing?
+		int64_t revID = 0;
+		if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID))
+		{
+			THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted)
+		}
+	
+		if(revID == item->second->GetRevisionID())
+		{
+			// Looks good... return the cached object
+			return *(item->second);
+		}
+		
+		// Delete this cached object
+		delete item->second;
+		mDirectoryCache.erase(item);
+	}
+	
+	// Need to load it up
+	
+	// First check to see if the cache is too big
+	if(mDirectoryCache.size() > MAX_CACHE_SIZE)
+	{
+		// Very simple. Just delete everything!
+		for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i)
+		{
+			delete (i->second);
+		}
+		mDirectoryCache.clear();
+	}
+
+	// Get a RaidFileRead to read it
+	int64_t revID = 0;
+	std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID));
+	ASSERT(revID != 0);
+	
+	// New directory object
+	std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory);
+	
+	// Read it from the stream, then set it's revision ID
+	dir->ReadFromStream(*objectFile, IOStream::TimeOutInfinite);
+	dir->SetRevisionID(revID);
+			
+	// Make sure the size of the directory is available for writing the dir back
+	int64_t dirSize = objectFile->GetDiscUsageInBlocks();
+	ASSERT(dirSize > 0);
+	dir->SetUserInfo1_SizeInBlocks(dirSize);
+
+	// Store in cache
+	BackupStoreDirectory *pdir = dir.release();
+	try
+	{	
+		mDirectoryCache[ObjectID] = pdir;
+	}
+	catch(...)
+	{
+		delete pdir;
+		throw;
+	}
+	
+	// Return it
+	return *pdir;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::AllocateObjectID()
+//		Purpose: Allocate a new object ID, tolerant of failures to save store info
+//		Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+int64_t BackupContext::AllocateObjectID()
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+
+	// Given that the store info may not be saved for STORE_INFO_SAVE_DELAY
+	// times after it has been updated, this is a reasonable number of times
+	// to try for finding an unused ID.
+	// (Sizes used in the store info are fixed by the housekeeping process)
+	int retryLimit = (STORE_INFO_SAVE_DELAY * 2);
+	
+	while(retryLimit > 0)
+	{
+		// Attempt to allocate an ID from the store
+		int64_t id = mpStoreInfo->AllocateObjectID();
+		
+		// Generate filename
+		std::string filename;
+		MakeObjectFilename(id, filename);
+		// Check it doesn't exist
+		if(!RaidFileRead::FileExists(mStoreDiscSet, filename))
+		{
+			// Success!
+			return id;
+		}
+		
+		// Decrement retry count, and try again
+		--retryLimit;
+		
+		// Mark that the store info should be saved as soon as possible
+		mSaveStoreInfoDelay = 0;
+		
+		TRACE1("When allocating object ID, found that %lld is already in use\n", id);
+	}
+	
+	THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
+//		Purpose: Add a file to the store, from a given stream, into a specified directory.
+//				 Returns object ID of the new file.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime,
+		int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
+		bool MarkFileWithSameNameAsOldVersions)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+	
+	// This is going to be a bit complex to make sure it copes OK
+	// with things going wrong.
+	// The only thing which isn't safe is incrementing the object ID
+	// and keeping the blocks used entirely accurate -- but these
+	// aren't big problems if they go horribly wrong. The sizes will
+	// be corrected the next time the account has a housekeeping run,
+	// and the object ID allocation code is tolerant of missed IDs.
+	// (the info is written lazily, so these are necessary)
+	
+	// Get the directory we want to modify
+	BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+	
+	// Allocate the next ID
+	int64_t id = AllocateObjectID();
+	
+	// Stream the file to disc
+	std::string fn;
+	MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
+	int64_t blocksUsed = 0;
+	RaidFileWrite *ppreviousVerStoreFile = 0;
+	bool reversedDiffIsCompletelyDifferent = false;
+	int64_t oldVersionNewBlocksUsed = 0;
+	try
+	{
+		RaidFileWrite storeFile(mStoreDiscSet, fn);
+		storeFile.Open(false /* no overwriting */);
+		int64_t spaceAdjustFromDiff = 0;	// size adjustment from use of patch in old file
+
+		// Diff or full file?
+		if(DiffFromFileID == 0)
+		{
+			// A full file, just store to disc
+			if(!rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT))
+			{
+				THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut)
+			}
+		}
+		else
+		{
+			// Check that the diffed from ID actually exists in the directory
+			if(dir.FindEntryByID(DiffFromFileID) == 0)
+			{
+				THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory)
+			}
+		
+			// Diff file, needs to be recreated.
+			// Choose a temporary filename.
+			std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp",
+				1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */));
+			
+			try
+			{
+				// Open it twice
+				FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL);
+				FileStream diff2(tempFn.c_str(), O_RDONLY);
+				// Unlink it immediately, so it definately goes away
+				if(::unlink(tempFn.c_str()) != 0)
+				{
+					THROW_EXCEPTION(CommonException, OSFileError);
+				}
+				
+				// Stream the incoming diff to this temporary file
+				if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT))
+				{
+					THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut)
+				}
+				
+				// Verify the diff
+				diff.Seek(0, IOStream::SeekType_Absolute);
+				if(!BackupStoreFile::VerifyEncodedFileFormat(diff))
+				{
+					THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify)
+				}
+
+				// Seek to beginning of diff file
+				diff.Seek(0, IOStream::SeekType_Absolute);
+
+				// Filename of the old version
+				std::string oldVersionFilename;
+				MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */);
+				
+				// Reassemble that diff -- open previous file, and combine the patch and file
+				std::auto_ptr<RaidFileRead> from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename));
+				BackupStoreFile::CombineFile(diff, diff2, *from, storeFile);
+
+				// Then... reverse the patch back (open the from file again, and create a write file to overwrite it)
+				std::auto_ptr<RaidFileRead> from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename));
+				ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename);
+				ppreviousVerStoreFile->Open(true /* allow overwriting */);
+				from->Seek(0, IOStream::SeekType_Absolute);
+				diff.Seek(0, IOStream::SeekType_Absolute);
+				BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile,
+						DiffFromFileID, &reversedDiffIsCompletelyDifferent);
+				
+				// Store disc space used
+				oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks();
+				
+				// And make a space adjustment for the size calculation
+				spaceAdjustFromDiff = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed;
+
+				// Everything cleans up here...
+			}
+			catch(...)
+			{
+				// Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway
+				::unlink(tempFn.c_str());
+				throw;
+			}
+		}
+		
+		// Get the blocks used
+		blocksUsed = storeFile.GetDiscUsageInBlocks();
+		
+		// Exceeds the hard limit?
+		if((mpStoreInfo->GetBlocksUsed() + blocksUsed - spaceAdjustFromDiff) > mpStoreInfo->GetBlocksHardLimit())
+		{
+			THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit)
+			// The store file will be deleted automatically by the RaidFile object
+		}
+
+		// Commit the file
+		storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+	}
+	catch(...)
+	{
+		// Delete any previous version store file
+		if(ppreviousVerStoreFile != 0)
+		{
+			delete ppreviousVerStoreFile;
+			ppreviousVerStoreFile = 0;
+		}
+		
+		throw;
+	}
+
+	// Verify the file -- only necessary for non-diffed versions
+	// NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because
+	// in the non-diffed code path it's never allocated.
+	if(DiffFromFileID == 0)
+	{
+		std::auto_ptr<RaidFileRead> checkFile(RaidFileRead::Open(mStoreDiscSet, fn));
+		if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile))
+		{
+			// Error! Delete the file
+			RaidFileWrite del(mStoreDiscSet, fn);
+			del.Delete();
+			
+			// Exception
+			THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify)
+		}
+	}			
+	
+	// Modify the directory -- first make all files with the same name
+	// marked as an old version
+	int64_t blocksInOldFiles = 0;
+	try
+	{
+		if(MarkFileWithSameNameAsOldVersions)
+		{
+			BackupStoreDirectory::Iterator i(dir);
+
+			BackupStoreDirectory::Entry *e = 0;
+			while((e = i.Next()) != 0)
+			{
+				// First, check it's not an old version (cheaper comparison)
+				if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0)
+				{
+					// Compare name
+					if(e->GetName() == rFilename)
+					{
+						// Check that it's definately not an old version
+						ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0);
+						// Set old version flag
+						e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion);
+						// Can safely do this, because we know we won't be here if it's already 
+						// an old version
+						blocksInOldFiles += e->GetSizeInBlocks();
+					}
+				}
+			}
+		}
+		
+		// Then the new entry
+		BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename,
+				ModificationTime, id, blocksUsed, BackupStoreDirectory::Entry::Flags_File, AttributesHash);
+
+		// Adjust for the patch back stuff?
+		if(DiffFromFileID != 0)
+		{
+			// Get old version entry
+			BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID);
+			ASSERT(poldEntry != 0);
+		
+			// Adjust dependency info of file?
+			if(!reversedDiffIsCompletelyDifferent)
+			{
+				poldEntry->SetDependsNewer(id);
+				pnewEntry->SetDependsOlder(DiffFromFileID);
+			}
+			
+			// Adjust size of old entry
+			int64_t oldSize = poldEntry->GetSizeInBlocks();
+			poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed);
+			
+			// And adjust blocks used count, for later adjustment
+			blocksUsed += (oldVersionNewBlocksUsed - oldSize);
+			blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize);
+		}
+
+		// Write the directory back to disc
+		SaveDirectory(dir, InDirectory);
+
+		// Commit the old version's new patched version, now that the directory safely reflects
+		// the state of the files on disc.
+		if(ppreviousVerStoreFile != 0)
+		{
+			ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+			delete ppreviousVerStoreFile;
+			ppreviousVerStoreFile = 0;
+		}
+	}
+	catch(...)
+	{
+		// Back out on adding that file
+		RaidFileWrite del(mStoreDiscSet, fn);
+		del.Delete();
+		
+		// Remove this entry from the cache
+		RemoveDirectoryFromCache(InDirectory);
+		
+		// Delete any previous version store file
+		if(ppreviousVerStoreFile != 0)
+		{
+			delete ppreviousVerStoreFile;
+			ppreviousVerStoreFile = 0;
+		}
+		
+		// Don't worry about the incremented number in the store info
+		throw;
+	}
+	
+	// Check logic
+	ASSERT(ppreviousVerStoreFile == 0);
+	
+	// Modify the store info
+	mpStoreInfo->ChangeBlocksUsed(blocksUsed);
+	mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles);
+	
+	// Save the store info -- can cope if this exceptions because infomation
+	// will be rebuilt by housekeeping, and ID allocation can recover.
+	SaveStoreInfo();
+	
+	// Return the ID to the caller
+	return id;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &)
+//		Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found.
+//		Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+bool BackupContext::DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut)
+{
+	// Essential checks!
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+
+	// Find the directory the file is in (will exception if it fails)
+	BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+
+	// Setup flags
+	bool fileExisted = false;
+	bool madeChanges = false;
+	rObjectIDOut = 0;		// not found
+
+	// Count of deleted blocks
+	int64_t blocksDel = 0;
+
+	try
+	{
+		// Iterate through directory, only looking at files which haven't been deleted
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *e = 0;
+		while((e = i.Next(BackupStoreDirectory::Entry::Flags_File,
+			BackupStoreDirectory::Entry::Flags_Deleted)) != 0)
+		{
+			// Compare name
+			if(e->GetName() == rFilename)
+			{
+				// Check that it's definately not already deleted
+				ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0);
+				// Set deleted flag
+				e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+				// Mark as made a change
+				madeChanges = true;
+				// Can safely do this, because we know we won't be here if it's already 
+				// an old version
+				blocksDel += e->GetSizeInBlocks();
+				// Is this the last version?
+				if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0)
+				{
+					// Yes. It's been found.
+					rObjectIDOut = e->GetObjectID();
+					fileExisted = true;
+				}
+			}
+		}
+		
+		// Save changes?
+		if(madeChanges)
+		{
+			// Save the directory back
+			SaveDirectory(dir, InDirectory);
+			
+			// Modify the store info, and write
+			mpStoreInfo->ChangeBlocksInDeletedFiles(blocksDel);
+			
+			// Maybe postponed save of store info
+			SaveStoreInfo();
+		}
+	}
+	catch(...)
+	{
+		RemoveDirectoryFromCache(InDirectory);
+		throw;
+	}
+		
+
+	return fileExisted;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::RemoveDirectoryFromCache(int64_t)
+//		Purpose: Remove directory from cache
+//		Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+void BackupContext::RemoveDirectoryFromCache(int64_t ObjectID)
+{
+	std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID));
+	if(item != mDirectoryCache.end())
+	{
+		// Delete this cached object
+		delete item->second;
+		// Erase the entry form the map
+		mDirectoryCache.erase(item);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::SaveDirectory(BackupStoreDirectory &, int64_t)
+//		Purpose: Save directory back to disc, update time in cache
+//		Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+void BackupContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(rDir.GetObjectID() != ObjectID)
+	{
+		THROW_EXCEPTION(BackupStoreException, Internal)
+	}
+
+	try
+	{
+		// Write to disc, adjust size in store info
+		std::string dirfn;
+		MakeObjectFilename(ObjectID, dirfn);
+		{
+			RaidFileWrite writeDir(mStoreDiscSet, dirfn);
+			writeDir.Open(true /* allow overwriting */);
+			rDir.WriteToStream(writeDir);
+
+			// get the disc usage (must do this before commiting it)
+			int64_t dirSize = writeDir.GetDiscUsageInBlocks();
+
+			// Commit directory
+			writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+			
+			// Make sure the size of the directory is available for writing the dir back
+			ASSERT(dirSize > 0);
+			int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks();
+			mpStoreInfo->ChangeBlocksUsed(sizeAdjustment);
+			mpStoreInfo->ChangeBlocksInDirectories(sizeAdjustment);
+			// Update size stored in directory
+			rDir.SetUserInfo1_SizeInBlocks(dirSize);
+		}
+		// Refresh revision ID in cache
+		{
+			int64_t revid = 0;
+			if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid))
+			{
+				THROW_EXCEPTION(BackupStoreException, Internal)
+			}
+			rDir.SetRevisionID(revid);
+		}
+	}
+	catch(...)
+	{
+		// Remove it from the cache if anything went wrong
+		RemoveDirectoryFromCache(ObjectID);
+		throw;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &)
+//		Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely.
+//		Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+	
+	// Flags as not already existing
+	rAlreadyExists = false;
+	
+	// Get the directory we want to modify
+	BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+
+	// Scan the directory for the name (only looking for directories which already exist)
+	{
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING,
+			BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)	// Ignore deleted and old directories
+		{
+			if(en->GetName() == rFilename)
+			{
+				// Already exists
+				rAlreadyExists = true;
+				return en->GetObjectID();
+			}
+		}
+	}
+
+	// Allocate the next ID
+	int64_t id = AllocateObjectID();
+
+	// Create a blank directory with the given attributes on disc
+	std::string fn;
+	MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
+	{
+		BackupStoreDirectory emptyDir(id, InDirectory);
+		// add the atttribues
+		emptyDir.SetAttributes(Attributes, AttributesModTime);
+		
+		// Write...
+		RaidFileWrite dirFile(mStoreDiscSet, fn);
+		dirFile.Open(false /* no overwriting */);
+		emptyDir.WriteToStream(dirFile);
+		// Get disc usage, before it's commited
+		int64_t dirSize = dirFile.GetDiscUsageInBlocks();
+		// Commit the file
+		dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);		
+
+		// Make sure the size of the directory is added to the usage counts in the info
+		ASSERT(dirSize > 0);
+		mpStoreInfo->ChangeBlocksUsed(dirSize);
+		mpStoreInfo->ChangeBlocksInDirectories(dirSize);
+		// Not added to cache, so don't set the size in the directory
+	}
+	
+	// Then add it into the directory
+	try
+	{
+		dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */);
+		SaveDirectory(dir, InDirectory);
+	}
+	catch(...)
+	{
+		// Back out on adding that directory
+		RaidFileWrite del(mStoreDiscSet, fn);
+		del.Delete();
+		
+		// Remove this entry from the cache
+		RemoveDirectoryFromCache(InDirectory);
+		
+		// Don't worry about the incremented number in the store info
+		throw;	
+	}
+	
+	// Save the store info (may be postponed)
+	SaveStoreInfo();
+
+	// tell caller what the ID was
+	return id;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &, bool)
+//		Purpose: Recusively deletes a directory (or undeletes if Undelete = true)
+//		Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+void BackupContext::DeleteDirectory(int64_t ObjectID, bool Undelete)
+{
+	// Essential checks!
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+
+	// Containing directory
+	int64_t InDirectory = 0;
+	
+	// Count of blocks deleted
+	int64_t blocksDeleted = 0;
+
+	try
+	{
+		// Get the directory that's to be deleted
+		{
+			// In block, because dir may not be valid after the delete directory call
+			BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
+			
+			// Store the directory it's in for later
+			InDirectory = dir.GetContainerID();
+		
+			// Depth first delete of contents
+			DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete);
+		}
+		
+		// Remove the entry from the directory it's in
+		ASSERT(InDirectory != 0);
+		BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory));
+		
+		BackupStoreDirectory::Iterator i(parentDir);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING),
+			Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0)	// Ignore deleted directories (or not deleted if Undelete)
+		{
+			if(en->GetObjectID() == ObjectID)
+			{
+				// This is the one to delete
+				if(Undelete)
+				{
+					en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+				}
+				else
+				{
+					en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+				}
+							
+				// Save it
+				SaveDirectory(parentDir, InDirectory);
+				
+				// Done
+				break;
+			}
+		}
+		
+		// Update blocks deleted count
+		mpStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted));
+		
+		// Save store info, may be postponed
+		SaveStoreInfo();
+	}
+	catch(...)
+	{
+		RemoveDirectoryFromCache(InDirectory);
+		throw;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::DeleteDirectoryRecurse(BackupStoreDirectory &, int64_t)
+//		Purpose: Private. Deletes a directory depth-first recusively.
+//		Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+void BackupContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete)
+{
+	try
+	{
+		// Does things carefully to avoid using a directory in the cache after recursive call
+		// because it may have been deleted.
+		
+		// Do sub directories
+		{
+			// Get the directory...
+			BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
+			
+			// Then scan it for directories
+			std::vector<int64_t> subDirs;
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = 0;
+			if(Undelete)
+			{
+				while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted,	// deleted dirs
+					BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING)) != 0)
+				{
+					// Store the directory ID.
+					subDirs.push_back(en->GetObjectID());
+				}
+			}
+			else
+			{
+				while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir,	// dirs only
+					BackupStoreDirectory::Entry::Flags_Deleted)) != 0)		// but not deleted ones
+				{
+					// Store the directory ID.
+					subDirs.push_back(en->GetObjectID());
+				}
+			}
+			
+			// Done with the directory for now. Recurse to sub directories
+			for(std::vector<int64_t>::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i)
+			{
+				DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete);	
+			}
+		}
+		
+		// Then, delete the files. Will need to load the directory again because it might have
+		// been removed from the cache.
+		{
+			// Get the directory...
+			BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
+	
+			// Changes made?
+			bool changesMade = false;
+	
+			// Run through files		
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = 0;
+
+			while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING),
+				Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0)	// Ignore deleted directories (or not deleted if Undelete)
+			{
+				// Add/remove the deleted flags
+				if(Undelete)
+				{
+					en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+				}
+				else
+				{
+					en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+				}
+							
+				// Keep count of the deleted blocks
+				if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0)
+				{
+					rBlocksDeletedOut += en->GetSizeInBlocks();
+				}
+				
+				// Did something
+				changesMade = true;
+			}
+			
+			// Save the directory
+			if(changesMade)
+			{
+				SaveDirectory(dir, ObjectID);
+			}
+		}
+	}
+	catch(...)
+	{
+		RemoveDirectoryFromCache(ObjectID);
+		throw;
+	}
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::ChangeDirAttributes(int64_t, const StreamableMemBlock &, int64_t)
+//		Purpose: Change the attributes of a directory
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+void BackupContext::ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+
+	try
+	{	
+		// Get the directory we want to modify
+		BackupStoreDirectory &dir(GetDirectoryInternal(Directory));
+	
+		// Set attributes
+		dir.SetAttributes(Attributes, AttributesModTime);
+		
+		// Save back
+		SaveDirectory(dir, Directory);
+	}
+	catch(...)
+	{
+		RemoveDirectoryFromCache(Directory);
+		throw;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::ChangeFileAttributes(int64_t, int64_t, const StreamableMemBlock &, int64_t)
+//		Purpose: Sets the attributes on a directory entry. Returns true if the object existed, false if it didn't.
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+bool BackupContext::ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+	
+	try
+	{
+		// Get the directory we want to modify
+		BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+	
+		// Find the file entry
+		BackupStoreDirectory::Entry *en = 0;
+		// Iterate through current versions of files, only
+		BackupStoreDirectory::Iterator i(dir);
+		while((en = i.Next(
+			BackupStoreDirectory::Entry::Flags_File,
+			BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)
+			) != 0)
+		{
+			if(en->GetName() == rFilename)
+			{
+				// Set attributes
+				en->SetAttributes(Attributes, AttributesHash);
+				
+				// Tell caller the object ID
+				rObjectIDOut = en->GetObjectID();
+				
+				// Done
+				break;
+			}
+		}
+		if(en == 0)
+		{
+			// Didn't find it
+			return false;
+		}
+	
+		// Save back
+		SaveDirectory(dir, InDirectory);
+	}
+	catch(...)
+	{
+		RemoveDirectoryFromCache(InDirectory);
+		throw;
+	}
+	
+	// Changed, everything OK
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::ObjectExists(int64_t)
+//		Purpose: Test to see if an object of this ID exists in the store
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+bool BackupContext::ObjectExists(int64_t ObjectID, int MustBe)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	
+	// Note that we need to allow object IDs a little bit greater than the last one in the store info,
+	// because the store info may not have got saved in an error condition. Max greater ID is
+	// STORE_INFO_SAVE_DELAY in this case, *2 to be safe.
+	if(ObjectID <= 0 || ObjectID > (mpStoreInfo->GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2)))
+	{
+		// Obviously bad object ID
+		return false;
+	}
+	
+	// Test to see if it exists on the disc
+	std::string filename;
+	MakeObjectFilename(ObjectID, filename);
+	if(!RaidFileRead::FileExists(mStoreDiscSet, filename))
+	{
+		// RaidFile reports no file there
+		return false;
+	}
+	
+	// Do we need to be more specific?
+	if(MustBe != ObjectExists_Anything)
+	{
+		// Open the file
+		std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename));
+
+		// Read the first integer
+		u_int32_t magic;
+		if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */))
+		{
+			// Failed to get any bytes, must have failed
+			return false;
+		}
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+		if(MustBe == ObjectExists_File && ntohl(magic) == OBJECTMAGIC_FILE_MAGIC_VALUE_V0)
+		{
+			// Old version detected
+			return true;
+		}
+#endif
+
+		// Right one?
+		u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE;
+	
+		// Check
+		if(ntohl(magic) != requiredMagic)
+		{
+			return false;
+		}
+		
+		// File is implicitly closed
+	}
+	
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::OpenObject(int64_t)
+//		Purpose: Opens an object
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupContext::OpenObject(int64_t ObjectID)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	
+	// Attempt to open the file
+	std::string fn;
+	MakeObjectFilename(ObjectID, fn);
+	return std::auto_ptr<IOStream>(RaidFileRead::Open(mStoreDiscSet, fn).release());
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::GetClientStoreMarker()
+//		Purpose: Retrieve the client store marker
+//		Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+int64_t BackupContext::GetClientStoreMarker()
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	
+	return mpStoreInfo->GetClientStoreMarker();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::GetStoreDiscUsageInfo(int64_t &, int64_t &, int64_t &)
+//		Purpose: Get disc usage info from store info
+//		Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+void BackupContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+
+	rBlocksUsed = mpStoreInfo->GetBlocksUsed();
+	rBlocksSoftLimit = mpStoreInfo->GetBlocksSoftLimit();
+	rBlocksHardLimit = mpStoreInfo->GetBlocksHardLimit();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::HardLimitExceeded()
+//		Purpose: Returns true if the hard limit has been exceeded
+//		Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+bool BackupContext::HardLimitExceeded()
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+
+	return mpStoreInfo->GetBlocksUsed() > mpStoreInfo->GetBlocksHardLimit();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::SetClientStoreMarker(int64_t)
+//		Purpose: Sets the client store marker, and commits it to disc
+//		Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+void BackupContext::SetClientStoreMarker(int64_t ClientStoreMarker)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+	
+	mpStoreInfo->SetClientStoreMarker(ClientStoreMarker);
+	SaveStoreInfo(false /* don't delay saving this */);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::MoveObject(int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
+//		Purpose: Move an object (and all objects with the same name) from one directory to another
+//		Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+void BackupContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+
+	// Should deleted files be excluded when checking for the existance of objects with the target name?
+	int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject)
+		?(BackupStoreDirectory::Entry::Flags_Deleted)
+		:(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING);
+	
+	// Special case if the directories are the same...
+	if(MoveFromDirectory == MoveToDirectory)
+	{
+		try
+		{
+			// Get the first directory
+			BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory));
+		
+			// Find the file entry
+			BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID);
+	
+			// Error if not found
+			if(en == 0)
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+			}
+			
+			// Check the new name doens't already exist (optionally ignoring deleted files)
+			{
+				BackupStoreDirectory::Iterator i(dir);
+				BackupStoreDirectory::Entry *c = 0;
+				while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0)
+				{
+					if(c->GetName() == rNewFilename)
+					{
+						THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory)
+					}
+				}
+			}
+			
+			// Need to get all the entries with the same name?
+			if(MoveAllWithSameName)
+			{
+				// Iterate through the directory, copying all with matching names
+				BackupStoreDirectory::Iterator i(dir);
+				BackupStoreDirectory::Entry *c = 0;
+				while((c = i.Next()) != 0)
+				{
+					if(c->GetName() == en->GetName())
+					{
+						// Rename this one
+						c->SetName(rNewFilename);
+					}
+				}
+			}
+			else
+			{
+				// Just copy this one
+				en->SetName(rNewFilename);
+			}
+			
+			// Save the directory back
+			SaveDirectory(dir, MoveFromDirectory);
+		}
+		catch(...)
+		{
+			RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same
+			throw;
+		}
+	
+		return;
+	}
+
+	// Got to be careful how this is written, as we can't guarentte that if we have two
+	// directories open, the first won't be deleted as the second is opened. (cache)
+
+	// List of entries to move
+	std::vector<BackupStoreDirectory::Entry *> moving;
+	
+	// list of directory IDs which need to have containing dir id changed
+	std::vector<int64_t> dirsToChangeContainingID;
+
+	try
+	{
+		// First of all, get copies of the entries to move to the to directory.
+		
+		{
+			// Get the first directory
+			BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory));
+		
+			// Find the file entry
+			BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID);
+	
+			// Error if not found
+			if(en == 0)
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+			}
+			
+			// Need to get all the entries with the same name?
+			if(MoveAllWithSameName)
+			{
+				// Iterate through the directory, copying all with matching names
+				BackupStoreDirectory::Iterator i(from);
+				BackupStoreDirectory::Entry *c = 0;
+				while((c = i.Next()) != 0)
+				{
+					if(c->GetName() == en->GetName())
+					{
+						// Copy
+						moving.push_back(new BackupStoreDirectory::Entry(*c));
+						
+						// Check for containing directory correction
+						if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID());
+					}
+				}
+				ASSERT(!moving.empty());
+			}
+			else
+			{
+				// Just copy this one
+				moving.push_back(new BackupStoreDirectory::Entry(*en));
+
+				// Check for containing directory correction
+				if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID());
+			}
+		}
+		
+		// Secondly, insert them into the to directory, and save it
+		
+		{
+			// To directory
+			BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory));
+	
+			// Check the new name doens't already exist
+			{
+				BackupStoreDirectory::Iterator i(to);
+				BackupStoreDirectory::Entry *c = 0;
+				while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0)
+				{
+					if(c->GetName() == rNewFilename)
+					{
+						THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory)
+					}
+				}
+			}
+			
+			// Copy the entries into it, changing the name as we go
+			for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
+			{
+				BackupStoreDirectory::Entry *en = (*i);
+				en->SetName(rNewFilename);
+				to.AddEntry(*en);	// adds copy
+			}
+	
+			// Save back
+			SaveDirectory(to, MoveToDirectory);
+		}
+
+		// Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory
+		try
+		{
+			// Get directory
+			BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory));
+		
+			// Delete each one
+			for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
+			{
+				from.DeleteEntry((*i)->GetObjectID());
+			}
+	
+			// Save back
+			SaveDirectory(from, MoveFromDirectory);		
+		}
+		catch(...)
+		{
+			// UNDO modification to To directory
+					
+			// Get directory
+			BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory));
+		
+			// Delete each one
+			for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
+			{
+				to.DeleteEntry((*i)->GetObjectID());
+			}
+	
+			// Save back
+			SaveDirectory(to, MoveToDirectory);
+
+			// Throw the error
+			throw;
+		}
+		
+		// Finally... for all the directories we moved, modify their containing directory ID
+		for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i)
+		{
+			// Load the directory
+			BackupStoreDirectory &change(GetDirectoryInternal(*i));
+			
+			// Modify containing dir ID
+			change.SetContainerID(MoveToDirectory);
+			
+			// Save it back
+			SaveDirectory(change, *i);
+		}
+	}
+	catch(...)
+	{
+		// Make sure directories aren't in the cache, as they may have been modified		
+		RemoveDirectoryFromCache(MoveToDirectory);
+		RemoveDirectoryFromCache(MoveFromDirectory);
+		for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i)
+		{
+			RemoveDirectoryFromCache(*i);			
+		}
+
+		while(!moving.empty())
+		{
+			delete moving.back();
+			moving.pop_back();
+		}
+		throw;
+	}	
+
+	// Clean up
+	while(!moving.empty())
+	{
+		delete moving.back();
+		moving.pop_back();
+	}
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::GetBackupStoreInfo()
+//		Purpose: Return the backup store info object, exception if it isn't loaded
+//		Created: 19/4/04
+//
+// --------------------------------------------------------------------------
+const BackupStoreInfo &BackupContext::GetBackupStoreInfo() const
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	
+	return *(mpStoreInfo.get());
+}
+
+
diff --git a/bin/bbstored/BackupContext.h b/bin/bbstored/BackupContext.h
new file mode 100755
index 00000000..18f2f25c
--- /dev/null
+++ b/bin/bbstored/BackupContext.h
@@ -0,0 +1,149 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupContext.h
+//		Purpose: Context for backup store server
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCONTEXT__H
+#define BACKUPCONTEXT__H
+
+#include <string>
+#include <map>
+#include <memory>
+
+#include "NamedLock.h"
+#include "Utils.h"
+
+class BackupStoreDirectory;
+class BackupStoreFilename;
+class BackupStoreDaemon;
+class BackupStoreInfo;
+class IOStream;
+class StreamableMemBlock;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupContext
+//		Purpose: Context for backup store server
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+class BackupContext
+{
+public:
+	BackupContext(int32_t ClientID, BackupStoreDaemon &rDaemon);
+	~BackupContext();
+private:
+	BackupContext(const BackupContext &rToCopy);
+public:
+
+	void ReceivedFinishCommand();
+	void CleanUp();
+
+	int32_t GetClientID() {return mClientID;}
+
+	enum
+	{
+		Phase_START		= 0,
+		Phase_Version	= 0,
+		Phase_Login		= 1,
+		Phase_Commands	= 2
+	};
+	
+	int GetPhase() const {return mProtocolPhase;}
+	void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;}
+	
+	// Read only locking
+	bool SessionIsReadOnly() {return mReadOnly;}
+	bool AttemptToGetWriteLock();
+
+	void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;}
+	bool GetClientHasAccount() const {return mClientHasAccount;}
+	const std::string &GetStoreRoot() const {return mStoreRoot;}
+	int GetStoreDiscSet() const {return mStoreDiscSet;}
+
+	// Store info
+	void LoadStoreInfo();
+	void SaveStoreInfo(bool AllowDelay = true);
+	const BackupStoreInfo &GetBackupStoreInfo() const;
+
+	// Client marker
+	int64_t GetClientStoreMarker();
+	void SetClientStoreMarker(int64_t ClientStoreMarker);
+	
+	// Usage information
+	void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit);
+	bool HardLimitExceeded();
+
+	// Reading directories
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    BackupContext::GetDirectory(int64_t)
+	//		Purpose: Return a reference to a directory. Valid only until the 
+	//				 next time a function which affects directories is called.
+	//				 Mainly this funciton, and creation of files.
+	//		Created: 2003/09/02
+	//
+	// --------------------------------------------------------------------------
+	const BackupStoreDirectory &GetDirectory(int64_t ObjectID)
+	{
+		// External callers aren't allowed to change it -- this function
+		// merely turns the the returned directory const.
+		return GetDirectoryInternal(ObjectID);
+	}
+	
+	// Manipulating files/directories
+	int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions);
+	int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists);
+	void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime);
+	bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut);
+	bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut);
+	void DeleteDirectory(int64_t ObjectID, bool Undelete = false);
+	void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject);
+
+	// Manipulating objects
+	enum
+	{
+		ObjectExists_Anything = 0,
+		ObjectExists_File = 1,
+		ObjectExists_Directory = 2
+	};
+	bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything);
+	std::auto_ptr<IOStream> OpenObject(int64_t ObjectID);
+	
+	// Info
+	int32_t GetClientID() const {return mClientID;}
+
+private:
+	void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false);
+	BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID);
+	void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID);
+	void RemoveDirectoryFromCache(int64_t ObjectID);
+	void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete);
+	int64_t AllocateObjectID();
+
+private:
+	int32_t mClientID;
+	BackupStoreDaemon &mrDaemon;
+	int mProtocolPhase;
+	bool mClientHasAccount;
+	std::string mStoreRoot;	// has final directory separator
+	int mStoreDiscSet;
+	bool mReadOnly;
+	NamedLock mWriteLock;
+	int mSaveStoreInfoDelay; // how many times to delay saving the store info
+	
+	// Store info
+	std::auto_ptr<BackupStoreInfo> mpStoreInfo;
+	
+	// Directory cache
+	std::map<int64_t, BackupStoreDirectory*> mDirectoryCache;
+};
+
+#endif // BACKUPCONTEXT__H
+
diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp
new file mode 100755
index 00000000..0afdaa5d
--- /dev/null
+++ b/bin/bbstored/BackupStoreDaemon.cpp
@@ -0,0 +1,284 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreDaemon.cpp
+//		Purpose: Backup store daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <syslog.h>
+#include <signal.h>
+
+#include "BackupContext.h"
+#include "BackupStoreDaemon.h"
+#include "BackupStoreConfigVerify.h"
+#include "autogen_BackupProtocolServer.h"
+#include "RaidFileController.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreAccounts.h"
+#include "BannerText.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::BackupStoreDaemon()
+//		Purpose: Constructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreDaemon::BackupStoreDaemon()
+	: mpAccountDatabase(0),
+	  mpAccounts(0),
+	  mExtendedLogging(false),
+	  mHaveForkedHousekeeping(false),
+	  mIsHousekeepingProcess(false),
+	  mInterProcessComms(mInterProcessCommsSocket)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::~BackupStoreDaemon()
+//		Purpose: Destructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreDaemon::~BackupStoreDaemon()
+{
+	// Must delete this one before the database ...
+	if(mpAccounts != 0)
+	{
+		delete mpAccounts;
+		mpAccounts = 0;
+	}
+	// ... as the mpAccounts object has a reference to it
+	if(mpAccountDatabase != 0)
+	{
+		delete mpAccountDatabase;
+		mpAccountDatabase = 0;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::DaemonName()
+//		Purpose: Name of daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+const char *BackupStoreDaemon::DaemonName() const
+{
+	return "bbstored";
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::DaemonBanner()
+//		Purpose: Daemon banner
+//		Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+const char *BackupStoreDaemon::DaemonBanner() const
+{
+#ifndef NDEBUG
+	// Don't display banner in debug builds
+	return 0;
+#else
+	return BANNER_TEXT("Backup Store Server");
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::GetConfigVerify()
+//		Purpose: Configuration definition
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+const ConfigurationVerify *BackupStoreDaemon::GetConfigVerify() const
+{
+	return &BackupConfigFileVerify;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::SetupInInitialProcess()
+//		Purpose: Setup before we fork -- get raid file controller going
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::SetupInInitialProcess()
+{
+	const Configuration &config(GetConfiguration());
+	
+	// Initialise the raid files controller
+	RaidFileController &rcontroller = RaidFileController::GetController();
+	rcontroller.Initialise(config.GetKeyValue("RaidFileConf").c_str());
+	
+	// Load the account database
+	std::auto_ptr<BackupStoreAccountDatabase> pdb(BackupStoreAccountDatabase::Read(config.GetKeyValue("AccountDatabase").c_str()));
+	mpAccountDatabase = pdb.release();
+	
+	// Create a accounts object
+	mpAccounts = new BackupStoreAccounts(*mpAccountDatabase);
+	
+	// Ready to go!
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::Run()
+//		Purpose: Run shim for the store daemon -- read some config details
+//		Created: 2003/10/24
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::Run()
+{
+	// Get extended logging flag
+	mExtendedLogging = false;
+	const Configuration &config(GetConfiguration());
+	mExtendedLogging = config.GetKeyValueBool("ExtendedLogging");
+	
+	// Fork off housekeeping daemon -- must only do this the first time Run() is called
+	if(!mHaveForkedHousekeeping)
+	{
+		// Open a socket pair for communication
+		int sv[2] = {-1,-1};
+		if(::socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, sv) != 0)
+		{
+			THROW_EXCEPTION(ServerException, SocketPairFailed)
+		}
+		int whichSocket = 0;
+		
+		// Fork
+		switch(::fork())
+		{
+		case -1:
+			{
+				// Error
+				THROW_EXCEPTION(ServerException, ServerForkError)
+			}
+			break;
+		case 0:
+			{
+				// In child process
+				mIsHousekeepingProcess = true;
+				SetProcessTitle("housekeeping, idle");
+				whichSocket = 1;
+				// Change the log name
+				::openlog("bbstored/hk", LOG_PID, LOG_LOCAL6);
+				// Log that housekeeping started
+				::syslog(LOG_INFO, "Housekeeping process started");
+				// Ignore term and hup
+				// Parent will handle these and alert the child via the socket, don't want to randomly die
+				::signal(SIGHUP, SIG_IGN);
+				::signal(SIGTERM, SIG_IGN);
+			}
+			break;
+		default:
+			{
+				// Parent process
+				whichSocket = 0;
+			}
+			break;
+		}
+		
+		// Mark that this has been, so -HUP doesn't try and do this again
+		mHaveForkedHousekeeping = true;
+		
+		// Attach the comms thing to the right socket, and close the other one
+		mInterProcessCommsSocket.Attach(sv[whichSocket]);
+		
+		if(::close(sv[(whichSocket == 0)?1:0]) != 0)
+		{
+			THROW_EXCEPTION(ServerException, SocketCloseError)
+		}
+	}
+
+	if(mIsHousekeepingProcess)
+	{
+		// Housekeeping process -- do other stuff
+		HousekeepingProcess();
+	}
+	else
+	{
+		// In server process -- use the base class to do the magic
+		ServerTLS<BOX_PORT_BBSTORED>::Run();
+		
+		// Why did it stop? Tell the housekeeping process to do the same
+		if(IsReloadConfigWanted())
+		{
+			mInterProcessCommsSocket.Write("h\n", 2);
+		}
+		if(IsTerminateWanted())
+		{
+			mInterProcessCommsSocket.Write("t\n", 2);
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::Connection(SocketStreamTLS &)
+//		Purpose: Handles a connection
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::Connection(SocketStreamTLS &rStream)
+{
+	// Get the common name from the certificate
+	std::string clientCommonName(rStream.GetPeerCommonName());
+	
+	// Log the name
+	::syslog(LOG_INFO, "Certificate CN: %s\n", clientCommonName.c_str());
+	
+	// Check it
+	int32_t id;
+	if(::sscanf(clientCommonName.c_str(), "BACKUP-%x", &id) != 1)
+	{
+		// Bad! Disconnect immediately
+		return;
+	}
+
+	// Make ps listings clearer
+	SetProcessTitle("client %08x", id);
+
+	// Create a context, using this ID
+	BackupContext context(id, *this);
+	
+	// See if the client has an account?
+	if(mpAccounts && mpAccounts->AccountExists(id))
+	{
+		std::string root;
+		int discSet;
+		mpAccounts->GetAccountRoot(id, root, discSet);
+		context.SetClientHasAccount(root, discSet);
+	}
+
+	// Handle a connection with the backup protocol
+	BackupProtocolServer server(rStream);
+	server.SetLogToSysLog(mExtendedLogging);
+	server.SetTimeout(BACKUP_STORE_TIMEOUT);
+	server.DoServer(context);
+	context.CleanUp();
+}
+
diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h
new file mode 100755
index 00000000..6c4a45bc
--- /dev/null
+++ b/bin/bbstored/BackupStoreDaemon.h
@@ -0,0 +1,77 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreDaemon.h
+//		Purpose: Backup store daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREDAEMON__H
+#define BACKUPSTOREDAEMON__H
+
+#include "ServerTLS.h"
+#include "BoxPortsAndFiles.h"
+#include "BackupConstants.h"
+#include "IOStreamGetLine.h"
+
+class BackupStoreAccounts;
+class BackupStoreAccountDatabase;
+class HousekeepStoreAccount;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreDaemon
+//		Purpose: Backup store daemon implementation
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+class BackupStoreDaemon : public ServerTLS<BOX_PORT_BBSTORED>
+{
+	friend class HousekeepStoreAccount;
+
+public:
+	BackupStoreDaemon();
+	~BackupStoreDaemon();
+private:
+	BackupStoreDaemon(const BackupStoreDaemon &rToCopy);
+public:
+
+	// For BackupContext to comminicate with housekeeping process
+	void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen)
+	{
+		mInterProcessCommsSocket.Write(Msg, MsgLen);
+	}
+
+protected:
+	
+	virtual void SetupInInitialProcess();
+
+	virtual void Run();
+
+	void Connection(SocketStreamTLS &rStream);
+	
+	virtual const char *DaemonName() const;
+	virtual const char *DaemonBanner() const;
+
+	const ConfigurationVerify *GetConfigVerify() const;
+	
+	// Housekeeping functions
+	void HousekeepingProcess();
+	bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0);
+
+private:
+	BackupStoreAccountDatabase *mpAccountDatabase;
+	BackupStoreAccounts *mpAccounts;
+	bool mExtendedLogging;
+	bool mHaveForkedHousekeeping;
+	bool mIsHousekeepingProcess;
+	
+	SocketStream mInterProcessCommsSocket;
+	IOStreamGetLine mInterProcessComms;
+};
+
+
+#endif // BACKUPSTOREDAEMON__H
+
diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp
new file mode 100755
index 00000000..4aa1999e
--- /dev/null
+++ b/bin/bbstored/HousekeepStoreAccount.cpp
@@ -0,0 +1,844 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    HousekeepStoreAccount.cpp
+//		Purpose: 
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <map>
+#include <stdio.h>
+
+#include "HousekeepStoreAccount.h"
+#include "BackupStoreDaemon.h"
+#include "StoreStructure.h"
+#include "BackupStoreConstants.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreInfo.h"
+#include "NamedLock.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreFile.h"
+
+#include "MemLeakFindOn.h"
+
+// check every 32 directories scanned/files deleted
+#define POLL_INTERPROCESS_MSG_CHECK_FREQUENCY	32
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::HousekeepStoreAccount(int, const std::string &, int, BackupStoreDaemon &)
+//		Purpose: Constructor
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon)
+	: mAccountID(AccountID),
+	  mStoreRoot(rStoreRoot),
+	  mStoreDiscSet(StoreDiscSet),
+	  mrDaemon(rDaemon),
+	  mDeletionSizeTarget(0),
+  	  mPotentialDeletionsTotalSize(0),
+	  mMaxSizeInPotentialDeletions(0),
+	  mBlocksUsed(0),
+	  mBlocksInOldFiles(0),
+	  mBlocksInDeletedFiles(0),
+	  mBlocksInDirectories(0),
+	  mBlocksUsedDelta(0),
+	  mBlocksInOldFilesDelta(0),
+	  mBlocksInDeletedFilesDelta(0),
+	  mBlocksInDirectoriesDelta(0),
+	  mFilesDeleted(0),
+	  mEmptyDirectoriesDeleted(0),
+	  mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::~HousekeepStoreAccount()
+//		Purpose: Destructor
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+HousekeepStoreAccount::~HousekeepStoreAccount()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DoHousekeeping()
+//		Purpose: Perform the housekeeping
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void HousekeepStoreAccount::DoHousekeeping()
+{
+	// Attempt to lock the account
+	std::string writeLockFilename;
+	StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFilename);
+	NamedLock writeLock;
+	if(!writeLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */))
+	{
+		// Couldn't lock the account -- just stop now
+		return;
+	}
+
+	// Load the store info to find necessary info for the housekeeping
+	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet, false /* Read/Write */));
+
+	// Calculate how much should be deleted
+	mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit();
+	if(mDeletionSizeTarget < 0)
+	{
+		mDeletionSizeTarget = 0;
+	}
+
+	// Scan the directory for potential things to delete
+	// This will also remove elegiable items marked with RemoveASAP
+	bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
+
+	// If scan directory stopped for some reason, probably parent instructed to teminate, stop now.
+	if(!continueHousekeeping)
+	{
+		// If any files were marked "delete now", then update the size of the store.
+		if(mBlocksUsedDelta != 0 || mBlocksInOldFilesDelta != 0 || mBlocksInDeletedFilesDelta != 0)
+		{
+			info->ChangeBlocksUsed(mBlocksUsedDelta);
+			info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+			info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+			
+			// Save the store info back
+			info->Save();
+		}
+	
+		return;
+	}
+
+	// Log any difference in opinion between the values recorded in the store info, and
+	// the values just calculated for space usage.
+	// BLOCK
+	{
+		int64_t used = info->GetBlocksUsed();
+		int64_t usedOld = info->GetBlocksInOldFiles();
+		int64_t usedDeleted = info->GetBlocksInDeletedFiles();
+		int64_t usedDirectories = info->GetBlocksInDirectories();
+
+		// If the counts were wrong, taking into account RemoveASAP items deleted, log a message
+		if((used + mBlocksUsedDelta) != mBlocksUsed || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
+			|| (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles || usedDirectories != mBlocksInDirectories)
+		{
+			// Log this
+			::syslog(LOG_ERR, "On housekeeping, sizes in store do not match calculated sizes, correcting");
+			::syslog(LOG_ERR, "different (store,calc): acc 0x%08x, used (%lld,%lld), old (%lld,%lld), deleted (%lld,%lld), dirs (%lld,%lld)",
+				mAccountID,
+				(used + mBlocksUsedDelta), mBlocksUsed, (usedOld + mBlocksInOldFilesDelta), mBlocksInOldFiles,
+				(usedDeleted + mBlocksInDeletedFilesDelta), mBlocksInDeletedFiles, usedDirectories, mBlocksInDirectories);
+		}
+		
+		// If the current values don't match, store them
+		if(used != mBlocksUsed || usedOld != mBlocksInOldFiles
+			|| usedDeleted != mBlocksInDeletedFiles || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
+		{	
+			// Set corrected values in store info
+			info->CorrectAllUsedValues(mBlocksUsed, mBlocksInOldFiles, mBlocksInDeletedFiles, mBlocksInDirectories + mBlocksInDirectoriesDelta);
+			info->Save();
+		}
+	}
+	
+	// Reset the delta counts for files, as they will include RemoveASAP flagged files deleted
+	// during the initial scan.
+	int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;	// keep for reporting
+	mBlocksUsedDelta = 0;
+	mBlocksInOldFilesDelta = 0;
+	mBlocksInDeletedFilesDelta = 0;
+	
+	// Go and delete items from the accounts
+	bool deleteInterrupted = DeleteFiles();
+	
+	// If that wasn't interrupted, remove any empty directories which are also marked as deleted in their containing directory
+	if(!deleteInterrupted)
+	{
+		deleteInterrupted = DeleteEmptyDirectories();
+	}
+	
+	// Log deletion if anything was deleted
+	if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0)
+	{
+		::syslog(LOG_INFO, "Account 0x%08x, removed %lld blocks (%lld files, %lld dirs)%s", mAccountID, 0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta),
+			mFilesDeleted, mEmptyDirectoriesDeleted,
+			deleteInterrupted?" was interrupted":"");
+	}
+	
+	// Make sure the delta's won't cause problems if the counts are really wrong, and
+	// it wasn't fixed because the store was updated during the scan.
+	if(mBlocksUsedDelta 			< (0 - info->GetBlocksUsed())) 				mBlocksUsedDelta = 			(0 - info->GetBlocksUsed());
+	if(mBlocksInOldFilesDelta 		< (0 - info->GetBlocksInOldFiles())) 		mBlocksInOldFilesDelta = 	(0 - info->GetBlocksInOldFiles());
+	if(mBlocksInDeletedFilesDelta 	< (0 - info->GetBlocksInDeletedFiles())) 	mBlocksInDeletedFilesDelta =(0 - info->GetBlocksInDeletedFiles());
+	if(mBlocksInDirectoriesDelta 	< (0 - info->GetBlocksInDirectories()))		mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories());
+	
+	// Update the usage counts in the store
+	info->ChangeBlocksUsed(mBlocksUsedDelta);
+	info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+	info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+	info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta);
+	
+	// Save the store info back
+	info->Save();
+	
+	// Explicity release the lock (would happen automatically on going out of scope, included for code clarity)
+	writeLock.ReleaseLock();
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::MakeObjectFilename(int64_t, std::string &)
+//		Purpose: Generate and place the filename for a given object ID
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut)
+{
+	// Delegate to utility function
+	StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rFilenameOut, false /* don't bother ensuring the directory exists */);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::ScanDirectory(int64_t)
+//		Purpose: Private. Scan a directory for potenitally deleteable items, and
+//				 add them to the list. Returns true if the scan should continue.
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
+{
+	if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+	{
+		mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+		// Check for having to stop
+		if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is locked
+		{
+			// Need to abort now
+			return false;
+		}
+	}
+
+	// Get the filename
+	std::string objectFilename;
+	MakeObjectFilename(ObjectID, objectFilename);
+
+	// Open it.
+	std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, objectFilename));
+	
+	// Add the size of the directory on disc to the size being calculated
+	int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
+	mBlocksInDirectories += originalDirSizeInBlocks;
+	mBlocksUsed += originalDirSizeInBlocks;
+	
+	// Read the directory in
+	BackupStoreDirectory dir;
+	dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+	
+	// Is it empty?
+	if(dir.GetNumberOfEntries() == 0)
+	{
+		// Add it to the list of directories to potentially delete
+		mEmptyDirectories.push_back(dir.GetObjectID());
+	}
+	
+	// BLOCK
+	{
+		// Remove any files which are marked for removal as soon as they become old
+		// or deleted.
+		bool deletedSomething = false;
+		do
+		{
+			// Iterate through the directory
+			deletedSomething = false;
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = 0;
+			while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+			{
+				int16_t enFlags = en->GetFlags();
+				if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0
+					&& (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+				{
+					// Delete this immediately.
+					DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks);
+					
+					// flag as having done something
+					deletedSomething = true;
+					
+					// Must start the loop from the beginning again, as iterator is now
+					// probably invalid.
+					break;
+				}
+			}
+		} while(deletedSomething);
+	}
+	
+	// BLOCK
+	{
+		// Add files to the list of potential deletions
+
+		// map to count the distance from the mark
+		std::map<std::pair<BackupStoreFilename, int32_t>, int32_t> markVersionAges;
+			// map of pair (filename, mark number) -> version age
+
+		// NOTE: use a reverse iterator to allow the distance from mark stuff to work
+		BackupStoreDirectory::ReverseIterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+
+		while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+		{
+			// Update recalculated usage sizes
+			int16_t enFlags = en->GetFlags();
+			int64_t enSizeInBlocks = en->GetSizeInBlocks();
+			mBlocksUsed += enSizeInBlocks;
+			if(enFlags & BackupStoreDirectory::Entry::Flags_OldVersion) mBlocksInOldFiles += enSizeInBlocks;
+			if(enFlags & BackupStoreDirectory::Entry::Flags_Deleted) mBlocksInDeletedFiles += enSizeInBlocks;
+					
+			// Work out ages of this version from the last mark
+			int32_t enVersionAge = 0;
+			std::map<std::pair<BackupStoreFilename, int32_t>, int32_t>::iterator enVersionAgeI(markVersionAges.find(std::pair<BackupStoreFilename, int32_t>(en->GetName(), en->GetMarkNumber())));
+			if(enVersionAgeI != markVersionAges.end())
+			{
+				enVersionAge = enVersionAgeI->second + 1;
+				enVersionAgeI->second = enVersionAge;
+			}
+			else
+			{
+				markVersionAges[std::pair<BackupStoreFilename, int32_t>(en->GetName(), en->GetMarkNumber())] = enVersionAge;
+			}
+			// enVersionAge is now the age of this version.
+			
+			// Potentially add it to the list if it's deleted, if it's an old version or deleted
+			if((enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+			{
+				// Is deleted / old version.
+				DelEn d;
+				d.mObjectID = en->GetObjectID();
+				d.mInDirectory = ObjectID;
+				d.mSizeInBlocks = en->GetSizeInBlocks();
+				d.mMarkNumber = en->GetMarkNumber();
+				d.mVersionAgeWithinMark = enVersionAge;
+				
+				// Add it to the list
+				mPotentialDeletions.insert(d);
+				
+				// Update various counts
+				mPotentialDeletionsTotalSize += d.mSizeInBlocks;
+				if(d.mSizeInBlocks > mMaxSizeInPotentialDeletions) mMaxSizeInPotentialDeletions = d.mSizeInBlocks;
+				
+				// Too much in the list of potential deletions?
+				// (check against the deletion target + the max size in deletions, so that we never delete things
+				// and take the total size below the deletion size target)
+				if(mPotentialDeletionsTotalSize > (mDeletionSizeTarget + mMaxSizeInPotentialDeletions))
+				{
+					int64_t sizeToRemove = mPotentialDeletionsTotalSize - (mDeletionSizeTarget + mMaxSizeInPotentialDeletions);
+					bool recalcMaxSize = false;
+					
+					while(sizeToRemove > 0)
+					{
+						// Make iterator for the last element, while checking that there's something there in the first place.
+						std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.end());
+						if(i != mPotentialDeletions.begin())
+						{
+							// Nothing left in set
+							break;
+						}
+						// Make this into an iterator pointing to the last element in the set
+						--i;
+						
+						// Delete this one?
+						if(sizeToRemove > i->mSizeInBlocks)
+						{
+							sizeToRemove -= i->mSizeInBlocks;
+							if(i->mSizeInBlocks >= mMaxSizeInPotentialDeletions)
+							{
+								// Will need to recalculate the maximum size now, because we've just deleted that element
+								recalcMaxSize = true;
+							}
+							mPotentialDeletions.erase(i);
+						}
+						else
+						{
+							// Over the size to remove, so stop now
+							break;
+						}
+					}
+					
+					if(recalcMaxSize)
+					{
+						// Because an object which was the maximum size recorded was deleted from the set
+						// it's necessary to recalculate this maximum.
+						mMaxSizeInPotentialDeletions = 0;
+						std::set<DelEn, DelEnCompare>::const_iterator i(mPotentialDeletions.begin());
+						for(; i != mPotentialDeletions.end(); ++i)
+						{
+							if(i->mSizeInBlocks > mMaxSizeInPotentialDeletions)
+							{
+								mMaxSizeInPotentialDeletions = i->mSizeInBlocks;
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	{
+		// Recurse into subdirectories
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
+		{
+			// Next level
+			ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir);
+			
+			if(!ScanDirectory(en->GetObjectID()))
+			{
+				// Halting operation
+				return false;
+			}
+		}
+	}
+	
+	return true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &, const HousekeepStoreAccount::DelEnd &)
+//		Purpose: Comparison function for set
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &x, const HousekeepStoreAccount::DelEn &y)
+{
+	// STL spec says this:
+	// A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. 
+
+	// The sort order here is intended to preserve the entries of most value, that is, the newest objects
+	// which are on a mark boundary.
+	
+	// Reverse order age, so oldest goes first
+	if(x.mVersionAgeWithinMark > y.mVersionAgeWithinMark)
+	{
+		return true;
+	}
+	else if(x.mVersionAgeWithinMark < y.mVersionAgeWithinMark)
+	{
+		return false;
+	}
+
+	// but mark number in ascending order, so that the oldest marks are deleted first
+	if(x.mMarkNumber < y.mMarkNumber)
+	{
+		return true;
+	}
+	else if(x.mMarkNumber > y.mMarkNumber)
+	{
+		return false;
+	}
+
+	// Just compare object ID now to put the oldest objects first
+	return x.mObjectID < y.mObjectID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DeleteFiles()
+//		Purpose: Delete the files targetted for deletion, returning true if the operation was interrupted
+//		Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DeleteFiles()
+{
+	// Only delete files if the deletion target is greater than zero
+	// (otherwise we delete one file each time round, which gradually deletes the old versions)
+	if(mDeletionSizeTarget <= 0)
+	{
+		// Not interrupted
+		return false;
+	}
+
+	// Iterate through the set of potential deletions, until enough has been deleted.
+	// (there is likely to be more in the set than should be actually deleted).
+	for(std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i)
+	{
+		if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+		{
+			mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+			// Check for having to stop
+			if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
+			{
+				// Need to abort now
+				return true;
+			}
+		}
+
+		// Load up the directory it's in
+		// Get the filename
+		std::string dirFilename;
+		BackupStoreDirectory dir;
+		int64_t dirSizeInBlocksOrig = 0;
+		{
+			MakeObjectFilename(i->mInDirectory, dirFilename);
+			std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename));
+			dirSizeInBlocksOrig = dirStream->GetDiscUsageInBlocks();
+			dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+		}
+		
+		// Delete the file
+		DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig);
+		
+		// Stop if the deletion target has been matched or exceeded
+		// (checking here rather than at the beginning will tend to reduce the
+		// space to slightly less than the soft limit, which will allow the backup
+		// client to start uploading files again)
+		if((0 - mBlocksUsedDelta) >= mDeletionSizeTarget)
+		{
+			break;
+		}
+	}
+
+	return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DeleteFile(int64_t, int64_t, BackupStoreDirectory &, const std::string &, int64_t)
+//		Purpose: Delete a file. Takes the directory already loaded in and the filename,
+//				 for efficiency in both the usage senarios.
+//		Created: 15/7/04
+//
+// --------------------------------------------------------------------------
+void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks)
+{
+	// Find the entry inside the directory
+	bool wasDeleted = false;
+	bool wasOldVersion = false;
+	int64_t deletedFileSizeInBlocks = 0;
+	// A pointer to an object which requires commiting if the directory save goes OK
+	std::auto_ptr<RaidFileWrite> padjustedEntry;
+	// BLOCK
+	{
+		BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID);
+		if(pentry == 0)
+		{
+			::syslog(LOG_ERR, "acc 0x%08x, object %lld not found in dir %lld, logic error/corruption? Run bbstoreaccounts check <accid> fix", mAccountID, ObjectID, InDirectory);
+			return;
+		}
+		
+		// Record the flags it's got set
+		wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0);
+		wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0);
+		// Check this should be deleted
+		if(!wasDeleted && !wasOldVersion)
+		{
+			// Things changed size we were last around
+			return;
+		}
+		
+		// Record size
+		deletedFileSizeInBlocks = pentry->GetSizeInBlocks();
+		
+		// If the entry is involved in a chain of patches, it needs to be handled
+		// a bit more carefully.
+		if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0)
+		{
+			// This entry is a patch from a newer entry. Just need to update the info on that entry.
+			BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
+			if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID)
+			{
+				THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+			}
+			// Change the info in the newer entry so that this no longer points to this entry
+			pnewer->SetDependsOlder(0);
+		}
+		else if(pentry->GetDependsOlder() != 0)
+		{
+			BackupStoreDirectory::Entry *polder = rDirectory.FindEntryByID(pentry->GetDependsOlder());
+			if(pentry->GetDependsNewer() == 0)
+			{
+				// There exists an older version which depends on this one. Need to combine the two over that one.
+
+				// Adjust the other entry in the directory
+				if(polder == 0 || polder->GetDependsNewer() != ObjectID)
+				{
+					THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+				}
+				// Change the info in the older entry so that this no longer points to this entry
+				polder->SetDependsNewer(0);
+			}
+			else
+			{
+				// This entry is in the middle of a chain, and two patches need combining.
+				
+				// First, adjust the directory entries
+				BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
+				if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID
+					|| polder == 0 || polder->GetDependsNewer() != ObjectID)
+				{
+					THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+				}
+				// Remove the middle entry from the linked list by simply using the values from this entry
+				pnewer->SetDependsOlder(pentry->GetDependsOlder());
+				polder->SetDependsNewer(pentry->GetDependsNewer());
+			}
+			
+			// COMMON CODE to both cases
+
+			// Generate the filename of the older version
+			std::string objFilenameOlder;
+			MakeObjectFilename(pentry->GetDependsOlder(), objFilenameOlder);
+			// Open it twice (it's the diff)
+			std::auto_ptr<RaidFileRead> pdiff(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
+			std::auto_ptr<RaidFileRead> pdiff2(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
+			// Open this file
+			std::string objFilename;
+			MakeObjectFilename(ObjectID, objFilename);
+			std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename));
+			// And open a write file to overwrite the other directory entry
+			padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, objFilenameOlder));
+			padjustedEntry->Open(true /* allow overwriting */);
+
+			if(pentry->GetDependsNewer() == 0)
+			{
+				// There exists an older version which depends on this one. Need to combine the two over that one.
+				BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, *padjustedEntry);
+			}
+			else
+			{
+				// This entry is in the middle of a chain, and two patches need combining.
+				BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry);
+			}
+			// The file will be committed later when the directory is safely commited.
+			
+			// Work out the adjusted size
+			int64_t newSize = padjustedEntry->GetDiscUsageInBlocks();
+			int64_t sizeDelta = newSize - polder->GetSizeInBlocks();
+			mBlocksUsedDelta += sizeDelta;
+			if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)
+			{
+				mBlocksInDeletedFilesDelta += sizeDelta;
+			}
+			if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0)
+			{
+				mBlocksInOldFilesDelta += sizeDelta;
+			}
+			polder->SetSizeInBlocks(newSize);
+		}
+		
+		// pentry no longer valid
+	}
+	
+	// Delete it from the directory
+	rDirectory.DeleteEntry(ObjectID);
+	
+	// Save directory back to disc
+	// BLOCK
+	int64_t dirRevisedSize = 0;
+	{
+		RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename);
+		writeDir.Open(true /* allow overwriting */);
+		rDirectory.WriteToStream(writeDir);
+
+		// get the disc usage (must do this before commiting it)
+		dirRevisedSize = writeDir.GetDiscUsageInBlocks();
+
+		// Commit directory
+		writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+		// adjust usage counts for this directory
+		if(dirRevisedSize > 0)
+		{
+			int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks;
+			mBlocksUsedDelta += adjust;
+			mBlocksInDirectoriesDelta += adjust;
+		}
+	}
+
+	// Commit any new adjusted entry
+	if(padjustedEntry.get() != 0)
+	{
+		padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+		padjustedEntry.reset();	// delete it now
+	}
+
+	// Delete from disc
+	{
+		std::string objFilename;
+		MakeObjectFilename(ObjectID, objFilename);
+		RaidFileWrite del(mStoreDiscSet, objFilename);
+		del.Delete();
+	}
+
+	// Adjust counts for the file
+	++mFilesDeleted;
+	mBlocksUsedDelta -= deletedFileSizeInBlocks;
+	if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
+	if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
+	
+	// Delete the directory?
+	// Do this if... dir has zero entries, and is marked as deleted in it's containing directory
+	if(rDirectory.GetNumberOfEntries() == 0)
+	{
+		// Candidate for deletion
+		mEmptyDirectories.push_back(InDirectory);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DeleteEmptyDirectories()
+//		Purpose: Remove any empty directories which are also marked as deleted in their containing directory,
+//				 returning true if the opertaion was interrupted
+//		Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DeleteEmptyDirectories()
+{
+	while(mEmptyDirectories.size() > 0)
+	{
+		std::vector<int64_t> toExamine;
+
+		// Go through list
+		for(std::vector<int64_t>::const_iterator i(mEmptyDirectories.begin()); i != mEmptyDirectories.end(); ++i)
+		{
+			if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+			{
+				mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+				// Check for having to stop
+				if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
+				{
+					// Need to abort now
+					return true;
+				}
+			}
+
+			// Do not delete the root directory
+			if(*i == BACKUPSTORE_ROOT_DIRECTORY_ID)
+			{
+				continue;
+			}
+
+			// Load up the directory to potentially delete
+			std::string dirFilename;
+			BackupStoreDirectory dir;
+			int64_t dirSizeInBlocks = 0;
+			{
+				MakeObjectFilename(*i, dirFilename);
+				// Check it actually exists (just in case it gets added twice to the list)
+				if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename))
+				{
+					// doesn't exist, next!
+					continue;
+				}
+				// load
+				std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename));
+				dirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
+				dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+			}
+
+			// Make sure this directory is actually empty
+			if(dir.GetNumberOfEntries() != 0)
+			{
+				// Not actually empty, try next one
+				continue;
+			}
+
+			// Candiate for deletion... open containing directory
+			std::string containingDirFilename;
+			BackupStoreDirectory containingDir;
+			int64_t containingDirSizeInBlocksOrig = 0;
+			{
+				MakeObjectFilename(dir.GetContainerID(), containingDirFilename);
+				std::auto_ptr<RaidFileRead> containingDirStream(RaidFileRead::Open(mStoreDiscSet, containingDirFilename));
+				containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks();
+				containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite);
+			}
+
+			// Find the entry
+			BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID());
+			if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0))
+			{
+				// Should be deleted
+				containingDir.DeleteEntry(dir.GetObjectID());
+
+				// Is the containing dir now a candidate for deletion?
+				if(containingDir.GetNumberOfEntries() == 0)
+				{
+					toExamine.push_back(containingDir.GetObjectID());
+				}
+
+				// Write revised parent directory
+				RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename);
+				writeDir.Open(true /* allow overwriting */);
+				containingDir.WriteToStream(writeDir);
+
+				// get the disc usage (must do this before commiting it)
+				int64_t dirSize = writeDir.GetDiscUsageInBlocks();
+
+				// Commit directory
+				writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+				// adjust usage counts for this directory
+				if(dirSize > 0)
+				{
+					int64_t adjust = dirSize - containingDirSizeInBlocksOrig;
+					mBlocksUsedDelta += adjust;
+					mBlocksInDirectoriesDelta += adjust;
+				}
+
+				// Delete the directory itself
+				{
+					RaidFileWrite del(mStoreDiscSet, dirFilename);
+					del.Delete();
+				}
+
+				// And adjust usage counts for the directory that's just been deleted
+				mBlocksUsedDelta -= dirSizeInBlocks;
+				mBlocksInDirectoriesDelta -= dirSizeInBlocks;
+
+				// Update count
+				++mEmptyDirectoriesDeleted;
+			}
+		}
+
+		// Remove contents of empty directories
+		mEmptyDirectories.clear();
+		// Swap in new, so it's examined next time round
+		mEmptyDirectories.swap(toExamine);
+	}
+	
+	// Not interrupted
+	return false;
+}
+
+
+
+
diff --git a/bin/bbstored/HousekeepStoreAccount.h b/bin/bbstored/HousekeepStoreAccount.h
new file mode 100755
index 00000000..6c8f251d
--- /dev/null
+++ b/bin/bbstored/HousekeepStoreAccount.h
@@ -0,0 +1,97 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    HousekeepStoreAccount.h
+//		Purpose: Action class to perform housekeeping on a store account
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef HOUSEKEEPSTOREACCOUNT__H
+#define HOUSEKEEPSTOREACCOUNT__H
+
+#include <string>
+#include <set>
+#include <vector>
+
+class BackupStoreDaemon;
+class BackupStoreDirectory;
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    HousekeepStoreAccount
+//		Purpose: Action class to perform housekeeping on a store account
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+class HousekeepStoreAccount
+{
+public:
+	HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon);
+	~HousekeepStoreAccount();
+	
+	void DoHousekeeping();
+	
+	
+private:
+	// utility functions
+	void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut);
+
+	bool ScanDirectory(int64_t ObjectID);
+	bool DeleteFiles();
+	bool DeleteEmptyDirectories();
+	void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks);
+
+private:
+	typedef struct
+	{
+		int64_t mObjectID;
+		int64_t mInDirectory;
+		int64_t mSizeInBlocks;
+		int32_t mMarkNumber;
+		int32_t mVersionAgeWithinMark;	// 0 == current, 1 latest old version, etc
+	} DelEn;
+	
+	struct DelEnCompare
+	{
+		bool operator()(const DelEn &x, const DelEn &y);
+	};
+	
+	int mAccountID;
+	std::string mStoreRoot;
+	int mStoreDiscSet;
+	BackupStoreDaemon &mrDaemon;
+	
+	int64_t mDeletionSizeTarget;
+	
+	std::set<DelEn, DelEnCompare> mPotentialDeletions;
+	int64_t mPotentialDeletionsTotalSize;
+	int64_t mMaxSizeInPotentialDeletions;
+	
+	// List of directories which are empty, and might be good for deleting
+	std::vector<int64_t> mEmptyDirectories;
+	
+	// The re-calculated blocks used stats
+	int64_t mBlocksUsed;
+	int64_t mBlocksInOldFiles;
+	int64_t mBlocksInDeletedFiles;
+	int64_t mBlocksInDirectories;
+
+	// Deltas from deletion
+	int64_t mBlocksUsedDelta;
+	int64_t mBlocksInOldFilesDelta;
+	int64_t mBlocksInDeletedFilesDelta;
+	int64_t mBlocksInDirectoriesDelta;
+	
+	// Deletion count
+	int64_t mFilesDeleted;
+	int64_t mEmptyDirectoriesDeleted;
+	
+	// Poll frequency
+	int mCountUntilNextInterprocessMsgCheck;
+};
+
+#endif // HOUSEKEEPSTOREACCOUNT__H
+
diff --git a/bin/bbstored/Makefile.extra b/bin/bbstored/Makefile.extra
new file mode 100755
index 00000000..187d53ef
--- /dev/null
+++ b/bin/bbstored/Makefile.extra
@@ -0,0 +1,9 @@
+
+MAKEPROTOCOL = ../../lib/server/makeprotocol.pl
+
+GEN_CMD_SRV = $(MAKEPROTOCOL) Server backupprotocol.txt
+
+# AUTOGEN SEEDING
+autogen_BackupProtocolServer.cpp autogen_BackupProtocolServer.h:	$(MAKEPROTOCOL) backupprotocol.txt
+	perl $(GEN_CMD_SRV)
+
diff --git a/bin/bbstored/backupprotocol.txt b/bin/bbstored/backupprotocol.txt
new file mode 100755
index 00000000..39cb1fb3
--- /dev/null
+++ b/bin/bbstored/backupprotocol.txt
@@ -0,0 +1,221 @@
+#
+# backup protocol definition
+#
+
+Name 				Backup
+IdentString			Box-Backup:v=B
+ServerContextClass	BackupContext	BackupContext.h
+
+ClientType		Filename	BackupStoreFilenameClear	BackupStoreFilenameClear.h
+ServerType		Filename	BackupStoreFilename			BackupStoreFilename.h
+
+ImplementLog	Server	syslog
+ImplementLog	Client	syslog
+ImplementLog	Client	file
+
+LogTypeToText	Client	Filename	\"%s\"	VAR.GetClearFilename().c_str()
+
+BEGIN_OBJECTS
+
+# -------------------------------------------------------------------------------------
+#  Session commands
+# -------------------------------------------------------------------------------------
+
+Error		0	IsError(Type,SubType)	Reply
+	int32		Type
+	int32		SubType
+	CONSTANT	ErrorType						1000
+	CONSTANT	Err_WrongVersion				1
+	CONSTANT	Err_NotInRightProtocolPhase		2
+	CONSTANT	Err_BadLogin					3
+	CONSTANT	Err_CannotLockStoreForWriting	4
+	CONSTANT	Err_SessionReadOnly				5
+	CONSTANT	Err_FileDoesNotVerify			6
+	CONSTANT	Err_DoesNotExist				7
+	CONSTANT	Err_DirectoryAlreadyExists		8
+	CONSTANT	Err_CannotDeleteRoot			9
+	CONSTANT	Err_TargetNameExists			10
+	CONSTANT	Err_StorageLimitExceeded		11
+	CONSTANT	Err_DiffFromFileDoesNotExist	12
+	CONSTANT	Err_DoesNotExistInDirectory		13
+	CONSTANT	Err_PatchConsistencyError		14
+
+Version		1	Command(Version)	Reply
+	int32	Version
+
+
+Login		2	Command(LoginConfirmed)
+	int32		ClientID
+	int32		Flags
+	CONSTANT	Flags_ReadOnly	1
+
+
+LoginConfirmed	3	Reply
+	int64		ClientStoreMarker
+	int64		BlocksUsed
+	int64		BlocksSoftLimit
+	int64		BlocksHardLimit
+
+
+Finished	4	Command(Finished)	Reply	EndsConversation
+
+
+# generic success object
+Success		5	Reply
+	int64		ObjectID
+
+
+SetClientStoreMarker	6	Command(Success)
+	int64		ClientStoreMarker
+
+
+# -------------------------------------------------------------------------------------
+#  Generic object commands
+# -------------------------------------------------------------------------------------
+
+GetObject	10	Command(Success)
+	int64		ObjectID
+	CONSTANT	NoObject	0
+	# reply has stream following, if ObjectID != NoObject
+
+
+MoveObject	11	Command(Success)
+	int64		ObjectID
+	int64		MoveFromDirectory
+	int64		MoveToDirectory
+	int32		Flags
+	Filename	NewFilename
+	
+	CONSTANT Flags_MoveAllWithSameName			1
+	CONSTANT Flags_AllowMoveOverDeletedObject	2
+
+# consider this an object command as, although it deals with directory entries,
+# it's not specific to either a file or a directory
+
+
+GetObjectName	12	Command(ObjectName)
+	int64		ObjectID
+	int64		ContainingDirectoryID
+	CONSTANT	ObjectID_DirectoryOnly	0
+
+	# set ObjectID to ObjectID_DirectoryOnly to only get info on the directory
+
+
+ObjectName		13	Reply
+	int32		NumNameElements
+	int64		ModificationTime
+	int64		AttributesHash
+	int16		Flags
+	# NumNameElements is zero if the object doesn't exist
+	CONSTANT	NumNameElements_ObjectDoesntExist	0
+	# a stream of Filename objects follows, if and only if NumNameElements > 0
+
+
+# -------------------------------------------------------------------------------------
+#  Directory commands
+# -------------------------------------------------------------------------------------
+
+CreateDirectory	20	Command(Success)	StreamWithCommand
+	int64		ContainingDirectoryID
+	int64		AttributesModTime
+	Filename	DirectoryName
+	# stream following containing attributes
+
+
+ListDirectory	21	Command(Success)
+	int64		ObjectID
+	int16		FlagsMustBeSet
+	int16		FlagsNotToBeSet
+	bool		SendAttributes
+	# make sure these flags are synced with those in BackupStoreDirectory
+	CONSTANT	Flags_INCLUDE_EVERYTHING 	-1
+	CONSTANT	Flags_EXCLUDE_NOTHING 		0
+	CONSTANT	Flags_EXCLUDE_EVERYTHING	15
+	CONSTANT	Flags_File			1
+	CONSTANT	Flags_Dir			2
+	CONSTANT	Flags_Deleted			4
+	CONSTANT	Flags_OldVersion		8
+	# make sure this is the same as in BackupStoreConstants.h
+	CONSTANT	RootDirectory			1
+
+	# reply has stream following Success object, containing a stored BackupStoreDirectory
+
+
+ChangeDirAttributes	22	Command(Success)	StreamWithCommand
+	int64		ObjectID
+	int64		AttributesModTime
+	# stream following containing attributes
+
+
+DeleteDirectory	23	Command(Success)
+	int64		ObjectID
+
+UndeleteDirectory	24	Command(Success)
+	int64		ObjectID
+	# may not have exactly the desired effect if files within in have been deleted before the directory was deleted.
+
+
+# -------------------------------------------------------------------------------------
+#  File commands
+# -------------------------------------------------------------------------------------
+
+StoreFile	30	Command(Success)	StreamWithCommand
+	int64		DirectoryObjectID
+	int64		ModificationTime
+	int64		AttributesHash
+	int64		DiffFromFileID		# 0 if the file is not a diff
+	Filename	Filename
+	# then send a stream containing the encoded file
+
+
+GetFile		31	Command(Success)
+	int64		InDirectory
+	int64		ObjectID
+	# error returned if not a file, or does not exist
+	# reply has stream following, containing an encoded file IN STREAM ORDER
+	# (use GetObject to get it in file order)
+
+
+SetReplacementFileAttributes	32	Command(Success)	StreamWithCommand
+	int64		InDirectory
+	int64		AttributesHash
+	Filename	Filename
+	# stream follows containing attributes
+
+
+DeleteFile	33	Command(Success)
+	int64		InDirectory
+	Filename	Filename
+	# will return 0 if the object couldn't be found in the specified directory
+
+
+GetBlockIndexByID	34	Command(Success)
+	int64		ObjectID
+
+	# stream of the block index follows the reply
+	# returns an error if the object didn't exist
+
+
+GetBlockIndexByName	35	Command(Success)
+	int64		InDirectory
+	Filename	Filename
+
+	# Success object contains the found ID -- or 0 if the entry wasn't found in the directory
+	# stream of the block index follows the reply if found ID != 0
+
+
+# -------------------------------------------------------------------------------------
+#  Information commands
+# -------------------------------------------------------------------------------------
+
+GetAccountUsage	40	Command(AccountUsage)
+	# no data members
+
+AccountUsage	41	Reply
+	int64	BlocksUsed
+	int64	BlocksInOldFiles
+	int64	BlocksInDeletedFiles
+	int64	BlocksInDirectories
+	int64	BlocksSoftLimit
+	int64	BlocksHardLimit
+	int32	BlockSize
diff --git a/bin/bbstored/bbstored-certs b/bin/bbstored/bbstored-certs
new file mode 100755
index 00000000..d1fa8dea
--- /dev/null
+++ b/bin/bbstored/bbstored-certs
@@ -0,0 +1,319 @@
+#!/usr/bin/perl
+use strict;
+
+# validity period for root certificates -- default is a very long time
+my $root_sign_period = '10000';
+
+# but less so for client certificates
+my $sign_period = '5000';
+
+# check and get command line parameters
+if($#ARGV < 1)
+{
+	print <<__E;
+
+bbstored certificates utility.
+
+Bad command line parameters.
+Usage:
+	bbstored-certs certs-dir command [arguments]
+
+certs-dir is the directory holding the root keys and certificates for the backup system
+command is the action to perform, taking parameters.
+
+Commands are
+
+	init
+		-- generate initial root certificates (certs-dir must not already exist)
+	sign certificate-name
+		-- sign a client certificate
+	sign-server certificate-name
+		-- sign a server certificate
+
+Signing requires confirmation that the certificate is correct and should be signed.
+
+__E
+	exit(1);
+}
+
+# check for OPENSSL_CONF environment var being set
+if(exists $ENV{'OPENSSL_CONF'})
+{
+	print <<__E;
+
+---------------------------------------
+
+WARNING:
+    You have the OPENSSL_CONF environment variable set.
+    Use of non-standard openssl configs may cause problems.
+
+---------------------------------------
+
+__E
+}
+
+# directory structure:
+#
+# roots/
+#	clientCA.pem -- root certificate for client (used on server)
+#	serverCA.pem -- root certificate for servers (used on clients)
+# keys/
+#   clientRootKey.pem -- root key for clients
+#   serverRootKey.pem -- root key for servers
+# servers/
+#   hostname.pem -- certificate for server 'hostname'
+# clients/
+#   account.pem -- certficiate for account 'account' (ID in hex)
+#
+
+
+# check parameters
+my ($cert_dir,$command,@args) = @ARGV;
+
+# check directory exists
+if($command ne 'init')
+{
+	if(!-d $cert_dir)
+	{
+		die "$cert_dir does not exist";
+	}
+}
+
+# run command
+if($command eq 'init') {&cmd_init;}
+elsif($command eq 'sign') {&cmd_sign;}
+elsif($command eq 'sign-server') {&cmd_sign_server;}
+else
+{
+	die "Unknown command $command"
+}
+
+sub cmd_init
+{
+	# create directories
+	unless(mkdir($cert_dir,0700)
+		&& mkdir($cert_dir.'/roots',0700)
+		&& mkdir($cert_dir.'/keys',0700)
+		&& mkdir($cert_dir.'/servers',0700)
+		&& mkdir($cert_dir.'/clients',0700))
+	{
+		die "Failed to create directory structure"
+	}
+
+	# create root keys and certrs
+	cmd_init_create_root('client');
+	cmd_init_create_root('server');
+}
+
+sub cmd_init_create_root
+{
+	my $entity = $_[0];
+
+	my $cert = "$cert_dir/roots/".$entity.'CA.pem';
+	my $serial = "$cert_dir/roots/".$entity.'CA.srl';
+	my $key = "$cert_dir/keys/".$entity.'RootKey.pem';
+	my $csr = "$cert_dir/keys/".$entity.'RootCSR.pem';
+
+	# generate key
+	if(system("openssl genrsa -out $key 2048") != 0)
+	{
+		die "Couldn't generate private key."
+	}
+	
+	# make CSR
+	die "Couldn't run openssl for CSR generation" unless
+		open(CSR,"|openssl req -new -key $key -sha1 -out $csr");
+	print CSR <<__E;
+.
+.
+.
+.
+.
+Backup system $entity root
+.
+.
+.
+
+__E
+	close CSR;
+	print "\n\n";
+	die "Certificate request wasn't created.\n" unless -f $csr;
+	
+	# sign it to make a self-signed root CA key
+	if(system("openssl x509 -req -in $csr -sha1 -extensions v3_ca -signkey $key -out $cert -days $root_sign_period") != 0)
+	{
+		die "Couldn't generate root certificate."
+	}
+	
+	# write the initial serial number
+	open SERIAL,">$serial" or die "Can't open $serial for writing";
+	print SERIAL "00\n";
+	close SERIAL;
+}
+
+sub cmd_sign
+{
+	my $csr = $args[0];
+	
+	if(!-f $csr)
+	{
+		die "$csr does not exist";
+	}
+	
+	# get the common name specified in this certificate
+	my $common_name = get_csr_common_name($csr);
+	
+	# look OK?
+	unless($common_name =~ m/\ABACKUP-([A-Fa-f0-9]+)\Z/)
+	{
+		die "The certificate presented does not appear to be a backup client certificate"
+	}
+	
+	my $acc = $1;
+	
+	# check against filename
+	if(!($csr =~ m/(\A|\/)([A-Fa-f0-9]+)-/) || $2 ne $acc)
+	{
+		die "Certificate request filename does not match name in certificate ($common_name)"
+	}
+		
+	print <<__E;
+
+This certificate is for backup account
+
+   $acc
+
+Ensure this matches the account number you are expecting. The filename is
+
+   $csr
+
+which should include this account number, and additionally, you should check
+that you received it from the right person.
+
+Signing the wrong certificate compromises the security of your backup system.
+
+Would you like to sign this certificate? (type 'yes' to confirm)
+__E
+
+	return unless get_confirmation();
+
+	# out certificate
+	my $out_cert = "$cert_dir/clients/$acc"."-cert.pem";
+
+	# sign it!
+	if(system("openssl x509 -req -in $csr -sha1 -extensions usr_crt -CA $cert_dir/roots/clientCA.pem -CAkey $cert_dir/keys/clientRootKey.pem -out $out_cert -days $sign_period") != 0)
+	{
+		die "Signing failed"
+	}
+	
+	# tell user what to do next
+	print <<__E;
+
+
+Certificate signed.
+
+Send the files
+
+   $out_cert
+   $cert_dir/roots/serverCA.pem
+
+to the client.
+
+__E
+}
+
+sub cmd_sign_server
+{
+	my $csr = $args[0];
+	
+	if(!-f $csr)
+	{
+		die "$csr does not exist";
+	}
+	
+	# get the common name specified in this certificate
+	my $common_name = get_csr_common_name($csr);
+	
+	# look OK?
+	if($common_name !~ m/\A[-a-zA-Z0-9.]+\Z/)
+	{
+		die "Invalid server name"
+	}
+	
+	print <<__E;
+
+This certificate is for backup server
+
+   $common_name
+
+Signing the wrong certificate compromises the security of your backup system.
+
+Would you like to sign this certificate? (type 'yes' to confirm)
+__E
+
+	return unless get_confirmation();
+
+	# out certificate
+	my $out_cert = "$cert_dir/servers/$common_name"."-cert.pem";
+
+	# sign it!
+	if(system("openssl x509 -req -in $csr -sha1 -extensions usr_crt -CA $cert_dir/roots/serverCA.pem -CAkey $cert_dir/keys/serverRootKey.pem -out $out_cert -days $sign_period") != 0)
+	{
+		die "Signing failed"
+	}
+	
+	# tell user what to do next
+	print <<__E;
+
+
+Certificate signed.
+
+Install the files
+
+   $out_cert
+   $cert_dir/roots/clientCA.pem
+
+on the server.
+
+__E
+}
+
+
+sub get_csr_common_name
+{
+	my $csr = $_[0];
+	
+	open CSRTEXT,"openssl req -text -in $csr |" or die "Can't open openssl for reading";
+	
+	my $subject;
+	while(<CSRTEXT>)
+	{
+		$subject = $1 if m/Subject:.+?CN=([-\.\w]+)/
+	}	
+	close CSRTEXT;
+
+	if($subject eq '')
+	{
+		die "No subject found in CSR $csr"
+	}
+	
+	return $subject
+}
+
+sub get_confirmation()
+{
+	my $line = <STDIN>;
+	chomp $line;
+	if(lc $line ne 'yes')
+	{
+		print "CANCELLED\n";
+		return 0;
+	}
+	
+	return 1;
+}
+
+
+
+
+
diff --git a/bin/bbstored/bbstored-config b/bin/bbstored/bbstored-config
new file mode 100755
index 00000000..0032f4c6
--- /dev/null
+++ b/bin/bbstored/bbstored-config
@@ -0,0 +1,242 @@
+#!/usr/bin/perl
+use strict;
+
+# should be running as root
+if($> != 0)
+{
+	printf "\nWARNING: this should be run as root\n\n"
+}
+
+# check and get command line parameters
+if($#ARGV < 2)
+{
+	print <<__E;
+
+Setup bbstored config utility.
+
+Bad command line parameters.
+Usage:
+	bbstored-config config-dir server-hostname username [raidfile-config]
+
+config-dir usually /etc/box
+server-hostname is the hostname used by clients to connect to this server
+username is the user to run the server under
+raidfile-config is optional. Use if you have a non-standard raidfile.conf file.
+
+__E
+	exit(1);
+}
+
+# check for OPENSSL_CONF environment var being set
+if(exists $ENV{'OPENSSL_CONF'})
+{
+	print <<__E;
+
+---------------------------------------
+
+WARNING:
+    You have the OPENSSL_CONF environment variable set.
+    Use of non-standard openssl configs may cause problems.
+
+---------------------------------------
+
+__E
+}
+
+# default locations
+my $default_config_location = '/etc/box/bbstored.conf';
+
+# command line parameters
+my ($config_dir,$server,$username,$raidfile_config) = @ARGV;
+
+$raidfile_config = $config_dir . '/raidfile.conf' unless $raidfile_config ne '';
+
+# check server exists, but don't bother checking that it's actually this machine.
+{
+	my @r = gethostbyname($server);
+	if($#r < 0)
+	{
+		die "Server '$server' not found. (check server name, test DNS lookup failed.)"
+	}
+}
+
+# check this exists
+if(!-f $raidfile_config)
+{
+	print "The RaidFile configuration file $raidfile_config doesn't exist.\nYou may need to create it with raidfile-config.\nWon't configure bbstored without it.\n";
+	exit(1);
+}
+
+# check that the user exists
+die "You shouldn't run bbstored as root" if $username eq 'root';
+my $user_uid = 0;
+(undef,undef,$user_uid) = getpwnam($username);
+if($user_uid == 0)
+{
+	die "User $username doesn't exist\n";
+}
+
+# check that directories are writeable
+open RAIDCONF,$raidfile_config or die "Can't open $raidfile_config";
+{
+	my %done = ();
+	while(<RAIDCONF>)
+	{
+		next unless m/Dir\d\s*=\s*(.+)/;
+		my $d = $1;
+		$d = $d.'/backup' if -e $d.'/backup';
+		print "Checking permissions on $d\n";
+		my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($d);
+		my $req_perms = ($uid == $user_uid)?0700:0007;
+		if(($mode & $req_perms) != $req_perms)
+		{
+			print "$username doesn't appear to have the necessary permissions on $d\n";
+			print "Either adjust permissions, or create a directory 'backup' inside the\n";
+			print "directory specified in raidfile.conf which is writable.\n";
+			exit(1);
+		}
+	}
+}
+close RAIDCONF;
+
+# ssl stuff
+my $private_key = "$config_dir/bbstored/$server-key.pem";
+my $certificate_request = "$config_dir/bbstored/$server-csr.pem";
+my $certificate = "$config_dir/bbstored/$server-cert.pem";
+my $ca_root_cert = "$config_dir/bbstored/clientCA.pem";
+
+# other files
+my $config_file = "$config_dir/bbstored.conf";
+my $accounts_file = "$config_dir/bbstored/accounts.txt";
+
+# summarise configuration
+
+print <<__E;
+
+Setup bbstored config utility.
+
+Configuration:
+   Writing configuration file: $config_file
+   Writing empty accounts file: $accounts_file
+   Server hostname: $server
+   RaidFile config: $raidfile_config
+
+__E
+
+# create directories
+if(!-d $config_dir)
+{
+	print "Creating $config_dir...\n";
+	mkdir $config_dir,0755 or die "Can't create $config_dir";
+}
+
+if(!-d "$config_dir/bbstored")
+{
+	print "Creating $config_dir/bbstored\n";
+	mkdir "$config_dir/bbstored",0755 or die "Can't create $config_dir/bbstored";
+}
+
+# create blank accounts file
+if(!-f $accounts_file)
+{
+	print "Creating blank accounts file\n";
+	open ACC,">$accounts_file";
+	close ACC;
+}
+
+# generate the private key for the server
+if(!-f $private_key)
+{
+	print "Generating private key...\n";
+	if(system("openssl genrsa -out $private_key 2048") != 0)
+	{
+		die "Couldn't generate private key."
+	}
+}
+
+# generate a certificate request
+if(!-f $certificate_request)
+{
+	die "Couldn't run openssl for CSR generation" unless
+		open(CSR,"|openssl req -new -key $private_key -sha1 -out $certificate_request");
+	print CSR <<__E;
+.
+.
+.
+.
+.
+$server
+.
+.
+.
+
+__E
+	close CSR;
+	print "\n\n";
+	die "Certificate request wasn't created.\n" unless -f $certificate_request
+}
+
+# write the configuration file
+print "Writing configuration file $config_file\n";
+open CONFIG,">$config_file" or die "Can't open config file for writing";
+print CONFIG <<__E;
+
+RaidFileConf = $raidfile_config
+AccountDatabase = $accounts_file
+
+# Uncomment this line to see exactly what commands are being received from clients.
+# ExtendedLogging = yes
+
+# scan all accounts for files which need deleting every 15 minutes.
+
+TimeBetweenHousekeeping = 900
+
+Server
+{
+	PidFile = /var/run/bbstored.pid
+	User = $username
+	ListenAddresses = inet:$server
+	CertificateFile = $certificate
+	PrivateKeyFile = $private_key
+	TrustedCAsFile = $ca_root_cert
+}
+
+
+__E
+
+close CONFIG;
+
+# explain to the user what they need to do next
+my $daemon_args = ($config_file eq $default_config_location)?'':" $config_file";
+
+print <<__E;
+
+===================================================================
+
+bbstored basic configuration complete.
+
+What you need to do now...
+
+1) Sign $certificate_request
+   using the bbstored-certs utility.
+
+2) Install the server certificate and root CA certificate as
+      $certificate
+      $ca_root_cert
+
+3) You may wish to read the configuration file
+      $config_file
+   and adjust as appropraite.
+
+4) Create accounts with bbstoreaccounts
+
+5) Start the backup store daemon with the command
+      /usr/local/bin/bbstored$daemon_args
+   in /etc/rc.local, or your local equivalent.
+
+===================================================================
+
+__E
+
+
+
diff --git a/bin/bbstored/bbstored.cpp b/bin/bbstored/bbstored.cpp
new file mode 100755
index 00000000..3eaf2639
--- /dev/null
+++ b/bin/bbstored/bbstored.cpp
@@ -0,0 +1,25 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    bbstored.cpp
+//		Purpose: main file for backup store daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreDaemon.h"
+#include "MainHelper.h"
+
+#include "MemLeakFindOn.h"
+
+int main(int argc, const char *argv[])
+{
+	MAINHELPER_START
+
+	BackupStoreDaemon daemon;
+	return daemon.Main(BOX_FILE_BBSTORED_DEFAULT_CONFIG, argc, argv);
+	
+	MAINHELPER_END
+}
+